From b1883ac2207bff7839f154bb2df6d49037407fe3 Mon Sep 17 00:00:00 2001 From: Pedro Rodrigues Date: Tue, 21 Jul 2026 12:47:30 +0100 Subject: [PATCH 01/15] feat: upload benchmark results to Braintrust alongside eval-results.json MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit One Braintrust experiment per repo experiment, one trace per eval: root eval row (scores from passed+checks, scenario metadata, token metrics) with llm/tool/task/score child spans rebuilt from the persisted transcript. Dual-write only — eval-results.json is untouched (export refactored onto a shared results scan, output byte-identical). Runners now also persist whole-run usage (Claude Code result line, Codex turn.completed, ai-sdk totalUsage) plus startedAt/durationMs; extra raw-result keys only, dropped by the web export. Co-Authored-By: Claude Fable 5 --- .env.example | 5 + .github/workflows/eval-refresh.yml | 17 + apps/framework/harness/run-eval.ts | 20 + apps/framework/harness/types.ts | 1 + apps/framework/lib/result-files.ts | 164 +++++ apps/framework/package.json | 6 +- apps/framework/scripts/export-results.ts | 159 +---- apps/framework/scripts/upload-braintrust.ts | 310 ++++++++++ package.json | 1 + .../src/agents/claude-code/runner.test.ts | 36 ++ .../core/src/agents/claude-code/runner.ts | 26 +- packages/core/src/agents/codex/runner.test.ts | 45 ++ packages/core/src/agents/codex/runner.ts | 28 +- packages/core/src/agents/engine.ts | 2 + packages/core/src/agents/types.ts | 7 +- packages/core/src/braintrust-spans.test.ts | 230 +++++++ packages/core/src/braintrust-spans.ts | 300 ++++++++++ packages/core/src/index.ts | 37 ++ pnpm-lock.yaml | 561 +++++++++++++++++- 19 files changed, 1811 insertions(+), 144 deletions(-) create mode 100644 apps/framework/lib/result-files.ts create mode 100644 apps/framework/scripts/upload-braintrust.ts create mode 100644 packages/core/src/agents/codex/runner.test.ts create mode 100644 packages/core/src/braintrust-spans.test.ts create mode 100644 packages/core/src/braintrust-spans.ts diff --git a/.env.example b/.env.example index 53fb4854..1dd322c2 100644 --- a/.env.example +++ b/.env.example @@ -8,3 +8,8 @@ OPENAI_API_KEY= # Vercel AI Gateway. The opencode harness routes through this. AI_GATEWAY_API_KEY= + +# Braintrust benchmark-results upload (optional — upload-braintrust skips +# itself with a warning when unset). +BRAINTRUST_API_KEY= +BRAINTRUST_PROJECT_ID= diff --git a/.github/workflows/eval-refresh.yml b/.github/workflows/eval-refresh.yml index 98b7e215..4e10ba48 100644 --- a/.github/workflows/eval-refresh.yml +++ b/.github/workflows/eval-refresh.yml @@ -400,6 +400,23 @@ jobs: pnpm --filter @supabase-evals/framework export-results -- "${export_args[@]}" fi + - name: Upload benchmark results to Braintrust + # Additive dual-write next to eval-results.json; never blocks the + # export/commit path (continue-on-error + the script itself exits 0 + # when credentials are missing). + continue-on-error: true + env: + BRAINTRUST_API_KEY: ${{ secrets.BRAINTRUST_API_KEY }} + BRAINTRUST_PROJECT_ID: ${{ secrets.BRAINTRUST_PROJECT_ID }} + shell: bash + run: | + set -euo pipefail + + pairs='${{ needs.prepare.outputs.pairs }}' + if jq -e 'any(.[]; .eval_suite == "benchmark")' <<< "$pairs" > /dev/null; then + pnpm --filter @supabase-evals/framework upload-braintrust -- --suite benchmark + fi + - name: Upload exported results if: github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 diff --git a/apps/framework/harness/run-eval.ts b/apps/framework/harness/run-eval.ts index 0f758e53..4b05e0ee 100644 --- a/apps/framework/harness/run-eval.ts +++ b/apps/framework/harness/run-eval.ts @@ -34,6 +34,7 @@ import { getExperimentDisplayMetadata, } from '@supabase-evals/core'; import type { + AgentUsage, ExperimentConfig, EvalInterface, EvalManifest, @@ -360,6 +361,8 @@ async function runOne( transcript: TranscriptPart[]; agentReport: string; stoppedReason: string; + steps: number; + usage?: AgentUsage; } > { const prompt = parseEvalMarkdown( @@ -388,6 +391,8 @@ async function runOne( let lastTranscript: TranscriptPart[] = []; let lastAgentReport = ''; let lastStoppedReason = 'not_started'; + let lastSteps = 0; + let lastUsage: AgentUsage | undefined; for (let attempt = 1; attempt <= RUNS; attempt += 1) { if (ev.mode === 'local-stack') { @@ -457,6 +462,8 @@ async function runOne( lastTranscript = run.transcript; lastAgentReport = run.agentReport; lastStoppedReason = run.stoppedReason; + lastSteps = run.steps; + lastUsage = run.usage; // Export the agent's workspace to the host so scorers can run host // tooling (vite/vitest from the repo root) against the produced files @@ -495,6 +502,8 @@ async function runOne( transcript: run.transcript, agentReport: run.agentReport, stoppedReason: run.stoppedReason, + steps: run.steps, + usage: run.usage, }; } logRetryAttempt(expName, ev, attempt, last); @@ -544,6 +553,8 @@ async function runOne( lastTranscript = run.transcript; lastAgentReport = run.agentReport; lastStoppedReason = run.stoppedReason; + lastSteps = run.steps; + lastUsage = run.usage; last = await (scorer as ToolScorer)({ ...session.scoringContext, toolCalls: run.toolCalls, @@ -561,6 +572,8 @@ async function runOne( transcript: run.transcript, agentReport: run.agentReport, stoppedReason: run.stoppedReason, + steps: run.steps, + usage: run.usage, }; } logRetryAttempt(expName, ev, attempt, last); @@ -575,6 +588,8 @@ async function runOne( transcript: lastTranscript, agentReport: lastAgentReport, stoppedReason: lastStoppedReason, + steps: lastSteps, + usage: lastUsage, }; } @@ -778,6 +793,11 @@ async function main() { eval: ev.id, ...ev.metadata, ...res, + // Wall-clock timing across all attempts. Extra keys on the raw + // file only: the export schema is loose and drops them, so + // eval-results.json is unaffected. + startedAt: new Date(start).toISOString(), + durationMs: Date.now() - start, }, null, 2 diff --git a/apps/framework/harness/types.ts b/apps/framework/harness/types.ts index d4cfc677..0930acf0 100644 --- a/apps/framework/harness/types.ts +++ b/apps/framework/harness/types.ts @@ -6,6 +6,7 @@ import type { } from '@supabase-evals/core/eval-metadata'; export type { + AgentUsage, ScoreResult, CheckResult, ToolCallRecord, diff --git a/apps/framework/lib/result-files.ts b/apps/framework/lib/result-files.ts new file mode 100644 index 00000000..b23f0798 --- /dev/null +++ b/apps/framework/lib/result-files.ts @@ -0,0 +1,164 @@ +/** + * Shared access to the raw result files under `results/` — the scan, the + * per-experiment metadata from `experiments/*.ts`, and the PROMPT.md lookup — + * used by both `scripts/export-results.ts` (eval-results.json) and + * `scripts/upload-braintrust.ts` (Braintrust experiments), so the two + * pipelines can never disagree about which files exist. + */ + +import { readdir, readFile, stat } from "node:fs/promises"; +import { existsSync } from "node:fs"; +import { dirname, join, relative, resolve, sep } from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; +import { parseEvalMarkdown } from "@supabase-evals/core/eval-markdown"; +import { + getExperimentDisplayMetadata, + type ExperimentConfig, + type ExperimentDisplayMetadata, +} from "@supabase-evals/core"; +import type { + EvalMetadata, + ExperimentSuite, +} from "@supabase-evals/core/eval-metadata"; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +export const ROOT = resolve(__dirname, "..", "..", ".."); +export const RESULTS_DIR = join(ROOT, "results"); +export const EVALS_DIR = join(ROOT, "evals"); +export const EXPERIMENTS_DIR = join(ROOT, "experiments"); + +export type ExperimentExportMetadata = { + display: ExperimentDisplayMetadata; + experimentSuite?: ExperimentSuite; +}; + +export async function loadExperimentMetadata(): Promise< + Map +> { + const map = new Map(); + for (const f of (await readdir(EXPERIMENTS_DIR)).filter((f) => + f.endsWith(".ts"), + )) { + const mod = await import(pathToFileURL(join(EXPERIMENTS_DIR, f)).href); + const config = mod.default as ExperimentConfig; + map.set(f.replace(/\.ts$/, ""), { + display: getExperimentDisplayMetadata(config), + experimentSuite: config.suite?.[0], + }); + } + return map; +} + +export type PromptData = EvalMetadata & { + prompt: string; + promptSourcePath: string; +}; + +export async function readPrompt( + evalId: string, +): Promise { + const promptPath = resolve(EVALS_DIR, evalId, "PROMPT.md"); + const normalizedEvalsDir = resolve(EVALS_DIR); + + if (!promptPath.startsWith(`${normalizedEvalsDir}${sep}`)) { + return undefined; + } + + if (!existsSync(promptPath)) { + return undefined; + } + + const parsed = parseEvalMarkdown( + await readFile(promptPath, "utf8"), + promptPath, + ); + + return { + ...parsed.metadata, + prompt: parsed.body, + promptSourcePath: relative(ROOT, promptPath).split(sep).join("/"), + }; +} + +export interface ResultFileRef { + experiment: string; + evalId: string; + /** Absolute path to the JSON file. */ + filePath: string; + /** Path relative to `results/`, POSIX separators. */ + sourcePath: string; +} + +/** + * Every raw result file under `results/`, honoring the layout the runner + * writes: `results//.json`, or + * `results///summary.json` for directory-shaped results. + */ +export async function collectResultFiles(filter: { + includeExperiment: (experiment: string) => boolean; + includeEval: (evalId: string) => boolean; +}): Promise { + if (!existsSync(RESULTS_DIR)) { + return []; + } + + const refs: ResultFileRef[] = []; + for (const experiment of await readdir(RESULTS_DIR)) { + if (experiment.startsWith(".") || experiment.startsWith("_")) { + continue; + } + + if (!filter.includeExperiment(experiment)) { + continue; + } + + const experimentDir = join(RESULTS_DIR, experiment); + if (!(await stat(experimentDir)).isDirectory()) { + continue; + } + + for (const entry of await readdir(experimentDir)) { + const entryPath = join(experimentDir, entry); + const entryStat = await stat(entryPath); + const relativeEntryPath = relative(RESULTS_DIR, entryPath) + .split(sep) + .join("/"); + + if (entryStat.isFile() && entry.endsWith(".json")) { + const evalId = entry.replace(/\.json$/, ""); + if (!filter.includeEval(evalId)) { + continue; + } + refs.push({ + experiment, + evalId, + filePath: entryPath, + sourcePath: relativeEntryPath, + }); + continue; + } + + if (!entryStat.isDirectory()) { + continue; + } + + if (!filter.includeEval(entry)) { + continue; + } + + const summaryPath = join(entryPath, "summary.json"); + if (!existsSync(summaryPath)) { + continue; + } + + refs.push({ + experiment, + evalId: entry, + filePath: summaryPath, + sourcePath: `${relativeEntryPath}/summary.json`, + }); + } + } + + return refs; +} diff --git a/apps/framework/package.json b/apps/framework/package.json index c726bc37..20d27369 100644 --- a/apps/framework/package.json +++ b/apps/framework/package.json @@ -11,6 +11,7 @@ "typecheck": "tsc --noEmit", "test:framework": "node --env-file-if-exists=../../.env --import tsx/esm scripts/smoke-framework.ts", "export-results": "node --import tsx/esm scripts/export-results.ts", + "upload-braintrust": "node --env-file-if-exists=../../.env --import tsx/esm scripts/upload-braintrust.ts", "demo:mcp": "node --env-file=../../.env --import tsx/esm scripts/mcp-demo.ts", "demo:executor": "node --env-file=../../.env --import tsx/esm scripts/executor-demo.ts" }, @@ -18,17 +19,18 @@ "@ai-sdk/anthropic": "catalog:", "@ai-sdk/mcp": "catalog:", "@ai-sdk/openai": "catalog:", - "@supabase-evals/platform-lite": "workspace:*", "@electric-sql/pglite": "catalog:", "@supabase-evals/core": "workspace:*", + "@supabase-evals/platform-lite": "workspace:*", "@supabase-evals/sandbox": "workspace:*", + "@supabase/lite": "catalog:", "@supabase/supabase-js": "catalog:", "@testing-library/jest-dom": "^6.9.1", "@testing-library/react": "^16.3.2", "@vitejs/plugin-react": "catalog:", "ai": "catalog:", + "braintrust": "^3.24.0", "happy-dom": "^20.9.0", - "@supabase/lite": "catalog:", "react": "^19.2.5", "react-dom": "^19.2.5", "vite": "catalog:", diff --git a/apps/framework/scripts/export-results.ts b/apps/framework/scripts/export-results.ts index 3b32923f..c0a6a4ba 100644 --- a/apps/framework/scripts/export-results.ts +++ b/apps/framework/scripts/export-results.ts @@ -1,16 +1,8 @@ #!/usr/bin/env tsx -import { mkdir, readdir, readFile, stat, writeFile } from 'node:fs/promises'; +import { mkdir, readFile, writeFile } from 'node:fs/promises'; import { existsSync } from 'node:fs'; -import { dirname, join, relative, resolve, sep } from 'node:path'; -import { pathToFileURL } from 'node:url'; -import { fileURLToPath } from 'node:url'; -import { parseEvalMarkdown } from '@supabase-evals/core/eval-markdown'; +import { dirname, join, relative, resolve } from 'node:path'; import { rawEvalResultSchema } from '@supabase-evals/core/eval-metadata'; -import { - getExperimentDisplayMetadata, - type ExperimentConfig, - type ExperimentDisplayMetadata, -} from '@supabase-evals/core'; import type { EvalResult, EvalSuite, @@ -22,12 +14,14 @@ import { readRepeatedFlag, readSuiteFilters, } from '../lib/cli-args.js'; +import { + ROOT, + collectResultFiles, + loadExperimentMetadata, + readPrompt, + type ExperimentExportMetadata, +} from '../lib/result-files.js'; -const __dirname = dirname(fileURLToPath(import.meta.url)); -const ROOT = resolve(__dirname, '..', '..', '..'); -const RESULTS_DIR = join(ROOT, 'results'); -const EVALS_DIR = join(ROOT, 'evals'); -const EXPERIMENTS_DIR = join(ROOT, 'experiments'); const OUTPUT_PATH = join( ROOT, 'apps', @@ -37,27 +31,6 @@ const OUTPUT_PATH = join( 'eval-results.json' ); -type ExperimentExportMetadata = { - display: ExperimentDisplayMetadata; - experimentSuite?: ExperimentSuite; -}; - -async function loadExperimentMetadata(): Promise< - Map -> { - const map = new Map(); - for (const f of (await readdir(EXPERIMENTS_DIR)).filter((f) => - f.endsWith('.ts') - )) { - const mod = await import(pathToFileURL(join(EXPERIMENTS_DIR, f)).href); - const config = mod.default as ExperimentConfig; - map.set(f.replace(/\.ts$/, ''), { - display: getExperimentDisplayMetadata(config), - experimentSuite: config.suite?.[0], - }); - } - return map; -} const rawArgs = process.argv.slice(2); const EXPERIMENT_FILTERS = readRepeatedFlag(rawArgs, 'experiment').map( normalizeExperimentName @@ -70,30 +43,6 @@ const MERGE = rawArgs.includes('--merge'); const OUTPUT_FLAG = readRepeatedFlag(rawArgs, 'output')[0]; const outputPath = OUTPUT_FLAG ? resolve(ROOT, OUTPUT_FLAG) : OUTPUT_PATH; -async function readPrompt(evalId: string) { - const promptPath = resolve(EVALS_DIR, evalId, 'PROMPT.md'); - const normalizedEvalsDir = resolve(EVALS_DIR); - - if (!promptPath.startsWith(`${normalizedEvalsDir}${sep}`)) { - return undefined; - } - - if (!existsSync(promptPath)) { - return undefined; - } - - const parsed = parseEvalMarkdown( - await readFile(promptPath, 'utf8'), - promptPath - ); - - return { - ...parsed.metadata, - prompt: parsed.body, - promptSourcePath: relative(ROOT, promptPath).split(sep).join('/'), - }; -} - async function readResultFile( filePath: string, sourcePath: string, @@ -174,81 +123,25 @@ function shouldIncludeExperimentSuite( } async function loadEvalResults(): Promise { - if (!existsSync(RESULTS_DIR)) { - return []; - } - const experimentMetadata = await loadExperimentMetadata(); - const results: EvalResult[] = []; - const experiments = await readdir(RESULTS_DIR); - - for (const experiment of experiments) { - if (experiment.startsWith('.') || experiment.startsWith('_')) { - continue; - } - - if (!shouldIncludeExperiment(experiment)) { - continue; - } - - const experimentDir = join(RESULTS_DIR, experiment); - if (!(await stat(experimentDir)).isDirectory()) { - continue; - } + const refs = await collectResultFiles({ + includeExperiment: shouldIncludeExperiment, + includeEval: shouldIncludeEval, + }); - for (const entry of await readdir(experimentDir)) { - const entryPath = join(experimentDir, entry); - const entryStat = await stat(entryPath); - const relativeEntryPath = relative(RESULTS_DIR, entryPath) - .split(sep) - .join('/'); - - if (entryStat.isFile() && entry.endsWith('.json')) { - const evalId = entry.replace(/\.json$/, ''); - if (!shouldIncludeEval(evalId)) { - continue; - } - - const result = await readResultFile( - entryPath, - relativeEntryPath, - experimentMetadata - ); - if ( - result && - shouldIncludeSuite(result.suite) && - shouldIncludeExperimentSuite(result.experimentSuite) - ) { - results.push(result); - } - continue; - } - - if (!entryStat.isDirectory()) { - continue; - } - - if (!shouldIncludeEval(entry)) { - continue; - } - - const summaryPath = join(entryPath, 'summary.json'); - if (!existsSync(summaryPath)) { - continue; - } - - const result = await readResultFile( - summaryPath, - `${relativeEntryPath}/summary.json`, - experimentMetadata - ); - if ( - result && - shouldIncludeSuite(result.suite) && - shouldIncludeExperimentSuite(result.experimentSuite) - ) { - results.push(result); - } + const results: EvalResult[] = []; + for (const ref of refs) { + const result = await readResultFile( + ref.filePath, + ref.sourcePath, + experimentMetadata + ); + if ( + result && + shouldIncludeSuite(result.suite) && + shouldIncludeExperimentSuite(result.experimentSuite) + ) { + results.push(result); } } diff --git a/apps/framework/scripts/upload-braintrust.ts b/apps/framework/scripts/upload-braintrust.ts new file mode 100644 index 00000000..66899848 --- /dev/null +++ b/apps/framework/scripts/upload-braintrust.ts @@ -0,0 +1,310 @@ +#!/usr/bin/env tsx +/** + * Uploads raw eval results to Braintrust as one experiment per repo + * experiment (model) and one trace per eval: root "eval" row (scores + + * metadata + metrics) with llm/tool/task/score child spans rebuilt from the + * persisted transcript (see `buildEvalTrace` in @supabase-evals/core). + * + * A second, additive destination: it only reads `results/` and never touches + * eval-results.json — the dual-write agreed in Slack while Braintrust is + * evaluated as the storage layer. Missing credentials are a warning, not an + * error, so this can run unconditionally in CI without gating the export. + * + * Env: BRAINTRUST_API_KEY + BRAINTRUST_PROJECT_ID (or BRAINTRUST_PROJECT for + * a project name, or --project). + * + * Flags mirror export-results (--experiment/--eval/--suite/--experiment-suite) + * plus: + * --dry build and print the traces, no network + * --project Braintrust project name (overrides env) + * --name-suffix extra suffix on the Braintrust experiment names + * --update append to existing same-named Braintrust experiments + */ + +import { readFile } from "node:fs/promises"; +import { execFileSync } from "node:child_process"; +import { + buildEvalTrace, + uploadableEvalResultSchema, + type BraintrustEvalTrace, +} from "@supabase-evals/core"; +import type { + EvalSuite, + ExperimentSuite, +} from "@supabase-evals/core/eval-metadata"; +import { + normalizeExperimentName, + readExperimentSuiteFilters, + readRepeatedFlag, + readSuiteFilters, +} from "../lib/cli-args.js"; +import { + ROOT, + collectResultFiles, + loadExperimentMetadata, + readPrompt, + type PromptData, +} from "../lib/result-files.js"; + +const rawArgs = process.argv.slice(2); +const EXPERIMENT_FILTERS = readRepeatedFlag(rawArgs, "experiment").map( + normalizeExperimentName, +); +const EVAL_FILTERS = readRepeatedFlag(rawArgs, "eval"); +const SUITE_FILTERS = readSuiteFilters(rawArgs); +const EXPERIMENT_SUITE_FILTERS = readExperimentSuiteFilters(rawArgs); +const DRY = rawArgs.includes("--dry"); +const UPDATE = rawArgs.includes("--update"); +const PROJECT_FLAG = readRepeatedFlag(rawArgs, "project")[0]; +const NAME_SUFFIX = readRepeatedFlag(rawArgs, "name-suffix")[0]; + +function shouldIncludeSuite(suite: EvalSuite | undefined): boolean { + if (SUITE_FILTERS.length === 0) return true; + return suite !== undefined && SUITE_FILTERS.includes(suite); +} + +function shouldIncludeExperimentSuite( + experimentSuite: ExperimentSuite | undefined, +): boolean { + if (EXPERIMENT_SUITE_FILTERS.length === 0) return true; + return ( + experimentSuite !== undefined && + EXPERIMENT_SUITE_FILTERS.includes(experimentSuite) + ); +} + +function gitShortSha(): string | undefined { + const fromEnv = process.env.GITHUB_SHA; + if (fromEnv) return fromEnv.slice(0, 7); + try { + return execFileSync("git", ["rev-parse", "--short", "HEAD"], { + cwd: ROOT, + encoding: "utf8", + }).trim(); + } catch { + return undefined; + } +} + +function ciRunUrl(): string | undefined { + const { GITHUB_SERVER_URL, GITHUB_REPOSITORY, GITHUB_RUN_ID } = process.env; + if (!GITHUB_SERVER_URL || !GITHUB_REPOSITORY || !GITHUB_RUN_ID) { + return undefined; + } + return `${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}`; +} + +interface PendingTrace { + trace: BraintrustEvalTrace; + /** Raw result file, attached to the root span for full-fidelity forensics. */ + filePath: string; +} + +async function collectTraces(): Promise> { + const refs = await collectResultFiles({ + includeExperiment: (experiment) => + EXPERIMENT_FILTERS.length === 0 || + EXPERIMENT_FILTERS.includes(normalizeExperimentName(experiment)), + includeEval: (evalId) => + EVAL_FILTERS.length === 0 || EVAL_FILTERS.includes(evalId), + }); + + const promptCache = new Map(); + const readPromptCached = async (evalId: string) => { + if (!promptCache.has(evalId)) { + promptCache.set(evalId, await readPrompt(evalId)); + } + return promptCache.get(evalId); + }; + + const experimentMetadata = await loadExperimentMetadata(); + const byExperiment = new Map(); + let skipped = 0; + + for (const ref of refs) { + const parsed = uploadableEvalResultSchema.safeParse( + JSON.parse(await readFile(ref.filePath, "utf8")), + ); + if (!parsed.success) { + skipped += 1; + console.warn(`SKIP ${ref.sourcePath}: does not match the result schema`); + continue; + } + const result = parsed.data; + const promptData = await readPromptCached(result.eval); + const experimentSuite = + result.experimentSuite ?? + result.profile ?? + experimentMetadata.get(result.experiment)?.experimentSuite; + const suite = promptData?.suite ?? result.suite; + if (!shouldIncludeSuite(suite) || !shouldIncludeExperimentSuite(experimentSuite)) { + continue; + } + + // Fill dimensions from PROMPT.md like the JSON export does, so both + // destinations describe a scenario identically. + const trace = buildEvalTrace({ + experiment: result.experiment, + result: { + ...result, + experimentSuite, + experimentDisplay: + result.experimentDisplay ?? + experimentMetadata.get(result.experiment)?.display, + stage: promptData?.stage ?? result.stage, + product: promptData?.product ?? result.product, + topic: promptData?.topic ?? result.topic, + suite, + interface: promptData?.interface ?? result.interface, + cliVersion: promptData?.cliVersion ?? result.cliVersion, + }, + prompt: promptData?.prompt, + extraMetadata: { + sourcePath: ref.sourcePath, + ...(promptData?.promptSourcePath + ? { promptSourcePath: promptData.promptSourcePath } + : {}), + }, + baseTimeMs: Date.now(), + }); + const group = byExperiment.get(result.experiment) ?? []; + group.push({ trace, filePath: ref.filePath }); + byExperiment.set(result.experiment, group); + } + + if (skipped > 0) { + console.warn(`Skipped ${skipped} unparseable result file(s).`); + } + return byExperiment; +} + +function printDry(byExperiment: Map): void { + for (const [experiment, pending] of byExperiment) { + console.log(`\n${experiment}: ${pending.length} eval trace(s)`); + for (const { trace } of pending) { + const spanSummary = trace.spans + .map((s) => `${s.type}:${s.name}`) + .join(", "); + console.log( + ` ${trace.evalId} — scores=${JSON.stringify(trace.scores)} spans=[${spanSummary}]`, + ); + } + } + const first = byExperiment.values().next().value?.[0]; + if (first) { + console.log("\nFirst trace, full shape:"); + console.log(JSON.stringify(first.trace, null, 2)); + } +} + +async function upload(byExperiment: Map) { + // Imported lazily so --dry works without braintrust needing credentials. + const { init, Attachment, flush } = await import("braintrust"); + + const projectName = PROJECT_FLAG ?? process.env.BRAINTRUST_PROJECT; + const projectId = process.env.BRAINTRUST_PROJECT_ID; + const sha = gitShortSha(); + const suffix = NAME_SUFFIX ? `-${NAME_SUFFIX}` : ""; + const experimentMetadata = await loadExperimentMetadata(); + + for (const [experiment, pending] of byExperiment) { + const name = `${experiment}${sha ? `@${sha}` : ""}${suffix}`; + const btExperiment = init({ + ...(projectName ? { project: projectName } : {}), + ...(projectId ? { projectId } : {}), + experiment: name, + update: UPDATE, + metadata: { + source: "supabase-evals", + experiment, + ...(experimentMetadata.get(experiment)?.experimentSuite + ? { experimentSuite: experimentMetadata.get(experiment)?.experimentSuite } + : {}), + ...(experimentMetadata.get(experiment)?.display ?? {}), + ...(sha ? { gitShortSha: sha } : {}), + ...(ciRunUrl() ? { ciRunUrl: ciRunUrl() } : {}), + }, + }); + + for (const { trace, filePath } of pending) { + const root = btExperiment.startSpan({ + name: trace.evalId, + type: "eval", + startTime: trace.startMs / 1000, + }); + root.log({ + input: trace.input, + ...(trace.output !== undefined ? { output: trace.output } : {}), + scores: trace.scores, + metadata: { + ...trace.metadata, + rawResultFile: new Attachment({ + data: filePath, + filename: `${trace.experiment}__${trace.evalId}.json`, + contentType: "application/json", + }), + }, + ...(Object.keys(trace.metrics).length > 0 + ? { metrics: trace.metrics } + : {}), + ...(trace.tags.length > 0 ? { tags: trace.tags } : {}), + }); + for (const span of trace.spans) { + const child = root.startSpan({ + name: span.name, + type: span.type, + startTime: span.startMs / 1000, + }); + child.log({ + ...(span.input !== undefined ? { input: span.input } : {}), + ...(span.output !== undefined ? { output: span.output } : {}), + ...(span.error !== undefined ? { error: span.error } : {}), + ...(span.metadata ? { metadata: span.metadata } : {}), + ...(span.scores ? { scores: span.scores } : {}), + }); + child.end({ endTime: span.endMs / 1000 }); + } + root.end({ endTime: trace.endMs / 1000 }); + } + + const summary = await btExperiment.summarize({ summarizeScores: false }); + console.log( + `Uploaded ${pending.length} eval trace(s) to Braintrust experiment "${name}"` + + (summary.experimentUrl ? `\n → ${summary.experimentUrl}` : ""), + ); + } + + await flush(); +} + +async function main() { + const byExperiment = await collectTraces(); + const total = [...byExperiment.values()].reduce((n, t) => n + t.length, 0); + if (total === 0) { + console.log("No result files matched — nothing to upload."); + return; + } + + if (DRY) { + printDry(byExperiment); + return; + } + + const hasProject = + PROJECT_FLAG || process.env.BRAINTRUST_PROJECT || process.env.BRAINTRUST_PROJECT_ID; + if (!process.env.BRAINTRUST_API_KEY || !hasProject) { + console.warn( + "Braintrust upload skipped: set BRAINTRUST_API_KEY and " + + "BRAINTRUST_PROJECT_ID (or BRAINTRUST_PROJECT / --project).", + ); + return; + } + + await upload(byExperiment); + console.log(`Done: ${total} eval trace(s) across ${byExperiment.size} experiment(s).`); +} + +main().catch((error: unknown) => { + console.error(error); + process.exit(1); +}); diff --git a/package.json b/package.json index ab7f505e..04bb8231 100644 --- a/package.json +++ b/package.json @@ -11,6 +11,7 @@ "eval:force": "pnpm --filter @supabase-evals/framework eval:force", "test:framework": "pnpm --filter @supabase-evals/framework test:framework", "export-results": "pnpm --filter @supabase-evals/framework export-results", + "upload-braintrust": "pnpm --filter @supabase-evals/framework upload-braintrust", "typecheck": "pnpm --filter @supabase-evals/framework typecheck && pnpm --filter @supabase-evals/web typecheck", "web": "pnpm --filter @supabase-evals/web dev", "web:build": "pnpm --filter @supabase-evals/web build", diff --git a/packages/core/src/agents/claude-code/runner.test.ts b/packages/core/src/agents/claude-code/runner.test.ts index 07450363..b2fcf3f6 100644 --- a/packages/core/src/agents/claude-code/runner.test.ts +++ b/packages/core/src/agents/claude-code/runner.test.ts @@ -55,3 +55,39 @@ describe('claudeCodeRunner.deriveStopReason', () => { expect(derive(undefined, ok)).toBe('stop'); }); }); + +describe("claudeCodeRunner.extractUsage", () => { + const extract = claudeCodeRunner.extractUsage!; + + it("reads the terminal result line's usage and cost", () => { + const raw = [ + JSON.stringify({ type: "system", subtype: "init" }), + JSON.stringify({ + type: "result", + subtype: "success", + total_cost_usd: 0.42, + usage: { + input_tokens: 10, + cache_creation_input_tokens: 200, + cache_read_input_tokens: 3000, + output_tokens: 40, + }, + }), + ].join("\n"); + expect(extract(raw)).toEqual({ + inputTokens: 10, + outputTokens: 40, + cachedInputTokens: 3000, + cacheCreationInputTokens: 200, + costUsd: 0.42, + }); + }); + + it("returns undefined without a result line or usage fields", () => { + expect(extract(undefined)).toBeUndefined(); + expect(extract("not json\n")).toBeUndefined(); + expect( + extract(JSON.stringify({ type: "result", subtype: "success" })), + ).toBeUndefined(); + }); +}); diff --git a/packages/core/src/agents/claude-code/runner.ts b/packages/core/src/agents/claude-code/runner.ts index 37129ca1..38df4682 100644 --- a/packages/core/src/agents/claude-code/runner.ts +++ b/packages/core/src/agents/claude-code/runner.ts @@ -5,8 +5,8 @@ */ import type { Model as AnthropicModel } from '@anthropic-ai/sdk/resources/messages'; -import type { McpServerConfig } from '../../index.js'; -import { parseJsonlRecords } from '../../json.js'; +import type { AgentUsage, McpServerConfig } from '../../index.js'; +import { isRecord, parseJsonlRecords } from '../../json.js'; import type { AgentRunner } from '../types.js'; import { npmGlobalBin, @@ -97,6 +97,28 @@ export const claudeCodeRunner: AgentRunner = { if (subtype) return subtype; // e.g. error_max_turns — surface verbatim return processStopReason(command); }, + + extractUsage(raw) { + // The terminal `result` line carries the run's aggregate accounting: + // `usage: { input_tokens, cache_creation_input_tokens, + // cache_read_input_tokens, output_tokens }` plus `total_cost_usd`. + const result = lastResultEvent(raw); + if (!result) return undefined; + const usage = isRecord(result.usage) ? result.usage : undefined; + const num = (value: unknown): number | undefined => + typeof value === 'number' && Number.isFinite(value) ? value : undefined; + + const extracted: AgentUsage = { + inputTokens: num(usage?.input_tokens), + outputTokens: num(usage?.output_tokens), + cachedInputTokens: num(usage?.cache_read_input_tokens), + cacheCreationInputTokens: num(usage?.cache_creation_input_tokens), + costUsd: num(result.total_cost_usd), + }; + return Object.values(extracted).some((v) => v !== undefined) + ? extracted + : undefined; + }, }; /** Claude Code's `--mcp-config` schema: `{ mcpServers: { name: {type, command, args, env} } }`. */ diff --git a/packages/core/src/agents/codex/runner.test.ts b/packages/core/src/agents/codex/runner.test.ts new file mode 100644 index 00000000..b81bdf1b --- /dev/null +++ b/packages/core/src/agents/codex/runner.test.ts @@ -0,0 +1,45 @@ +import { describe, expect, it } from "vitest"; +import { codexRunner } from "./runner.js"; + +describe("codexRunner.extractUsage", () => { + const extract = codexRunner.extractUsage!; + + it("reads usage from turn.completed", () => { + const raw = [ + JSON.stringify({ type: "thread.started", thread_id: "t1" }), + JSON.stringify({ + type: "turn.completed", + usage: { input_tokens: 100, cached_input_tokens: 30, output_tokens: 25 }, + }), + ].join("\n"); + expect(extract(raw)).toEqual({ + inputTokens: 100, + cachedInputTokens: 30, + outputTokens: 25, + }); + }); + + it("sums usage across multiple turns", () => { + const raw = [ + JSON.stringify({ + type: "turn.completed", + usage: { input_tokens: 100, cached_input_tokens: 0, output_tokens: 25 }, + }), + JSON.stringify({ + type: "turn.completed", + usage: { input_tokens: 50, cached_input_tokens: 10, output_tokens: 5 }, + }), + ].join("\n"); + expect(extract(raw)).toEqual({ + inputTokens: 150, + cachedInputTokens: 10, + outputTokens: 30, + }); + }); + + it("returns undefined without a turn.completed usage payload", () => { + expect(extract(undefined)).toBeUndefined(); + expect(extract("not json\n")).toBeUndefined(); + expect(extract(JSON.stringify({ type: "turn.completed" }))).toBeUndefined(); + }); +}); diff --git a/packages/core/src/agents/codex/runner.ts b/packages/core/src/agents/codex/runner.ts index f4e3837e..135ff2d2 100644 --- a/packages/core/src/agents/codex/runner.ts +++ b/packages/core/src/agents/codex/runner.ts @@ -8,8 +8,8 @@ */ import type { ChatModel } from 'openai/resources/shared'; -import type { McpServerConfig } from '../../index.js'; -import { parseJsonlRecords } from '../../json.js'; +import type { AgentUsage, McpServerConfig } from '../../index.js'; +import { isRecord, parseJsonlRecords } from '../../json.js'; import type { AgentRunner } from '../types.js'; import { npmGlobalBin, @@ -115,6 +115,30 @@ export const codexRunner: AgentRunner = { return processStopReason(command); } }, + + extractUsage(raw) { + // Each `turn.completed` carries `usage: { input_tokens, + // cached_input_tokens, output_tokens }`. `codex exec` runs a single turn, + // but sum across all of them to stay correct if that ever changes. + if (!raw) return undefined; + const { records } = parseJsonlRecords(raw); + let sawUsage = false; + let inputTokens = 0; + let cachedInputTokens = 0; + let outputTokens = 0; + for (const record of records) { + if (record.type !== 'turn.completed' || !isRecord(record.usage)) continue; + sawUsage = true; + const num = (value: unknown): number => + typeof value === 'number' && Number.isFinite(value) ? value : 0; + inputTokens += num(record.usage.input_tokens); + cachedInputTokens += num(record.usage.cached_input_tokens); + outputTokens += num(record.usage.output_tokens); + } + if (!sawUsage) return undefined; + const usage: AgentUsage = { inputTokens, cachedInputTokens, outputTokens }; + return usage; + }, }; /** The last turn-level outcome in a `codex exec --json` stream, if any. */ diff --git a/packages/core/src/agents/engine.ts b/packages/core/src/agents/engine.ts index de61c1fa..a47d4571 100644 --- a/packages/core/src/agents/engine.ts +++ b/packages/core/src/agents/engine.ts @@ -108,6 +108,7 @@ export function createCliAgent( const { events } = raw ? parser.parseTranscript(raw) : { events: [] }; const adapted = adaptTranscript(events); + const usage = runner.extractUsage?.(raw); // Surface run failures that would otherwise be invisible in results // (visible under --debug): the CLI's own error events, or a run that @@ -131,6 +132,7 @@ export function createCliAgent( steps: adapted.steps, stoppedReason: runner.deriveStopReason?.(raw, command) ?? processStopReason(command), + ...(usage ? { usage } : {}), }; }, }; diff --git a/packages/core/src/agents/types.ts b/packages/core/src/agents/types.ts index baf91962..1838cfe2 100644 --- a/packages/core/src/agents/types.ts +++ b/packages/core/src/agents/types.ts @@ -11,7 +11,7 @@ * concern that diverges per agent lives in exactly one place. */ -import type { CommandResult, McpServerConfig } from '../index.js'; +import type { AgentUsage, CommandResult, McpServerConfig } from '../index.js'; import type { AgentHarnessId, ModelProvider, @@ -107,6 +107,11 @@ export interface AgentRunner { * result. Falls back to a process-exit-based reason when omitted. */ deriveStopReason?(raw: string | undefined, command: CommandResult): string; + /** + * Optional: whole-run token/cost usage from the raw transcript's own + * accounting events. Undefined when the transcript doesn't carry any. + */ + extractUsage?(raw: string | undefined): AgentUsage | undefined; } /** diff --git a/packages/core/src/braintrust-spans.test.ts b/packages/core/src/braintrust-spans.test.ts new file mode 100644 index 00000000..ef1eb602 --- /dev/null +++ b/packages/core/src/braintrust-spans.test.ts @@ -0,0 +1,230 @@ +import { describe, expect, it } from "vitest"; +import { + buildEvalTrace, + uploadableEvalResultSchema, + type UploadableEvalResult, +} from "./braintrust-spans.js"; + +const BASE = Date.parse("2026-07-21T10:00:00.000Z"); + +/** A modern raw result: transcript + toolCalls + usage + timing. */ +const modernResult = { + experiment: "claude-code-sonnet-5", + experimentSuite: "benchmark", + experimentDisplay: { + agent: "claude-code", + modelProvider: "anthropic", + modelId: "claude-sonnet-5", + }, + eval: "investigate-security-001-public-table", + stage: "investigate", + product: ["database"], + topic: ["security", "rls"], + suite: "benchmark", + passed: true, + checks: [ + { name: "named the vulnerable table", passed: true }, + { name: "proposed a concrete fix", passed: false, notes: "missed RLS" }, + ], + attempts: 1, + skills: { available: ["supabase"], loaded: ["supabase"] }, + transcript: [ + { type: "tool_call", name: "Skill", input: { skill: "supabase" }, output: "Launching skill: supabase" }, + { type: "message", role: "user", content: "skill body" }, + { type: "message", role: "assistant", content: "Looking at the tables." }, + { type: "tool_call", name: "Bash", input: { command: "psql" }, error: "exit 1" }, + { type: "message", role: "assistant", content: "Final report." }, + ], + toolCalls: [ + { endpoint: "Skill", body: { skill: "supabase" }, loadedSkill: "supabase", result: "Launching skill: supabase", ts: 0 }, + { endpoint: "Bash", body: { command: "psql" }, name: "shell", command: "psql", error: "exit 1", ts: 0 }, + ], + agentReport: "Final report.", + stoppedReason: "stop", + steps: 2, + usage: { inputTokens: 100, outputTokens: 50, cachedInputTokens: 20, costUsd: 0.12 }, + startedAt: "2026-07-21T09:00:00.000Z", + durationMs: 60_000, +} satisfies Record; + +function parse(result: Record): UploadableEvalResult { + return uploadableEvalResultSchema.parse(result); +} + +describe("uploadableEvalResultSchema", () => { + it("accepts a legacy result without transcript fields", () => { + const parsed = parse({ + experiment: "claude-code-sonnet-4.6", + eval: "some-eval", + passed: false, + checks: [{ name: "a check", passed: false }], + }); + expect(parsed.transcript).toBeUndefined(); + expect(parsed.usage).toBeUndefined(); + }); + + it("accepts the modern shape", () => { + const parsed = parse(modernResult); + expect(parsed.transcript).toHaveLength(5); + expect(parsed.usage?.costUsd).toBe(0.12); + }); +}); + +describe("buildEvalTrace", () => { + const trace = buildEvalTrace({ + experiment: "claude-code-sonnet-5", + result: parse(modernResult), + prompt: "Find the security issue.", + extraMetadata: { sourcePath: "claude-code-sonnet-5/x.json" }, + baseTimeMs: BASE, + }); + + it("uses the prompt as row input and the report as output", () => { + expect(trace.input).toBe("Find the security issue."); + expect(trace.output).toBe("Final report."); + }); + + it("scores passed plus one score per check", () => { + expect(trace.scores).toEqual({ + passed: 1, + "named the vulnerable table": 1, + "proposed a concrete fix": 0, + }); + }); + + it("emits spans in transcript order, then score spans", () => { + expect(trace.spans.map((s) => `${s.type}:${s.name}`)).toEqual([ + "tool:Skill", + "task:user", + "llm:assistant", + "tool:Bash", + "llm:assistant", + "score:named the vulnerable table", + "score:proposed a concrete fix", + ]); + }); + + it("pairs tool spans with the normalized toolCalls record", () => { + const bash = trace.spans.find((s) => s.name === "Bash"); + expect(bash?.metadata).toEqual({ tool: "shell", command: "psql" }); + expect(bash?.error).toBe("exit 1"); + expect(bash?.output).toBeUndefined(); + const skill = trace.spans.find((s) => s.name === "Skill"); + expect(skill?.metadata).toEqual({ loadedSkill: "supabase" }); + expect(skill?.output).toBe("Launching skill: supabase"); + }); + + it("uses the recorded run window and spreads spans across it", () => { + const start = Date.parse("2026-07-21T09:00:00.000Z"); + expect(trace.startMs).toBe(start); + expect(trace.endMs).toBe(start + 60_000); + const starts = trace.spans.map((s) => s.startMs); + expect(starts).toEqual([...starts].sort((a, b) => a - b)); + expect(starts[0]).toBe(start); + expect(trace.spans.at(-1)!.endMs).toBeCloseTo(start + 60_000, 5); + }); + + it("maps usage onto Braintrust metric names", () => { + expect(trace.metrics).toEqual({ + prompt_tokens: 100, + completion_tokens: 50, + prompt_cached_tokens: 20, + tokens: 150, + }); + expect(trace.metadata.costUsd).toBe(0.12); + }); + + it("carries the scenario dimensions in metadata and tags", () => { + expect(trace.metadata).toMatchObject({ + eval: "investigate-security-001-public-table", + experiment: "claude-code-sonnet-5", + experimentSuite: "benchmark", + agent: "claude-code", + modelId: "claude-sonnet-5", + stage: "investigate", + suite: "benchmark", + attempts: 1, + steps: 2, + stoppedReason: "stop", + stepTiming: "synthetic-order-only", + sourcePath: "claude-code-sonnet-5/x.json", + }); + expect(trace.tags).toEqual(["investigate", "database", "security", "rls"]); + }); + + it("attaches check notes to score spans", () => { + const failedCheck = trace.spans.find( + (s) => s.name === "proposed a concrete fix", + ); + expect(failedCheck?.scores).toEqual({ "proposed a concrete fix": 0 }); + expect(failedCheck?.output).toEqual({ passed: false, notes: "missed RLS" }); + }); + + it("falls back to synthetic timing and eval-id input for legacy results", () => { + const legacy = buildEvalTrace({ + experiment: "old", + result: parse({ + experiment: "old", + eval: "legacy-eval", + passed: false, + checks: [{ name: "a check", passed: false }], + }), + baseTimeMs: BASE, + }); + expect(legacy.input).toEqual({ eval: "legacy-eval" }); + expect(legacy.output).toBeUndefined(); + expect(legacy.startMs).toBe(BASE); + expect(legacy.endMs).toBe(BASE + 1000); + expect(legacy.spans).toHaveLength(1); + expect(legacy.spans[0]!.type).toBe("score"); + expect(legacy.metrics).toEqual({}); + }); + + it("de-duplicates and reserves score names", () => { + const clashing = buildEvalTrace({ + experiment: "x", + result: parse({ + experiment: "x", + eval: "y", + passed: true, + checks: [ + { name: "passed", passed: true }, + { name: "same", passed: true }, + { name: "same", passed: false }, + ], + }), + baseTimeMs: BASE, + }); + expect(Object.keys(clashing.scores)).toEqual([ + "passed", + "check: passed", + "same", + "same (2)", + ]); + expect(clashing.scores["same (2)"]).toBe(0); + }); + + it("truncates oversized tool outputs", () => { + const big = buildEvalTrace({ + experiment: "x", + result: parse({ + experiment: "x", + eval: "y", + passed: true, + transcript: [ + { + type: "tool_call", + name: "Read", + input: {}, + output: "x".repeat(150_000), + }, + ], + toolCalls: [{ endpoint: "Read", body: {}, ts: 0 }], + }), + baseTimeMs: BASE, + }); + const output = big.spans[0]!.output as string; + expect(output.length).toBeLessThan(30_000); + expect(output).toContain("[truncated 150000 chars]"); + }); +}); diff --git a/packages/core/src/braintrust-spans.ts b/packages/core/src/braintrust-spans.ts new file mode 100644 index 00000000..5f3f26ae --- /dev/null +++ b/packages/core/src/braintrust-spans.ts @@ -0,0 +1,300 @@ +/** + * Pure mapping from a raw eval-result file to a Braintrust-shaped trace: one + * root ("eval") row per result plus an ordered list of child spans — `llm` + * for assistant turns, `tool` for tool calls, `task` for user/system + * messages, `score` for scorer checks. + * + * Deliberately dependency-free (no braintrust import): this module only + * builds a serializable tree; `apps/framework/scripts/upload-braintrust.ts` + * walks it with the Braintrust SDK. That keeps the mapping unit-testable and + * identical across agents (Claude Code, Codex, ai-sdk), since it reads the + * canonical transcript the harness already persists. + * + * Timing: CLI transcripts carry no per-event timestamps (every persisted + * `toolCalls[].ts` is 0 today), so child spans get synthetic, evenly spread + * times across the run window purely to preserve ordering in the Braintrust + * UI. The root advertises this via `metadata.stepTiming = + * "synthetic-order-only"`. The window itself is real when the result carries + * `startedAt`/`durationMs` (new runs), else anchored at `baseTimeMs`. + */ + +import { z } from "zod"; +import { rawEvalResultSchema } from "./eval-metadata.js"; + +const transcriptPartSchema = z.discriminatedUnion("type", [ + z.object({ + type: z.literal("message"), + role: z.enum(["system", "user", "assistant"]), + content: z.string(), + }), + z.object({ + type: z.literal("tool_call"), + name: z.string(), + input: z.record(z.string(), z.unknown()).default({}), + output: z.unknown().optional(), + error: z.string().optional(), + }), +]); + +const toolCallRecordSchema = z.looseObject({ + endpoint: z.string(), + body: z.record(z.string(), z.unknown()).default({}), + name: z.string().optional(), + path: z.string().optional(), + command: z.string().optional(), + url: z.string().optional(), + loadedSkill: z.string().optional(), + result: z.unknown().optional(), + error: z.string().optional(), + ts: z.number().default(0), +}); + +const agentUsageSchema = z + .object({ + inputTokens: z.number(), + outputTokens: z.number(), + cachedInputTokens: z.number(), + cacheCreationInputTokens: z.number(), + reasoningTokens: z.number(), + totalTokens: z.number(), + costUsd: z.number(), + }) + .partial(); + +/** + * A raw result file as the uploader consumes it: the exported shape plus the + * transcript fields the web export drops. Everything beyond the base shape is + * optional so results written before these fields existed still upload. + */ +export const uploadableEvalResultSchema = rawEvalResultSchema.extend({ + transcript: z.array(transcriptPartSchema).optional(), + toolCalls: z.array(toolCallRecordSchema).optional(), + agentReport: z.string().optional(), + stoppedReason: z.string().optional(), + steps: z.number().optional(), + usage: agentUsageSchema.optional(), + startedAt: z.string().optional(), + durationMs: z.number().optional(), +}); +export type UploadableEvalResult = z.infer; + +export interface BraintrustSpanNode { + name: string; + type: "llm" | "tool" | "task" | "score"; + input?: unknown; + output?: unknown; + error?: string; + metadata?: Record; + scores?: Record; + startMs: number; + endMs: number; +} + +export interface BraintrustEvalTrace { + evalId: string; + experiment: string; + input: unknown; + output?: unknown; + scores: Record; + metadata: Record; + metrics: Record; + tags: string[]; + startMs: number; + endMs: number; + spans: BraintrustSpanNode[]; +} + +/** + * Braintrust rejects oversized events; cap any single logged value well below + * that so one giant tool output can't sink the whole row. + */ +const MAX_FIELD_CHARS = 100_000; +const TRUNCATION_PREVIEW_CHARS = 20_000; + +function capForUpload(value: unknown): unknown { + if (value === undefined || value === null) return value; + if (typeof value === "string") { + if (value.length <= MAX_FIELD_CHARS) return value; + return `${value.slice(0, TRUNCATION_PREVIEW_CHARS)}\n…[truncated ${value.length} chars]`; + } + let serialized: string; + try { + serialized = JSON.stringify(value) ?? ""; + } catch { + return String(value); + } + if (serialized.length <= MAX_FIELD_CHARS) return value; + return { + truncated: true, + chars: serialized.length, + preview: serialized.slice(0, TRUNCATION_PREVIEW_CHARS), + }; +} + +/** Unique, human-readable score name per check; `passed` is reserved. */ +function scoreNameFor(checkName: string, taken: Set): string { + const base = checkName.trim() || "check"; + let name = base === "passed" ? "check: passed" : base; + for (let n = 2; taken.has(name); n += 1) name = `${base} (${n})`; + taken.add(name); + return name; +} + +function usageMetrics( + usage: UploadableEvalResult["usage"], +): Record { + if (!usage) return {}; + const metrics: Record = {}; + // Braintrust's conventional metric names, so its token columns light up. + if (usage.inputTokens !== undefined) metrics.prompt_tokens = usage.inputTokens; + if (usage.outputTokens !== undefined) metrics.completion_tokens = usage.outputTokens; + if (usage.cachedInputTokens !== undefined) { + metrics.prompt_cached_tokens = usage.cachedInputTokens; + } + if (usage.cacheCreationInputTokens !== undefined) { + metrics.prompt_cache_creation_tokens = usage.cacheCreationInputTokens; + } + if (usage.reasoningTokens !== undefined) { + metrics.completion_reasoning_tokens = usage.reasoningTokens; + } + const total = + usage.totalTokens ?? + (usage.inputTokens !== undefined || usage.outputTokens !== undefined + ? (usage.inputTokens ?? 0) + (usage.outputTokens ?? 0) + : undefined); + if (total !== undefined) metrics.tokens = total; + return metrics; +} + +export function buildEvalTrace(args: { + experiment: string; + result: UploadableEvalResult; + /** Scenario prompt (PROMPT.md body), when resolvable. */ + prompt?: string; + /** Extra root metadata from the caller (source paths, git info, …). */ + extraMetadata?: Record; + /** Anchor for synthetic timing when the result carries no `startedAt`. */ + baseTimeMs: number; +}): BraintrustEvalTrace { + const { result } = args; + + const parsedStart = result.startedAt ? Date.parse(result.startedAt) : NaN; + const startMs = Number.isNaN(parsedStart) ? args.baseTimeMs : parsedStart; + + const transcript = result.transcript ?? []; + const toolCalls = result.toolCalls ?? []; + const checks = result.checks ?? []; + const spanCount = transcript.length + checks.length; + // Real window when the run recorded one; otherwise one synthetic second per + // span, purely so ordering survives. + const durationMs = result.durationMs ?? Math.max(spanCount, 1) * 1000; + const endMs = startMs + durationMs; + const slotMs = durationMs / Math.max(spanCount, 1); + const slot = (index: number): { startMs: number; endMs: number } => ({ + startMs: startMs + index * slotMs, + endMs: startMs + (index + 1) * slotMs, + }); + + const spans: BraintrustSpanNode[] = []; + // The k-th tool_call transcript part and toolCalls[k] describe the same + // call (the adapter pushes both in lockstep); the record carries the + // normalized views the part lacks. + let toolCallIndex = 0; + for (const part of transcript) { + const timing = slot(spans.length); + if (part.type === "message") { + spans.push({ + name: part.role, + type: part.role === "assistant" ? "llm" : "task", + output: capForUpload(part.content), + ...timing, + }); + continue; + } + const record = toolCalls.at(toolCallIndex); + toolCallIndex += 1; + const normalized: Record = {}; + if (record?.name) normalized.tool = record.name; + if (record?.path) normalized.path = record.path; + if (record?.command) normalized.command = record.command; + if (record?.url) normalized.url = record.url; + if (record?.loadedSkill) normalized.loadedSkill = record.loadedSkill; + spans.push({ + name: part.name, + type: "tool", + input: capForUpload(part.input), + output: part.error === undefined ? capForUpload(part.output) : undefined, + ...(part.error !== undefined ? { error: part.error } : {}), + ...(Object.keys(normalized).length > 0 ? { metadata: normalized } : {}), + ...timing, + }); + } + + const scores: Record = { + passed: result.passed === true ? 1 : 0, + }; + const takenScoreNames = new Set(Object.keys(scores)); + for (const check of checks) { + const scoreName = scoreNameFor(check.name, takenScoreNames); + const value = check.passed ? 1 : 0; + scores[scoreName] = value; + spans.push({ + name: scoreName, + type: "score", + output: { + passed: check.passed, + ...(check.notes ? { notes: check.notes } : {}), + ...(check.judgeNotes ? { judgeNotes: check.judgeNotes } : {}), + }, + scores: { [scoreName]: value }, + ...slot(spans.length), + }); + } + + const display = result.experimentDisplay; + const metadata: Record = { + eval: result.eval, + experiment: args.experiment, + ...(result.experimentSuite ?? result.profile + ? { experimentSuite: result.experimentSuite ?? result.profile } + : {}), + ...(display ? { ...display } : {}), + ...(result.stage ? { stage: result.stage } : {}), + ...(result.product ? { product: result.product } : {}), + ...(result.topic ? { topic: result.topic } : {}), + ...(result.suite ? { suite: result.suite } : {}), + ...(result.interface ? { interface: result.interface } : {}), + ...(result.cliVersion ? { cliVersion: result.cliVersion } : {}), + ...(result.attempts !== undefined ? { attempts: result.attempts } : {}), + ...(result.steps !== undefined ? { steps: result.steps } : {}), + ...(result.stoppedReason ? { stoppedReason: result.stoppedReason } : {}), + ...(result.skills ? { skills: result.skills } : {}), + ...(result.docs ? { docs: result.docs } : {}), + ...(result.usage?.costUsd !== undefined + ? { costUsd: result.usage.costUsd } + : {}), + stepTiming: "synthetic-order-only", + ...args.extraMetadata, + }; + + return { + evalId: result.eval, + experiment: args.experiment, + // The prompt is the comparison key across experiments (Braintrust matches + // rows by input); fall back to the eval id so old results still line up. + input: args.prompt ?? { eval: result.eval }, + ...(result.agentReport ? { output: capForUpload(result.agentReport) } : {}), + scores, + metadata, + metrics: usageMetrics(result.usage), + tags: [ + ...(result.stage ? [result.stage] : []), + ...(result.product ?? []), + ...(result.topic ?? []), + ...(result.interface ? [result.interface] : []), + ], + startMs, + endMs, + spans, + }; +} diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index af04d674..4838a540 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -116,6 +116,15 @@ export type { export { createParser, supportedParsers } from './agents/registry.js'; export { adaptTranscript } from './parsers/adapt.js'; export type { AdaptedTranscript } from './parsers/adapt.js'; +export { + buildEvalTrace, + uploadableEvalResultSchema, +} from './braintrust-spans.js'; +export type { + BraintrustEvalTrace, + BraintrustSpanNode, + UploadableEvalResult, +} from './braintrust-spans.js'; export type { AgentTranscriptParser } from './parsers/types.js'; export type { ToolName, @@ -373,12 +382,31 @@ export type AgentRunArgs = { timeoutSec: number; }; +/** + * Whole-run token/cost usage, when the agent's harness reports it (Claude + * Code's terminal `result` line, Codex's `turn.completed`, ai-sdk's + * `totalUsage`). All fields optional — absent means the harness didn't say. + */ +export interface AgentUsage { + inputTokens?: number; + outputTokens?: number; + /** Prompt tokens served from cache (cache reads). */ + cachedInputTokens?: number; + /** Prompt tokens written to cache (Claude Code only). */ + cacheCreationInputTokens?: number; + reasoningTokens?: number; + totalTokens?: number; + /** Run cost in USD, when the harness reports one (Claude Code only). */ + costUsd?: number; +} + export type AgentRunResult = { agentReport: string; toolCalls: ToolCallRecord[]; transcript: TranscriptPart[]; steps: number; stoppedReason: string; + usage?: AgentUsage; }; export type AgentHarness = { @@ -722,6 +750,14 @@ export function aiSdkAgent(options: { const agentReport = result.text.trim(); + const usage: AgentUsage = { + inputTokens: result.totalUsage.inputTokens, + outputTokens: result.totalUsage.outputTokens, + cachedInputTokens: result.totalUsage.cachedInputTokens, + reasoningTokens: result.totalUsage.reasoningTokens, + totalTokens: result.totalUsage.totalTokens, + }; + return { agentReport, toolCalls, @@ -731,6 +767,7 @@ export function aiSdkAgent(options: { result.steps.length >= MAX_STEPS ? 'max_steps' : result.finishReason, + usage, }; } finally { await closeMcpHandles(mcpHandles); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index de9e3fa6..c420e9f0 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -132,6 +132,9 @@ importers: ai: specifier: 'catalog:' version: 6.0.199(zod@4.4.3) + braintrust: + specifier: ^3.24.0 + version: 3.24.0(zod@4.4.3) happy-dom: specifier: ^20.9.0 version: 20.10.2 @@ -189,7 +192,7 @@ importers: version: 1.17.0(react@19.2.7) nuqs: specifier: ^2.9.2 - version: 2.9.2(react@19.2.7) + version: 2.9.4(react@19.2.7) radix-ui: specifier: ^1.4.3 version: 1.5.0(@types/react-dom@19.2.3(@types/react@19.2.17))(@types/react@19.2.17)(react-dom@19.2.7(react@19.2.7))(react@19.2.7) @@ -626,6 +629,45 @@ packages: cpu: [x64] os: [win32] + '@braintrust/bt-darwin-arm64@0.12.0': + resolution: {integrity: sha512-mY6VW/3VwcOQOGN8sYHS6F0xzHTFwgZcNlj7zlQttI6OXOCGt/bhonGIqd03QBhxmj0M31ymSS7TqSeCK/RYIQ==} + cpu: [arm64] + os: [darwin] + + '@braintrust/bt-darwin-x64@0.12.0': + resolution: {integrity: sha512-woyRyDv2DfCF8+von+3X9f1cddAbFnKLSfCbEkrBQVc0ZsAM60as8nehYv7DkafJF/67yKFx/sUraV65sukesQ==} + cpu: [x64] + os: [darwin] + + '@braintrust/bt-linux-arm64@0.12.0': + resolution: {integrity: sha512-J9/7f3EIMKmFmSSQHQnAQLpUgB8YJENrOlkABjlTprBgsIYVgz7tSH7yg2P3ur+2Jem7PpGnrDxHGh/UjRfjsg==} + cpu: [arm64] + os: [linux] + + '@braintrust/bt-linux-x64-musl@0.12.0': + resolution: {integrity: sha512-KnLgENOoztBXcH+mLFJe4bYzi67kSOuELOQrm4Hler35GjgaBMI1fFLIUjKRPAmyT9VIhBMhy1ICfGEFLueMEQ==} + cpu: [x64] + os: [linux] + + '@braintrust/bt-linux-x64@0.12.0': + resolution: {integrity: sha512-HJFbUl3HYYY1Ivw8OYBeIMcIsU9r7+fC9BzLWB5FdH7881ToKee2wbbLZssMCxFbMVeUxzEo+SzGD/1Z9Gk9CA==} + cpu: [x64] + os: [linux] + + '@braintrust/bt-win32-arm64@0.12.0': + resolution: {integrity: sha512-+7PkdBAmiqEJsWteGHP/Zs62F75PkHPA8m/ej4TOz/mHGKulUU0bZibw91KRqIB7J7jGi5ItvCiqkiGaLKLnyw==} + cpu: [arm64] + os: [win32] + + '@braintrust/bt-win32-x64@0.12.0': + resolution: {integrity: sha512-Q0c+pVUGm82n/7N8QJ5s6j/G/Q0GFzqOaTipHjkmElLbAOOEcfRH/uljdtIPNBiEQcrzqirS0GmOgiFkeQ3Txg==} + cpu: [x64] + os: [win32] + + '@colors/colors@1.5.0': + resolution: {integrity: sha512-ooWCrlZP11i8GImSjTHYHLkvFDP48nS4+204nGb1RiX/WXYHmJA2III9/e2DWVabCESdW7hBAEzHRqUn9OUVvQ==} + engines: {node: '>=0.1.90'} + '@commander-js/extra-typings@14.0.0': resolution: {integrity: sha512-hIn0ncNaJRLkZrxBIp5AsW/eXEHNKYQBh0aPdoUqNgD+Io3NIykQqpKFyKcuasZhicGaEZJX/JBSIkZ4e5x8Dg==} peerDependencies: @@ -662,6 +704,12 @@ packages: cpu: [ppc64] os: [aix] + '@esbuild/aix-ppc64@0.28.1': + resolution: {integrity: sha512-Svl7tq8k/08+p6CXPpRjQ1fKX+1odH/BQbb48fV6fj3CWHhsoIOoY87w1oHXm0qEpkIK3ZfVgp0hed3XBXzXMQ==} + engines: {node: '>=18'} + cpu: [ppc64] + os: [aix] + '@esbuild/android-arm64@0.27.7': resolution: {integrity: sha512-62dPZHpIXzvChfvfLJow3q5dDtiNMkwiRzPylSCfriLvZeq0a1bWChrGx/BbUbPwOrsWKMn8idSllklzBy+dgQ==} engines: {node: '>=18'} @@ -674,6 +722,12 @@ packages: cpu: [arm64] os: [android] + '@esbuild/android-arm64@0.28.1': + resolution: {integrity: sha512-34EGEbCIAgosYz6goLcopX6Mo7NyGv9tfwEM2/7Ce2VcVRk568iSvniGWcUXIy7wEDR1wzolcxcriFVrWYcwBg==} + engines: {node: '>=18'} + cpu: [arm64] + os: [android] + '@esbuild/android-arm@0.27.7': resolution: {integrity: sha512-jbPXvB4Yj2yBV7HUfE2KHe4GJX51QplCN1pGbYjvsyCZbQmies29EoJbkEc+vYuU5o45AfQn37vZlyXy4YJ8RQ==} engines: {node: '>=18'} @@ -686,6 +740,12 @@ packages: cpu: [arm] os: [android] + '@esbuild/android-arm@0.28.1': + resolution: {integrity: sha512-0k2F129Xdio1TdJfzJ8sy1Q47vUD2NnwdhiAf7drUN1EBTfPf4hsFCtmMgu/6m8JSzsBrlmVjudMBQqOfG8usQ==} + engines: {node: '>=18'} + cpu: [arm] + os: [android] + '@esbuild/android-x64@0.27.7': resolution: {integrity: sha512-x5VpMODneVDb70PYV2VQOmIUUiBtY3D3mPBG8NxVk5CogneYhkR7MmM3yR/uMdITLrC1ml/NV1rj4bMJuy9MCg==} engines: {node: '>=18'} @@ -698,6 +758,12 @@ packages: cpu: [x64] os: [android] + '@esbuild/android-x64@0.28.1': + resolution: {integrity: sha512-dbwY7ltSMDWsRatcRpCnES4F+im88OCUgGZjy52shC7GqHRE/cYlxNbB4Z4UpJswpcc4Qxd2oE/ufM0p61IKng==} + engines: {node: '>=18'} + cpu: [x64] + os: [android] + '@esbuild/darwin-arm64@0.27.7': resolution: {integrity: sha512-5lckdqeuBPlKUwvoCXIgI2D9/ABmPq3Rdp7IfL70393YgaASt7tbju3Ac+ePVi3KDH6N2RqePfHnXkaDtY9fkw==} engines: {node: '>=18'} @@ -710,6 +776,12 @@ packages: cpu: [arm64] os: [darwin] + '@esbuild/darwin-arm64@0.28.1': + resolution: {integrity: sha512-TZbWkQY7kvTAXbXUT7uVACR5cMHsDiSz9z7ZKAX/RTq/WJEk3QyRr0wZpNhBDX+/0CtdqUIJlOiodQcta6tY3Q==} + engines: {node: '>=18'} + cpu: [arm64] + os: [darwin] + '@esbuild/darwin-x64@0.27.7': resolution: {integrity: sha512-rYnXrKcXuT7Z+WL5K980jVFdvVKhCHhUwid+dDYQpH+qu+TefcomiMAJpIiC2EM3Rjtq0sO3StMV/+3w3MyyqQ==} engines: {node: '>=18'} @@ -722,6 +794,12 @@ packages: cpu: [x64] os: [darwin] + '@esbuild/darwin-x64@0.28.1': + resolution: {integrity: sha512-zfdzgK9ACBNZLI/CyHTOx81SyNbM6YXn7rxSgX97VjyiPl9W1i4Ka4fgKECEoFCKGpvBj5qArWIGgQjOwkgskQ==} + engines: {node: '>=18'} + cpu: [x64] + os: [darwin] + '@esbuild/freebsd-arm64@0.27.7': resolution: {integrity: sha512-B48PqeCsEgOtzME2GbNM2roU29AMTuOIN91dsMO30t+Ydis3z/3Ngoj5hhnsOSSwNzS+6JppqWsuhTp6E82l2w==} engines: {node: '>=18'} @@ -734,6 +812,12 @@ packages: cpu: [arm64] os: [freebsd] + '@esbuild/freebsd-arm64@0.28.1': + resolution: {integrity: sha512-wG2EA8ENdEI0qhkSZMjfqrdY+ziCYCPMmtZjjIwOmXFjmyzEHn+UUxk5of+SYsjtfs3VpnlC7QLzSI5hY/rOAw==} + engines: {node: '>=18'} + cpu: [arm64] + os: [freebsd] + '@esbuild/freebsd-x64@0.27.7': resolution: {integrity: sha512-jOBDK5XEjA4m5IJK3bpAQF9/Lelu/Z9ZcdhTRLf4cajlB+8VEhFFRjWgfy3M1O4rO2GQ/b2dLwCUGpiF/eATNQ==} engines: {node: '>=18'} @@ -746,6 +830,12 @@ packages: cpu: [x64] os: [freebsd] + '@esbuild/freebsd-x64@0.28.1': + resolution: {integrity: sha512-i7dZ9vQgnvSCzi/rYCXNgtF/U+eKZNJBzu3eTQbRgHnM7tNSizLOkRFAl3qzVc/Op/u5YkHHa4pf/3DOYHthLQ==} + engines: {node: '>=18'} + cpu: [x64] + os: [freebsd] + '@esbuild/linux-arm64@0.27.7': resolution: {integrity: sha512-RZPHBoxXuNnPQO9rvjh5jdkRmVizktkT7TCDkDmQ0W2SwHInKCAV95GRuvdSvA7w4VMwfCjUiPwDi0ZO6Nfe9A==} engines: {node: '>=18'} @@ -758,6 +848,12 @@ packages: cpu: [arm64] os: [linux] + '@esbuild/linux-arm64@0.28.1': + resolution: {integrity: sha512-yHs+0uc8+nvEAfAfxrWQKK5peSNzBc4PegcMO0EJ2hT71uA7vB8Ihg2e77R2P7SG5uYjPbHlLLmve4LLLRCf0g==} + engines: {node: '>=18'} + cpu: [arm64] + os: [linux] + '@esbuild/linux-arm@0.27.7': resolution: {integrity: sha512-RkT/YXYBTSULo3+af8Ib0ykH8u2MBh57o7q/DAs3lTJlyVQkgQvlrPTnjIzzRPQyavxtPtfg0EopvDyIt0j1rA==} engines: {node: '>=18'} @@ -770,6 +866,12 @@ packages: cpu: [arm] os: [linux] + '@esbuild/linux-arm@0.28.1': + resolution: {integrity: sha512-qVXBOHQS+d5Y722GwJzJUtOLlX7km3CraOaGormF1pDtPd2C/l1SHRPgjLunLGe51Sh5YYWKMFDyV4SxgMQYTQ==} + engines: {node: '>=18'} + cpu: [arm] + os: [linux] + '@esbuild/linux-ia32@0.27.7': resolution: {integrity: sha512-GA48aKNkyQDbd3KtkplYWT102C5sn/EZTY4XROkxONgruHPU72l+gW+FfF8tf2cFjeHaRbWpOYa/uRBz/Xq1Pg==} engines: {node: '>=18'} @@ -782,6 +884,12 @@ packages: cpu: [ia32] os: [linux] + '@esbuild/linux-ia32@0.28.1': + resolution: {integrity: sha512-d1z4ZuP0ajrfz/FhGT4vv278rX8KnPPJx8i5+AtK7TYbx9Le9F1hyzurZpkEyjkGa9dUGhQow4C1NmeGvqxN2w==} + engines: {node: '>=18'} + cpu: [ia32] + os: [linux] + '@esbuild/linux-loong64@0.27.7': resolution: {integrity: sha512-a4POruNM2oWsD4WKvBSEKGIiWQF8fZOAsycHOt6JBpZ+JN2n2JH9WAv56SOyu9X5IqAjqSIPTaJkqN8F7XOQ5Q==} engines: {node: '>=18'} @@ -794,6 +902,12 @@ packages: cpu: [loong64] os: [linux] + '@esbuild/linux-loong64@0.28.1': + resolution: {integrity: sha512-M5sRjUVZrkm1OAPR3dlOYzNmN+loZKGVi1VUQGrwuqLcbR6qeAz+famMhjASeH3YVKvZz+zT1jlh/keC3Rj/lg==} + engines: {node: '>=18'} + cpu: [loong64] + os: [linux] + '@esbuild/linux-mips64el@0.27.7': resolution: {integrity: sha512-KabT5I6StirGfIz0FMgl1I+R1H73Gp0ofL9A3nG3i/cYFJzKHhouBV5VWK1CSgKvVaG4q1RNpCTR2LuTVB3fIw==} engines: {node: '>=18'} @@ -806,6 +920,12 @@ packages: cpu: [mips64el] os: [linux] + '@esbuild/linux-mips64el@0.28.1': + resolution: {integrity: sha512-mRObBZeHh2OxcBFPWE/FjylkRgZdYuiTR3vaTozquCGOH14iP9oN4x4Ge81CoIDYQrXmIxpFumJBu5MtZpnQJQ==} + engines: {node: '>=18'} + cpu: [mips64el] + os: [linux] + '@esbuild/linux-ppc64@0.27.7': resolution: {integrity: sha512-gRsL4x6wsGHGRqhtI+ifpN/vpOFTQtnbsupUF5R5YTAg+y/lKelYR1hXbnBdzDjGbMYjVJLJTd2OFmMewAgwlQ==} engines: {node: '>=18'} @@ -818,6 +938,12 @@ packages: cpu: [ppc64] os: [linux] + '@esbuild/linux-ppc64@0.28.1': + resolution: {integrity: sha512-slScBsMAb3GFDcdrCgLwZtPYRoH2H/youv10QiZyRjmsP48fznoveWytSgCI/R0ZcUgpc0ZhIUEx6LHts8yrfQ==} + engines: {node: '>=18'} + cpu: [ppc64] + os: [linux] + '@esbuild/linux-riscv64@0.27.7': resolution: {integrity: sha512-hL25LbxO1QOngGzu2U5xeXtxXcW+/GvMN3ejANqXkxZ/opySAZMrc+9LY/WyjAan41unrR3YrmtTsUpwT66InQ==} engines: {node: '>=18'} @@ -830,6 +956,12 @@ packages: cpu: [riscv64] os: [linux] + '@esbuild/linux-riscv64@0.28.1': + resolution: {integrity: sha512-kw0owk1o0GFETUJyW0jc0G4Yzs0BHZn0JDZ8JRT088vjJYX777BAs1fDGxAC+q831qOs2DTC96mNsG2opdfyyQ==} + engines: {node: '>=18'} + cpu: [riscv64] + os: [linux] + '@esbuild/linux-s390x@0.27.7': resolution: {integrity: sha512-2k8go8Ycu1Kb46vEelhu1vqEP+UeRVj2zY1pSuPdgvbd5ykAw82Lrro28vXUrRmzEsUV0NzCf54yARIK8r0fdw==} engines: {node: '>=18'} @@ -842,6 +974,12 @@ packages: cpu: [s390x] os: [linux] + '@esbuild/linux-s390x@0.28.1': + resolution: {integrity: sha512-/lAIjX8aYFRByhh6L5rYtPEDRqa9de/4V/juOXcta5frjvzXO4/sqEtyytse0g3zZFuWu5cDN0MkLz2qRDD2Ag==} + engines: {node: '>=18'} + cpu: [s390x] + os: [linux] + '@esbuild/linux-x64@0.27.7': resolution: {integrity: sha512-hzznmADPt+OmsYzw1EE33ccA+HPdIqiCRq7cQeL1Jlq2gb1+OyWBkMCrYGBJ+sxVzve2ZJEVeePbLM2iEIZSxA==} engines: {node: '>=18'} @@ -854,6 +992,12 @@ packages: cpu: [x64] os: [linux] + '@esbuild/linux-x64@0.28.1': + resolution: {integrity: sha512-u/anNYF2mmVOEDwLtnQ1wOr3EZ9sTNGLWrsYGYwHWzGA3Si84IOkHXlbWTD1NB+9/1lcnweYKO54uhxZydNzfA==} + engines: {node: '>=18'} + cpu: [x64] + os: [linux] + '@esbuild/netbsd-arm64@0.27.7': resolution: {integrity: sha512-b6pqtrQdigZBwZxAn1UpazEisvwaIDvdbMbmrly7cDTMFnw/+3lVxxCTGOrkPVnsYIosJJXAsILG9XcQS+Yu6w==} engines: {node: '>=18'} @@ -866,6 +1010,12 @@ packages: cpu: [arm64] os: [netbsd] + '@esbuild/netbsd-arm64@0.28.1': + resolution: {integrity: sha512-oks0DYbLwWMmaakTsCb+zL4E+aHRVLom9IJZOAthMQEPiQmydXHkziYEsGYRx0uNV/IjEKGAV941JzH02pflqw==} + engines: {node: '>=18'} + cpu: [arm64] + os: [netbsd] + '@esbuild/netbsd-x64@0.27.7': resolution: {integrity: sha512-OfatkLojr6U+WN5EDYuoQhtM+1xco+/6FSzJJnuWiUw5eVcicbyK3dq5EeV/QHT1uy6GoDhGbFpprUiHUYggrw==} engines: {node: '>=18'} @@ -878,6 +1028,12 @@ packages: cpu: [x64] os: [netbsd] + '@esbuild/netbsd-x64@0.28.1': + resolution: {integrity: sha512-aeL6lAnN89Hz43Mlh1G8ARasbuoYvSITDEx0tHh5b7jJnHcssqgjy9Yx430GDpmCa6OyrKoS0aNRjKundRizGg==} + engines: {node: '>=18'} + cpu: [x64] + os: [netbsd] + '@esbuild/openbsd-arm64@0.27.7': resolution: {integrity: sha512-AFuojMQTxAz75Fo8idVcqoQWEHIXFRbOc1TrVcFSgCZtQfSdc1RXgB3tjOn/krRHENUB4j00bfGjyl2mJrU37A==} engines: {node: '>=18'} @@ -890,6 +1046,12 @@ packages: cpu: [arm64] os: [openbsd] + '@esbuild/openbsd-arm64@0.28.1': + resolution: {integrity: sha512-MEFJe5C3R8pwXdZ5Y21oo6m7ePiS0d9pWucn99O/wvyJZChoIQKrQDxKrGeW8F5+T0okTHesAmDeiHDTIq0V/Q==} + engines: {node: '>=18'} + cpu: [arm64] + os: [openbsd] + '@esbuild/openbsd-x64@0.27.7': resolution: {integrity: sha512-+A1NJmfM8WNDv5CLVQYJ5PshuRm/4cI6WMZRg1by1GwPIQPCTs1GLEUHwiiQGT5zDdyLiRM/l1G0Pv54gvtKIg==} engines: {node: '>=18'} @@ -902,6 +1064,12 @@ packages: cpu: [x64] os: [openbsd] + '@esbuild/openbsd-x64@0.28.1': + resolution: {integrity: sha512-i/ZLIOafE0Z8cI/XANJAixoJL/uRAoS2xOA3rb0xN+KK0K177cMAsQYkzHtBrtMXAKuAc7HGgcWiZ/sRC1Nxgw==} + engines: {node: '>=18'} + cpu: [x64] + os: [openbsd] + '@esbuild/openharmony-arm64@0.27.7': resolution: {integrity: sha512-+KrvYb/C8zA9CU/g0sR6w2RBw7IGc5J2BPnc3dYc5VJxHCSF1yNMxTV5LQ7GuKteQXZtspjFbiuW5/dOj7H4Yw==} engines: {node: '>=18'} @@ -914,6 +1082,12 @@ packages: cpu: [arm64] os: [openharmony] + '@esbuild/openharmony-arm64@0.28.1': + resolution: {integrity: sha512-ge+Z7EXFNt2BO1oAMsVpiQ8EwndV9i1xXerAeTIK7AtPs3bKFXQM7nlRxDSIUIMeueR1CNXxqztLzdNeReKBJg==} + engines: {node: '>=18'} + cpu: [arm64] + os: [openharmony] + '@esbuild/sunos-x64@0.27.7': resolution: {integrity: sha512-ikktIhFBzQNt/QDyOL580ti9+5mL/YZeUPKU2ivGtGjdTYoqz6jObj6nOMfhASpS4GU4Q/Clh1QtxWAvcYKamA==} engines: {node: '>=18'} @@ -926,6 +1100,12 @@ packages: cpu: [x64] os: [sunos] + '@esbuild/sunos-x64@0.28.1': + resolution: {integrity: sha512-BEjgtECkL3vY+SaSQ6nzVfiALUeFxpawyp8Jmf5PtYhf1Ug40N1h/hxlhts+f1FvSvarEigdxS3BlSMI2PJLcQ==} + engines: {node: '>=18'} + cpu: [x64] + os: [sunos] + '@esbuild/win32-arm64@0.27.7': resolution: {integrity: sha512-7yRhbHvPqSpRUV7Q20VuDwbjW5kIMwTHpptuUzV+AA46kiPze5Z7qgt6CLCK3pWFrHeNfDd1VKgyP4O+ng17CA==} engines: {node: '>=18'} @@ -938,6 +1118,12 @@ packages: cpu: [arm64] os: [win32] + '@esbuild/win32-arm64@0.28.1': + resolution: {integrity: sha512-lCv9eK/H6ZJWbE7bh2nw54CZ9M2nupBxJcTsdk/QQnWkdSjKGuxmmH8/GWrlT1eMmZfn4dGcCjRte397WqfQXA==} + engines: {node: '>=18'} + cpu: [arm64] + os: [win32] + '@esbuild/win32-ia32@0.27.7': resolution: {integrity: sha512-SmwKXe6VHIyZYbBLJrhOoCJRB/Z1tckzmgTLfFYOfpMAx63BJEaL9ExI8x7v0oAO3Zh6D/Oi1gVxEYr5oUCFhw==} engines: {node: '>=18'} @@ -950,6 +1136,12 @@ packages: cpu: [ia32] os: [win32] + '@esbuild/win32-ia32@0.28.1': + resolution: {integrity: sha512-zvb/mB2bSCoJOpoCBgYKKpX6YM6mJBlBUVUtVj41DlZJVEB6/0CKlRYxP5wWl1C1ILiCoAU5wZZ4q1P3qeS6Eg==} + engines: {node: '>=18'} + cpu: [ia32] + os: [win32] + '@esbuild/win32-x64@0.27.7': resolution: {integrity: sha512-56hiAJPhwQ1R4i+21FVF7V8kSD5zZTdHcVuRFMW0hn753vVfQN8xlx4uOPT4xoGH0Z/oVATuR82AiqSTDIpaHg==} engines: {node: '>=18'} @@ -962,6 +1154,12 @@ packages: cpu: [x64] os: [win32] + '@esbuild/win32-x64@0.28.1': + resolution: {integrity: sha512-bm4Mowrv+GXMlpWX++EcXw/iLyd1o3+bJkC2DkWXYVvgZCqD/bSj9ctZeAMC3cIxgjRVR2Dufaiu4YPxr5gW1A==} + engines: {node: '>=18'} + cpu: [x64] + os: [win32] + '@eslint-community/eslint-utils@4.9.1': resolution: {integrity: sha512-phrYmNiYppR7znFEdqgfWHXR6NCkZEK7hwWDHZUjit/2/U0r6XvkDl0SYnoM51Hq7FhCGdLDT6zxCCOY1hexsQ==} engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} @@ -1066,6 +1264,12 @@ packages: '@jridgewell/trace-mapping@0.3.31': resolution: {integrity: sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==} + '@kwsites/file-exists@1.1.1': + resolution: {integrity: sha512-m9/5YGR18lIwxSFDwfE3oA7bWuq9kdau6ugN4H2rJeyhFQZcG9AgSHkQtSD15a8WvTgfz9aikZMrKPHvbpqFiw==} + + '@kwsites/promise-deferred@1.1.1': + resolution: {integrity: sha512-GaHYm+c0O9MjZRu0ongGBRbinu8gVAMd2UZjji6jVmqKtZluZnptXGWhz1E8j8D2HJ3f/yMxKAUC0b+57wncIw==} + '@launchql/protobufjs@7.2.6': resolution: {integrity: sha512-vwi1nG2/heVFsIMHQU1KxTjUp5c757CTtRAZn/jutApCkFlle1iv8tzM/DHlSZJKDldxaYqnNYTg0pTyp8Bbtg==} engines: {node: '>=12.0.0'} @@ -1089,6 +1293,9 @@ packages: '@cfworker/json-schema': optional: true + '@next/env@14.2.35': + resolution: {integrity: sha512-DuhvCtj4t9Gwrx80dmz2F4t/zKQ4ktN8WrMwOuVzkJfBilwAwGr6v16M5eI8yCuZ63H9TTuEU09Iu2HqkzFPVQ==} + '@noble/ciphers@1.3.0': resolution: {integrity: sha512-2I0gnIVPtfnMw9ee9h1dJG7tp81+8Ob3OJb3Mv37rx5L40/b0i7djjCVvGOVqc9AEIQyvyu1i6ypKdFw8R8gQw==} engines: {node: ^14.21.3 || >=16} @@ -2067,6 +2274,12 @@ packages: '@opentelemetry/sdk-trace-base': ^1.30.1 || ^2.1.0 '@opentelemetry/semantic-conventions': ^1.39.0 + '@simple-git/args-pathspec@1.0.3': + resolution: {integrity: sha512-ngJMaHlsWDTfjyq9F3VIQ8b7NXbBLq5j9i5bJ6XLYtD6qlDXT7fdKY2KscWWUF8t18xx052Y/PUO1K1TRc9yKA==} + + '@simple-git/argv-parser@1.1.1': + resolution: {integrity: sha512-Q9lBcfQ+VQCpQqGJFHe5yooOS5hGdLFFbJ5R+R5aDsnkPCahtn1hSkMcORX65J2Z5lxSkD0lQorMsncuBQxYUw==} + '@sindresorhus/merge-streams@4.0.0': resolution: {integrity: sha512-tlqY9xq5ukxTUZBmoOp+m61cqwQD5pHJtFY3Mn8CA8ps6yghLH/Hw8UPdqg4OLmFW3IFlcXnQNmo/dh8HzXYIQ==} engines: {node: '>=18'} @@ -2411,6 +2624,15 @@ packages: resolution: {integrity: sha512-VYNCgUc0nOmC4WJmWw9GkrKdfr8Zl4/rxhC5SvgacBgxiW9W/9NRttUoHHXV8xdII3MaRgkZZVX8Ikzc/Jmjag==} engines: {node: '>=14'} + '@vercel/functions@1.6.0': + resolution: {integrity: sha512-R6FKQrYT5MZs5IE1SqeCJWxMuBdHawFcCZboKKw8p7s+6/mcd55Gx6tWmyKnQTyrSEA04NH73Tc9CbqpEle8RA==} + engines: {node: '>= 16'} + peerDependencies: + '@aws-sdk/credential-provider-web-identity': '*' + peerDependenciesMeta: + '@aws-sdk/credential-provider-web-identity': + optional: true + '@vercel/oidc@3.2.0': resolution: {integrity: sha512-UycprH3T6n3jH0k44NHMa7pnFHGu/N05MjojYr+Mc6I7obkoLIJujSWwin1pCvdy/eOxrI/l3uDLQsmcrOb4ug==} engines: {node: '>= 20'} @@ -2538,6 +2760,10 @@ packages: resolution: {integrity: sha512-6t10qk83GOG8p0vKmaCr8eiilZwO171AvbROMtvvNiwrTly62t+7XkA8RdIIVbpMhCASAsxgAzdRSwh6nw/5Dg==} engines: {node: '>=4'} + astring@1.9.0: + resolution: {integrity: sha512-LElXdjswlqjWrPpJFg1Fx4wpkOCxj1TDHlSV4PlaRxHGWko024xICaa97ZkMfs6DRKlCguiAI+rbXv5GWwXIkg==} + hasBin: true + balanced-match@1.0.2: resolution: {integrity: sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==} @@ -2572,6 +2798,12 @@ packages: resolution: {integrity: sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==} engines: {node: '>=8'} + braintrust@3.24.0: + resolution: {integrity: sha512-jF2XK1AImY2jI6uPxxU/KNGyYIIsr4PhaUD1IsbdggSNLIBkwzmluKvCzs1OUVy6LbpL4T2ekv8t+83UQ64hEg==} + hasBin: true + peerDependencies: + zod: ^3.25.34 || ^4.0 + browserslist@4.28.2: resolution: {integrity: sha512-48xSriZYYg+8qXna9kwqjIVzuQxi+KYWp2+5nCYnYKPTr0LvD89Jqk2Or5ogxz0NUMfIjhh2lIUX/LyX9B4oIg==} engines: {node: ^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7} @@ -2637,10 +2869,18 @@ packages: resolution: {integrity: sha512-aCj4O5wKyszjMmDT4tZj93kxyydN/K5zPWSCe6/0AV/AA1pqe5ZBIw0a2ZfPQV7lL5/yb5HsUreJ6UFAF1tEQw==} engines: {node: '>=18'} + cli-progress@3.12.0: + resolution: {integrity: sha512-tRkV3HJ1ASwm19THiiLIXLO7Im7wlTuKnvkYaTkyoAPefqjNg7W7DHKUlGRxy9vxDvbyCYQkQozvptuMkGCg8A==} + engines: {node: '>=4'} + cli-spinners@2.9.2: resolution: {integrity: sha512-ywqV+5MmyL4E7ybXgKys4DugZbX0FC6LnwrhjuykIjnK9k8OQacQ7axGKnjDXWNhns0xot3bZI5h55H8yo9cJg==} engines: {node: '>=6'} + cli-table3@0.6.5: + resolution: {integrity: sha512-+W/5efTR7y5HRD7gACw9yQjqMVvEMLBHmboM/kPWam+H+Hmyrgjh6YncVKK122YZkXrLudzTuAukUw9FnMf7IQ==} + engines: {node: 10.* || >= 12.*} + clsx@2.1.1: resolution: {integrity: sha512-eYm0QWBtUrBWZWG0d386OGAw16Z995PiOVo2B7bjWSbHedGl5e0ZWaq65kOGgUSNesEIDkB9ISbTg/JK9dhCZA==} engines: {node: '>=6'} @@ -2728,6 +2968,9 @@ packages: resolution: {integrity: sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==} engines: {node: '>= 12'} + dc-browser@1.0.4: + resolution: {integrity: sha512-7oEtnzNlcE+hr4OvO3GR6Gndgw8BhW+wKOEwMqSleyY7N29jbAxzyW5BaJl7qBCw+6OIxfMWtY0T+6dxq8RWLw==} + debug@4.4.3: resolution: {integrity: sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==} engines: {node: '>=6.0'} @@ -2789,6 +3032,10 @@ packages: dom-accessibility-api@0.6.3: resolution: {integrity: sha512-7ZgogeTnjuHbo+ct10G9Ffp0mif17idi0IyWNVA/wcwcm7NPOD/WEHVP3n7n3MhXqxoIYm8d6MuZohYWIZ4T3w==} + dotenv@16.6.1: + resolution: {integrity: sha512-uBq4egWHTcTt33a72vpSG0z3HnPuIl6NqYcTrKEg2azoEyl2hpW0zqlxysq2pK9HlDIHyHyakeYaYnSAwd8bow==} + engines: {node: '>=12'} + dotenv@17.4.2: resolution: {integrity: sha512-nI4U3TottKAcAD9LLud4Cb7b2QztQMUEfHbvhTH09bqXTxnSie8WnjPALV/WMCrJZ6UV/qHJ6L03OqO3LcdYZw==} engines: {node: '>=12'} @@ -2810,6 +3057,9 @@ packages: emoji-regex@10.6.0: resolution: {integrity: sha512-toUI84YS5YmxW219erniWD0CIVOo46xGKColeNQRgOzDorgBi1v4D71/OFzgD9GO2UGKIv1C3Sp8DAn0+j5w7A==} + emoji-regex@8.0.0: + resolution: {integrity: sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==} + encodeurl@2.0.0: resolution: {integrity: sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg==} engines: {node: '>= 0.8'} @@ -2858,6 +3108,11 @@ packages: engines: {node: '>=18'} hasBin: true + esbuild@0.28.1: + resolution: {integrity: sha512-HrJrvZv5ayxBzPfwphOoNzkzOIIlifzk0KJrGK2c8R4+LKpMtpYLQeUdjnwjWv/LZlkH2laZk+4w78pi99D4Vw==} + engines: {node: '>=18'} + hasBin: true + escalade@3.2.0: resolution: {integrity: sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==} engines: {node: '>=6'} @@ -2938,6 +3193,10 @@ packages: resolution: {integrity: sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==} engines: {node: '>= 0.6'} + eventsource-parser@1.1.2: + resolution: {integrity: sha512-v0eOBUbiaFojBu2s2NPBfYUoRR9GjcDNvCXVaqEf5vVfpIAh9f8RCo4vXTP8c63QRKCFwoLpMpTdPwwhEKVgzA==} + engines: {node: '>=14.18'} + eventsource-parser@3.1.0: resolution: {integrity: sha512-kJezFj9YFAMLeORyi7aCLxLbD5/qWMQnoMVlVPyHIll7lgRJCc3JVln9Vgl9nwQi0YkMnhdGTMNn7CkRRAptMg==} engines: {node: '>=18.0.0'} @@ -3278,6 +3537,10 @@ packages: resolution: {integrity: sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==} engines: {node: '>=0.10.0'} + is-fullwidth-code-point@3.0.0: + resolution: {integrity: sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==} + engines: {node: '>=8'} + is-glob@4.0.3: resolution: {integrity: sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==} engines: {node: '>=0.10.0'} @@ -3567,6 +3830,10 @@ packages: resolution: {integrity: sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==} engines: {node: '>= 8'} + meriyah@6.1.4: + resolution: {integrity: sha512-Sz8FzjzI0kN13GK/6MVEsVzMZEPvOhnmmI1lU5+/1cGOiK3QUahntrNNtdVeihrO7t9JpoH75iMNXg6R6uWflQ==} + engines: {node: '>=18.0.0'} + micromatch@4.0.8: resolution: {integrity: sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==} engines: {node: '>=8.6'} @@ -3614,6 +3881,10 @@ packages: ms@2.1.3: resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} + mustache@4.2.0: + resolution: {integrity: sha512-71ippSywq5Yb7/tVYyGbkBggbU8H3u5Rz56fH60jGFgr8uHwxs+aSKeqmluIVzM0m0kB7xQjKS6qPfd0b2ZoqQ==} + hasBin: true + nanoid@3.3.12: resolution: {integrity: sha512-ZB9RH/39qpq5Vu6Y+NmUaFhQR6pp+M2Xt76XBnEwDaGcVAqhlvxrl3B2bKS5D3NH3QR76v3aSrKaF/Kiy7lEtQ==} engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1} @@ -3650,8 +3921,8 @@ packages: resolution: {integrity: sha512-9qny7Z9DsQU8Ou39ERsPU4OZQlSTP47ShQzuKZ6PRXpYLtIFgl/DEBYEXKlvcEa+9tHVcK8CF81Y2V72qaZhWA==} engines: {node: '>=18'} - nuqs@2.9.2: - resolution: {integrity: sha512-SUrBU5PfLckv2SAquHkYs76s603vbjl2OLEc+jlL3sITwvJDbE/fg3vN7l9Rdp4WIUs6E0QD+m3DfH9QUQfLkg==} + nuqs@2.9.4: + resolution: {integrity: sha512-lsz3NyCOKmuNAyW052i9RWqcTntoYb2Qm6FxSWnkTDwOJnGS6fzpXDAp0VcwTevw3xgnWebYpDr9rm6+o4DHbw==} peerDependencies: '@remix-run/react': '>=2' '@tanstack/react-router': ^1 @@ -4098,6 +4369,9 @@ packages: resolution: {integrity: sha512-vfD3pmTzGpufjScBh50YHKzEu2lxBWhVEHsNGoEXmCmn2hKGfeNLYMzCJpe8cD7gqX7TJluOVpBkAequ6dgMmA==} engines: {node: '>=4'} + semifies@1.0.0: + resolution: {integrity: sha512-xXR3KGeoxTNWPD4aBvL5NUpMTT7WMANr3EWnaS190QVkY52lqqcVRD7Q05UVbBhiWDGWMlJEUam9m7uFFGVScw==} + semver@6.3.1: resolution: {integrity: sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==} hasBin: true @@ -4156,6 +4430,9 @@ packages: resolution: {integrity: sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==} engines: {node: '>=14'} + simple-git@3.36.0: + resolution: {integrity: sha512-cGQjLjK8bxJw4QuYT7gxHw3/IouVESbhahSsHrX97MzCL1gu2u7oy38W6L2ZIGECEfIBG4BabsWDPjBxJENv9Q==} + sisteransi@1.0.5: resolution: {integrity: sha512-bLGGlR1QxBcynn2d5YmDX4MGjlZvy2MRBDRNHLJ8VI6l6+9FUiyTFNJ0IveOSP0bcXgVDPRcfGqA0pjaqUpfVg==} @@ -4167,6 +4444,10 @@ packages: resolution: {integrity: sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==} engines: {node: '>=0.10.0'} + source-map@0.7.6: + resolution: {integrity: sha512-i5uvt8C3ikiWeNZSVZNWcfZPItFQOsYTUAOkcUPGd8DqDy1uOUikjt5dG+uRlwyvR108Fb9DOd4GvXfT0N2/uQ==} + engines: {node: '>= 12'} + split2@4.2.0: resolution: {integrity: sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==} engines: {node: '>= 10.x'} @@ -4194,6 +4475,10 @@ packages: strfy-js@3.2.2: resolution: {integrity: sha512-hUgJ5k2PR1ivhq4uObxnin5j6GcOr0Y0N1lzi3z6SRhxNqu4rzpDfyoC2ToUAyM8yXNXM0zs6f4KIiqj8NqheQ==} + string-width@4.2.3: + resolution: {integrity: sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==} + engines: {node: '>=8'} + string-width@7.2.0: resolution: {integrity: sha512-tsaTIkKW9b4N+AEj+SVA+WhJzV7/zMhcSu78mLKWSk7cXMOSHsBKFWUs0fWwq8QyK3MgJBQRX6Gbi4kYbdvGkQ==} engines: {node: '>=18'} @@ -4252,6 +4537,10 @@ packages: resolution: {integrity: sha512-uxc/zpqFg6x7C8vOE7lh6Lbda8eEL9zmVm/PLeTPBRhh1xCgdWaQ+J1CUieGpIfm2HdtsUpRv+HshiasBMcc6A==} engines: {node: '>=6'} + termi-link@1.1.0: + resolution: {integrity: sha512-2qSN6TnomHgVLtk+htSWbaYs4Rd2MH/RU7VpHTy6MBstyNyWbM4yKd1DCYpE3fDg8dmGWojXCngNi/MHCzGuAA==} + engines: {node: '>=12'} + tiny-invariant@1.3.3: resolution: {integrity: sha512-+FbBPE1o9QAYvviau/qC5SE3caw21q3xkvWKBtja5vgqOWIHHJ3ioaq1VPfn/Szqctz2bU/oYeKd9/z5BL+PVg==} @@ -4347,6 +4636,10 @@ packages: resolution: {integrity: sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==} engines: {node: '>= 0.8'} + unplugin@2.3.11: + resolution: {integrity: sha512-5uKD0nqiYVzlmCRs01Fhs2BdkEgBS3SAVP6ndrBsuK42iC2+JHyxM05Rm9G8+5mkmRtzMZGY8Ct5+mliZxU/Ww==} + engines: {node: '>=18.12.0'} + update-browserslist-db@1.2.3: resolution: {integrity: sha512-Js0m9cx+qOgDxo0eMiFGEueWztz+d4+M3rGlmKPT+T4IS/jP4ylw3Nwpu6cpTTP8R1MAC1kF4VbdLt3ARf209w==} hasBin: true @@ -4382,6 +4675,10 @@ packages: util-deprecate@1.0.2: resolution: {integrity: sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==} + uuid@11.1.1: + resolution: {integrity: sha512-vIYxrBCC/N/K+Js3qSN88go7kIfNPssr/hHCesKCQNAjmgvYS2oqr69kIufEG+O4+PfezOH4EbIeHCfFov8ZgQ==} + hasBin: true + uuid@13.0.2: resolution: {integrity: sha512-vzi9uRZ926x4XV73S/4qQaTwPXM2JBj6/6lI/byHH1jOpCzb0zDbfytgA9LcN/hzb2l7WQSQnxITOVx5un/wGw==} hasBin: true @@ -4479,6 +4776,9 @@ packages: resolution: {integrity: sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==} engines: {node: '>= 8'} + webpack-virtual-modules@0.6.2: + resolution: {integrity: sha512-66/V2i5hQanC51vBQKPH4aI8NMAcBW59FVBs+rC7eGHupMyfn34q7rZIE+ETlJ+XTevqfUhVVBgSUNSW2flEUQ==} + whatwg-mimetype@3.0.0: resolution: {integrity: sha512-nt+N2dzIutVRxARx1nghPKGv1xHikU7HKdfafKkLNLindmPU/ch3U31NOCGGA/dmPcmb1VlofO0vnKAcsm0o/Q==} engines: {node: '>=12'} @@ -4862,6 +5162,30 @@ snapshots: '@biomejs/cli-win32-x64@1.9.4': optional: true + '@braintrust/bt-darwin-arm64@0.12.0': + optional: true + + '@braintrust/bt-darwin-x64@0.12.0': + optional: true + + '@braintrust/bt-linux-arm64@0.12.0': + optional: true + + '@braintrust/bt-linux-x64-musl@0.12.0': + optional: true + + '@braintrust/bt-linux-x64@0.12.0': + optional: true + + '@braintrust/bt-win32-arm64@0.12.0': + optional: true + + '@braintrust/bt-win32-x64@0.12.0': + optional: true + + '@colors/colors@1.5.0': + optional: true + '@commander-js/extra-typings@14.0.0(commander@14.0.3)': dependencies: commander: 14.0.3 @@ -4896,156 +5220,234 @@ snapshots: '@esbuild/aix-ppc64@0.28.0': optional: true + '@esbuild/aix-ppc64@0.28.1': + optional: true + '@esbuild/android-arm64@0.27.7': optional: true '@esbuild/android-arm64@0.28.0': optional: true + '@esbuild/android-arm64@0.28.1': + optional: true + '@esbuild/android-arm@0.27.7': optional: true '@esbuild/android-arm@0.28.0': optional: true + '@esbuild/android-arm@0.28.1': + optional: true + '@esbuild/android-x64@0.27.7': optional: true '@esbuild/android-x64@0.28.0': optional: true + '@esbuild/android-x64@0.28.1': + optional: true + '@esbuild/darwin-arm64@0.27.7': optional: true '@esbuild/darwin-arm64@0.28.0': optional: true + '@esbuild/darwin-arm64@0.28.1': + optional: true + '@esbuild/darwin-x64@0.27.7': optional: true '@esbuild/darwin-x64@0.28.0': optional: true + '@esbuild/darwin-x64@0.28.1': + optional: true + '@esbuild/freebsd-arm64@0.27.7': optional: true '@esbuild/freebsd-arm64@0.28.0': optional: true + '@esbuild/freebsd-arm64@0.28.1': + optional: true + '@esbuild/freebsd-x64@0.27.7': optional: true '@esbuild/freebsd-x64@0.28.0': optional: true + '@esbuild/freebsd-x64@0.28.1': + optional: true + '@esbuild/linux-arm64@0.27.7': optional: true '@esbuild/linux-arm64@0.28.0': optional: true + '@esbuild/linux-arm64@0.28.1': + optional: true + '@esbuild/linux-arm@0.27.7': optional: true '@esbuild/linux-arm@0.28.0': optional: true + '@esbuild/linux-arm@0.28.1': + optional: true + '@esbuild/linux-ia32@0.27.7': optional: true '@esbuild/linux-ia32@0.28.0': optional: true + '@esbuild/linux-ia32@0.28.1': + optional: true + '@esbuild/linux-loong64@0.27.7': optional: true '@esbuild/linux-loong64@0.28.0': optional: true + '@esbuild/linux-loong64@0.28.1': + optional: true + '@esbuild/linux-mips64el@0.27.7': optional: true '@esbuild/linux-mips64el@0.28.0': optional: true + '@esbuild/linux-mips64el@0.28.1': + optional: true + '@esbuild/linux-ppc64@0.27.7': optional: true '@esbuild/linux-ppc64@0.28.0': optional: true + '@esbuild/linux-ppc64@0.28.1': + optional: true + '@esbuild/linux-riscv64@0.27.7': optional: true '@esbuild/linux-riscv64@0.28.0': optional: true + '@esbuild/linux-riscv64@0.28.1': + optional: true + '@esbuild/linux-s390x@0.27.7': optional: true '@esbuild/linux-s390x@0.28.0': optional: true + '@esbuild/linux-s390x@0.28.1': + optional: true + '@esbuild/linux-x64@0.27.7': optional: true '@esbuild/linux-x64@0.28.0': optional: true + '@esbuild/linux-x64@0.28.1': + optional: true + '@esbuild/netbsd-arm64@0.27.7': optional: true '@esbuild/netbsd-arm64@0.28.0': optional: true + '@esbuild/netbsd-arm64@0.28.1': + optional: true + '@esbuild/netbsd-x64@0.27.7': optional: true '@esbuild/netbsd-x64@0.28.0': optional: true + '@esbuild/netbsd-x64@0.28.1': + optional: true + '@esbuild/openbsd-arm64@0.27.7': optional: true '@esbuild/openbsd-arm64@0.28.0': optional: true + '@esbuild/openbsd-arm64@0.28.1': + optional: true + '@esbuild/openbsd-x64@0.27.7': optional: true '@esbuild/openbsd-x64@0.28.0': optional: true + '@esbuild/openbsd-x64@0.28.1': + optional: true + '@esbuild/openharmony-arm64@0.27.7': optional: true '@esbuild/openharmony-arm64@0.28.0': optional: true + '@esbuild/openharmony-arm64@0.28.1': + optional: true + '@esbuild/sunos-x64@0.27.7': optional: true '@esbuild/sunos-x64@0.28.0': optional: true + '@esbuild/sunos-x64@0.28.1': + optional: true + '@esbuild/win32-arm64@0.27.7': optional: true '@esbuild/win32-arm64@0.28.0': optional: true + '@esbuild/win32-arm64@0.28.1': + optional: true + '@esbuild/win32-ia32@0.27.7': optional: true '@esbuild/win32-ia32@0.28.0': optional: true + '@esbuild/win32-ia32@0.28.1': + optional: true + '@esbuild/win32-x64@0.27.7': optional: true '@esbuild/win32-x64@0.28.0': optional: true + '@esbuild/win32-x64@0.28.1': + optional: true + '@eslint-community/eslint-utils@4.9.1(eslint@9.39.4(jiti@2.7.0))': dependencies: eslint: 9.39.4(jiti@2.7.0) @@ -5154,6 +5556,14 @@ snapshots: '@jridgewell/resolve-uri': 3.1.2 '@jridgewell/sourcemap-codec': 1.5.5 + '@kwsites/file-exists@1.1.1': + dependencies: + debug: 4.4.3(supports-color@10.2.2) + transitivePeerDependencies: + - supports-color + + '@kwsites/promise-deferred@1.1.1': {} + '@launchql/protobufjs@7.2.6': dependencies: '@protobufjs/aspromise': 1.1.2 @@ -5224,6 +5634,8 @@ snapshots: transitivePeerDependencies: - supports-color + '@next/env@14.2.35': {} + '@noble/ciphers@1.3.0': {} '@noble/curves@1.9.7': @@ -6205,6 +6617,12 @@ snapshots: '@opentelemetry/semantic-conventions': 1.41.1 '@sentry/core': 10.57.0 + '@simple-git/args-pathspec@1.0.3': {} + + '@simple-git/argv-parser@1.1.1': + dependencies: + '@simple-git/args-pathspec': 1.0.3 + '@sindresorhus/merge-streams@4.0.0': {} '@stablelib/base64@1.0.1': {} @@ -6574,6 +6992,8 @@ snapshots: '@vercel/detect-agent@1.2.3': {} + '@vercel/functions@1.6.0': {} + '@vercel/oidc@3.2.0': {} '@vitejs/plugin-react@5.2.0(vite@7.3.5(@types/node@22.19.20)(jiti@2.7.0)(lightningcss@1.32.0)(tsx@4.22.4))': @@ -6714,6 +7134,8 @@ snapshots: dependencies: tslib: 2.8.1 + astring@1.9.0: {} + balanced-match@1.0.2: {} balanced-match@4.0.4: {} @@ -6753,6 +7175,51 @@ snapshots: dependencies: fill-range: 7.1.1 + braintrust@3.24.0(zod@4.4.3): + dependencies: + '@next/env': 14.2.35 + '@vercel/functions': 1.6.0 + acorn: 8.16.0 + acorn-import-attributes: 1.9.5(acorn@8.16.0) + ajv: 8.20.0 + argparse: 2.0.1 + astring: 1.9.0 + cjs-module-lexer: 2.2.0 + cli-progress: 3.12.0 + cli-table3: 0.6.5 + cors: 2.8.6 + dc-browser: 1.0.4 + dotenv: 16.6.1 + esbuild: 0.28.1 + esquery: 1.7.0 + eventsource-parser: 1.1.2 + express: 5.2.1 + http-errors: 2.0.1 + meriyah: 6.1.4 + minimatch: 10.2.5 + module-details-from-path: 1.0.4 + mustache: 4.2.0 + pluralize: 8.0.0 + semifies: 1.0.0 + simple-git: 3.36.0 + source-map: 0.7.6 + termi-link: 1.1.0 + unplugin: 2.3.11 + uuid: 11.1.1 + zod: 4.4.3 + zod-to-json-schema: 3.25.2(zod@4.4.3) + optionalDependencies: + '@braintrust/bt-darwin-arm64': 0.12.0 + '@braintrust/bt-darwin-x64': 0.12.0 + '@braintrust/bt-linux-arm64': 0.12.0 + '@braintrust/bt-linux-x64': 0.12.0 + '@braintrust/bt-linux-x64-musl': 0.12.0 + '@braintrust/bt-win32-arm64': 0.12.0 + '@braintrust/bt-win32-x64': 0.12.0 + transitivePeerDependencies: + - '@aws-sdk/credential-provider-web-identity' + - supports-color + browserslist@4.28.2: dependencies: baseline-browser-mapping: 2.10.35 @@ -6812,8 +7279,18 @@ snapshots: dependencies: restore-cursor: 5.1.0 + cli-progress@3.12.0: + dependencies: + string-width: 4.2.3 + cli-spinners@2.9.2: {} + cli-table3@0.6.5: + dependencies: + string-width: 4.2.3 + optionalDependencies: + '@colors/colors': 1.5.0 + clsx@2.1.1: {} code-block-writer@13.0.3: {} @@ -6874,6 +7351,8 @@ snapshots: data-uri-to-buffer@4.0.1: {} + dc-browser@1.0.4: {} + debug@4.4.3(supports-color@10.2.2): dependencies: ms: 2.1.3 @@ -6909,6 +7388,8 @@ snapshots: dom-accessibility-api@0.6.3: {} + dotenv@16.6.1: {} + dotenv@17.4.2: {} dunder-proto@1.0.1: @@ -6930,6 +7411,8 @@ snapshots: emoji-regex@10.6.0: {} + emoji-regex@8.0.0: {} + encodeurl@2.0.0: {} enhanced-resolve@5.23.0: @@ -7018,6 +7501,35 @@ snapshots: '@esbuild/win32-ia32': 0.28.0 '@esbuild/win32-x64': 0.28.0 + esbuild@0.28.1: + optionalDependencies: + '@esbuild/aix-ppc64': 0.28.1 + '@esbuild/android-arm': 0.28.1 + '@esbuild/android-arm64': 0.28.1 + '@esbuild/android-x64': 0.28.1 + '@esbuild/darwin-arm64': 0.28.1 + '@esbuild/darwin-x64': 0.28.1 + '@esbuild/freebsd-arm64': 0.28.1 + '@esbuild/freebsd-x64': 0.28.1 + '@esbuild/linux-arm': 0.28.1 + '@esbuild/linux-arm64': 0.28.1 + '@esbuild/linux-ia32': 0.28.1 + '@esbuild/linux-loong64': 0.28.1 + '@esbuild/linux-mips64el': 0.28.1 + '@esbuild/linux-ppc64': 0.28.1 + '@esbuild/linux-riscv64': 0.28.1 + '@esbuild/linux-s390x': 0.28.1 + '@esbuild/linux-x64': 0.28.1 + '@esbuild/netbsd-arm64': 0.28.1 + '@esbuild/netbsd-x64': 0.28.1 + '@esbuild/openbsd-arm64': 0.28.1 + '@esbuild/openbsd-x64': 0.28.1 + '@esbuild/openharmony-arm64': 0.28.1 + '@esbuild/sunos-x64': 0.28.1 + '@esbuild/win32-arm64': 0.28.1 + '@esbuild/win32-ia32': 0.28.1 + '@esbuild/win32-x64': 0.28.1 + escalade@3.2.0: {} escape-html@1.0.3: {} @@ -7117,6 +7629,8 @@ snapshots: etag@1.8.1: {} + eventsource-parser@1.1.2: {} + eventsource-parser@3.1.0: {} eventsource@3.0.7: @@ -7469,6 +7983,8 @@ snapshots: is-extglob@2.1.1: {} + is-fullwidth-code-point@3.0.0: {} + is-glob@4.0.3: dependencies: is-extglob: 2.1.1 @@ -7674,6 +8190,8 @@ snapshots: merge2@1.4.1: {} + meriyah@6.1.4: {} + micromatch@4.0.8: dependencies: braces: 3.0.3 @@ -7711,6 +8229,8 @@ snapshots: ms@2.1.3: {} + mustache@4.2.0: {} + nanoid@3.3.12: {} natural-compare@1.4.0: {} @@ -7738,7 +8258,7 @@ snapshots: path-key: 4.0.0 unicorn-magic: 0.3.0 - nuqs@2.9.2(react@19.2.7): + nuqs@2.9.4(react@19.2.7): dependencies: '@standard-schema/spec': 1.1.0 react: 19.2.7 @@ -8202,6 +8722,8 @@ snapshots: extend-shallow: 2.0.1 kind-of: 6.0.3 + semifies@1.0.0: {} + semver@6.3.1: {} semver@7.8.3: {} @@ -8314,12 +8836,24 @@ snapshots: signal-exit@4.1.0: {} + simple-git@3.36.0: + dependencies: + '@kwsites/file-exists': 1.1.1 + '@kwsites/promise-deferred': 1.1.1 + '@simple-git/args-pathspec': 1.0.3 + '@simple-git/argv-parser': 1.1.1 + debug: 4.4.3(supports-color@10.2.2) + transitivePeerDependencies: + - supports-color + sisteransi@1.0.5: {} source-map-js@1.2.1: {} source-map@0.6.1: {} + source-map@0.7.6: {} + split2@4.2.0: {} sprintf-js@1.0.3: {} @@ -8341,6 +8875,12 @@ snapshots: dependencies: minimatch: 10.2.5 + string-width@4.2.3: + dependencies: + emoji-regex: 8.0.0 + is-fullwidth-code-point: 3.0.0 + strip-ansi: 6.0.1 + string-width@7.2.0: dependencies: emoji-regex: 10.6.0 @@ -8387,6 +8927,8 @@ snapshots: tapable@2.3.3: {} + termi-link@1.1.0: {} + tiny-invariant@1.3.3: {} tinybench@2.9.0: {} @@ -8468,6 +9010,13 @@ snapshots: unpipe@1.0.0: {} + unplugin@2.3.11: + dependencies: + '@jridgewell/remapping': 2.3.5 + acorn: 8.16.0 + picomatch: 4.0.4 + webpack-virtual-modules: 0.6.2 + update-browserslist-db@1.2.3(browserslist@4.28.2): dependencies: browserslist: 4.28.2 @@ -8497,6 +9046,8 @@ snapshots: util-deprecate@1.0.2: {} + uuid@11.1.1: {} + uuid@13.0.2: {} validate-npm-package-name@7.0.2: {} @@ -8593,6 +9144,8 @@ snapshots: web-streams-polyfill@3.3.3: {} + webpack-virtual-modules@0.6.2: {} + whatwg-mimetype@3.0.0: {} which@2.0.2: From 4cc42595336b0ab03a77305ff30e466c94991999 Mon Sep 17 00:00:00 2001 From: Pedro Rodrigues Date: Tue, 21 Jul 2026 13:20:03 +0100 Subject: [PATCH 02/15] fix: address review findings on Braintrust upload MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Guard transcript↔toolCalls pairing by tool name (ai-sdk completion order can differ from call order under concurrent tools) - Normalize Claude Code inputTokens to include cache reads/writes so token counts compare across agents (OpenAI-style accounting) - Tolerate corrupt raw JSON files: skip + warn instead of aborting - Load experiment metadata once; hoist repeated lookups Co-Authored-By: Claude Fable 5 --- apps/framework/scripts/upload-braintrust.ts | 38 ++++++++++++------- .../src/agents/claude-code/runner.test.ts | 4 +- .../core/src/agents/claude-code/runner.ts | 17 +++++++-- packages/core/src/braintrust-spans.ts | 11 ++++-- 4 files changed, 50 insertions(+), 20 deletions(-) diff --git a/apps/framework/scripts/upload-braintrust.ts b/apps/framework/scripts/upload-braintrust.ts index 66899848..df206ff7 100644 --- a/apps/framework/scripts/upload-braintrust.ts +++ b/apps/framework/scripts/upload-braintrust.ts @@ -100,7 +100,9 @@ interface PendingTrace { filePath: string; } -async function collectTraces(): Promise> { +async function collectTraces( + experimentMetadata: Awaited>, +): Promise> { const refs = await collectResultFiles({ includeExperiment: (experiment) => EXPERIMENT_FILTERS.length === 0 || @@ -117,14 +119,19 @@ async function collectTraces(): Promise> { return promptCache.get(evalId); }; - const experimentMetadata = await loadExperimentMetadata(); const byExperiment = new Map(); let skipped = 0; for (const ref of refs) { - const parsed = uploadableEvalResultSchema.safeParse( - JSON.parse(await readFile(ref.filePath, "utf8")), - ); + let json: unknown; + try { + json = JSON.parse(await readFile(ref.filePath, "utf8")); + } catch { + skipped += 1; + console.warn(`SKIP ${ref.sourcePath}: not valid JSON`); + continue; + } + const parsed = uploadableEvalResultSchema.safeParse(json); if (!parsed.success) { skipped += 1; console.warn(`SKIP ${ref.sourcePath}: does not match the result schema`); @@ -197,7 +204,10 @@ function printDry(byExperiment: Map): void { } } -async function upload(byExperiment: Map) { +async function upload( + byExperiment: Map, + experimentMetadata: Awaited>, +) { // Imported lazily so --dry works without braintrust needing credentials. const { init, Attachment, flush } = await import("braintrust"); @@ -205,10 +215,11 @@ async function upload(byExperiment: Map) { const projectId = process.env.BRAINTRUST_PROJECT_ID; const sha = gitShortSha(); const suffix = NAME_SUFFIX ? `-${NAME_SUFFIX}` : ""; - const experimentMetadata = await loadExperimentMetadata(); + const runUrl = ciRunUrl(); for (const [experiment, pending] of byExperiment) { const name = `${experiment}${sha ? `@${sha}` : ""}${suffix}`; + const meta = experimentMetadata.get(experiment); const btExperiment = init({ ...(projectName ? { project: projectName } : {}), ...(projectId ? { projectId } : {}), @@ -217,12 +228,12 @@ async function upload(byExperiment: Map) { metadata: { source: "supabase-evals", experiment, - ...(experimentMetadata.get(experiment)?.experimentSuite - ? { experimentSuite: experimentMetadata.get(experiment)?.experimentSuite } + ...(meta?.experimentSuite + ? { experimentSuite: meta.experimentSuite } : {}), - ...(experimentMetadata.get(experiment)?.display ?? {}), + ...(meta?.display ?? {}), ...(sha ? { gitShortSha: sha } : {}), - ...(ciRunUrl() ? { ciRunUrl: ciRunUrl() } : {}), + ...(runUrl ? { ciRunUrl: runUrl } : {}), }, }); @@ -278,7 +289,8 @@ async function upload(byExperiment: Map) { } async function main() { - const byExperiment = await collectTraces(); + const experimentMetadata = await loadExperimentMetadata(); + const byExperiment = await collectTraces(experimentMetadata); const total = [...byExperiment.values()].reduce((n, t) => n + t.length, 0); if (total === 0) { console.log("No result files matched — nothing to upload."); @@ -300,7 +312,7 @@ async function main() { return; } - await upload(byExperiment); + await upload(byExperiment, experimentMetadata); console.log(`Done: ${total} eval trace(s) across ${byExperiment.size} experiment(s).`); } diff --git a/packages/core/src/agents/claude-code/runner.test.ts b/packages/core/src/agents/claude-code/runner.test.ts index b2fcf3f6..0c77a015 100644 --- a/packages/core/src/agents/claude-code/runner.test.ts +++ b/packages/core/src/agents/claude-code/runner.test.ts @@ -75,7 +75,9 @@ describe("claudeCodeRunner.extractUsage", () => { }), ].join("\n"); expect(extract(raw)).toEqual({ - inputTokens: 10, + // 10 raw + 3000 cache-read + 200 cache-creation: OpenAI-style totals + // so counts compare across agents. + inputTokens: 3210, outputTokens: 40, cachedInputTokens: 3000, cacheCreationInputTokens: 200, diff --git a/packages/core/src/agents/claude-code/runner.ts b/packages/core/src/agents/claude-code/runner.ts index 38df4682..8b60dd77 100644 --- a/packages/core/src/agents/claude-code/runner.ts +++ b/packages/core/src/agents/claude-code/runner.ts @@ -108,11 +108,22 @@ export const claudeCodeRunner: AgentRunner = { const num = (value: unknown): number | undefined => typeof value === 'number' && Number.isFinite(value) ? value : undefined; + // Anthropic's `input_tokens` excludes cache reads/writes; report + // OpenAI-style totals (cached ⊆ input) so token counts compare across + // agents. The cache splits are still reported separately. + const rawInput = num(usage?.input_tokens); + const cacheRead = num(usage?.cache_read_input_tokens); + const cacheCreation = num(usage?.cache_creation_input_tokens); + const inputTokens = + rawInput === undefined && cacheRead === undefined && cacheCreation === undefined + ? undefined + : (rawInput ?? 0) + (cacheRead ?? 0) + (cacheCreation ?? 0); + const extracted: AgentUsage = { - inputTokens: num(usage?.input_tokens), + inputTokens, outputTokens: num(usage?.output_tokens), - cachedInputTokens: num(usage?.cache_read_input_tokens), - cacheCreationInputTokens: num(usage?.cache_creation_input_tokens), + cachedInputTokens: cacheRead, + cacheCreationInputTokens: cacheCreation, costUsd: num(result.total_cost_usd), }; return Object.values(extracted).some((v) => v !== undefined) diff --git a/packages/core/src/braintrust-spans.ts b/packages/core/src/braintrust-spans.ts index 5f3f26ae..58c2d3f6 100644 --- a/packages/core/src/braintrust-spans.ts +++ b/packages/core/src/braintrust-spans.ts @@ -197,8 +197,8 @@ export function buildEvalTrace(args: { const spans: BraintrustSpanNode[] = []; // The k-th tool_call transcript part and toolCalls[k] describe the same - // call (the adapter pushes both in lockstep); the record carries the - // normalized views the part lacks. + // call for CLI agents (the adapter pushes both in lockstep); the record + // carries the normalized views the part lacks. let toolCallIndex = 0; for (const part of transcript) { const timing = slot(spans.length); @@ -211,8 +211,13 @@ export function buildEvalTrace(args: { }); continue; } - const record = toolCalls.at(toolCallIndex); + const candidate = toolCalls.at(toolCallIndex); toolCallIndex += 1; + // ai-sdk populates toolCalls in completion order, which can differ from + // the transcript's call order when tools run concurrently — only trust + // the record when it names the same tool. + const record = + candidate && candidate.endpoint === part.name ? candidate : undefined; const normalized: Record = {}; if (record?.name) normalized.tool = record.name; if (record?.path) normalized.path = record.path; From bc354a4271271a2ceda3e8dda23ea1ca6441c234 Mon Sep 17 00:00:00 2001 From: Pedro Rodrigues Date: Tue, 21 Jul 2026 15:42:05 +0100 Subject: [PATCH 03/15] feat: overall passed verdict as a score span in Braintrust traces Trace-view counterpart of eval-results.json's headline passed flag, emitted ahead of the per-check score spans with a checksPassed summary. Co-Authored-By: Claude Fable 5 --- packages/core/src/braintrust-spans.test.ts | 14 ++++++++++---- packages/core/src/braintrust-spans.ts | 15 ++++++++++++++- 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/packages/core/src/braintrust-spans.test.ts b/packages/core/src/braintrust-spans.test.ts index ef1eb602..4a215e7f 100644 --- a/packages/core/src/braintrust-spans.test.ts +++ b/packages/core/src/braintrust-spans.test.ts @@ -92,16 +92,20 @@ describe("buildEvalTrace", () => { }); }); - it("emits spans in transcript order, then score spans", () => { + it("emits spans in transcript order, then the verdict and check score spans", () => { expect(trace.spans.map((s) => `${s.type}:${s.name}`)).toEqual([ "tool:Skill", "task:user", "llm:assistant", "tool:Bash", "llm:assistant", + "score:passed", "score:named the vulnerable table", "score:proposed a concrete fix", ]); + const verdict = trace.spans.find((s) => s.name === "passed"); + expect(verdict?.scores).toEqual({ passed: 1 }); + expect(verdict?.output).toEqual({ passed: true, checksPassed: "1/2" }); }); it("pairs tool spans with the normalized toolCalls record", () => { @@ -174,9 +178,11 @@ describe("buildEvalTrace", () => { expect(legacy.input).toEqual({ eval: "legacy-eval" }); expect(legacy.output).toBeUndefined(); expect(legacy.startMs).toBe(BASE); - expect(legacy.endMs).toBe(BASE + 1000); - expect(legacy.spans).toHaveLength(1); - expect(legacy.spans[0]!.type).toBe("score"); + expect(legacy.endMs).toBe(BASE + 2000); + expect(legacy.spans.map((s) => `${s.type}:${s.name}`)).toEqual([ + "score:passed", + "score:a check", + ]); expect(legacy.metrics).toEqual({}); }); diff --git a/packages/core/src/braintrust-spans.ts b/packages/core/src/braintrust-spans.ts index 58c2d3f6..f6065591 100644 --- a/packages/core/src/braintrust-spans.ts +++ b/packages/core/src/braintrust-spans.ts @@ -184,7 +184,8 @@ export function buildEvalTrace(args: { const transcript = result.transcript ?? []; const toolCalls = result.toolCalls ?? []; const checks = result.checks ?? []; - const spanCount = transcript.length + checks.length; + // +1 for the overall `passed` score span emitted ahead of the checks. + const spanCount = transcript.length + checks.length + 1; // Real window when the run recorded one; otherwise one synthetic second per // span, purely so ordering survives. const durationMs = result.durationMs ?? Math.max(spanCount, 1) * 1000; @@ -239,6 +240,18 @@ export function buildEvalTrace(args: { passed: result.passed === true ? 1 : 0, }; const takenScoreNames = new Set(Object.keys(scores)); + // The overall verdict as its own score span — the trace-view counterpart of + // eval-results.json's headline `passed`, ahead of the per-check spans. + spans.push({ + name: "passed", + type: "score", + output: { + passed: result.passed === true, + checksPassed: `${checks.filter((c) => c.passed).length}/${checks.length}`, + }, + scores: { passed: scores.passed! }, + ...slot(spans.length), + }); for (const check of checks) { const scoreName = scoreNameFor(check.name, takenScoreNames); const value = check.passed ? 1 : 0; From be011191dc59dc68d6101d49aee2bf5af4bb1657 Mon Sep 17 00:00:00 2001 From: Pedro Rodrigues Date: Tue, 21 Jul 2026 16:02:50 +0100 Subject: [PATCH 04/15] feat: structured AgentTrace transcript + hierarchical Braintrust spans Agent-agnostic turn model assembled from the canonical events: one turn per model API call with tool executions paired inside it, per-turn token usage and message ids (Claude Code stamps them from each assistant stream-json line), subagent sidechains nested under their Task call via parent_tool_use_id, codex grouped by closure rule, ai-sdk turns built straight from steps. Additive: the flat transcript/toolCalls scorer surface and eval-results.json are unchanged; raw files gain a trace field the Braintrust uploader prefers (llm turn spans owning their tool spans) with the flat mapping kept for older results. Co-Authored-By: Claude Fable 5 --- apps/framework/harness/run-eval.ts | 8 + apps/framework/harness/types.ts | 1 + apps/framework/scripts/upload-braintrust.ts | 14 +- .../core/src/agents/claude-code/parser.ts | 52 +++- packages/core/src/agents/engine.ts | 2 + packages/core/src/braintrust-spans.test.ts | 81 ++++++ packages/core/src/braintrust-spans.ts | 234 +++++++++++++----- packages/core/src/index.ts | 63 +++++ .../core/src/transcript/agent-trace.test.ts | 171 +++++++++++++ packages/core/src/transcript/agent-trace.ts | 233 +++++++++++++++++ packages/core/src/transcript/types.ts | 17 ++ 11 files changed, 813 insertions(+), 63 deletions(-) create mode 100644 packages/core/src/transcript/agent-trace.test.ts create mode 100644 packages/core/src/transcript/agent-trace.ts diff --git a/apps/framework/harness/run-eval.ts b/apps/framework/harness/run-eval.ts index 4b05e0ee..ddbe9417 100644 --- a/apps/framework/harness/run-eval.ts +++ b/apps/framework/harness/run-eval.ts @@ -34,6 +34,7 @@ import { getExperimentDisplayMetadata, } from '@supabase-evals/core'; import type { + AgentTrace, AgentUsage, ExperimentConfig, EvalInterface, @@ -363,6 +364,7 @@ async function runOne( stoppedReason: string; steps: number; usage?: AgentUsage; + trace?: AgentTrace; } > { const prompt = parseEvalMarkdown( @@ -393,6 +395,7 @@ async function runOne( let lastStoppedReason = 'not_started'; let lastSteps = 0; let lastUsage: AgentUsage | undefined; + let lastTrace: AgentTrace | undefined; for (let attempt = 1; attempt <= RUNS; attempt += 1) { if (ev.mode === 'local-stack') { @@ -464,6 +467,7 @@ async function runOne( lastStoppedReason = run.stoppedReason; lastSteps = run.steps; lastUsage = run.usage; + lastTrace = run.trace; // Export the agent's workspace to the host so scorers can run host // tooling (vite/vitest from the repo root) against the produced files @@ -504,6 +508,7 @@ async function runOne( stoppedReason: run.stoppedReason, steps: run.steps, usage: run.usage, + trace: run.trace, }; } logRetryAttempt(expName, ev, attempt, last); @@ -555,6 +560,7 @@ async function runOne( lastStoppedReason = run.stoppedReason; lastSteps = run.steps; lastUsage = run.usage; + lastTrace = run.trace; last = await (scorer as ToolScorer)({ ...session.scoringContext, toolCalls: run.toolCalls, @@ -574,6 +580,7 @@ async function runOne( stoppedReason: run.stoppedReason, steps: run.steps, usage: run.usage, + trace: run.trace, }; } logRetryAttempt(expName, ev, attempt, last); @@ -590,6 +597,7 @@ async function runOne( stoppedReason: lastStoppedReason, steps: lastSteps, usage: lastUsage, + trace: lastTrace, }; } diff --git a/apps/framework/harness/types.ts b/apps/framework/harness/types.ts index 0930acf0..c4586768 100644 --- a/apps/framework/harness/types.ts +++ b/apps/framework/harness/types.ts @@ -6,6 +6,7 @@ import type { } from '@supabase-evals/core/eval-metadata'; export type { + AgentTrace, AgentUsage, ScoreResult, CheckResult, diff --git a/apps/framework/scripts/upload-braintrust.ts b/apps/framework/scripts/upload-braintrust.ts index df206ff7..8415cb3a 100644 --- a/apps/framework/scripts/upload-braintrust.ts +++ b/apps/framework/scripts/upload-braintrust.ts @@ -260,8 +260,11 @@ async function upload( : {}), ...(trace.tags.length > 0 ? { tags: trace.tags } : {}), }); - for (const span of trace.spans) { - const child = root.startSpan({ + const logSpanTree = ( + parent: typeof root, + span: (typeof trace.spans)[number], + ): void => { + const child = parent.startSpan({ name: span.name, type: span.type, startTime: span.startMs / 1000, @@ -271,9 +274,16 @@ async function upload( ...(span.output !== undefined ? { output: span.output } : {}), ...(span.error !== undefined ? { error: span.error } : {}), ...(span.metadata ? { metadata: span.metadata } : {}), + ...(span.metrics ? { metrics: span.metrics } : {}), ...(span.scores ? { scores: span.scores } : {}), }); + for (const grandchild of span.children ?? []) { + logSpanTree(child, grandchild); + } child.end({ endTime: span.endMs / 1000 }); + }; + for (const span of trace.spans) { + logSpanTree(root, span); } root.end({ endTime: trace.endMs / 1000 }); } diff --git a/packages/core/src/agents/claude-code/parser.ts b/packages/core/src/agents/claude-code/parser.ts index d0aef98a..b55cbf86 100644 --- a/packages/core/src/agents/claude-code/parser.ts +++ b/packages/core/src/agents/claude-code/parser.ts @@ -165,6 +165,52 @@ function loadedSkillsFromClaudeCodeCall( return []; } +/** + * Per-API-call context stamped onto every event a line produces, so the + * trace assembler can group a line's message/thinking/tool_use blocks into + * one turn, attach the call's own token usage, and nest subagent sidechains + * (`parent_tool_use_id`) under their Task call. + */ +function lineContext( + data: Record, + lineIndex: number, +): Pick { + const context: ReturnType = {}; + const parent = data.parent_tool_use_id; + if (typeof parent === "string") context.parentToolUseId = parent; + + if (data.type !== "assistant" && data.role !== "assistant") return context; + const message = isRecord(data.message) ? data.message : undefined; + const messageId = typeof message?.id === "string" ? message.id : undefined; + // One assistant stream-json line = one API message; the id groups a + // message's blocks (and any continuation lines for the same id) into one + // turn. Lines without an id still get a unique per-line key. + context.turnKey = messageId ?? `line-${lineIndex}`; + if (messageId) context.messageId = messageId; + + const usage = isRecord(message?.usage) ? message.usage : undefined; + if (usage) { + const num = (value: unknown): number | undefined => + typeof value === "number" && Number.isFinite(value) ? value : undefined; + const rawInput = num(usage.input_tokens); + const cacheRead = num(usage.cache_read_input_tokens); + const cacheCreation = num(usage.cache_creation_input_tokens); + const outputTokens = num(usage.output_tokens); + if (rawInput !== undefined || cacheRead !== undefined || cacheCreation !== undefined || outputTokens !== undefined) { + context.usage = { + // OpenAI-style totals (cached ⊆ input), matching extractUsage. + inputTokens: (rawInput ?? 0) + (cacheRead ?? 0) + (cacheCreation ?? 0), + ...(outputTokens !== undefined ? { outputTokens } : {}), + ...(cacheRead !== undefined ? { cachedInputTokens: cacheRead } : {}), + ...(cacheCreation !== undefined + ? { cacheCreationInputTokens: cacheCreation } + : {}), + }; + } + } + return context; +} + function recordToEvents(data: Record): TranscriptEvent[] { const events: TranscriptEvent[] = []; const timestamp = @@ -289,9 +335,11 @@ export const claudeCodeParser: AgentTranscriptParser = { const { records, errors } = parseJsonlRecords(raw); const events: TranscriptEvent[] = []; let lastAssistantText: string | undefined; - for (const record of records) { + for (const [lineIndex, record] of records.entries()) { try { - for (const event of recordToEvents(record)) { + const context = lineContext(record, lineIndex); + for (const bare of recordToEvents(record)) { + const event: TranscriptEvent = { ...context, ...bare }; const isAssistantMessage = event.type === 'message' && event.role === 'assistant'; // The terminal `result` line repeats the final assistant message that diff --git a/packages/core/src/agents/engine.ts b/packages/core/src/agents/engine.ts index a47d4571..0781e7b2 100644 --- a/packages/core/src/agents/engine.ts +++ b/packages/core/src/agents/engine.ts @@ -22,6 +22,7 @@ import type { AgentHarness, AgentRunResult } from '../index.js'; import type { ModelProvider, ReasoningEffortLevel } from '../eval-metadata.js'; import { adaptTranscript } from '../parsers/adapt.js'; +import { assembleAgentTrace } from '../transcript/agent-trace.js'; import type { AgentTranscriptParser } from '../parsers/types.js'; import type { AgentRunner } from './types.js'; import { @@ -133,6 +134,7 @@ export function createCliAgent( stoppedReason: runner.deriveStopReason?.(raw, command) ?? processStopReason(command), ...(usage ? { usage } : {}), + trace: assembleAgentTrace(events), }; }, }; diff --git a/packages/core/src/braintrust-spans.test.ts b/packages/core/src/braintrust-spans.test.ts index 4a215e7f..1c1aac27 100644 --- a/packages/core/src/braintrust-spans.test.ts +++ b/packages/core/src/braintrust-spans.test.ts @@ -210,6 +210,87 @@ describe("buildEvalTrace", () => { expect(clashing.scores["same (2)"]).toBe(0); }); + it("builds hierarchical spans from a structured trace", () => { + const structured = buildEvalTrace({ + experiment: "x", + result: parse({ + experiment: "x", + eval: "y", + passed: true, + checks: [{ name: "a check", passed: true }], + startedAt: "2026-07-21T09:00:00.000Z", + durationMs: 40_000, + trace: { + turns: [ + { + index: 0, + messageId: "msg_1", + usage: { inputTokens: 100, outputTokens: 10 }, + thinking: "hmm", + text: "Looking.", + toolCalls: [ + { + name: "Bash", + canonicalName: "shell", + args: { command: "ls" }, + command: "ls", + output: "files", + }, + { + name: "Task", + canonicalName: "agent_task", + args: { prompt: "explore" }, + children: [ + { + index: 0, + toolCalls: [ + { name: "Read", canonicalName: "file_read", args: {}, output: "code" }, + ], + }, + ], + }, + ], + }, + { index: 1, text: "Done.", toolCalls: [] }, + ], + interjections: [{ afterTurnIndex: 0, role: "user", content: "tool says hi" }], + errors: [], + }, + }), + baseTimeMs: BASE, + }); + + expect(structured.spans.map((s) => `${s.type}:${s.name}`)).toEqual([ + "llm:turn 1", + "task:user", + "llm:turn 2", + "score:passed", + "score:a check", + ]); + const turn1 = structured.spans[0]!; + expect(turn1.metrics).toEqual({ + prompt_tokens: 100, + completion_tokens: 10, + tokens: 110, + }); + expect(turn1.metadata).toEqual({ messageId: "msg_1" }); + expect(turn1.output).toEqual({ thinking: "hmm", text: "Looking." }); + expect(turn1.children!.map((s) => `${s.type}:${s.name}`)).toEqual([ + "tool:Bash", + "tool:Task", + ]); + const task = turn1.children![1]!; + expect(task.children!.map((s) => `${s.type}:${s.name}`)).toEqual(["llm:turn 1"]); + expect(task.children![0]!.children!.map((s) => s.name)).toEqual(["Read"]); + + // Children subdivide their parent's slot; everything stays inside the window. + const start = Date.parse("2026-07-21T09:00:00.000Z"); + expect(turn1.startMs).toBe(start); + expect(task.startMs).toBeGreaterThanOrEqual(turn1.startMs); + expect(task.endMs).toBeLessThanOrEqual(turn1.endMs); + expect(structured.spans.at(-1)!.endMs).toBeCloseTo(start + 40_000, 5); + }); + it("truncates oversized tool outputs", () => { const big = buildEvalTrace({ experiment: "x", diff --git a/packages/core/src/braintrust-spans.ts b/packages/core/src/braintrust-spans.ts index f6065591..8f53dc86 100644 --- a/packages/core/src/braintrust-spans.ts +++ b/packages/core/src/braintrust-spans.ts @@ -1,25 +1,31 @@ /** * Pure mapping from a raw eval-result file to a Braintrust-shaped trace: one - * root ("eval") row per result plus an ordered list of child spans — `llm` - * for assistant turns, `tool` for tool calls, `task` for user/system - * messages, `score` for scorer checks. + * root ("eval") row per result plus child spans — `llm` for assistant turns, + * `tool` for tool calls, `task` for user/system messages, `score` for the + * overall verdict and each scorer check. + * + * When the result carries the structured `trace` (AgentTrace, written by + * runs since it landed), spans are hierarchical: turn llm spans own their + * tool spans, and subagent sidechains nest under the tool call that spawned + * them. Older results fall back to a flat sequence rebuilt from + * `transcript[]`/`toolCalls[]`. * * Deliberately dependency-free (no braintrust import): this module only * builds a serializable tree; `apps/framework/scripts/upload-braintrust.ts` * walks it with the Braintrust SDK. That keeps the mapping unit-testable and - * identical across agents (Claude Code, Codex, ai-sdk), since it reads the - * canonical transcript the harness already persists. + * identical across agents (Claude Code, Codex, ai-sdk). * - * Timing: CLI transcripts carry no per-event timestamps (every persisted - * `toolCalls[].ts` is 0 today), so child spans get synthetic, evenly spread - * times across the run window purely to preserve ordering in the Braintrust - * UI. The root advertises this via `metadata.stepTiming = - * "synthetic-order-only"`. The window itself is real when the result carries - * `startedAt`/`durationMs` (new runs), else anchored at `baseTimeMs`. + * Timing: CLI transcripts carry no per-event timestamps, so spans get + * synthetic, evenly spread times across the run window purely to preserve + * ordering in the Braintrust UI (children subdivide their parent's slot). + * The root advertises this via `metadata.stepTiming = "synthetic-order-only"`. + * The window itself is real when the result carries `startedAt`/`durationMs` + * (new runs), else anchored at `baseTimeMs`. */ import { z } from "zod"; import { rawEvalResultSchema } from "./eval-metadata.js"; +import type { AgentTrace, AgentTurn, ToolExecution } from "./transcript/agent-trace.js"; const transcriptPartSchema = z.discriminatedUnion("type", [ z.object({ @@ -61,6 +67,16 @@ const agentUsageSchema = z }) .partial(); +// Shallow guard for the structured trace: the runner one directory over is +// the only writer, so full recursive validation buys little — the mapper +// reads every field defensively. +const agentTraceSchema = z.custom( + (value) => + typeof value === "object" && + value !== null && + Array.isArray((value as AgentTrace).turns), +); + /** * A raw result file as the uploader consumes it: the exported shape plus the * transcript fields the web export drops. Everything beyond the base shape is @@ -75,6 +91,7 @@ export const uploadableEvalResultSchema = rawEvalResultSchema.extend({ usage: agentUsageSchema.optional(), startedAt: z.string().optional(), durationMs: z.number().optional(), + trace: agentTraceSchema.optional(), }); export type UploadableEvalResult = z.infer; @@ -85,11 +102,18 @@ export interface BraintrustSpanNode { output?: unknown; error?: string; metadata?: Record; + metrics?: Record; scores?: Record; + children?: BraintrustSpanNode[]; startMs: number; endMs: number; } +/** A span node before the layout pass assigns its timing. */ +type UnplacedSpan = Omit & { + children?: UnplacedSpan[]; +}; + export interface BraintrustEvalTrace { evalId: string; experiment: string; @@ -166,49 +190,83 @@ function usageMetrics( return metrics; } -export function buildEvalTrace(args: { - experiment: string; - result: UploadableEvalResult; - /** Scenario prompt (PROMPT.md body), when resolvable. */ - prompt?: string; - /** Extra root metadata from the caller (source paths, git info, …). */ - extraMetadata?: Record; - /** Anchor for synthetic timing when the result carries no `startedAt`. */ - baseTimeMs: number; -}): BraintrustEvalTrace { - const { result } = args; +/** One llm span per turn, its tool spans (and subagent turns) nested. */ +function turnSpans(trace: AgentTrace): UnplacedSpan[] { + const spans: UnplacedSpan[] = []; + const interjectionsAfter = new Map(); + for (const interjection of trace.interjections ?? []) { + const group = interjectionsAfter.get(interjection.afterTurnIndex) ?? []; + group.push({ + name: interjection.role, + type: "task", + output: capForUpload(interjection.content), + }); + interjectionsAfter.set(interjection.afterTurnIndex, group); + } - const parsedStart = result.startedAt ? Date.parse(result.startedAt) : NaN; - const startMs = Number.isNaN(parsedStart) ? args.baseTimeMs : parsedStart; + spans.push(...(interjectionsAfter.get(-1) ?? [])); + for (const turn of trace.turns) { + spans.push(turnSpan(turn)); + spans.push(...(interjectionsAfter.get(turn.index) ?? [])); + } + return spans; +} +function turnSpan(turn: AgentTurn): UnplacedSpan { + const output: Record = {}; + if (turn.thinking) output.thinking = capForUpload(turn.thinking); + if (turn.text) output.text = capForUpload(turn.text); + const metadata: Record = {}; + if (turn.messageId) metadata.messageId = turn.messageId; + return { + name: `turn ${turn.index + 1}`, + type: "llm", + ...(Object.keys(output).length > 0 ? { output } : {}), + ...(Object.keys(metadata).length > 0 ? { metadata } : {}), + ...(turn.usage ? { metrics: usageMetrics(turn.usage) } : {}), + children: (turn.toolCalls ?? []).map(toolSpan), + }; +} + +function toolSpan(execution: ToolExecution): UnplacedSpan { + const metadata: Record = {}; + if (execution.canonicalName && execution.canonicalName !== "tool_use") { + metadata.tool = execution.canonicalName; + } + if (execution.path) metadata.path = execution.path; + if (execution.command) metadata.command = execution.command; + if (execution.url) metadata.url = execution.url; + if (execution.loadedSkill) metadata.loadedSkill = execution.loadedSkill; + return { + name: execution.name, + type: "tool", + input: capForUpload(execution.args ?? {}), + ...(execution.error === undefined + ? { output: capForUpload(execution.output) } + : { error: execution.error }), + ...(Object.keys(metadata).length > 0 ? { metadata } : {}), + // A subagent sidechain renders as the Task call's own turn subtree. + ...(execution.children?.length + ? { children: execution.children.map(turnSpan) } + : {}), + }; +} + +/** Legacy flat fallback for results that predate the structured trace. */ +function flatSpans(result: UploadableEvalResult): UnplacedSpan[] { const transcript = result.transcript ?? []; const toolCalls = result.toolCalls ?? []; - const checks = result.checks ?? []; - // +1 for the overall `passed` score span emitted ahead of the checks. - const spanCount = transcript.length + checks.length + 1; - // Real window when the run recorded one; otherwise one synthetic second per - // span, purely so ordering survives. - const durationMs = result.durationMs ?? Math.max(spanCount, 1) * 1000; - const endMs = startMs + durationMs; - const slotMs = durationMs / Math.max(spanCount, 1); - const slot = (index: number): { startMs: number; endMs: number } => ({ - startMs: startMs + index * slotMs, - endMs: startMs + (index + 1) * slotMs, - }); - - const spans: BraintrustSpanNode[] = []; + const spans: UnplacedSpan[] = []; // The k-th tool_call transcript part and toolCalls[k] describe the same // call for CLI agents (the adapter pushes both in lockstep); the record // carries the normalized views the part lacks. let toolCallIndex = 0; for (const part of transcript) { - const timing = slot(spans.length); if (part.type === "message") { spans.push({ name: part.role, type: part.role === "assistant" ? "llm" : "task", output: capForUpload(part.content), - ...timing, }); continue; } @@ -219,44 +277,90 @@ export function buildEvalTrace(args: { // the record when it names the same tool. const record = candidate && candidate.endpoint === part.name ? candidate : undefined; - const normalized: Record = {}; - if (record?.name) normalized.tool = record.name; - if (record?.path) normalized.path = record.path; - if (record?.command) normalized.command = record.command; - if (record?.url) normalized.url = record.url; - if (record?.loadedSkill) normalized.loadedSkill = record.loadedSkill; + const metadata: Record = {}; + if (record?.name) metadata.tool = record.name; + if (record?.path) metadata.path = record.path; + if (record?.command) metadata.command = record.command; + if (record?.url) metadata.url = record.url; + if (record?.loadedSkill) metadata.loadedSkill = record.loadedSkill; spans.push({ name: part.name, type: "tool", input: capForUpload(part.input), output: part.error === undefined ? capForUpload(part.output) : undefined, ...(part.error !== undefined ? { error: part.error } : {}), - ...(Object.keys(normalized).length > 0 ? { metadata: normalized } : {}), - ...timing, + ...(Object.keys(metadata).length > 0 ? { metadata } : {}), }); } + return spans; +} +/** + * Assign synthetic timing: siblings split their window into equal contiguous + * slots, children subdivide their parent's slot. Order → position; durations + * are not meaningful (see module comment). + */ +function layout( + spans: UnplacedSpan[], + startMs: number, + endMs: number, +): BraintrustSpanNode[] { + const slotMs = (endMs - startMs) / Math.max(spans.length, 1); + return spans.map((span, index) => { + const slotStart = startMs + index * slotMs; + const slotEnd = startMs + (index + 1) * slotMs; + const { children, ...rest } = span; + return { + ...rest, + ...(children?.length + ? { children: layout(children, slotStart, slotEnd) } + : {}), + startMs: slotStart, + endMs: slotEnd, + }; + }); +} + +export function buildEvalTrace(args: { + experiment: string; + result: UploadableEvalResult; + /** Scenario prompt (PROMPT.md body), when resolvable. */ + prompt?: string; + /** Extra root metadata from the caller (source paths, git info, …). */ + extraMetadata?: Record; + /** Anchor for synthetic timing when the result carries no `startedAt`. */ + baseTimeMs: number; +}): BraintrustEvalTrace { + const { result } = args; + + const hasStructuredTrace = (result.trace?.turns?.length ?? 0) > 0; + const activity = hasStructuredTrace + ? turnSpans(result.trace!) + : flatSpans(result); + + const checks = result.checks ?? []; const scores: Record = { passed: result.passed === true ? 1 : 0, }; const takenScoreNames = new Set(Object.keys(scores)); // The overall verdict as its own score span — the trace-view counterpart of // eval-results.json's headline `passed`, ahead of the per-check spans. - spans.push({ - name: "passed", - type: "score", - output: { - passed: result.passed === true, - checksPassed: `${checks.filter((c) => c.passed).length}/${checks.length}`, + const scoreSpans: UnplacedSpan[] = [ + { + name: "passed", + type: "score", + output: { + passed: result.passed === true, + checksPassed: `${checks.filter((c) => c.passed).length}/${checks.length}`, + }, + scores: { passed: scores.passed! }, }, - scores: { passed: scores.passed! }, - ...slot(spans.length), - }); + ]; for (const check of checks) { const scoreName = scoreNameFor(check.name, takenScoreNames); const value = check.passed ? 1 : 0; scores[scoreName] = value; - spans.push({ + scoreSpans.push({ name: scoreName, type: "score", output: { @@ -265,10 +369,19 @@ export function buildEvalTrace(args: { ...(check.judgeNotes ? { judgeNotes: check.judgeNotes } : {}), }, scores: { [scoreName]: value }, - ...slot(spans.length), }); } + const unplaced = [...activity, ...scoreSpans]; + + const parsedStart = result.startedAt ? Date.parse(result.startedAt) : NaN; + const startMs = Number.isNaN(parsedStart) ? args.baseTimeMs : parsedStart; + // Real window when the run recorded one; otherwise one synthetic second per + // top-level span, purely so ordering survives. + const durationMs = result.durationMs ?? Math.max(unplaced.length, 1) * 1000; + const endMs = startMs + durationMs; + const spans = layout(unplaced, startMs, endMs); + const display = result.experimentDisplay; const metadata: Record = { eval: result.eval, @@ -291,6 +404,9 @@ export function buildEvalTrace(args: { ...(result.usage?.costUsd !== undefined ? { costUsd: result.usage.costUsd } : {}), + ...(hasStructuredTrace && result.trace?.errors?.length + ? { traceErrors: result.trace.errors } + : {}), stepTiming: "synthetic-order-only", ...args.extraMetadata, }; diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 4838a540..5d19e349 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -9,6 +9,11 @@ import { basename, dirname, join } from 'node:path'; import { tmpdir } from 'node:os'; import { fileURLToPath } from 'node:url'; import type { ToolName } from './transcript/types.js'; +import type { + AgentTrace, + AgentTurn, + ToolExecution, +} from './transcript/agent-trace.js'; import { createClient, type SupabaseClient } from '@supabase/supabase-js'; import { createMCPClient } from '@ai-sdk/mcp'; import { Experimental_StdioMCPTransport as StdioMCPTransport } from '@ai-sdk/mcp/mcp-stdio'; @@ -131,6 +136,13 @@ export type { TranscriptEvent, ParsedTranscript, } from './transcript/types.js'; +export { assembleAgentTrace } from './transcript/agent-trace.js'; +export type { + AgentTrace, + AgentTurn, + ToolExecution, + TraceInterjection, +} from './transcript/agent-trace.js'; export type { AgentHarnessId, CheckResult, @@ -407,6 +419,12 @@ export type AgentRunResult = { steps: number; stoppedReason: string; usage?: AgentUsage; + /** + * Structured view of the same run — turns with nested tool executions + * (see transcript/agent-trace.ts). Additive: the flat transcript/toolCalls + * surface scorers consume is unchanged. + */ + trace?: AgentTrace; }; export type AgentHarness = { @@ -758,6 +776,50 @@ export function aiSdkAgent(options: { totalTokens: result.totalUsage.totalTokens, }; + // Structured trace straight from the steps: one turn per step, tool + // executions paired within it (`toolOutputs` above), per-step usage. + const turns: AgentTurn[] = result.steps.map((step, index) => { + const turn: AgentTurn = { index, toolCalls: [] }; + for (const part of step.content) { + if (part.type === "text") { + const content = part.text.trim(); + if (content) { + turn.text = turn.text ? `${turn.text}\n${content}` : content; + } + } else if (part.type === "reasoning") { + const content = part.text.trim(); + if (content) { + turn.thinking = turn.thinking + ? `${turn.thinking}\n${content}` + : content; + } + } else if (part.type === "tool-call") { + const resolved = toolOutputs.get(part.toolCallId); + const execution: ToolExecution = { + id: part.toolCallId, + name: part.toolName, + canonicalName: "tool_use", + args: isRecord(part.input) ? part.input : {}, + ...(resolved?.error === undefined + ? { output: resolved?.output } + : { error: resolved.error }), + }; + turn.toolCalls.push(execution); + } + } + if (step.usage) { + turn.usage = { + inputTokens: step.usage.inputTokens, + outputTokens: step.usage.outputTokens, + cachedInputTokens: step.usage.cachedInputTokens, + reasoningTokens: step.usage.reasoningTokens, + totalTokens: step.usage.totalTokens, + }; + } + return turn; + }); + const trace: AgentTrace = { turns, interjections: [], errors: [] }; + return { agentReport, toolCalls, @@ -768,6 +830,7 @@ export function aiSdkAgent(options: { ? 'max_steps' : result.finishReason, usage, + trace, }; } finally { await closeMcpHandles(mcpHandles); diff --git a/packages/core/src/transcript/agent-trace.test.ts b/packages/core/src/transcript/agent-trace.test.ts new file mode 100644 index 00000000..51e4b693 --- /dev/null +++ b/packages/core/src/transcript/agent-trace.test.ts @@ -0,0 +1,171 @@ +import { describe, expect, it } from "vitest"; +import { assembleAgentTrace } from "./agent-trace.js"; +import type { TranscriptEvent } from "./types.js"; + +describe("assembleAgentTrace with turn keys (Claude Code style)", () => { + const events: TranscriptEvent[] = [ + // Turn 1: one assistant API message with thinking + text + two tool uses. + { + type: "thinking", + content: "let me look", + turnKey: "msg_1", + messageId: "msg_1", + usage: { inputTokens: 100, outputTokens: 10 }, + }, + { + type: "message", + role: "assistant", + content: "Checking the tables.", + turnKey: "msg_1", + messageId: "msg_1", + usage: { inputTokens: 100, outputTokens: 10 }, + }, + { + type: "tool_call", + turnKey: "msg_1", + tool: { name: "shell", originalName: "Bash", id: "t1", args: { command: "ls" }, command: "ls" }, + }, + { + type: "tool_call", + turnKey: "msg_1", + tool: { name: "agent_task", originalName: "Task", id: "t2", args: { prompt: "explore" } }, + }, + // Results arrive on later lines. + { type: "tool_result", tool: { name: "unknown", originalName: "t1", id: "t1", result: "files", success: true } }, + // Subagent sidechain spawned by the Task call. + { + type: "message", + role: "assistant", + content: "sub work", + turnKey: "msg_sub", + parentToolUseId: "t2", + }, + { + type: "tool_call", + turnKey: "msg_sub", + parentToolUseId: "t2", + tool: { name: "file_read", originalName: "Read", id: "t3", args: { file_path: "a.ts" }, path: "a.ts" }, + }, + { type: "tool_result", parentToolUseId: "t2", tool: { name: "unknown", originalName: "t3", id: "t3", result: "code", success: true } }, + // Turn 2: a second API message. + { + type: "message", + role: "assistant", + content: "Done.", + turnKey: "msg_2", + messageId: "msg_2", + usage: { inputTokens: 200, outputTokens: 20 }, + }, + ]; + const trace = assembleAgentTrace(events); + + it("groups one API message into one turn", () => { + expect(trace.turns).toHaveLength(2); + const [first, second] = trace.turns; + expect(first!.text).toBe("Checking the tables."); + expect(first!.thinking).toBe("let me look"); + expect(first!.toolCalls.map((t) => t.name)).toEqual(["Bash", "Task"]); + expect(second!.text).toBe("Done."); + expect(second!.toolCalls).toHaveLength(0); + }); + + it("carries per-turn usage and message ids", () => { + expect(trace.turns[0]!.messageId).toBe("msg_1"); + expect(trace.turns[0]!.usage).toEqual({ inputTokens: 100, outputTokens: 10 }); + expect(trace.turns[1]!.usage).toEqual({ inputTokens: 200, outputTokens: 20 }); + }); + + it("pairs tool results onto their executions", () => { + const bash = trace.turns[0]!.toolCalls[0]!; + expect(bash.output).toBe("files"); + expect(bash.success).toBe(true); + expect(bash.command).toBe("ls"); + }); + + it("nests the subagent sidechain under the Task call", () => { + const task = trace.turns[0]!.toolCalls[1]!; + expect(task.children).toHaveLength(1); + const sub = task.children![0]!; + expect(sub.text).toBe("sub work"); + expect(sub.toolCalls[0]!.name).toBe("Read"); + expect(sub.toolCalls[0]!.output).toBe("code"); + // Sidechain turns don't leak into the main thread. + expect(trace.turns).toHaveLength(2); + }); +}); + +describe("assembleAgentTrace closure rule (Codex style, no turn keys)", () => { + const events: TranscriptEvent[] = [ + { type: "thinking", content: "plan" }, + { + type: "tool_call", + tool: { name: "shell", originalName: "command_execution", id: "c1", args: { command: "psql" }, command: "psql" }, + }, + { type: "tool_result", tool: { name: "shell", originalName: "command_execution", id: "c1", result: "rows", success: true } }, + { type: "message", role: "assistant", content: "Found it." }, + { + type: "tool_call", + tool: { name: "shell", originalName: "command_execution", id: "c2", args: { command: "fix" }, command: "fix" }, + }, + { type: "message", role: "assistant", content: "Fixed." }, + ]; + const trace = assembleAgentTrace(events); + + it("closes a turn on each assistant message", () => { + expect(trace.turns).toHaveLength(2); + expect(trace.turns[0]!.thinking).toBe("plan"); + expect(trace.turns[0]!.text).toBe("Found it."); + expect(trace.turns[0]!.toolCalls.map((t) => t.args.command)).toEqual(["psql"]); + expect(trace.turns[1]!.text).toBe("Fixed."); + expect(trace.turns[1]!.toolCalls.map((t) => t.args.command)).toEqual(["fix"]); + }); +}); + +describe("assembleAgentTrace edges", () => { + it("records user/system messages as interjections", () => { + const trace = assembleAgentTrace([ + { type: "message", role: "user", content: "skill body" }, + { type: "message", role: "assistant", content: "ok", turnKey: "m1" }, + { type: "message", role: "user", content: "tool feedback" }, + ]); + expect(trace.turns).toHaveLength(1); + expect(trace.interjections).toEqual([ + { afterTurnIndex: -1, role: "user", content: "skill body" }, + { afterTurnIndex: 0, role: "user", content: "tool feedback" }, + ]); + }); + + it("collects agent errors and flags orphaned sidechains", () => { + const trace = assembleAgentTrace([ + { type: "error", content: "boom" }, + { + type: "message", + role: "assistant", + content: "orphan", + turnKey: "m1", + parentToolUseId: "missing", + }, + ]); + expect(trace.turns).toHaveLength(0); + expect(trace.errors[0]).toBe("boom"); + expect(trace.errors[1]).toContain("orphaned subagent transcript"); + }); + + it("stores failed tool results as errors", () => { + const trace = assembleAgentTrace([ + { + type: "tool_call", + tool: { name: "shell", originalName: "Bash", id: "x", args: {} }, + }, + { + type: "tool_result", + tool: { name: "unknown", originalName: "x", id: "x", result: "exit 1", success: false }, + }, + { type: "message", role: "assistant", content: "hm" }, + ]); + const call = trace.turns[0]!.toolCalls[0]!; + expect(call.error).toBe("exit 1"); + expect(call.output).toBeUndefined(); + expect(call.success).toBe(false); + }); +}); diff --git a/packages/core/src/transcript/agent-trace.ts b/packages/core/src/transcript/agent-trace.ts new file mode 100644 index 00000000..2738301b --- /dev/null +++ b/packages/core/src/transcript/agent-trace.ts @@ -0,0 +1,233 @@ +/** + * Structured, agent-agnostic transcript: turns (one per model API call) with + * their tool executions nested, results paired at assembly time, and subagent + * sidechains attached under the tool call that spawned them. + * + * Assembled from the same canonical `TranscriptEvent[]` the flat scorer + * surface (`adaptTranscript`) consumes, so the two views can never disagree + * about what happened; this one just stops throwing the structure away. + * Turn grouping uses `event.turnKey` when the parser stamped one (Claude + * Code: one assistant line = one API message; ai-sdk builds turns directly + * from steps) and otherwise falls back to a closure rule: thinking and tool + * calls accumulate into the open turn, an assistant message closes it + * (Codex's item stream). + */ + +import type { AgentUsage } from "../index.js"; +import type { ToolName, TranscriptEvent } from "./types.js"; + +export interface ToolExecution { + /** Correlation id, when the harness provides one. */ + id?: string; + /** Original tool name as the agent emitted it. */ + name: string; + /** Canonical cross-agent tool category. */ + canonicalName: ToolName; + args: Record; + /** Normalized arg views (see TranscriptEvent.tool). */ + path?: string; + command?: string; + url?: string; + loadedSkill?: string; + output?: unknown; + error?: string; + success?: boolean; + /** Epoch-ms timing when the transcript carries real timestamps. */ + startMs?: number; + /** Subagent sidechain (e.g. Claude Code Task) spawned by this call. */ + children?: AgentTurn[]; +} + +export interface AgentTurn { + index: number; + /** Provider message id of the API call, when exposed. */ + messageId?: string; + /** Per-turn token usage, when the transcript carries it. */ + usage?: AgentUsage; + startMs?: number; + thinking?: string; + /** Assistant prose (absent on pure-tool turns). */ + text?: string; + toolCalls: ToolExecution[]; +} + +/** A user/system message arriving between turns (e.g. an injected skill body). */ +export interface TraceInterjection { + /** Index of the last turn opened before this message; -1 = before any turn. */ + afterTurnIndex: number; + role: "user" | "system"; + content: string; +} + +export interface AgentTrace { + turns: AgentTurn[]; + interjections: TraceInterjection[]; + /** Agent-reported error events, in order. */ + errors: string[]; +} + +function parseMs(timestamp: string | undefined): number | undefined { + if (!timestamp) return undefined; + const ms = Date.parse(timestamp); + return Number.isNaN(ms) ? undefined : ms; +} + +export function assembleAgentTrace(events: TranscriptEvent[]): AgentTrace { + // Split sidechain events (subagent activity) from the main thread; each + // group is assembled recursively and attached to its spawning tool call. + const main: TranscriptEvent[] = []; + const sidechains = new Map(); + for (const event of events) { + if (event.parentToolUseId) { + const group = sidechains.get(event.parentToolUseId) ?? []; + group.push({ ...event, parentToolUseId: undefined }); + sidechains.set(event.parentToolUseId, group); + } else { + main.push(event); + } + } + + const turns: AgentTurn[] = []; + const interjections: TraceInterjection[] = []; + const errors: string[] = []; + const executionsById = new Map(); + const turnsByKey = new Map(); + // Closure-rule state for events without a turnKey: an assistant message + // marks the open turn closed, so the next event starts a fresh one. + let openTurn: AgentTurn | undefined; + let openTurnClosed = false; + + const turnFor = (event: TranscriptEvent): AgentTurn => { + if (event.turnKey) { + const existing = turnsByKey.get(event.turnKey); + if (existing) return existing; + const turn = newTurn(event); + turnsByKey.set(event.turnKey, turn); + return turn; + } + if (!openTurn || openTurnClosed) { + openTurn = newTurn(event); + openTurnClosed = false; + } + return openTurn; + }; + + const newTurn = (event: TranscriptEvent): AgentTurn => { + const turn: AgentTurn = { index: turns.length, toolCalls: [] }; + const startMs = parseMs(event.timestamp); + if (startMs !== undefined) turn.startMs = startMs; + turns.push(turn); + return turn; + }; + + for (const event of events) { + if (event.parentToolUseId) continue; + + if (event.type === "error") { + if (event.content) errors.push(event.content); + continue; + } + + if (event.type === "message" && event.role && event.role !== "assistant") { + const content = event.content?.trim(); + if (content) { + interjections.push({ + afterTurnIndex: turns.length - 1, + role: event.role, + content, + }); + } + continue; + } + + if (event.type === "tool_result") { + const execution = event.tool?.id + ? executionsById.get(event.tool.id) + : undefined; + if (execution && event.tool) { + if (event.tool.success === false) { + execution.error = + typeof event.tool.result === "string" + ? event.tool.result + : JSON.stringify(event.tool.result); + } else { + execution.output = event.tool.result; + } + if (event.tool.success !== undefined) { + execution.success = event.tool.success; + } + } + continue; + } + + if (event.type === "message" && event.role === "assistant") { + const turn = turnFor(event); + const content = event.content?.trim(); + if (content) { + turn.text = turn.text ? `${turn.text}\n${content}` : content; + } + annotate(turn, event); + if (!event.turnKey) openTurnClosed = true; + continue; + } + + if (event.type === "thinking") { + const turn = turnFor(event); + const content = event.content?.trim(); + if (content) { + turn.thinking = turn.thinking ? `${turn.thinking}\n${content}` : content; + } + annotate(turn, event); + continue; + } + + if (event.type === "tool_call" && event.tool) { + const turn = turnFor(event); + const execution: ToolExecution = { + ...(event.tool.id ? { id: event.tool.id } : {}), + name: event.tool.originalName, + canonicalName: event.tool.name, + args: event.tool.args ?? {}, + ...(event.tool.path ? { path: event.tool.path } : {}), + ...(event.tool.command ? { command: event.tool.command } : {}), + ...(event.tool.url ? { url: event.tool.url } : {}), + ...(event.tool.loadedSkill + ? { loadedSkill: event.tool.loadedSkill } + : {}), + }; + const startMs = parseMs(event.timestamp); + if (startMs !== undefined) execution.startMs = startMs; + turn.toolCalls.push(execution); + if (execution.id) executionsById.set(execution.id, execution); + annotate(turn, event); + } + } + + // Attach sidechains to their spawning tool call; groups whose parent call + // never appeared (truncated transcript) surface as an error note instead + // of vanishing. + for (const [parentId, group] of sidechains) { + const parent = executionsById.get(parentId); + const subTrace = assembleAgentTrace(group); + errors.push(...subTrace.errors); + if (parent) { + parent.children = subTrace.turns; + } else if (subTrace.turns.length > 0) { + errors.push( + `orphaned subagent transcript (parent tool_use ${parentId} not found)`, + ); + } + } + + return { turns, interjections, errors }; +} + +/** Fold an event's per-call identifiers/usage onto its turn. */ +function annotate(turn: AgentTurn, event: TranscriptEvent): void { + if (event.messageId && !turn.messageId) turn.messageId = event.messageId; + if (event.usage && !turn.usage) turn.usage = event.usage; + if (turn.startMs === undefined) { + const startMs = parseMs(event.timestamp); + if (startMs !== undefined) turn.startMs = startMs; + } +} diff --git a/packages/core/src/transcript/types.ts b/packages/core/src/transcript/types.ts index 01c73b64..5e9024b6 100644 --- a/packages/core/src/transcript/types.ts +++ b/packages/core/src/transcript/types.ts @@ -32,6 +32,23 @@ export type ToolName = export interface TranscriptEvent { /** ISO timestamp of the event, when the agent records one. */ timestamp?: string; + /** + * Groups events belonging to one model API call (a turn). Parsers stamp it + * when the raw transcript exposes the boundary (Claude Code: one assistant + * stream-json line = one API message). Absent = the trace assembler falls + * back to its closure rule (an assistant message ends the open turn). + */ + turnKey?: string; + /** Provider message id of the API call this event belongs to, if exposed. */ + messageId?: string; + /** Per-API-call token usage, when the raw transcript carries it. */ + usage?: import("../index.js").AgentUsage; + /** + * Tool-use id of the parent Task/subagent call for sidechain events + * (Claude Code's `parent_tool_use_id`). The assembler nests these under + * that tool execution instead of the main thread. + */ + parentToolUseId?: string; /** Event kind. */ type: 'message' | 'tool_call' | 'tool_result' | 'thinking' | 'error'; /** For `message` events: the speaker. */ From cb8578758cd914d37103710c110abf7e113cf6f3 Mon Sep 17 00:00:00 2001 From: Pedro Rodrigues Date: Tue, 21 Jul 2026 17:52:07 +0100 Subject: [PATCH 05/15] feat: turn/step trace semantics, single passed scorer, judge evidence spans A turn now means everything between user prompts (one per eval is normal); model API calls are steps inside it. Braintrust rows carry one scorer (passed) with the checks array in its output instead of a score per check. judge() calls are recorded via an AsyncLocalStorage collector (no per-eval wiring), persisted as judgeCalls in raw results, and rendered as llm children under the passed span with the rubric, the exact input the judge evaluated, model, duration, and tokens. Co-Authored-By: Claude Fable 5 --- apps/framework/harness/run-eval.ts | 51 ++-- packages/core/src/braintrust-spans.test.ts | 262 ++++++++++-------- packages/core/src/braintrust-spans.ts | 137 +++++---- packages/core/src/index.ts | 56 +++- packages/core/src/judge-recorder.ts | 40 +++ .../core/src/transcript/agent-trace.test.ts | 80 +++--- packages/core/src/transcript/agent-trace.ts | 173 +++++++----- 7 files changed, 493 insertions(+), 306 deletions(-) create mode 100644 packages/core/src/judge-recorder.ts diff --git a/apps/framework/harness/run-eval.ts b/apps/framework/harness/run-eval.ts index ddbe9417..1fff4689 100644 --- a/apps/framework/harness/run-eval.ts +++ b/apps/framework/harness/run-eval.ts @@ -30,9 +30,11 @@ import { viteBuild, vitestRun } from './project-runner.js'; import { buildDocsResult, buildSkillResult, + collectJudgeCalls, rehydrateTruncatedDocsResults, getExperimentDisplayMetadata, } from '@supabase-evals/core'; +import type { JudgeCallRecord } from '@supabase-evals/core'; import type { AgentTrace, AgentUsage, @@ -365,6 +367,7 @@ async function runOne( steps: number; usage?: AgentUsage; trace?: AgentTrace; + judgeCalls?: JudgeCallRecord[]; } > { const prompt = parseEvalMarkdown( @@ -396,6 +399,7 @@ async function runOne( let lastSteps = 0; let lastUsage: AgentUsage | undefined; let lastTrace: AgentTrace | undefined; + let lastJudgeCalls: JudgeCallRecord[] = []; for (let attempt = 1; attempt <= RUNS; attempt += 1) { if (ev.mode === 'local-stack') { @@ -483,18 +487,22 @@ async function runOne( copiedWithheldTests = true; }; - last = await (scorer as LocalStackScorer)({ - ...session.scoringContext, - toolCalls: run.toolCalls, - transcript: run.transcript, - agentReport: run.agentReport, - hostWorkspace, - runViteBuild: () => viteBuild(hostWorkspace), - runVitest: () => { - ensureWithheldTests(); - return vitestRun(hostWorkspace); - }, - }); + const scored = await collectJudgeCalls(() => + (scorer as LocalStackScorer)({ + ...session.scoringContext, + toolCalls: run.toolCalls, + transcript: run.transcript, + agentReport: run.agentReport, + hostWorkspace, + runViteBuild: () => viteBuild(hostWorkspace), + runVitest: () => { + ensureWithheldTests(); + return vitestRun(hostWorkspace); + }, + }), + ); + last = scored.result; + lastJudgeCalls = scored.judgeCalls; if (STOP_ON_PASS && last.passed) { return { @@ -509,6 +517,7 @@ async function runOne( steps: run.steps, usage: run.usage, trace: run.trace, + judgeCalls: lastJudgeCalls, }; } logRetryAttempt(expName, ev, attempt, last); @@ -561,12 +570,16 @@ async function runOne( lastSteps = run.steps; lastUsage = run.usage; lastTrace = run.trace; - last = await (scorer as ToolScorer)({ - ...session.scoringContext, - toolCalls: run.toolCalls, - transcript: run.transcript, - agentReport: run.agentReport, - }); + const scored = await collectJudgeCalls(() => + (scorer as ToolScorer)({ + ...session.scoringContext, + toolCalls: run.toolCalls, + transcript: run.transcript, + agentReport: run.agentReport, + }), + ); + last = scored.result; + lastJudgeCalls = scored.judgeCalls; if (STOP_ON_PASS && last.passed) { return { @@ -581,6 +594,7 @@ async function runOne( steps: run.steps, usage: run.usage, trace: run.trace, + judgeCalls: lastJudgeCalls, }; } logRetryAttempt(expName, ev, attempt, last); @@ -598,6 +612,7 @@ async function runOne( steps: lastSteps, usage: lastUsage, trace: lastTrace, + judgeCalls: lastJudgeCalls, }; } diff --git a/packages/core/src/braintrust-spans.test.ts b/packages/core/src/braintrust-spans.test.ts index 1c1aac27..9dd93821 100644 --- a/packages/core/src/braintrust-spans.test.ts +++ b/packages/core/src/braintrust-spans.test.ts @@ -7,7 +7,7 @@ import { const BASE = Date.parse("2026-07-21T10:00:00.000Z"); -/** A modern raw result: transcript + toolCalls + usage + timing. */ +/** A modern raw result: transcript + toolCalls + usage + timing (no trace). */ const modernResult = { experiment: "claude-code-sonnet-5", experimentSuite: "benchmark", @@ -70,7 +70,7 @@ describe("uploadableEvalResultSchema", () => { }); }); -describe("buildEvalTrace", () => { +describe("buildEvalTrace (flat fallback)", () => { const trace = buildEvalTrace({ experiment: "claude-code-sonnet-5", result: parse(modernResult), @@ -84,15 +84,21 @@ describe("buildEvalTrace", () => { expect(trace.output).toBe("Final report."); }); - it("scores passed plus one score per check", () => { - expect(trace.scores).toEqual({ - passed: 1, - "named the vulnerable table": 1, - "proposed a concrete fix": 0, + it("exposes a single passed score with the checks in its output", () => { + expect(trace.scores).toEqual({ passed: 1 }); + const verdict = trace.spans.find((s) => s.name === "passed")!; + expect(verdict.scores).toEqual({ passed: 1 }); + expect(verdict.output).toEqual({ + passed: true, + checksPassed: "1/2", + checks: [ + { name: "named the vulnerable table", passed: true }, + { name: "proposed a concrete fix", passed: false, notes: "missed RLS" }, + ], }); }); - it("emits spans in transcript order, then the verdict and check score spans", () => { + it("emits spans in transcript order, then the verdict span", () => { expect(trace.spans.map((s) => `${s.type}:${s.name}`)).toEqual([ "tool:Skill", "task:user", @@ -100,12 +106,7 @@ describe("buildEvalTrace", () => { "tool:Bash", "llm:assistant", "score:passed", - "score:named the vulnerable table", - "score:proposed a concrete fix", ]); - const verdict = trace.spans.find((s) => s.name === "passed"); - expect(verdict?.scores).toEqual({ passed: 1 }); - expect(verdict?.output).toEqual({ passed: true, checksPassed: "1/2" }); }); it("pairs tool spans with the normalized toolCalls record", () => { @@ -156,14 +157,6 @@ describe("buildEvalTrace", () => { expect(trace.tags).toEqual(["investigate", "database", "security", "rls"]); }); - it("attaches check notes to score spans", () => { - const failedCheck = trace.spans.find( - (s) => s.name === "proposed a concrete fix", - ); - expect(failedCheck?.scores).toEqual({ "proposed a concrete fix": 0 }); - expect(failedCheck?.output).toEqual({ passed: false, notes: "missed RLS" }); - }); - it("falls back to synthetic timing and eval-id input for legacy results", () => { const legacy = buildEvalTrace({ experiment: "old", @@ -178,140 +171,169 @@ describe("buildEvalTrace", () => { expect(legacy.input).toEqual({ eval: "legacy-eval" }); expect(legacy.output).toBeUndefined(); expect(legacy.startMs).toBe(BASE); - expect(legacy.endMs).toBe(BASE + 2000); + expect(legacy.endMs).toBe(BASE + 1000); expect(legacy.spans.map((s) => `${s.type}:${s.name}`)).toEqual([ "score:passed", - "score:a check", ]); expect(legacy.metrics).toEqual({}); }); - it("de-duplicates and reserves score names", () => { - const clashing = buildEvalTrace({ + it("truncates oversized tool outputs", () => { + const big = buildEvalTrace({ experiment: "x", result: parse({ experiment: "x", eval: "y", passed: true, - checks: [ - { name: "passed", passed: true }, - { name: "same", passed: true }, - { name: "same", passed: false }, + transcript: [ + { + type: "tool_call", + name: "Read", + input: {}, + output: "x".repeat(150_000), + }, ], + toolCalls: [{ endpoint: "Read", body: {}, ts: 0 }], }), baseTimeMs: BASE, }); - expect(Object.keys(clashing.scores)).toEqual([ - "passed", - "check: passed", - "same", - "same (2)", - ]); - expect(clashing.scores["same (2)"]).toBe(0); + const output = big.spans[0]!.output as string; + expect(output.length).toBeLessThan(30_000); + expect(output).toContain("[truncated 150000 chars]"); }); +}); - it("builds hierarchical spans from a structured trace", () => { - const structured = buildEvalTrace({ +describe("buildEvalTrace (structured trace)", () => { + const structured = buildEvalTrace({ + experiment: "x", + result: parse({ experiment: "x", - result: parse({ - experiment: "x", - eval: "y", - passed: true, - checks: [{ name: "a check", passed: true }], - startedAt: "2026-07-21T09:00:00.000Z", - durationMs: 40_000, - trace: { - turns: [ - { - index: 0, - messageId: "msg_1", - usage: { inputTokens: 100, outputTokens: 10 }, - thinking: "hmm", - text: "Looking.", - toolCalls: [ - { - name: "Bash", - canonicalName: "shell", - args: { command: "ls" }, - command: "ls", - output: "files", - }, - { - name: "Task", - canonicalName: "agent_task", - args: { prompt: "explore" }, - children: [ - { - index: 0, - toolCalls: [ - { name: "Read", canonicalName: "file_read", args: {}, output: "code" }, - ], - }, - ], - }, - ], - }, - { index: 1, text: "Done.", toolCalls: [] }, - ], - interjections: [{ afterTurnIndex: 0, role: "user", content: "tool says hi" }], - errors: [], + eval: "y", + passed: true, + checks: [{ name: "a check", passed: true, judgeNotes: "solid work" }], + startedAt: "2026-07-21T09:00:00.000Z", + durationMs: 40_000, + trace: { + turns: [ + { + index: 0, + steps: [ + { + index: 0, + messageId: "msg_1", + usage: { inputTokens: 100, outputTokens: 10 }, + thinking: "hmm", + text: "Looking.", + toolCalls: [ + { + name: "Bash", + canonicalName: "shell", + args: { command: "ls" }, + command: "ls", + output: "files", + }, + { + name: "Task", + canonicalName: "agent_task", + args: { prompt: "explore" }, + children: [ + { + index: 0, + toolCalls: [ + { name: "Read", canonicalName: "file_read", args: {}, output: "code" }, + ], + }, + ], + }, + ], + }, + { index: 1, text: "Done.", toolCalls: [] }, + ], + }, + { + index: 1, + userMessage: "one more thing", + steps: [{ index: 2, text: "Sure.", toolCalls: [] }], + }, + ], + errors: [], + }, + judgeCalls: [ + { + rubric: "Pass if the work is solid.", + input: "the files the judge saw", + passed: true, + notes: "solid work", + modelId: "gpt-5.5", + durationMs: 1234, + usage: { inputTokens: 700, outputTokens: 30 }, }, - }), - baseTimeMs: BASE, - }); + ], + }), + baseTimeMs: BASE, + }); + it("nests steps inside turns and tools inside steps", () => { expect(structured.spans.map((s) => `${s.type}:${s.name}`)).toEqual([ - "llm:turn 1", - "task:user", - "llm:turn 2", + "task:turn 1", + "task:turn 2", "score:passed", - "score:a check", ]); const turn1 = structured.spans[0]!; - expect(turn1.metrics).toEqual({ + expect(turn1.children!.map((s) => `${s.type}:${s.name}`)).toEqual([ + "llm:step 1", + "llm:step 2", + ]); + const step1 = turn1.children![0]!; + expect(step1.metrics).toEqual({ prompt_tokens: 100, completion_tokens: 10, tokens: 110, }); - expect(turn1.metadata).toEqual({ messageId: "msg_1" }); - expect(turn1.output).toEqual({ thinking: "hmm", text: "Looking." }); - expect(turn1.children!.map((s) => `${s.type}:${s.name}`)).toEqual([ + expect(step1.metadata).toEqual({ messageId: "msg_1" }); + expect(step1.output).toEqual({ thinking: "hmm", text: "Looking." }); + expect(step1.children!.map((s) => `${s.type}:${s.name}`)).toEqual([ "tool:Bash", "tool:Task", ]); - const task = turn1.children![1]!; - expect(task.children!.map((s) => `${s.type}:${s.name}`)).toEqual(["llm:turn 1"]); + const task = step1.children![1]!; + expect(task.children!.map((s) => `${s.type}:${s.name}`)).toEqual(["llm:step 1"]); expect(task.children![0]!.children!.map((s) => s.name)).toEqual(["Read"]); + }); - // Children subdivide their parent's slot; everything stays inside the window. - const start = Date.parse("2026-07-21T09:00:00.000Z"); - expect(turn1.startMs).toBe(start); - expect(task.startMs).toBeGreaterThanOrEqual(turn1.startMs); - expect(task.endMs).toBeLessThanOrEqual(turn1.endMs); - expect(structured.spans.at(-1)!.endMs).toBeCloseTo(start + 40_000, 5); + it("carries the opening user message on its turn", () => { + const turn2 = structured.spans[1]!; + expect(turn2.input).toBe("one more thing"); + expect(turn2.children!.map((s) => s.name)).toEqual(["step 3"]); }); - it("truncates oversized tool outputs", () => { - const big = buildEvalTrace({ - experiment: "x", - result: parse({ - experiment: "x", - eval: "y", - passed: true, - transcript: [ - { - type: "tool_call", - name: "Read", - input: {}, - output: "x".repeat(150_000), - }, - ], - toolCalls: [{ endpoint: "Read", body: {}, ts: 0 }], - }), - baseTimeMs: BASE, + it("nests judge evidence under the passed score span", () => { + const verdict = structured.spans[2]!; + expect(verdict.scores).toEqual({ passed: 1 }); + expect(verdict.children!.map((s) => `${s.type}:${s.name}`)).toEqual([ + "llm:judge: a check", + ]); + const judgeSpan = verdict.children![0]!; + expect(judgeSpan.input).toEqual({ + rubric: "Pass if the work is solid.", + input: "the files the judge saw", }); - const output = big.spans[0]!.output as string; - expect(output.length).toBeLessThan(30_000); - expect(output).toContain("[truncated 150000 chars]"); + expect(judgeSpan.output).toEqual({ passed: true, notes: "solid work" }); + expect(judgeSpan.metadata).toEqual({ modelId: "gpt-5.5", durationMs: 1234 }); + expect(judgeSpan.metrics).toEqual({ + prompt_tokens: 700, + completion_tokens: 30, + tokens: 730, + }); + }); + + it("keeps everything inside the run window", () => { + const start = Date.parse("2026-07-21T09:00:00.000Z"); + const turn1 = structured.spans[0]!; + expect(turn1.startMs).toBe(start); + const step1 = turn1.children![0]!; + expect(step1.startMs).toBeGreaterThanOrEqual(turn1.startMs); + expect(step1.endMs).toBeLessThanOrEqual(turn1.endMs); + expect(structured.spans.at(-1)!.endMs).toBeCloseTo(start + 40_000, 5); }); }); diff --git a/packages/core/src/braintrust-spans.ts b/packages/core/src/braintrust-spans.ts index 8f53dc86..e57b9422 100644 --- a/packages/core/src/braintrust-spans.ts +++ b/packages/core/src/braintrust-spans.ts @@ -25,7 +25,7 @@ import { z } from "zod"; import { rawEvalResultSchema } from "./eval-metadata.js"; -import type { AgentTrace, AgentTurn, ToolExecution } from "./transcript/agent-trace.js"; +import type { AgentStep, AgentTrace, ToolExecution } from "./transcript/agent-trace.js"; const transcriptPartSchema = z.discriminatedUnion("type", [ z.object({ @@ -82,6 +82,16 @@ const agentTraceSchema = z.custom( * transcript fields the web export drops. Everything beyond the base shape is * optional so results written before these fields existed still upload. */ +const judgeCallSchema = z.looseObject({ + rubric: z.string(), + input: z.string(), + passed: z.boolean(), + notes: z.string().optional(), + modelId: z.string().optional(), + durationMs: z.number().optional(), + usage: agentUsageSchema.optional(), +}); + export const uploadableEvalResultSchema = rawEvalResultSchema.extend({ transcript: z.array(transcriptPartSchema).optional(), toolCalls: z.array(toolCallRecordSchema).optional(), @@ -92,6 +102,7 @@ export const uploadableEvalResultSchema = rawEvalResultSchema.extend({ startedAt: z.string().optional(), durationMs: z.number().optional(), trace: agentTraceSchema.optional(), + judgeCalls: z.array(judgeCallSchema).optional(), }); export type UploadableEvalResult = z.infer; @@ -155,15 +166,6 @@ function capForUpload(value: unknown): unknown { }; } -/** Unique, human-readable score name per check; `passed` is reserved. */ -function scoreNameFor(checkName: string, taken: Set): string { - const base = checkName.trim() || "check"; - let name = base === "passed" ? "check: passed" : base; - for (let n = 2; taken.has(name); n += 1) name = `${base} (${n})`; - taken.add(name); - return name; -} - function usageMetrics( usage: UploadableEvalResult["usage"], ): Record { @@ -190,41 +192,35 @@ function usageMetrics( return metrics; } -/** One llm span per turn, its tool spans (and subagent turns) nested. */ +/** + * One task span per turn (everything between user inputs — usually exactly + * one in an eval), its per-API-call step llm spans nested inside, tool spans + * under their step. + */ function turnSpans(trace: AgentTrace): UnplacedSpan[] { - const spans: UnplacedSpan[] = []; - const interjectionsAfter = new Map(); - for (const interjection of trace.interjections ?? []) { - const group = interjectionsAfter.get(interjection.afterTurnIndex) ?? []; - group.push({ - name: interjection.role, - type: "task", - output: capForUpload(interjection.content), - }); - interjectionsAfter.set(interjection.afterTurnIndex, group); - } - - spans.push(...(interjectionsAfter.get(-1) ?? [])); - for (const turn of trace.turns) { - spans.push(turnSpan(turn)); - spans.push(...(interjectionsAfter.get(turn.index) ?? [])); - } - return spans; + return trace.turns.map((turn) => ({ + name: `turn ${turn.index + 1}`, + type: "task" as const, + ...(turn.userMessage !== undefined + ? { input: capForUpload(turn.userMessage) } + : {}), + children: (turn.steps ?? []).map(stepSpan), + })); } -function turnSpan(turn: AgentTurn): UnplacedSpan { +function stepSpan(step: AgentStep): UnplacedSpan { const output: Record = {}; - if (turn.thinking) output.thinking = capForUpload(turn.thinking); - if (turn.text) output.text = capForUpload(turn.text); + if (step.thinking) output.thinking = capForUpload(step.thinking); + if (step.text) output.text = capForUpload(step.text); const metadata: Record = {}; - if (turn.messageId) metadata.messageId = turn.messageId; + if (step.messageId) metadata.messageId = step.messageId; return { - name: `turn ${turn.index + 1}`, + name: `step ${step.index + 1}`, type: "llm", ...(Object.keys(output).length > 0 ? { output } : {}), ...(Object.keys(metadata).length > 0 ? { metadata } : {}), - ...(turn.usage ? { metrics: usageMetrics(turn.usage) } : {}), - children: (turn.toolCalls ?? []).map(toolSpan), + ...(step.usage ? { metrics: usageMetrics(step.usage) } : {}), + children: (step.toolCalls ?? []).map(toolSpan), }; } @@ -245,9 +241,9 @@ function toolSpan(execution: ToolExecution): UnplacedSpan { ? { output: capForUpload(execution.output) } : { error: execution.error }), ...(Object.keys(metadata).length > 0 ? { metadata } : {}), - // A subagent sidechain renders as the Task call's own turn subtree. + // A subagent sidechain renders as the Task call's own step subtree. ...(execution.children?.length - ? { children: execution.children.map(turnSpan) } + ? { children: execution.children.map(stepSpan) } : {}), }; } @@ -295,6 +291,45 @@ function flatSpans(result: UploadableEvalResult): UnplacedSpan[] { return spans; } +type ParsedCheck = NonNullable[number]; +type ParsedJudgeCall = z.infer; + +/** + * One llm span per judge call, named after the check whose verdict it + * produced (matched by verbatim notes; a judge call is consumed at most + * once). Its input is the rubric plus the exact text the judge evaluated — + * the part that is otherwise discarded after scoring. + */ +function judgeCallSpans( + checks: ParsedCheck[], + judgeCalls: ParsedJudgeCall[], +): UnplacedSpan[] { + const unmatched = [...checks]; + return judgeCalls.map((call) => { + const index = unmatched.findIndex( + (check) => + check.judgeNotes !== undefined && + check.judgeNotes === call.notes && + check.passed === call.passed, + ); + const check = index >= 0 ? unmatched.splice(index, 1)[0] : undefined; + const metadata: Record = {}; + if (call.modelId) metadata.modelId = call.modelId; + if (call.durationMs !== undefined) metadata.durationMs = call.durationMs; + return { + name: check ? `judge: ${check.name}` : "judge", + type: "llm" as const, + input: { rubric: call.rubric, input: capForUpload(call.input) }, + output: { + passed: call.passed, + ...(call.notes ? { notes: call.notes } : {}), + }, + ...(Object.keys(metadata).length > 0 ? { metadata } : {}), + ...(call.usage ? { metrics: usageMetrics(call.usage) } : {}), + }; + }); +} + /** * Assign synthetic timing: siblings split their window into equal contiguous * slots, children subdivide their parent's slot. Order → position; durations @@ -339,12 +374,15 @@ export function buildEvalTrace(args: { : flatSpans(result); const checks = result.checks ?? []; + // A single Braintrust scorer (`passed`), mirroring eval-results.json's + // headline flag; the checks array rides along as the scorer's output + // instead of one scorer per check, keeping the experiment UI to one score + // column. Judge calls made during scoring nest under it as llm spans — the + // evidence trail of exactly what each judge saw and what the verdict cost. const scores: Record = { passed: result.passed === true ? 1 : 0, }; - const takenScoreNames = new Set(Object.keys(scores)); - // The overall verdict as its own score span — the trace-view counterpart of - // eval-results.json's headline `passed`, ahead of the per-check spans. + const judgeSpans = judgeCallSpans(checks, result.judgeCalls ?? []); const scoreSpans: UnplacedSpan[] = [ { name: "passed", @@ -352,25 +390,12 @@ export function buildEvalTrace(args: { output: { passed: result.passed === true, checksPassed: `${checks.filter((c) => c.passed).length}/${checks.length}`, + checks, }, scores: { passed: scores.passed! }, + ...(judgeSpans.length > 0 ? { children: judgeSpans } : {}), }, ]; - for (const check of checks) { - const scoreName = scoreNameFor(check.name, takenScoreNames); - const value = check.passed ? 1 : 0; - scores[scoreName] = value; - scoreSpans.push({ - name: scoreName, - type: "score", - output: { - passed: check.passed, - ...(check.notes ? { notes: check.notes } : {}), - ...(check.judgeNotes ? { judgeNotes: check.judgeNotes } : {}), - }, - scores: { [scoreName]: value }, - }); - } const unplaced = [...activity, ...scoreSpans]; diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 5d19e349..57b15d2f 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -10,10 +10,12 @@ import { tmpdir } from 'node:os'; import { fileURLToPath } from 'node:url'; import type { ToolName } from './transcript/types.js'; import type { + AgentStep, AgentTrace, AgentTurn, ToolExecution, } from './transcript/agent-trace.js'; +import { recordJudgeCall } from './judge-recorder.js'; import { createClient, type SupabaseClient } from '@supabase/supabase-js'; import { createMCPClient } from '@ai-sdk/mcp'; import { Experimental_StdioMCPTransport as StdioMCPTransport } from '@ai-sdk/mcp/mcp-stdio'; @@ -138,11 +140,13 @@ export type { } from './transcript/types.js'; export { assembleAgentTrace } from './transcript/agent-trace.js'; export type { + AgentStep, AgentTrace, AgentTurn, ToolExecution, - TraceInterjection, } from './transcript/agent-trace.js'; +export { collectJudgeCalls, recordJudgeCall } from './judge-recorder.js'; +export type { JudgeCallRecord } from './judge-recorder.js'; export type { AgentHarnessId, CheckResult, @@ -619,7 +623,8 @@ export async function judge(args: JudgeInput): Promise { const providerOptions = args.providerOptions ?? DEFAULT_JUDGE_PROVIDER_OPTIONS; assertProviderReady(model.provider); - const { output } = await generateText({ + const startedAt = Date.now(); + const result = await generateText({ model, system: 'You are a strict eval judge. Return only the requested structured judgment.', @@ -628,6 +633,25 @@ export async function judge(args: JudgeInput): Promise { maxOutputTokens: MAX_OUTPUT_TOKENS, providerOptions: withProviderDefaults(model.provider, providerOptions), }); + const output = result.output; + + // Evidence trail for the Braintrust upload: what this judge actually saw + // and what the verdict cost. No-op unless a collector is active. + recordJudgeCall({ + rubric: args.rubric, + input: args.input, + passed: output.passed, + notes: output.notes, + modelId: model.modelId, + durationMs: Date.now() - startedAt, + usage: { + inputTokens: result.totalUsage.inputTokens, + outputTokens: result.totalUsage.outputTokens, + cachedInputTokens: result.totalUsage.cachedInputTokens, + reasoningTokens: result.totalUsage.reasoningTokens, + totalTokens: result.totalUsage.totalTokens, + }, + }); return { passed: output.passed, @@ -776,21 +800,24 @@ export function aiSdkAgent(options: { totalTokens: result.totalUsage.totalTokens, }; - // Structured trace straight from the steps: one turn per step, tool - // executions paired within it (`toolOutputs` above), per-step usage. - const turns: AgentTurn[] = result.steps.map((step, index) => { - const turn: AgentTurn = { index, toolCalls: [] }; + // Structured trace straight from the steps: one model call per step, + // tool executions paired within it (`toolOutputs` above), per-step + // usage. One prompt in, everything autonomous → a single turn. + const traceSteps: AgentStep[] = result.steps.map((step, index) => { + const traceStep: AgentStep = { index, toolCalls: [] }; for (const part of step.content) { if (part.type === "text") { const content = part.text.trim(); if (content) { - turn.text = turn.text ? `${turn.text}\n${content}` : content; + traceStep.text = traceStep.text + ? `${traceStep.text}\n${content}` + : content; } } else if (part.type === "reasoning") { const content = part.text.trim(); if (content) { - turn.thinking = turn.thinking - ? `${turn.thinking}\n${content}` + traceStep.thinking = traceStep.thinking + ? `${traceStep.thinking}\n${content}` : content; } } else if (part.type === "tool-call") { @@ -804,11 +831,11 @@ export function aiSdkAgent(options: { ? { output: resolved?.output } : { error: resolved.error }), }; - turn.toolCalls.push(execution); + traceStep.toolCalls.push(execution); } } if (step.usage) { - turn.usage = { + traceStep.usage = { inputTokens: step.usage.inputTokens, outputTokens: step.usage.outputTokens, cachedInputTokens: step.usage.cachedInputTokens, @@ -816,9 +843,12 @@ export function aiSdkAgent(options: { totalTokens: step.usage.totalTokens, }; } - return turn; + return traceStep; }); - const trace: AgentTrace = { turns, interjections: [], errors: [] }; + const trace: AgentTrace = { + turns: [{ index: 0, steps: traceSteps }], + errors: [], + }; return { agentReport, diff --git a/packages/core/src/judge-recorder.ts b/packages/core/src/judge-recorder.ts new file mode 100644 index 00000000..8007ea0a --- /dev/null +++ b/packages/core/src/judge-recorder.ts @@ -0,0 +1,40 @@ +/** + * Records every `judge()` call made while a scorer runs — rubric, input, + * verdict, model, tokens — without any per-eval wiring: the eval runner + * wraps the scorer invocation in `collectJudgeCalls`, and `judge()` appends + * to whichever collection is active on the async context. The records are + * persisted with the raw result so the Braintrust upload can show exactly + * what each judge saw (its input is otherwise discarded after the call). + * + * AsyncLocalStorage keeps concurrent eval runs in one process from + * interleaving their judge calls. + */ + +import { AsyncLocalStorage } from "node:async_hooks"; +import type { AgentUsage } from "./index.js"; + +export interface JudgeCallRecord { + rubric: string; + input: string; + passed: boolean; + notes?: string; + modelId: string; + durationMs: number; + usage?: AgentUsage; +} + +const storage = new AsyncLocalStorage(); + +/** Append a judge call to the active collection, if any (no-op outside one). */ +export function recordJudgeCall(record: JudgeCallRecord): void { + storage.getStore()?.push(record); +} + +/** Run `fn` (typically a scorer) and collect the judge calls it made. */ +export async function collectJudgeCalls( + fn: () => Promise, +): Promise<{ result: T; judgeCalls: JudgeCallRecord[] }> { + const judgeCalls: JudgeCallRecord[] = []; + const result = await storage.run(judgeCalls, fn); + return { result, judgeCalls }; +} diff --git a/packages/core/src/transcript/agent-trace.test.ts b/packages/core/src/transcript/agent-trace.test.ts index 51e4b693..5a1464e3 100644 --- a/packages/core/src/transcript/agent-trace.test.ts +++ b/packages/core/src/transcript/agent-trace.test.ts @@ -4,7 +4,7 @@ import type { TranscriptEvent } from "./types.js"; describe("assembleAgentTrace with turn keys (Claude Code style)", () => { const events: TranscriptEvent[] = [ - // Turn 1: one assistant API message with thinking + text + two tool uses. + // Step 1: one assistant API message with thinking + text + two tool uses. { type: "thinking", content: "let me look", @@ -47,7 +47,7 @@ describe("assembleAgentTrace with turn keys (Claude Code style)", () => { tool: { name: "file_read", originalName: "Read", id: "t3", args: { file_path: "a.ts" }, path: "a.ts" }, }, { type: "tool_result", parentToolUseId: "t2", tool: { name: "unknown", originalName: "t3", id: "t3", result: "code", success: true } }, - // Turn 2: a second API message. + // Step 2: a second API message. { type: "message", role: "assistant", @@ -59,38 +59,40 @@ describe("assembleAgentTrace with turn keys (Claude Code style)", () => { ]; const trace = assembleAgentTrace(events); - it("groups one API message into one turn", () => { - expect(trace.turns).toHaveLength(2); - const [first, second] = trace.turns; - expect(first!.text).toBe("Checking the tables."); - expect(first!.thinking).toBe("let me look"); - expect(first!.toolCalls.map((t) => t.name)).toEqual(["Bash", "Task"]); - expect(second!.text).toBe("Done."); - expect(second!.toolCalls).toHaveLength(0); + it("keeps one autonomous run as a single turn with one step per API message", () => { + expect(trace.turns).toHaveLength(1); + const steps = trace.turns[0]!.steps; + expect(steps).toHaveLength(2); + expect(steps[0]!.text).toBe("Checking the tables."); + expect(steps[0]!.thinking).toBe("let me look"); + expect(steps[0]!.toolCalls.map((t) => t.name)).toEqual(["Bash", "Task"]); + expect(steps[1]!.text).toBe("Done."); + expect(steps[1]!.toolCalls).toHaveLength(0); }); - it("carries per-turn usage and message ids", () => { - expect(trace.turns[0]!.messageId).toBe("msg_1"); - expect(trace.turns[0]!.usage).toEqual({ inputTokens: 100, outputTokens: 10 }); - expect(trace.turns[1]!.usage).toEqual({ inputTokens: 200, outputTokens: 20 }); + it("carries per-step usage and message ids", () => { + const steps = trace.turns[0]!.steps; + expect(steps[0]!.messageId).toBe("msg_1"); + expect(steps[0]!.usage).toEqual({ inputTokens: 100, outputTokens: 10 }); + expect(steps[1]!.usage).toEqual({ inputTokens: 200, outputTokens: 20 }); }); it("pairs tool results onto their executions", () => { - const bash = trace.turns[0]!.toolCalls[0]!; + const bash = trace.turns[0]!.steps[0]!.toolCalls[0]!; expect(bash.output).toBe("files"); expect(bash.success).toBe(true); expect(bash.command).toBe("ls"); }); it("nests the subagent sidechain under the Task call", () => { - const task = trace.turns[0]!.toolCalls[1]!; + const task = trace.turns[0]!.steps[0]!.toolCalls[1]!; expect(task.children).toHaveLength(1); const sub = task.children![0]!; expect(sub.text).toBe("sub work"); expect(sub.toolCalls[0]!.name).toBe("Read"); expect(sub.toolCalls[0]!.output).toBe("code"); - // Sidechain turns don't leak into the main thread. - expect(trace.turns).toHaveLength(2); + // Sidechain steps don't leak into the main thread. + expect(trace.turns[0]!.steps).toHaveLength(2); }); }); @@ -111,28 +113,40 @@ describe("assembleAgentTrace closure rule (Codex style, no turn keys)", () => { ]; const trace = assembleAgentTrace(events); - it("closes a turn on each assistant message", () => { - expect(trace.turns).toHaveLength(2); - expect(trace.turns[0]!.thinking).toBe("plan"); - expect(trace.turns[0]!.text).toBe("Found it."); - expect(trace.turns[0]!.toolCalls.map((t) => t.args.command)).toEqual(["psql"]); - expect(trace.turns[1]!.text).toBe("Fixed."); - expect(trace.turns[1]!.toolCalls.map((t) => t.args.command)).toEqual(["fix"]); + it("closes a step on each assistant message, all within one turn", () => { + expect(trace.turns).toHaveLength(1); + const steps = trace.turns[0]!.steps; + expect(steps).toHaveLength(2); + expect(steps[0]!.thinking).toBe("plan"); + expect(steps[0]!.text).toBe("Found it."); + expect(steps[0]!.toolCalls.map((t) => t.args.command)).toEqual(["psql"]); + expect(steps[1]!.text).toBe("Fixed."); + expect(steps[1]!.toolCalls.map((t) => t.args.command)).toEqual(["fix"]); }); }); -describe("assembleAgentTrace edges", () => { - it("records user/system messages as interjections", () => { +describe("assembleAgentTrace turn boundaries and edges", () => { + it("opens a new turn on a mid-run user message", () => { + const trace = assembleAgentTrace([ + { type: "message", role: "assistant", content: "first", turnKey: "m1" }, + { type: "message", role: "user", content: "extra instruction" }, + { type: "message", role: "assistant", content: "second", turnKey: "m2" }, + ]); + expect(trace.turns).toHaveLength(2); + expect(trace.turns[0]!.userMessage).toBeUndefined(); + expect(trace.turns[0]!.steps.map((s) => s.text)).toEqual(["first"]); + expect(trace.turns[1]!.userMessage).toBe("extra instruction"); + expect(trace.turns[1]!.steps.map((s) => s.text)).toEqual(["second"]); + }); + + it("folds a leading user message into the first turn", () => { const trace = assembleAgentTrace([ { type: "message", role: "user", content: "skill body" }, { type: "message", role: "assistant", content: "ok", turnKey: "m1" }, - { type: "message", role: "user", content: "tool feedback" }, ]); expect(trace.turns).toHaveLength(1); - expect(trace.interjections).toEqual([ - { afterTurnIndex: -1, role: "user", content: "skill body" }, - { afterTurnIndex: 0, role: "user", content: "tool feedback" }, - ]); + expect(trace.turns[0]!.userMessage).toBe("skill body"); + expect(trace.turns[0]!.steps.map((s) => s.text)).toEqual(["ok"]); }); it("collects agent errors and flags orphaned sidechains", () => { @@ -163,7 +177,7 @@ describe("assembleAgentTrace edges", () => { }, { type: "message", role: "assistant", content: "hm" }, ]); - const call = trace.turns[0]!.toolCalls[0]!; + const call = trace.turns[0]!.steps[0]!.toolCalls[0]!; expect(call.error).toBe("exit 1"); expect(call.output).toBeUndefined(); expect(call.success).toBe(false); diff --git a/packages/core/src/transcript/agent-trace.ts b/packages/core/src/transcript/agent-trace.ts index 2738301b..5ccbd9b9 100644 --- a/packages/core/src/transcript/agent-trace.ts +++ b/packages/core/src/transcript/agent-trace.ts @@ -1,16 +1,22 @@ /** - * Structured, agent-agnostic transcript: turns (one per model API call) with - * their tool executions nested, results paired at assembly time, and subagent - * sidechains attached under the tool call that spawned them. + * Structured, agent-agnostic transcript. + * + * A **turn** is everything the agent does between user inputs — one user + * prompt in, all autonomous work until the next user input. Evals send a + * single prompt, so most traces have exactly one turn (a mid-run injected + * user message, e.g. a skill body, opens a new one). Inside a turn, a + * **step** is one model API call: its thinking, assistant text, and the tool + * executions it requested, with results paired at assembly time and subagent + * sidechains nested under the tool call that spawned them. * * Assembled from the same canonical `TranscriptEvent[]` the flat scorer * surface (`adaptTranscript`) consumes, so the two views can never disagree * about what happened; this one just stops throwing the structure away. - * Turn grouping uses `event.turnKey` when the parser stamped one (Claude - * Code: one assistant line = one API message; ai-sdk builds turns directly - * from steps) and otherwise falls back to a closure rule: thinking and tool - * calls accumulate into the open turn, an assistant message closes it - * (Codex's item stream). + * Step grouping uses `event.turnKey` when the parser stamped one (Claude + * Code: one assistant line = one API message; ai-sdk builds steps directly) + * and otherwise falls back to a closure rule: thinking and tool calls + * accumulate into the open step, an assistant message closes it (Codex's + * item stream). */ import type { AgentUsage } from "../index.js"; @@ -35,33 +41,36 @@ export interface ToolExecution { /** Epoch-ms timing when the transcript carries real timestamps. */ startMs?: number; /** Subagent sidechain (e.g. Claude Code Task) spawned by this call. */ - children?: AgentTurn[]; + children?: AgentStep[]; } -export interface AgentTurn { +/** One model API call within a turn. */ +export interface AgentStep { index: number; /** Provider message id of the API call, when exposed. */ messageId?: string; - /** Per-turn token usage, when the transcript carries it. */ + /** Per-call token usage, when the transcript carries it. */ usage?: AgentUsage; startMs?: number; thinking?: string; - /** Assistant prose (absent on pure-tool turns). */ + /** Assistant prose (absent on pure-tool steps). */ text?: string; toolCalls: ToolExecution[]; } -/** A user/system message arriving between turns (e.g. an injected skill body). */ -export interface TraceInterjection { - /** Index of the last turn opened before this message; -1 = before any turn. */ - afterTurnIndex: number; - role: "user" | "system"; - content: string; +/** Everything the agent did between two user inputs. */ +export interface AgentTurn { + index: number; + /** + * The user/system message that opened this turn. Absent on the first turn + * — the eval prompt itself lives on the result, not in the transcript. + */ + userMessage?: string; + steps: AgentStep[]; } export interface AgentTrace { turns: AgentTurn[]; - interjections: TraceInterjection[]; /** Agent-reported error events, in order. */ errors: string[]; } @@ -88,41 +97,59 @@ export function assembleAgentTrace(events: TranscriptEvent[]): AgentTrace { } const turns: AgentTurn[] = []; - const interjections: TraceInterjection[] = []; const errors: string[] = []; const executionsById = new Map(); - const turnsByKey = new Map(); + const stepsByKey = new Map(); + let stepCount = 0; // Closure-rule state for events without a turnKey: an assistant message - // marks the open turn closed, so the next event starts a fresh one. - let openTurn: AgentTurn | undefined; - let openTurnClosed = false; + // marks the open step closed, so the next event starts a fresh one. + let openStep: AgentStep | undefined; + let openStepClosed = false; + // A user/system message between steps opens the next turn; consecutive + // boundary messages fold into one. + let pendingUserMessage: string | undefined; - const turnFor = (event: TranscriptEvent): AgentTurn => { - if (event.turnKey) { - const existing = turnsByKey.get(event.turnKey); - if (existing) return existing; - const turn = newTurn(event); - turnsByKey.set(event.turnKey, turn); + const currentTurn = (): AgentTurn => { + if (turns.length === 0 || pendingUserMessage !== undefined) { + const turn: AgentTurn = { + index: turns.length, + ...(pendingUserMessage !== undefined + ? { userMessage: pendingUserMessage } + : {}), + steps: [], + }; + pendingUserMessage = undefined; + turns.push(turn); return turn; } - if (!openTurn || openTurnClosed) { - openTurn = newTurn(event); - openTurnClosed = false; - } - return openTurn; + return turns[turns.length - 1]!; }; - const newTurn = (event: TranscriptEvent): AgentTurn => { - const turn: AgentTurn = { index: turns.length, toolCalls: [] }; + const newStep = (event: TranscriptEvent): AgentStep => { + const step: AgentStep = { index: stepCount, toolCalls: [] }; + stepCount += 1; const startMs = parseMs(event.timestamp); - if (startMs !== undefined) turn.startMs = startMs; - turns.push(turn); - return turn; + if (startMs !== undefined) step.startMs = startMs; + currentTurn().steps.push(step); + return step; }; - for (const event of events) { - if (event.parentToolUseId) continue; + const stepFor = (event: TranscriptEvent): AgentStep => { + if (event.turnKey) { + const existing = stepsByKey.get(event.turnKey); + if (existing) return existing; + const step = newStep(event); + stepsByKey.set(event.turnKey, step); + return step; + } + if (!openStep || openStepClosed) { + openStep = newStep(event); + openStepClosed = false; + } + return openStep; + }; + for (const event of main) { if (event.type === "error") { if (event.content) errors.push(event.content); continue; @@ -131,11 +158,12 @@ export function assembleAgentTrace(events: TranscriptEvent[]): AgentTrace { if (event.type === "message" && event.role && event.role !== "assistant") { const content = event.content?.trim(); if (content) { - interjections.push({ - afterTurnIndex: turns.length - 1, - role: event.role, - content, - }); + pendingUserMessage = + pendingUserMessage === undefined + ? content + : `${pendingUserMessage}\n\n${content}`; + // A user input also ends whatever keyless step was open. + openStepClosed = true; } continue; } @@ -161,28 +189,30 @@ export function assembleAgentTrace(events: TranscriptEvent[]): AgentTrace { } if (event.type === "message" && event.role === "assistant") { - const turn = turnFor(event); + const step = stepFor(event); const content = event.content?.trim(); if (content) { - turn.text = turn.text ? `${turn.text}\n${content}` : content; + step.text = step.text ? `${step.text}\n${content}` : content; } - annotate(turn, event); - if (!event.turnKey) openTurnClosed = true; + annotate(step, event); + if (!event.turnKey) openStepClosed = true; continue; } if (event.type === "thinking") { - const turn = turnFor(event); + const step = stepFor(event); const content = event.content?.trim(); if (content) { - turn.thinking = turn.thinking ? `${turn.thinking}\n${content}` : content; + step.thinking = step.thinking + ? `${step.thinking}\n${content}` + : content; } - annotate(turn, event); + annotate(step, event); continue; } if (event.type === "tool_call" && event.tool) { - const turn = turnFor(event); + const step = stepFor(event); const execution: ToolExecution = { ...(event.tool.id ? { id: event.tool.id } : {}), name: event.tool.originalName, @@ -197,12 +227,22 @@ export function assembleAgentTrace(events: TranscriptEvent[]): AgentTrace { }; const startMs = parseMs(event.timestamp); if (startMs !== undefined) execution.startMs = startMs; - turn.toolCalls.push(execution); + step.toolCalls.push(execution); if (execution.id) executionsById.set(execution.id, execution); - annotate(turn, event); + annotate(step, event); } } + // A trailing user message with no assistant activity after it still opens + // an (empty) turn, so it isn't silently dropped. + if (pendingUserMessage !== undefined) { + turns.push({ + index: turns.length, + userMessage: pendingUserMessage, + steps: [], + }); + } + // Attach sidechains to their spawning tool call; groups whose parent call // never appeared (truncated transcript) surface as an error note instead // of vanishing. @@ -210,24 +250,25 @@ export function assembleAgentTrace(events: TranscriptEvent[]): AgentTrace { const parent = executionsById.get(parentId); const subTrace = assembleAgentTrace(group); errors.push(...subTrace.errors); + const subSteps = subTrace.turns.flatMap((turn) => turn.steps); if (parent) { - parent.children = subTrace.turns; - } else if (subTrace.turns.length > 0) { + parent.children = subSteps; + } else if (subSteps.length > 0) { errors.push( `orphaned subagent transcript (parent tool_use ${parentId} not found)`, ); } } - return { turns, interjections, errors }; + return { turns, errors }; } -/** Fold an event's per-call identifiers/usage onto its turn. */ -function annotate(turn: AgentTurn, event: TranscriptEvent): void { - if (event.messageId && !turn.messageId) turn.messageId = event.messageId; - if (event.usage && !turn.usage) turn.usage = event.usage; - if (turn.startMs === undefined) { +/** Fold an event's per-call identifiers/usage onto its step. */ +function annotate(step: AgentStep, event: TranscriptEvent): void { + if (event.messageId && !step.messageId) step.messageId = event.messageId; + if (event.usage && !step.usage) step.usage = event.usage; + if (step.startMs === undefined) { const startMs = parseMs(event.timestamp); - if (startMs !== undefined) turn.startMs = startMs; + if (startMs !== undefined) step.startMs = startMs; } } From 9c74d7993b73707e61d26d544526895722755952 Mon Sep 17 00:00:00 2001 From: Pedro Rodrigues Date: Tue, 21 Jul 2026 18:06:11 +0100 Subject: [PATCH 06/15] feat: attach the agent CLI's verbatim transcript to Braintrust rows MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The runner persists stream-json/exec-json stdout as a sibling .transcript.jsonl (never inside the result JSON — it can be megabytes; .jsonl keeps it out of the results scan and the web export). The uploader attaches it to the root span via Braintrust Attachments, the documented mechanism for large documents (object storage, no span size limits), next to the existing raw-result attachment. Co-Authored-By: Claude Fable 5 --- apps/framework/harness/run-eval.ts | 23 ++++++++++++++++- apps/framework/scripts/upload-braintrust.ts | 28 +++++++++++++++++++-- packages/core/src/agents/engine.ts | 1 + packages/core/src/index.ts | 7 ++++++ 4 files changed, 56 insertions(+), 3 deletions(-) diff --git a/apps/framework/harness/run-eval.ts b/apps/framework/harness/run-eval.ts index 1fff4689..bc9185e1 100644 --- a/apps/framework/harness/run-eval.ts +++ b/apps/framework/harness/run-eval.ts @@ -277,6 +277,15 @@ function resultPath(modelName: string, ev: Pick) { return join(ROOT, 'results', modelName, `${ev.id}.json`); } +/** + * Sibling of a result file holding the agent CLI's verbatim JSONL transcript + * (`.transcript.jsonl`). Not `.json`, so the results scan and the + * web export never pick it up. + */ +function rawTranscriptPath(resultFilePath: string) { + return `${resultFilePath.slice(0, -".json".length)}.transcript.jsonl`; +} + function workspacePath(modelName: string, evalId: string, attempt: number) { return join( ROOT, @@ -368,6 +377,7 @@ async function runOne( usage?: AgentUsage; trace?: AgentTrace; judgeCalls?: JudgeCallRecord[]; + rawTranscript?: string; } > { const prompt = parseEvalMarkdown( @@ -400,6 +410,7 @@ async function runOne( let lastUsage: AgentUsage | undefined; let lastTrace: AgentTrace | undefined; let lastJudgeCalls: JudgeCallRecord[] = []; + let lastRawTranscript: string | undefined; for (let attempt = 1; attempt <= RUNS; attempt += 1) { if (ev.mode === 'local-stack') { @@ -472,6 +483,7 @@ async function runOne( lastSteps = run.steps; lastUsage = run.usage; lastTrace = run.trace; + lastRawTranscript = run.rawTranscript; // Export the agent's workspace to the host so scorers can run host // tooling (vite/vitest from the repo root) against the produced files @@ -518,6 +530,7 @@ async function runOne( usage: run.usage, trace: run.trace, judgeCalls: lastJudgeCalls, + rawTranscript: run.rawTranscript, }; } logRetryAttempt(expName, ev, attempt, last); @@ -570,6 +583,7 @@ async function runOne( lastSteps = run.steps; lastUsage = run.usage; lastTrace = run.trace; + lastRawTranscript = run.rawTranscript; const scored = await collectJudgeCalls(() => (scorer as ToolScorer)({ ...session.scoringContext, @@ -595,6 +609,7 @@ async function runOne( usage: run.usage, trace: run.trace, judgeCalls: lastJudgeCalls, + rawTranscript: run.rawTranscript, }; } logRetryAttempt(expName, ev, attempt, last); @@ -613,6 +628,7 @@ async function runOne( usage: lastUsage, trace: lastTrace, judgeCalls: lastJudgeCalls, + rawTranscript: lastRawTranscript, }; } @@ -803,8 +819,13 @@ async function main() { console.log(`⏳ RUN ${name} x ${ev.id}`); const run = async () => { try { - const res = await runOne(name, config, ev); + const { rawTranscript, ...res } = await runOne(name, config, ev); mkdirSync(dirname(out), { recursive: true }); + // The CLI's verbatim transcript is a sibling file, never part of the + // result JSON (it can be megabytes); the Braintrust upload attaches it. + if (rawTranscript) { + writeFileSync(rawTranscriptPath(out), rawTranscript); + } const experimentDisplay = getExperimentDisplayMetadata(config); writeFileSync( out, diff --git a/apps/framework/scripts/upload-braintrust.ts b/apps/framework/scripts/upload-braintrust.ts index 8415cb3a..45e82ae8 100644 --- a/apps/framework/scripts/upload-braintrust.ts +++ b/apps/framework/scripts/upload-braintrust.ts @@ -22,6 +22,8 @@ */ import { readFile } from "node:fs/promises"; +import { existsSync } from "node:fs"; +import { basename } from "node:path"; import { execFileSync } from "node:child_process"; import { buildEvalTrace, @@ -98,6 +100,11 @@ interface PendingTrace { trace: BraintrustEvalTrace; /** Raw result file, attached to the root span for full-fidelity forensics. */ filePath: string; + /** + * The agent CLI's verbatim JSONL transcript (sibling + * `.transcript.jsonl` written by the runner), attached alongside. + */ + rawTranscriptPath?: string; } async function collectTraces( @@ -174,8 +181,13 @@ async function collectTraces( }, baseTimeMs: Date.now(), }); + const transcriptPath = ref.filePath.replace(/\.json$/, ".transcript.jsonl"); const group = byExperiment.get(result.experiment) ?? []; - group.push({ trace, filePath: ref.filePath }); + group.push({ + trace, + filePath: ref.filePath, + ...(existsSync(transcriptPath) ? { rawTranscriptPath: transcriptPath } : {}), + }); byExperiment.set(result.experiment, group); } @@ -237,7 +249,7 @@ async function upload( }, }); - for (const { trace, filePath } of pending) { + for (const { trace, filePath, rawTranscriptPath } of pending) { const root = btExperiment.startSpan({ name: trace.evalId, type: "eval", @@ -254,6 +266,18 @@ async function upload( filename: `${trace.experiment}__${trace.evalId}.json`, contentType: "application/json", }), + // The agent CLI's own transcript, verbatim — attachments are + // Braintrust's mechanism for large documents (span fields cap out; + // attachments go to object storage and stay downloadable). + ...(rawTranscriptPath + ? { + rawTranscript: new Attachment({ + data: rawTranscriptPath, + filename: basename(rawTranscriptPath), + contentType: "application/jsonl", + }), + } + : {}), }, ...(Object.keys(trace.metrics).length > 0 ? { metrics: trace.metrics } diff --git a/packages/core/src/agents/engine.ts b/packages/core/src/agents/engine.ts index 0781e7b2..185bad94 100644 --- a/packages/core/src/agents/engine.ts +++ b/packages/core/src/agents/engine.ts @@ -135,6 +135,7 @@ export function createCliAgent( runner.deriveStopReason?.(raw, command) ?? processStopReason(command), ...(usage ? { usage } : {}), trace: assembleAgentTrace(events), + ...(raw ? { rawTranscript: raw } : {}), }; }, }; diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 57b15d2f..e41aa255 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -429,6 +429,13 @@ export type AgentRunResult = { * surface scorers consume is unchanged. */ trace?: AgentTrace; + /** + * The agent CLI's own transcript, verbatim (stream-json / exec-json + * JSONL). Absent for in-process agents (ai-sdk), which have no CLI + * transcript. The runner persists it as a sibling file next to the result + * JSON, never inside it. + */ + rawTranscript?: string; }; export type AgentHarness = { From 6cc2804e36bdf282c97bb8159c749088f43a556f Mon Sep 17 00:00:00 2001 From: Pedro Rodrigues Date: Wed, 22 Jul 2026 12:55:23 +0100 Subject: [PATCH 07/15] refactor: review-pass cleanups on the Braintrust upload branch - Anthropic usage accounting extracted to agents/claude-code/usage.ts, shared by the runner (run totals) and parser (per-message) so the OpenAI-style convention can't drift; finiteNumber joins json.ts and replaces the codex runner's inline coercer - Stale .transcript.jsonl siblings removed on re-runs that produce no raw transcript, so an old transcript can't attach to a new result - Dead per-event startMs capture deleted from the trace assembler (nothing consumed it; span timing is synthetic by design) - turnKey renamed stepKey to match the turn/step model; stale module and test wording corrected - rawTranscriptPathFor and makeFilterPredicate centralized in lib/result-files.ts / lib/cli-args.ts, deleting the duplicated sibling-path convention and suite filters - judge() no longer records an all-undefined usage object; internal recordJudgeCall and BraintrustSpanNode dropped from the public core exports Co-Authored-By: Claude Fable 5 --- apps/framework/harness/run-eval.ts | 18 +++----- apps/framework/lib/cli-args.ts | 11 +++++ apps/framework/lib/result-files.ts | 10 ++++ apps/framework/scripts/export-results.ts | 25 ++-------- apps/framework/scripts/upload-braintrust.ts | 22 +++------ .../core/src/agents/claude-code/parser.ts | 31 +++---------- .../core/src/agents/claude-code/runner.ts | 31 ++----------- packages/core/src/agents/claude-code/usage.ts | 46 +++++++++++++++++++ packages/core/src/agents/codex/runner.ts | 10 ++-- packages/core/src/braintrust-spans.ts | 27 ++++++----- packages/core/src/index.ts | 20 ++++---- packages/core/src/json.ts | 7 +++ .../core/src/transcript/agent-trace.test.ts | 26 +++++------ packages/core/src/transcript/agent-trace.ts | 35 ++++---------- packages/core/src/transcript/types.ts | 6 +-- 15 files changed, 159 insertions(+), 166 deletions(-) create mode 100644 packages/core/src/agents/claude-code/usage.ts diff --git a/apps/framework/harness/run-eval.ts b/apps/framework/harness/run-eval.ts index bc9185e1..b881445f 100644 --- a/apps/framework/harness/run-eval.ts +++ b/apps/framework/harness/run-eval.ts @@ -25,6 +25,7 @@ import { readRepeatedFlag, readSuiteFilters, } from '../lib/cli-args.js'; +import { rawTranscriptPathFor } from '../lib/result-files.js'; import { bootPlatformBackend } from './platform-backend.js'; import { viteBuild, vitestRun } from './project-runner.js'; import { @@ -277,15 +278,6 @@ function resultPath(modelName: string, ev: Pick) { return join(ROOT, 'results', modelName, `${ev.id}.json`); } -/** - * Sibling of a result file holding the agent CLI's verbatim JSONL transcript - * (`.transcript.jsonl`). Not `.json`, so the results scan and the - * web export never pick it up. - */ -function rawTranscriptPath(resultFilePath: string) { - return `${resultFilePath.slice(0, -".json".length)}.transcript.jsonl`; -} - function workspacePath(modelName: string, evalId: string, attempt: number) { return join( ROOT, @@ -822,9 +814,13 @@ async function main() { const { rawTranscript, ...res } = await runOne(name, config, ev); mkdirSync(dirname(out), { recursive: true }); // The CLI's verbatim transcript is a sibling file, never part of the - // result JSON (it can be megabytes); the Braintrust upload attaches it. + // result JSON (it can be megabytes); the Braintrust upload attaches + // it. Remove any stale sibling on runs that produce none, so a + // re-run can't pair an old transcript with a new result. if (rawTranscript) { - writeFileSync(rawTranscriptPath(out), rawTranscript); + writeFileSync(rawTranscriptPathFor(out), rawTranscript); + } else { + rmSync(rawTranscriptPathFor(out), { force: true }); } const experimentDisplay = getExperimentDisplayMetadata(config); writeFileSync( diff --git a/apps/framework/lib/cli-args.ts b/apps/framework/lib/cli-args.ts index 23f4e36d..8c2a481e 100644 --- a/apps/framework/lib/cli-args.ts +++ b/apps/framework/lib/cli-args.ts @@ -54,6 +54,17 @@ export function readSuiteFilters(rawArgs: string[]): EvalSuite[] { }); } +/** + * Predicate for a repeated-filter flag: no filters means include everything, + * otherwise the value must be present (and match one of them). + */ +export function makeFilterPredicate( + filters: readonly T[], +): (value: T | undefined) => boolean { + return (value) => + filters.length === 0 || (value !== undefined && filters.includes(value)); +} + export function readExperimentSuiteFilters( rawArgs: string[] ): ExperimentSuite[] { diff --git a/apps/framework/lib/result-files.ts b/apps/framework/lib/result-files.ts index b23f0798..d5a25209 100644 --- a/apps/framework/lib/result-files.ts +++ b/apps/framework/lib/result-files.ts @@ -80,6 +80,16 @@ export async function readPrompt( }; } +/** + * Sibling of a result file holding the agent CLI's verbatim JSONL transcript + * (`.transcript.jsonl`). Not `.json`, so the results scan and the + * web export never pick it up. Single definition so the runner (writer) and + * the Braintrust uploader (reader) can't drift. + */ +export function rawTranscriptPathFor(resultFilePath: string): string { + return `${resultFilePath.slice(0, -".json".length)}.transcript.jsonl`; +} + export interface ResultFileRef { experiment: string; evalId: string; diff --git a/apps/framework/scripts/export-results.ts b/apps/framework/scripts/export-results.ts index c0a6a4ba..c0229bcd 100644 --- a/apps/framework/scripts/export-results.ts +++ b/apps/framework/scripts/export-results.ts @@ -9,6 +9,7 @@ import type { ExperimentSuite, } from '@supabase-evals/core/eval-metadata'; import { + makeFilterPredicate, normalizeExperimentName, readExperimentSuiteFilters, readRepeatedFlag, @@ -101,26 +102,10 @@ function shouldIncludeEval(evalId: string): boolean { return EVAL_FILTERS.includes(evalId); } -function shouldIncludeSuite(suite: EvalSuite | undefined): boolean { - if (SUITE_FILTERS.length === 0) { - return true; - } - - return suite !== undefined && SUITE_FILTERS.includes(suite); -} - -function shouldIncludeExperimentSuite( - experimentSuite: ExperimentSuite | undefined -): boolean { - if (EXPERIMENT_SUITE_FILTERS.length === 0) { - return true; - } - - return ( - experimentSuite !== undefined && - EXPERIMENT_SUITE_FILTERS.includes(experimentSuite) - ); -} +const shouldIncludeSuite = makeFilterPredicate(SUITE_FILTERS); +const shouldIncludeExperimentSuite = makeFilterPredicate( + EXPERIMENT_SUITE_FILTERS +); async function loadEvalResults(): Promise { const experimentMetadata = await loadExperimentMetadata(); diff --git a/apps/framework/scripts/upload-braintrust.ts b/apps/framework/scripts/upload-braintrust.ts index 45e82ae8..0c6eec6c 100644 --- a/apps/framework/scripts/upload-braintrust.ts +++ b/apps/framework/scripts/upload-braintrust.ts @@ -35,6 +35,7 @@ import type { ExperimentSuite, } from "@supabase-evals/core/eval-metadata"; import { + makeFilterPredicate, normalizeExperimentName, readExperimentSuiteFilters, readRepeatedFlag, @@ -44,6 +45,7 @@ import { ROOT, collectResultFiles, loadExperimentMetadata, + rawTranscriptPathFor, readPrompt, type PromptData, } from "../lib/result-files.js"; @@ -60,20 +62,10 @@ const UPDATE = rawArgs.includes("--update"); const PROJECT_FLAG = readRepeatedFlag(rawArgs, "project")[0]; const NAME_SUFFIX = readRepeatedFlag(rawArgs, "name-suffix")[0]; -function shouldIncludeSuite(suite: EvalSuite | undefined): boolean { - if (SUITE_FILTERS.length === 0) return true; - return suite !== undefined && SUITE_FILTERS.includes(suite); -} - -function shouldIncludeExperimentSuite( - experimentSuite: ExperimentSuite | undefined, -): boolean { - if (EXPERIMENT_SUITE_FILTERS.length === 0) return true; - return ( - experimentSuite !== undefined && - EXPERIMENT_SUITE_FILTERS.includes(experimentSuite) - ); -} +const shouldIncludeSuite = makeFilterPredicate(SUITE_FILTERS); +const shouldIncludeExperimentSuite = makeFilterPredicate( + EXPERIMENT_SUITE_FILTERS, +); function gitShortSha(): string | undefined { const fromEnv = process.env.GITHUB_SHA; @@ -181,7 +173,7 @@ async function collectTraces( }, baseTimeMs: Date.now(), }); - const transcriptPath = ref.filePath.replace(/\.json$/, ".transcript.jsonl"); + const transcriptPath = rawTranscriptPathFor(ref.filePath); const group = byExperiment.get(result.experiment) ?? []; group.push({ trace, diff --git a/packages/core/src/agents/claude-code/parser.ts b/packages/core/src/agents/claude-code/parser.ts index b55cbf86..15a808ab 100644 --- a/packages/core/src/agents/claude-code/parser.ts +++ b/packages/core/src/agents/claude-code/parser.ts @@ -26,6 +26,7 @@ import { extractLoadedSkillsFromText, type ArgFieldMap, } from '../../parsers/shared/extract.js'; +import { anthropicUsage } from './usage.js'; /** Claude Code's tool names → canonical names (case-sensitive). Owned here, not in shared. */ const CLAUDE_CODE_TOOLS: AgentToolMap = { @@ -168,13 +169,13 @@ function loadedSkillsFromClaudeCodeCall( /** * Per-API-call context stamped onto every event a line produces, so the * trace assembler can group a line's message/thinking/tool_use blocks into - * one turn, attach the call's own token usage, and nest subagent sidechains + * one step, attach the call's own token usage, and nest subagent sidechains * (`parent_tool_use_id`) under their Task call. */ function lineContext( data: Record, lineIndex: number, -): Pick { +): Pick { const context: ReturnType = {}; const parent = data.parent_tool_use_id; if (typeof parent === "string") context.parentToolUseId = parent; @@ -184,30 +185,12 @@ function lineContext( const messageId = typeof message?.id === "string" ? message.id : undefined; // One assistant stream-json line = one API message; the id groups a // message's blocks (and any continuation lines for the same id) into one - // turn. Lines without an id still get a unique per-line key. - context.turnKey = messageId ?? `line-${lineIndex}`; + // step. Lines without an id still get a unique per-line key. + context.stepKey = messageId ?? `line-${lineIndex}`; if (messageId) context.messageId = messageId; - const usage = isRecord(message?.usage) ? message.usage : undefined; - if (usage) { - const num = (value: unknown): number | undefined => - typeof value === "number" && Number.isFinite(value) ? value : undefined; - const rawInput = num(usage.input_tokens); - const cacheRead = num(usage.cache_read_input_tokens); - const cacheCreation = num(usage.cache_creation_input_tokens); - const outputTokens = num(usage.output_tokens); - if (rawInput !== undefined || cacheRead !== undefined || cacheCreation !== undefined || outputTokens !== undefined) { - context.usage = { - // OpenAI-style totals (cached ⊆ input), matching extractUsage. - inputTokens: (rawInput ?? 0) + (cacheRead ?? 0) + (cacheCreation ?? 0), - ...(outputTokens !== undefined ? { outputTokens } : {}), - ...(cacheRead !== undefined ? { cachedInputTokens: cacheRead } : {}), - ...(cacheCreation !== undefined - ? { cacheCreationInputTokens: cacheCreation } - : {}), - }; - } - } + const usage = anthropicUsage(message?.usage); + if (usage) context.usage = usage; return context; } diff --git a/packages/core/src/agents/claude-code/runner.ts b/packages/core/src/agents/claude-code/runner.ts index 8b60dd77..05ba2a2a 100644 --- a/packages/core/src/agents/claude-code/runner.ts +++ b/packages/core/src/agents/claude-code/runner.ts @@ -5,8 +5,9 @@ */ import type { Model as AnthropicModel } from '@anthropic-ai/sdk/resources/messages'; -import type { AgentUsage, McpServerConfig } from '../../index.js'; -import { isRecord, parseJsonlRecords } from '../../json.js'; +import type { McpServerConfig } from '../../index.js'; +import { parseJsonlRecords } from '../../json.js'; +import { anthropicUsage } from './usage.js'; import type { AgentRunner } from '../types.js'; import { npmGlobalBin, @@ -104,31 +105,7 @@ export const claudeCodeRunner: AgentRunner = { // cache_read_input_tokens, output_tokens }` plus `total_cost_usd`. const result = lastResultEvent(raw); if (!result) return undefined; - const usage = isRecord(result.usage) ? result.usage : undefined; - const num = (value: unknown): number | undefined => - typeof value === 'number' && Number.isFinite(value) ? value : undefined; - - // Anthropic's `input_tokens` excludes cache reads/writes; report - // OpenAI-style totals (cached ⊆ input) so token counts compare across - // agents. The cache splits are still reported separately. - const rawInput = num(usage?.input_tokens); - const cacheRead = num(usage?.cache_read_input_tokens); - const cacheCreation = num(usage?.cache_creation_input_tokens); - const inputTokens = - rawInput === undefined && cacheRead === undefined && cacheCreation === undefined - ? undefined - : (rawInput ?? 0) + (cacheRead ?? 0) + (cacheCreation ?? 0); - - const extracted: AgentUsage = { - inputTokens, - outputTokens: num(usage?.output_tokens), - cachedInputTokens: cacheRead, - cacheCreationInputTokens: cacheCreation, - costUsd: num(result.total_cost_usd), - }; - return Object.values(extracted).some((v) => v !== undefined) - ? extracted - : undefined; + return anthropicUsage(result.usage, result.total_cost_usd); }, }; diff --git a/packages/core/src/agents/claude-code/usage.ts b/packages/core/src/agents/claude-code/usage.ts new file mode 100644 index 00000000..80885f90 --- /dev/null +++ b/packages/core/src/agents/claude-code/usage.ts @@ -0,0 +1,46 @@ +/** + * Anthropic usage accounting, shared by the runner (whole-run totals from + * the terminal `result` line) and the parser (per-message usage on assistant + * lines) so the convention can't drift between them: Anthropic's + * `input_tokens` excludes cache reads/writes, so `inputTokens` is reported + * OpenAI-style (cached ⊆ input) to keep token counts comparable across + * agents. The cache splits stay reported separately. + */ + +import type { AgentUsage } from "../../index.js"; +import { finiteNumber, isRecord } from "../../json.js"; + +export function anthropicUsage( + usage: unknown, + costUsd?: unknown, +): AgentUsage | undefined { + const fields = isRecord(usage) ? usage : undefined; + const rawInput = finiteNumber(fields?.input_tokens); + const cacheRead = finiteNumber(fields?.cache_read_input_tokens); + const cacheCreation = finiteNumber(fields?.cache_creation_input_tokens); + const outputTokens = finiteNumber(fields?.output_tokens); + const cost = finiteNumber(costUsd); + + const hasInput = + rawInput !== undefined || + cacheRead !== undefined || + cacheCreation !== undefined; + if (!hasInput && outputTokens === undefined && cost === undefined) { + return undefined; + } + + return { + ...(hasInput + ? { + inputTokens: + (rawInput ?? 0) + (cacheRead ?? 0) + (cacheCreation ?? 0), + } + : {}), + ...(outputTokens !== undefined ? { outputTokens } : {}), + ...(cacheRead !== undefined ? { cachedInputTokens: cacheRead } : {}), + ...(cacheCreation !== undefined + ? { cacheCreationInputTokens: cacheCreation } + : {}), + ...(cost !== undefined ? { costUsd: cost } : {}), + }; +} diff --git a/packages/core/src/agents/codex/runner.ts b/packages/core/src/agents/codex/runner.ts index 135ff2d2..e83ac802 100644 --- a/packages/core/src/agents/codex/runner.ts +++ b/packages/core/src/agents/codex/runner.ts @@ -9,7 +9,7 @@ import type { ChatModel } from 'openai/resources/shared'; import type { AgentUsage, McpServerConfig } from '../../index.js'; -import { isRecord, parseJsonlRecords } from '../../json.js'; +import { finiteNumber, isRecord, parseJsonlRecords } from '../../json.js'; import type { AgentRunner } from '../types.js'; import { npmGlobalBin, @@ -129,11 +129,9 @@ export const codexRunner: AgentRunner = { for (const record of records) { if (record.type !== 'turn.completed' || !isRecord(record.usage)) continue; sawUsage = true; - const num = (value: unknown): number => - typeof value === 'number' && Number.isFinite(value) ? value : 0; - inputTokens += num(record.usage.input_tokens); - cachedInputTokens += num(record.usage.cached_input_tokens); - outputTokens += num(record.usage.output_tokens); + inputTokens += finiteNumber(record.usage.input_tokens) ?? 0; + cachedInputTokens += finiteNumber(record.usage.cached_input_tokens) ?? 0; + outputTokens += finiteNumber(record.usage.output_tokens) ?? 0; } if (!sawUsage) return undefined; const usage: AgentUsage = { inputTokens, cachedInputTokens, outputTokens }; diff --git a/packages/core/src/braintrust-spans.ts b/packages/core/src/braintrust-spans.ts index e57b9422..450d23a8 100644 --- a/packages/core/src/braintrust-spans.ts +++ b/packages/core/src/braintrust-spans.ts @@ -1,14 +1,14 @@ /** * Pure mapping from a raw eval-result file to a Braintrust-shaped trace: one - * root ("eval") row per result plus child spans — `llm` for assistant turns, - * `tool` for tool calls, `task` for user/system messages, `score` for the - * overall verdict and each scorer check. + * root ("eval") row per result plus child spans — `task` for turns (and, in + * the flat fallback, user/system messages), `llm` for steps (model API + * calls) and judge calls, `tool` for tool calls, `score` for the verdict. * * When the result carries the structured `trace` (AgentTrace, written by - * runs since it landed), spans are hierarchical: turn llm spans own their - * tool spans, and subagent sidechains nest under the tool call that spawned - * them. Older results fall back to a flat sequence rebuilt from - * `transcript[]`/`toolCalls[]`. + * runs since it landed), spans are hierarchical: turn task spans own their + * step llm spans, steps own their tool spans, and subagent sidechains nest + * under the tool call that spawned them. Older results fall back to a flat + * sequence rebuilt from `transcript[]`/`toolCalls[]`. * * Deliberately dependency-free (no braintrust import): this module only * builds a serializable tree; `apps/framework/scripts/upload-braintrust.ts` @@ -55,6 +55,8 @@ const toolCallRecordSchema = z.looseObject({ ts: z.number().default(0), }); +// Read-side twin of the AgentUsage interface (index.ts) — deliberately all- +// optional so results written by older runners still parse. Keep in sync. const agentUsageSchema = z .object({ inputTokens: z.number(), @@ -77,11 +79,7 @@ const agentTraceSchema = z.custom( Array.isArray((value as AgentTrace).turns), ); -/** - * A raw result file as the uploader consumes it: the exported shape plus the - * transcript fields the web export drops. Everything beyond the base shape is - * optional so results written before these fields existed still upload. - */ +// Read-side twin of JudgeCallRecord (judge-recorder.ts). Keep in sync. const judgeCallSchema = z.looseObject({ rubric: z.string(), input: z.string(), @@ -92,6 +90,11 @@ const judgeCallSchema = z.looseObject({ usage: agentUsageSchema.optional(), }); +/** + * A raw result file as the uploader consumes it: the exported shape plus the + * transcript fields the web export drops. Everything beyond the base shape is + * optional so results written before these fields existed still upload. + */ export const uploadableEvalResultSchema = rawEvalResultSchema.extend({ transcript: z.array(transcriptPartSchema).optional(), toolCalls: z.array(toolCallRecordSchema).optional(), diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index e41aa255..4fd3be68 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -129,7 +129,6 @@ export { } from './braintrust-spans.js'; export type { BraintrustEvalTrace, - BraintrustSpanNode, UploadableEvalResult, } from './braintrust-spans.js'; export type { AgentTranscriptParser } from './parsers/types.js'; @@ -145,7 +144,7 @@ export type { AgentTurn, ToolExecution, } from './transcript/agent-trace.js'; -export { collectJudgeCalls, recordJudgeCall } from './judge-recorder.js'; +export { collectJudgeCalls } from './judge-recorder.js'; export type { JudgeCallRecord } from './judge-recorder.js'; export type { AgentHarnessId, @@ -644,6 +643,13 @@ export async function judge(args: JudgeInput): Promise { // Evidence trail for the Braintrust upload: what this judge actually saw // and what the verdict cost. No-op unless a collector is active. + const judgeUsage: AgentUsage = { + inputTokens: result.totalUsage.inputTokens, + outputTokens: result.totalUsage.outputTokens, + cachedInputTokens: result.totalUsage.cachedInputTokens, + reasoningTokens: result.totalUsage.reasoningTokens, + totalTokens: result.totalUsage.totalTokens, + }; recordJudgeCall({ rubric: args.rubric, input: args.input, @@ -651,13 +657,9 @@ export async function judge(args: JudgeInput): Promise { notes: output.notes, modelId: model.modelId, durationMs: Date.now() - startedAt, - usage: { - inputTokens: result.totalUsage.inputTokens, - outputTokens: result.totalUsage.outputTokens, - cachedInputTokens: result.totalUsage.cachedInputTokens, - reasoningTokens: result.totalUsage.reasoningTokens, - totalTokens: result.totalUsage.totalTokens, - }, + ...(Object.values(judgeUsage).some((v) => v !== undefined) + ? { usage: judgeUsage } + : {}), }); return { diff --git a/packages/core/src/json.ts b/packages/core/src/json.ts index b8da5d0d..d0e9d06f 100644 --- a/packages/core/src/json.ts +++ b/packages/core/src/json.ts @@ -4,6 +4,13 @@ export function isRecord(value: unknown): value is Record { return typeof value === 'object' && value !== null && !Array.isArray(value); } +/** The value if it is a finite number, else undefined. */ +export function finiteNumber(value: unknown): number | undefined { + return typeof value === "number" && Number.isFinite(value) + ? value + : undefined; +} + export interface JsonlRecords { records: Record[]; errors: string[]; diff --git a/packages/core/src/transcript/agent-trace.test.ts b/packages/core/src/transcript/agent-trace.test.ts index 5a1464e3..745da5dd 100644 --- a/packages/core/src/transcript/agent-trace.test.ts +++ b/packages/core/src/transcript/agent-trace.test.ts @@ -2,13 +2,13 @@ import { describe, expect, it } from "vitest"; import { assembleAgentTrace } from "./agent-trace.js"; import type { TranscriptEvent } from "./types.js"; -describe("assembleAgentTrace with turn keys (Claude Code style)", () => { +describe("assembleAgentTrace with step keys (Claude Code style)", () => { const events: TranscriptEvent[] = [ // Step 1: one assistant API message with thinking + text + two tool uses. { type: "thinking", content: "let me look", - turnKey: "msg_1", + stepKey: "msg_1", messageId: "msg_1", usage: { inputTokens: 100, outputTokens: 10 }, }, @@ -16,18 +16,18 @@ describe("assembleAgentTrace with turn keys (Claude Code style)", () => { type: "message", role: "assistant", content: "Checking the tables.", - turnKey: "msg_1", + stepKey: "msg_1", messageId: "msg_1", usage: { inputTokens: 100, outputTokens: 10 }, }, { type: "tool_call", - turnKey: "msg_1", + stepKey: "msg_1", tool: { name: "shell", originalName: "Bash", id: "t1", args: { command: "ls" }, command: "ls" }, }, { type: "tool_call", - turnKey: "msg_1", + stepKey: "msg_1", tool: { name: "agent_task", originalName: "Task", id: "t2", args: { prompt: "explore" } }, }, // Results arrive on later lines. @@ -37,12 +37,12 @@ describe("assembleAgentTrace with turn keys (Claude Code style)", () => { type: "message", role: "assistant", content: "sub work", - turnKey: "msg_sub", + stepKey: "msg_sub", parentToolUseId: "t2", }, { type: "tool_call", - turnKey: "msg_sub", + stepKey: "msg_sub", parentToolUseId: "t2", tool: { name: "file_read", originalName: "Read", id: "t3", args: { file_path: "a.ts" }, path: "a.ts" }, }, @@ -52,7 +52,7 @@ describe("assembleAgentTrace with turn keys (Claude Code style)", () => { type: "message", role: "assistant", content: "Done.", - turnKey: "msg_2", + stepKey: "msg_2", messageId: "msg_2", usage: { inputTokens: 200, outputTokens: 20 }, }, @@ -96,7 +96,7 @@ describe("assembleAgentTrace with turn keys (Claude Code style)", () => { }); }); -describe("assembleAgentTrace closure rule (Codex style, no turn keys)", () => { +describe("assembleAgentTrace closure rule (Codex style, no step keys)", () => { const events: TranscriptEvent[] = [ { type: "thinking", content: "plan" }, { @@ -128,9 +128,9 @@ describe("assembleAgentTrace closure rule (Codex style, no turn keys)", () => { describe("assembleAgentTrace turn boundaries and edges", () => { it("opens a new turn on a mid-run user message", () => { const trace = assembleAgentTrace([ - { type: "message", role: "assistant", content: "first", turnKey: "m1" }, + { type: "message", role: "assistant", content: "first", stepKey: "m1" }, { type: "message", role: "user", content: "extra instruction" }, - { type: "message", role: "assistant", content: "second", turnKey: "m2" }, + { type: "message", role: "assistant", content: "second", stepKey: "m2" }, ]); expect(trace.turns).toHaveLength(2); expect(trace.turns[0]!.userMessage).toBeUndefined(); @@ -142,7 +142,7 @@ describe("assembleAgentTrace turn boundaries and edges", () => { it("folds a leading user message into the first turn", () => { const trace = assembleAgentTrace([ { type: "message", role: "user", content: "skill body" }, - { type: "message", role: "assistant", content: "ok", turnKey: "m1" }, + { type: "message", role: "assistant", content: "ok", stepKey: "m1" }, ]); expect(trace.turns).toHaveLength(1); expect(trace.turns[0]!.userMessage).toBe("skill body"); @@ -156,7 +156,7 @@ describe("assembleAgentTrace turn boundaries and edges", () => { type: "message", role: "assistant", content: "orphan", - turnKey: "m1", + stepKey: "m1", parentToolUseId: "missing", }, ]); diff --git a/packages/core/src/transcript/agent-trace.ts b/packages/core/src/transcript/agent-trace.ts index 5ccbd9b9..f6603eae 100644 --- a/packages/core/src/transcript/agent-trace.ts +++ b/packages/core/src/transcript/agent-trace.ts @@ -12,7 +12,7 @@ * Assembled from the same canonical `TranscriptEvent[]` the flat scorer * surface (`adaptTranscript`) consumes, so the two views can never disagree * about what happened; this one just stops throwing the structure away. - * Step grouping uses `event.turnKey` when the parser stamped one (Claude + * Step grouping uses `event.stepKey` when the parser stamped one (Claude * Code: one assistant line = one API message; ai-sdk builds steps directly) * and otherwise falls back to a closure rule: thinking and tool calls * accumulate into the open step, an assistant message closes it (Codex's @@ -38,8 +38,6 @@ export interface ToolExecution { output?: unknown; error?: string; success?: boolean; - /** Epoch-ms timing when the transcript carries real timestamps. */ - startMs?: number; /** Subagent sidechain (e.g. Claude Code Task) spawned by this call. */ children?: AgentStep[]; } @@ -51,7 +49,6 @@ export interface AgentStep { messageId?: string; /** Per-call token usage, when the transcript carries it. */ usage?: AgentUsage; - startMs?: number; thinking?: string; /** Assistant prose (absent on pure-tool steps). */ text?: string; @@ -75,12 +72,6 @@ export interface AgentTrace { errors: string[]; } -function parseMs(timestamp: string | undefined): number | undefined { - if (!timestamp) return undefined; - const ms = Date.parse(timestamp); - return Number.isNaN(ms) ? undefined : ms; -} - export function assembleAgentTrace(events: TranscriptEvent[]): AgentTrace { // Split sidechain events (subagent activity) from the main thread; each // group is assembled recursively and attached to its spawning tool call. @@ -101,7 +92,7 @@ export function assembleAgentTrace(events: TranscriptEvent[]): AgentTrace { const executionsById = new Map(); const stepsByKey = new Map(); let stepCount = 0; - // Closure-rule state for events without a turnKey: an assistant message + // Closure-rule state for events without a stepKey: an assistant message // marks the open step closed, so the next event starts a fresh one. let openStep: AgentStep | undefined; let openStepClosed = false; @@ -125,25 +116,23 @@ export function assembleAgentTrace(events: TranscriptEvent[]): AgentTrace { return turns[turns.length - 1]!; }; - const newStep = (event: TranscriptEvent): AgentStep => { + const newStep = (): AgentStep => { const step: AgentStep = { index: stepCount, toolCalls: [] }; stepCount += 1; - const startMs = parseMs(event.timestamp); - if (startMs !== undefined) step.startMs = startMs; currentTurn().steps.push(step); return step; }; const stepFor = (event: TranscriptEvent): AgentStep => { - if (event.turnKey) { - const existing = stepsByKey.get(event.turnKey); + if (event.stepKey) { + const existing = stepsByKey.get(event.stepKey); if (existing) return existing; - const step = newStep(event); - stepsByKey.set(event.turnKey, step); + const step = newStep(); + stepsByKey.set(event.stepKey, step); return step; } if (!openStep || openStepClosed) { - openStep = newStep(event); + openStep = newStep(); openStepClosed = false; } return openStep; @@ -195,7 +184,7 @@ export function assembleAgentTrace(events: TranscriptEvent[]): AgentTrace { step.text = step.text ? `${step.text}\n${content}` : content; } annotate(step, event); - if (!event.turnKey) openStepClosed = true; + if (!event.stepKey) openStepClosed = true; continue; } @@ -225,8 +214,6 @@ export function assembleAgentTrace(events: TranscriptEvent[]): AgentTrace { ? { loadedSkill: event.tool.loadedSkill } : {}), }; - const startMs = parseMs(event.timestamp); - if (startMs !== undefined) execution.startMs = startMs; step.toolCalls.push(execution); if (execution.id) executionsById.set(execution.id, execution); annotate(step, event); @@ -267,8 +254,4 @@ export function assembleAgentTrace(events: TranscriptEvent[]): AgentTrace { function annotate(step: AgentStep, event: TranscriptEvent): void { if (event.messageId && !step.messageId) step.messageId = event.messageId; if (event.usage && !step.usage) step.usage = event.usage; - if (step.startMs === undefined) { - const startMs = parseMs(event.timestamp); - if (startMs !== undefined) step.startMs = startMs; - } } diff --git a/packages/core/src/transcript/types.ts b/packages/core/src/transcript/types.ts index 5e9024b6..f65ca01b 100644 --- a/packages/core/src/transcript/types.ts +++ b/packages/core/src/transcript/types.ts @@ -33,12 +33,12 @@ export interface TranscriptEvent { /** ISO timestamp of the event, when the agent records one. */ timestamp?: string; /** - * Groups events belonging to one model API call (a turn). Parsers stamp it + * Groups events belonging to one model API call (a step). Parsers stamp it * when the raw transcript exposes the boundary (Claude Code: one assistant * stream-json line = one API message). Absent = the trace assembler falls - * back to its closure rule (an assistant message ends the open turn). + * back to its closure rule (an assistant message ends the open step). */ - turnKey?: string; + stepKey?: string; /** Provider message id of the API call this event belongs to, if exposed. */ messageId?: string; /** Per-API-call token usage, when the raw transcript carries it. */ From 1d7be868d10c230c8a9e130610b3f8b554e2dea2 Mon Sep 17 00:00:00 2001 From: Pedro Rodrigues Date: Wed, 22 Jul 2026 14:22:54 +0100 Subject: [PATCH 08/15] feat: add custom suite mode to eval-refresh for ad hoc branch runs Lets workflow_dispatch pin exact eval/experiment pairs and publish them to Braintrust regardless of each eval's own suite: field, without touching the tracked eval-results.json/regression exports. Blocked on main since it's meant for in-progress branch work only. Co-Authored-By: Claude Sonnet 5 --- .github/workflows/eval-refresh.yml | 119 ++++++++++++++++++++++------- 1 file changed, 90 insertions(+), 29 deletions(-) diff --git a/.github/workflows/eval-refresh.yml b/.github/workflows/eval-refresh.yml index 4e10ba48..a882cf71 100644 --- a/.github/workflows/eval-refresh.yml +++ b/.github/workflows/eval-refresh.yml @@ -12,7 +12,7 @@ on: required: false default: "" suite: - description: "Comma-separated eval suites to run" + description: "Comma-separated eval suites to run, or 'custom' to run the exact eval/experiments pairs below regardless of each eval's own suite (blocked on main)" required: true default: "benchmark" experiment_suite: @@ -107,6 +107,25 @@ jobs: suite_json="$(jq -Rc 'split(",") | map(gsub("^\\s+|\\s+$"; "")) | map(select(length > 0))' <<< "$suite")" experiment_suite_json="$(jq -Rc 'split(",") | map(gsub("^\\s+|\\s+$"; "")) | map(select(length > 0))' <<< "$experiment_suite")" + # custom bypasses suite-based discovery entirely and runs the exact + # eval/experiments pairs given, so it needs both explicit and is + # restricted to branches — it's meant for ad hoc, in-progress work, + # never for main's tracked benchmark/regression results. + if jq -e 'index("custom") != null' <<< "$suite_json" > /dev/null 2>&1; then + if [ "${{ github.event_name }}" != "workflow_dispatch" ]; then + echo "::error::suite=custom is only supported for manual workflow_dispatch runs." >&2 + exit 1 + fi + if [ "${{ github.ref_name }}" = "main" ]; then + echo "::error::suite=custom is not allowed on main — dispatch this workflow from a branch." >&2 + exit 1 + fi + if [ -z "$eval_id" ] || [ -z "$experiments_override" ]; then + echo "::error::suite=custom requires both the 'eval' and 'experiments' inputs to be set." >&2 + exit 1 + fi + fi + # run-evals takes priority; run-evals-changed only filters when run-evals is absent. # filter_changed drives the changed-eval filter (PR-only, needs a PR diff). filter_changed="false" @@ -209,37 +228,57 @@ jobs: pairs='[]' experiments_override="${{ steps.inputs.outputs.experiments_override }}" - for id in "${matching[@]}"; do - eval_suite=$(sed -n 's/^suite:[[:space:]]*//p' "evals/$id/PROMPT.md" | head -n 1) - case "$eval_suite" in - benchmark) experiment_suites=(benchmark no-skills) ;; - regression) experiment_suites=(regression) ;; - *) continue ;; - esac - - for experiment_suite in "${experiment_suites[@]}"; do - if ! jq -e --arg suite "$experiment_suite" 'index($suite) != null' \ - <<< '${{ steps.inputs.outputs.experiment_suite }}' > /dev/null; then - continue - fi - - if [ -n "$experiments_override" ]; then - experiments_json="$(jq -Rc 'split(",") | map(gsub("^\\s+|\\s+$"; "")) | map(select(length > 0))' <<< "$experiments_override")" - else - experiments_json="$(pnpm --silent eval -- list --experiment-suite "$experiment_suite")" - fi + if jq -e 'index("custom") != null' <<< "$suite_json" > /dev/null 2>&1; then + # Custom mode: cross-product the explicit --eval and --experiments + # lists directly, ignoring each eval's own suite: field and the + # experiment_suite input entirely. eval_suite is tagged "custom" + # here only for CI routing (publish step below) — it never + # overrides the real suite metadata Braintrust/exports read from + # each eval's PROMPT.md. + experiments_json="$(jq -Rc 'split(",") | map(gsub("^\\s+|\\s+$"; "")) | map(select(length > 0))' <<< "$experiments_override")" + for id in "${matching[@]}"; do while IFS= read -r experiment; do pairs="$(jq -c \ --arg eval_id "$id" \ --arg experiment "$experiment" \ - --arg experiment_suite "$experiment_suite" \ - --arg eval_suite "$eval_suite" \ - '. + [{eval_id: $eval_id, experiment: $experiment, experiment_suite: $experiment_suite, eval_suite: $eval_suite}]' \ + '. + [{eval_id: $eval_id, experiment: $experiment, experiment_suite: "", eval_suite: "custom"}]' \ <<< "$pairs")" done < <(jq -r '.[]' <<< "$experiments_json") done - done + else + for id in "${matching[@]}"; do + eval_suite=$(sed -n 's/^suite:[[:space:]]*//p' "evals/$id/PROMPT.md" | head -n 1) + case "$eval_suite" in + benchmark) experiment_suites=(benchmark no-skills) ;; + regression) experiment_suites=(regression) ;; + *) continue ;; + esac + + for experiment_suite in "${experiment_suites[@]}"; do + if ! jq -e --arg suite "$experiment_suite" 'index($suite) != null' \ + <<< '${{ steps.inputs.outputs.experiment_suite }}' > /dev/null; then + continue + fi + + if [ -n "$experiments_override" ]; then + experiments_json="$(jq -Rc 'split(",") | map(gsub("^\\s+|\\s+$"; "")) | map(select(length > 0))' <<< "$experiments_override")" + else + experiments_json="$(pnpm --silent eval -- list --experiment-suite "$experiment_suite")" + fi + + while IFS= read -r experiment; do + pairs="$(jq -c \ + --arg eval_id "$id" \ + --arg experiment "$experiment" \ + --arg experiment_suite "$experiment_suite" \ + --arg eval_suite "$eval_suite" \ + '. + [{eval_id: $eval_id, experiment: $experiment, experiment_suite: $experiment_suite, eval_suite: $eval_suite}]' \ + <<< "$pairs")" + done < <(jq -r '.[]' <<< "$experiments_json") + done + done + fi if [ "$(jq 'length' <<< "$pairs")" -eq 0 ]; then echo "No experiment and eval pairs matched" >&2 @@ -295,12 +334,20 @@ jobs: run: | set -euo pipefail - pnpm eval -- \ - --experiment "${{ matrix.experiment }}" \ - --experiment-suite "${{ matrix.experiment_suite }}" \ - --eval "${{ matrix.eval_id }}" \ - --runs "${{ needs.prepare.outputs.runs }}" \ + args=( + --experiment "${{ matrix.experiment }}" + --eval "${{ matrix.eval_id }}" + --runs "${{ needs.prepare.outputs.runs }}" --timeout-sec "${{ needs.prepare.outputs.timeout_sec }}" + ) + # Custom-mode pairs carry no experiment_suite (it's not required to + # target an exact --experiment/--eval pair) — omit the flag rather + # than pass an empty value the schema would reject. + if [ -n "${{ matrix.experiment_suite }}" ]; then + args+=(--experiment-suite "${{ matrix.experiment_suite }}") + fi + + pnpm eval -- "${args[@]}" # A skipped experiment (e.g. missing API key) exits 0 without writing results. if [ ! -f "results/${{ matrix.experiment }}/${{ matrix.eval_id }}.json" ]; then @@ -417,6 +464,20 @@ jobs: pnpm --filter @supabase-evals/framework upload-braintrust -- --suite benchmark fi + # Custom-mode pairs aren't tagged benchmark/regression (that field + # is CI-routing-only, see the discover step), so route by the exact + # eval/experiment pairs instead of --suite, and suffix the + # Braintrust experiment name with the branch so these ad hoc runs + # are never mistaken for tracked benchmark/regression results. + if jq -e 'any(.[]; .eval_suite == "custom")' <<< "$pairs" > /dev/null; then + custom_evals="$(jq -r '[.[] | select(.eval_suite == "custom") | .eval_id] | unique | join(",")' <<< "$pairs")" + custom_experiments="$(jq -r '[.[] | select(.eval_suite == "custom") | .experiment] | unique | join(",")' <<< "$pairs")" + pnpm --filter @supabase-evals/framework upload-braintrust -- \ + --eval "$custom_evals" \ + --experiment "$custom_experiments" \ + --name-suffix "-custom-${{ github.ref_name }}" + fi + - name: Upload exported results if: github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 From 0a42f07ae2374d16b4c9bf594fdb251f64348abf Mon Sep 17 00:00:00 2001 From: Pedro Rodrigues Date: Wed, 22 Jul 2026 14:40:44 +0100 Subject: [PATCH 09/15] feat: baseline comparison + auto-updating PR comment for Braintrust uploads MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit init() now passes baseExperiment (most recent prior experiment for the same model, discovered via the REST API, best-effort) so summarize() and the UI diff against the previous refresh. --summary-md writes a markdown table (experiment links, passed %, Δ vs base); the workflow appends it to the step summary and upserts a single marker-matched PR comment — the braintrustdata/eval-action reviewer experience without adopting its execution model. Co-Authored-By: Claude Fable 5 --- .github/workflows/eval-refresh.yml | 37 +++++++- apps/framework/scripts/upload-braintrust.ts | 96 ++++++++++++++++++++- 2 files changed, 129 insertions(+), 4 deletions(-) diff --git a/.github/workflows/eval-refresh.yml b/.github/workflows/eval-refresh.yml index a882cf71..f067e0e6 100644 --- a/.github/workflows/eval-refresh.yml +++ b/.github/workflows/eval-refresh.yml @@ -460,8 +460,10 @@ jobs: set -euo pipefail pairs='${{ needs.prepare.outputs.pairs }}' + summary_md="$RUNNER_TEMP/braintrust-summary.md" if jq -e 'any(.[]; .eval_suite == "benchmark")' <<< "$pairs" > /dev/null; then - pnpm --filter @supabase-evals/framework upload-braintrust -- --suite benchmark + pnpm --filter @supabase-evals/framework upload-braintrust -- \ + --suite benchmark --summary-md "$summary_md" fi # Custom-mode pairs aren't tagged benchmark/regression (that field @@ -475,7 +477,38 @@ jobs: pnpm --filter @supabase-evals/framework upload-braintrust -- \ --eval "$custom_evals" \ --experiment "$custom_experiments" \ - --name-suffix "-custom-${{ github.ref_name }}" + --name-suffix "-custom-${{ github.ref_name }}" \ + --summary-md "$summary_md" + fi + + if [ -f "$summary_md" ]; then + cat "$summary_md" >> "$GITHUB_STEP_SUMMARY" + fi + + - name: Comment Braintrust results on PR + # Single auto-updating comment (matched by the hidden marker), same + # reviewer experience as braintrustdata/eval-action but fed by our + # own uploader. Never blocks the publish path. + if: github.event_name == 'pull_request' + continue-on-error: true + env: + GH_TOKEN: ${{ github.token }} + shell: bash + run: | + set -euo pipefail + + summary="$RUNNER_TEMP/braintrust-summary.md" + if [ ! -f "$summary" ]; then + echo "no Braintrust summary to post" + exit 0 + fi + marker='' + comment_id=$(gh api "repos/${{ github.repository }}/issues/${{ github.event.pull_request.number }}/comments" \ + --paginate -q ".[] | select(.body | startswith(\"$marker\")) | .id" | head -n 1) + if [ -n "$comment_id" ]; then + gh api -X PATCH "repos/${{ github.repository }}/issues/comments/$comment_id" -F body=@"$summary" + else + gh api "repos/${{ github.repository }}/issues/${{ github.event.pull_request.number }}/comments" -F body=@"$summary" fi - name: Upload exported results diff --git a/apps/framework/scripts/upload-braintrust.ts b/apps/framework/scripts/upload-braintrust.ts index 0c6eec6c..b866ad7f 100644 --- a/apps/framework/scripts/upload-braintrust.ts +++ b/apps/framework/scripts/upload-braintrust.ts @@ -19,9 +19,11 @@ * --project Braintrust project name (overrides env) * --name-suffix extra suffix on the Braintrust experiment names * --update append to existing same-named Braintrust experiments + * --summary-md

write a markdown summary (links + passed Δ vs base), + * used by CI to post/update a PR comment */ -import { readFile } from "node:fs/promises"; +import { readFile, writeFile } from "node:fs/promises"; import { existsSync } from "node:fs"; import { basename } from "node:path"; import { execFileSync } from "node:child_process"; @@ -61,6 +63,7 @@ const DRY = rawArgs.includes("--dry"); const UPDATE = rawArgs.includes("--update"); const PROJECT_FLAG = readRepeatedFlag(rawArgs, "project")[0]; const NAME_SUFFIX = readRepeatedFlag(rawArgs, "name-suffix")[0]; +const SUMMARY_MD = readRepeatedFlag(rawArgs, "summary-md")[0]; const shouldIncludeSuite = makeFilterPredicate(SUITE_FILTERS); const shouldIncludeExperimentSuite = makeFilterPredicate( @@ -88,6 +91,74 @@ function ciRunUrl(): string | undefined { return `${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}`; } +/** + * Most recent prior Braintrust experiment for the same repo experiment, used + * as the comparison base (score diffs in summarize() and in the UI). + * Experiments are named `@[-suffix]`; prefer plain-sha names + * (real refreshes) over suffixed ad-hoc ones. Best-effort: any failure means + * "no base", never a broken upload. + */ +async function findBaseExperimentName( + projectId: string | undefined, + experiment: string, + currentName: string, +): Promise { + const apiKey = process.env.BRAINTRUST_API_KEY; + if (!projectId || !apiKey) return undefined; + try { + const response = await fetch( + `https://api.braintrust.dev/v1/experiment?project_id=${encodeURIComponent(projectId)}&limit=100`, + { headers: { Authorization: `Bearer ${apiKey}` } }, + ); + if (!response.ok) return undefined; + const body = (await response.json()) as { + objects?: Array<{ name?: string }>; + }; + // Objects arrive newest-first. + const names = (body.objects ?? []) + .map((o) => o.name) + .filter( + (name): name is string => + typeof name === "string" && + name.startsWith(`${experiment}@`) && + name !== currentName, + ); + return names.find((name) => /^.+@[0-9a-f]{7,}$/.test(name)) ?? names[0]; + } catch { + return undefined; + } +} + +interface SummaryRow { + name: string; + url?: string; + evalCount: number; + passedScore?: number; + passedDiff?: number; + comparedTo?: string; +} + +function formatSummaryMarkdown(rows: SummaryRow[]): string { + const pct = (value: number) => `${Math.round(value * 100)}%`; + const lines = [ + "", + "### Braintrust benchmark results", + "", + "| Experiment | passed | Δ vs base | evals |", + "|---|---|---|---|", + ]; + for (const row of rows) { + const name = row.url ? `[${row.name}](${row.url})` : row.name; + const passed = row.passedScore !== undefined ? pct(row.passedScore) : "–"; + const diff = + row.passedDiff === undefined || !row.comparedTo + ? "–" + : `${row.passedDiff > 0 ? "🟢 +" : row.passedDiff < 0 ? "🔴 " : "⚪ "}${pct(row.passedDiff)} (vs ${row.comparedTo})`; + lines.push(`| ${name} | ${passed} | ${diff} | ${row.evalCount} |`); + } + return `${lines.join("\n")}\n`; +} + interface PendingTrace { trace: BraintrustEvalTrace; /** Raw result file, attached to the root span for full-fidelity forensics. */ @@ -220,15 +291,22 @@ async function upload( const sha = gitShortSha(); const suffix = NAME_SUFFIX ? `-${NAME_SUFFIX}` : ""; const runUrl = ciRunUrl(); + const summaryRows: SummaryRow[] = []; for (const [experiment, pending] of byExperiment) { const name = `${experiment}${sha ? `@${sha}` : ""}${suffix}`; const meta = experimentMetadata.get(experiment); + const baseExperiment = await findBaseExperimentName( + projectId, + experiment, + name, + ); const btExperiment = init({ ...(projectName ? { project: projectName } : {}), ...(projectId ? { projectId } : {}), experiment: name, update: UPDATE, + ...(baseExperiment ? { baseExperiment } : {}), metadata: { source: "supabase-evals", experiment, @@ -304,7 +382,16 @@ async function upload( root.end({ endTime: trace.endMs / 1000 }); } - const summary = await btExperiment.summarize({ summarizeScores: false }); + const summary = await btExperiment.summarize(); + const passed = summary.scores?.passed; + summaryRows.push({ + name, + url: summary.experimentUrl, + evalCount: pending.length, + passedScore: passed?.score, + passedDiff: passed?.diff, + comparedTo: summary.comparisonExperimentName, + }); console.log( `Uploaded ${pending.length} eval trace(s) to Braintrust experiment "${name}"` + (summary.experimentUrl ? `\n → ${summary.experimentUrl}` : ""), @@ -312,6 +399,11 @@ async function upload( } await flush(); + + if (SUMMARY_MD) { + await writeFile(SUMMARY_MD, formatSummaryMarkdown(summaryRows)); + console.log(`Wrote summary to ${SUMMARY_MD}`); + } } async function main() { From 6e10e6a58fa5d2b173d1f4ae9f00c43e97ea1296 Mon Sep 17 00:00:00 2001 From: Pedro Rodrigues Date: Wed, 22 Jul 2026 14:50:31 +0100 Subject: [PATCH 10/15] feat: distinguish scheduled vs PR runs in Braintrust + pre-filtered list links MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Experiments now carry trigger/branch/prNumber/ciRunId metadata and trigger/pr-N tags. The summary (and PR comment) links the experiments list pre-filtered by BTQL — all experiments from this CI run, this PR, or the scheduled main history — using the UI's search URL format. Co-Authored-By: Claude Fable 5 --- apps/framework/scripts/upload-braintrust.ts | 107 +++++++++++++++++++- 1 file changed, 105 insertions(+), 2 deletions(-) diff --git a/apps/framework/scripts/upload-braintrust.ts b/apps/framework/scripts/upload-braintrust.ts index b866ad7f..3fa6e2d1 100644 --- a/apps/framework/scripts/upload-braintrust.ts +++ b/apps/framework/scripts/upload-braintrust.ts @@ -91,6 +91,62 @@ function ciRunUrl(): string | undefined { return `${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}`; } +/** + * What produced this upload — the axis that separates scheduled main + * refreshes from PR runs (and local ad-hoc uploads) in Braintrust. Stored as + * experiment metadata/tags so the experiments list can be pre-filtered by + * BTQL, e.g. `metadata.trigger = 'schedule'` or `metadata.prNumber = '101'`. + */ +interface RunContext { + trigger: string; + branch?: string; + prNumber?: string; + runId?: string; +} + +function runContext(): RunContext { + const event = process.env.GITHUB_EVENT_NAME; + if (!event) { + let branch: string | undefined; + try { + branch = execFileSync("git", ["rev-parse", "--abbrev-ref", "HEAD"], { + cwd: ROOT, + encoding: "utf8", + }).trim(); + } catch { + branch = undefined; + } + return { trigger: "local", branch }; + } + // For PR events GITHUB_REF_NAME is "/merge". + const prMatch = /^(\d+)\/merge$/.exec(process.env.GITHUB_REF_NAME ?? ""); + return { + trigger: event, + branch: process.env.GITHUB_HEAD_REF || process.env.GITHUB_REF_NAME, + ...(event === "pull_request" && prMatch ? { prNumber: prMatch[1] } : {}), + runId: process.env.GITHUB_RUN_ID, + }; +} + +/** + * Experiments-list URL pre-filtered by BTQL expressions, mirroring the UI's + * own format: `?search={"filter":[""]}` with the expression + * URL-encoded inside the JSON and the JSON encoded again. + */ +function experimentsFilterUrl( + anyExperimentUrl: string, + exprs: string[], +): string | undefined { + const marker = "/experiments/"; + const index = anyExperimentUrl.lastIndexOf(marker); + if (index === -1) return undefined; + const base = anyExperimentUrl.slice(0, index + marker.length - 1); + const search = encodeURIComponent( + JSON.stringify({ filter: exprs.map((expr) => encodeURIComponent(expr)) }), + ); + return `${base}?search=${search}`; +} + /** * Most recent prior Braintrust experiment for the same repo experiment, used * as the comparison base (score diffs in summarize() and in the UI). @@ -138,7 +194,10 @@ interface SummaryRow { comparedTo?: string; } -function formatSummaryMarkdown(rows: SummaryRow[]): string { +function formatSummaryMarkdown( + rows: SummaryRow[], + filterLinks: Array<{ label: string; url: string }>, +): string { const pct = (value: number) => `${Math.round(value * 100)}%`; const lines = [ "", @@ -156,6 +215,12 @@ function formatSummaryMarkdown(rows: SummaryRow[]): string { : `${row.passedDiff > 0 ? "🟢 +" : row.passedDiff < 0 ? "🔴 " : "⚪ "}${pct(row.passedDiff)} (vs ${row.comparedTo})`; lines.push(`| ${name} | ${passed} | ${diff} | ${row.evalCount} |`); } + if (filterLinks.length > 0) { + lines.push( + "", + filterLinks.map(({ label, url }) => `[${label}](${url})`).join(" · "), + ); + } return `${lines.join("\n")}\n`; } @@ -291,6 +356,7 @@ async function upload( const sha = gitShortSha(); const suffix = NAME_SUFFIX ? `-${NAME_SUFFIX}` : ""; const runUrl = ciRunUrl(); + const context = runContext(); const summaryRows: SummaryRow[] = []; for (const [experiment, pending] of byExperiment) { @@ -307,6 +373,10 @@ async function upload( experiment: name, update: UPDATE, ...(baseExperiment ? { baseExperiment } : {}), + tags: [ + context.trigger, + ...(context.prNumber ? [`pr-${context.prNumber}`] : []), + ], metadata: { source: "supabase-evals", experiment, @@ -316,6 +386,10 @@ async function upload( ...(meta?.display ?? {}), ...(sha ? { gitShortSha: sha } : {}), ...(runUrl ? { ciRunUrl: runUrl } : {}), + trigger: context.trigger, + ...(context.branch ? { branch: context.branch } : {}), + ...(context.prNumber ? { prNumber: context.prNumber } : {}), + ...(context.runId ? { ciRunId: context.runId } : {}), }, }); @@ -400,8 +474,37 @@ async function upload( await flush(); + // Pre-filtered experiments-list links for this run's slice of the project + // (Braintrust bakes view filters into the URL). + const anyUrl = summaryRows.find((row) => row.url)?.url; + const filterLinks: Array<{ label: string; url: string }> = []; + if (anyUrl) { + const add = (label: string, exprs: string[]) => { + const url = experimentsFilterUrl(anyUrl, exprs); + if (url) filterLinks.push({ label, url }); + }; + if (context.runId) { + add("All experiments from this CI run", [ + `metadata.ciRunId = '${context.runId}'`, + ]); + } + if (context.prNumber) { + add(`All experiments for PR #${context.prNumber}`, [ + `metadata.prNumber = '${context.prNumber}'`, + ]); + } + if (context.trigger === "schedule") { + add("All scheduled main-branch experiments", [ + `metadata.trigger = 'schedule'`, + ]); + } + for (const { label, url } of filterLinks) { + console.log(`${label}: ${url}`); + } + } + if (SUMMARY_MD) { - await writeFile(SUMMARY_MD, formatSummaryMarkdown(summaryRows)); + await writeFile(SUMMARY_MD, formatSummaryMarkdown(summaryRows, filterLinks)); console.log(`Wrote summary to ${SUMMARY_MD}`); } } From 12665f466ba3e1c1ab10597e2d311d70c313a75e Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 22 Jul 2026 14:45:16 +0000 Subject: [PATCH 11/15] chore: refresh eval results --- apps/web/src/data/eval-results.json | 19349 ++++++---------- .../web/src/data/regression-eval-results.json | 216 +- 2 files changed, 7730 insertions(+), 11835 deletions(-) diff --git a/apps/web/src/data/eval-results.json b/apps/web/src/data/eval-results.json index 5b28a57e..265dba5c 100644 --- a/apps/web/src/data/eval-results.json +++ b/apps/web/src/data/eval-results.json @@ -1,11 +1,11 @@ [ { - "experiment": "claude-code-opus-5", + "experiment": "claude-code-opus-4.8", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "claude-code", "modelProvider": "anthropic", - "modelId": "claude-opus-5", + "modelId": "claude-opus-4-8", "reasoningEffort": "high" }, "eval": "build-cli-001-bootstrap-app", @@ -46,7 +46,7 @@ { "name": "REST API returns no todos to anonymous requests", "passed": true, - "notes": "error 42501: permission denied for table todos" + "notes": "0 rows" }, { "name": "REST API returns the todos to authenticated requests", @@ -64,43 +64,20 @@ ] }, "docs": { - "calls": [ - { - "source": "web_fetch", - "query": "List any entries tagged breaking-change, and anything relevant to: local development with the CLI, database migrations, RLS policies, the anon/authenticated roles, API keys (publishable/secret vs anon/service_role), seed files, or config.toml. Include dates and links.", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 8255 - }, - { - "source": "web_fetch", - "query": "Show the full recommended workflow and exact SQL for exposing a table in the public schema to the Data API, including GRANT statements for anon and authenticated roles, enabling RLS, and read-only access patterns. Quote the SQL verbatim.", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/api/securing-your-api.md" - } - ], - "resultChars": 1245 - } - ] + "calls": [] }, "prompt": "We're kicking off a todos app and I want the Supabase side ready for the team\nto build on. Set it up the way we'd run it in development, with schema changes\ntracked as migrations so they can be reviewed and replayed.\n\nFor the first slice we just need a `todos` table. Todos aren't public: anyone\nsigned in can read all of them, but nothing should be writable through the API\nfor now. Add a couple of sample todos so there's something to look at.\n\nBefore you hand it back, make sure the running API actually behaves that way —\nsigned-in users get the todos, signed-out requests get nothing.", "promptSourcePath": "evals/build-cli-001-bootstrap-app/PROMPT.md", "attempts": 1, - "sourcePath": "claude-code-opus-5/build-cli-001-bootstrap-app.json" + "sourcePath": "claude-code-opus-4.8/build-cli-001-bootstrap-app.json" }, { - "experiment": "claude-code-opus-5", + "experiment": "claude-code-opus-4.8", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "claude-code", "modelProvider": "anthropic", - "modelId": "claude-opus-5", + "modelId": "claude-opus-4-8", "reasoningEffort": "high" }, "eval": "build-cli-002-declarative-schema", @@ -143,45 +120,20 @@ ] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{ searchDocs(query: \"declarative database schemas add column generate migration\", limit: 4) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/local-development/declarative-database-schemas", - "title": "Declarative database schemas" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows", - "title": "Local development workflow" - }, - { - "url": "https://supabase.com/docs/guides/deployment/database-migrations", - "title": "Database Migrations" - }, - { - "url": "https://supabase.com/docs/guides/deployment/managing-environments", - "title": "Managing Environments" - } - ], - "resultChars": 66910 - } - ] + "calls": [] }, "prompt": "Add a description text column to the `products` table in my local Supabase stack", "promptSourcePath": "evals/build-cli-002-declarative-schema/PROMPT.md", "attempts": 1, - "sourcePath": "claude-code-opus-5/build-cli-002-declarative-schema.json" + "sourcePath": "claude-code-opus-4.8/build-cli-002-declarative-schema.json" }, { - "experiment": "claude-code-opus-5", + "experiment": "claude-code-opus-4.8", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "claude-code", "modelProvider": "anthropic", - "modelId": "claude-opus-5", + "modelId": "claude-opus-4-8", "reasoningEffort": "high" }, "eval": "build-cli-003-pg-cron-queue-workflow", @@ -213,7 +165,7 @@ { "name": "process-tasks function drains the queue", "passed": true, - "notes": "function removed the seeded message (id 10) from the queue" + "notes": "function removed the seeded message (id 6) from the queue" } ], "skills": { @@ -229,79 +181,78 @@ "calls": [ { "source": "search_docs", - "query": "{ searchDocs(query: \"pgmq queues send read pop archive pgmq_public API\", limit: 6) { nodes { title href content } } }", + "query": "{ q1: searchDocs(query: \"pg_cron schedule job queue pgmq send message\", limit: 4) { nodes { title href content } } q2: searchDocs(query: \"invoke edge function from pg_cron pg_net\", limit: 3) { nodes { title href content } } }", "hasContent": true, "pages": [ - { - "url": "https://supabase.com/docs/guides/queues/quickstart", - "title": "Quickstart" - }, { "url": "https://supabase.com/docs/guides/queues/pgmq", "title": "PGMQ Extension" }, { - "url": "https://supabase.com/docs/guides/queues", - "title": "Supabase Queues" + "url": "https://supabase.com/docs/guides/database/extensions/pg_net", + "title": "pg_net: Async Networking" }, { - "url": "https://supabase.com/docs/guides/queues/api", - "title": "API" + "url": "https://supabase.com/docs/guides/troubleshooting/pgcron-debugging-guide-n1KTaz", + "title": "pg_cron debugging guide" }, { - "url": "https://supabase.com/docs/guides/queues/expose-self-hosted-queues", - "title": "Expose Queues for local and self-hosted Supabase" - } - ], - "resultChars": 39340 - }, - { - "source": "web_fetch", - "query": "How to create/schedule a cron job named X running every minute, and how to schedule a job that calls a Supabase Queue send / edge function. Show exact SQL syntax for cron.schedule and cron.unschedule, and any notes about naming or the pg_cron version. Include the full code examples verbatim.", - "hasContent": true, - "pages": [ + "url": "https://supabase.com/docs/guides/cron", + "title": "Cron" + }, { - "url": "https://supabase.com/docs/guides/cron/quickstart.md" - } - ], - "resultChars": 1475 - }, - { - "source": "web_fetch", - "query": "List any entries tagged breaking-change, and any entries related to Queues/pgmq, pg_cron/Cron, or Edge Functions runtime/deployment. Include dates and links.", - "hasContent": true, - "pages": [ + "url": "https://supabase.com/docs/guides/functions/schedule-functions", + "title": "Scheduling Edge Functions" + }, { - "url": "https://supabase.com/changelog.md" + "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", + "title": "Automatic embeddings" } ], - "resultChars": 8295 + "resultChars": 88459 }, { - "source": "web_fetch", - "query": "What exactly changed about the delay parameter behavior in pgmq 1.5.1 versus 1.4.4? Quote the details precisely, including any impact on send()/send_batch() and what values of delay are affected.", - "hasContent": true, + "source": "search_docs", + "query": "{ q1: searchDocs(query: \"pg_cron schedule edge function pg_net queue drain\", limit: 5) { nodes { title href } } }", + "hasContent": false, "pages": [ { - "url": "https://supabase.com/changelog/39378-potential-breaking-change-in-pgmq-from-1-4-4-to-1-5-1-and-temporary-halt-on-upgrade-for-existing-projects" + "url": "https://supabase.com/docs/guides/database/extensions/pg_net", + "title": "pg_net: Async Networking" + }, + { + "url": "https://supabase.com/docs/guides/functions/schedule-functions", + "title": "Scheduling Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", + "title": "Automatic embeddings" + }, + { + "url": "https://supabase.com/docs/guides/cron/quickstart", + "title": "Quickstart" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/pgcron-debugging-guide-n1KTaz", + "title": "pg_cron debugging guide" } ], - "resultChars": 659 + "resultChars": 621 } ] }, "prompt": "I want to set up a recurring background workflow on my local Supabase stack.\n\nCan you set up a cron job called `enqueue-tasks` to run every minute and push a task into a queue called `tasks`? Then add a `process-tasks` edge function that reads messages off the `tasks` queue and removes them, so a scheduled worker can keep the backlog drained.", "promptSourcePath": "evals/build-cli-003-pg-cron-queue-workflow/PROMPT.md", "attempts": 1, - "sourcePath": "claude-code-opus-5/build-cli-003-pg-cron-queue-workflow.json" + "sourcePath": "claude-code-opus-4.8/build-cli-003-pg-cron-queue-workflow.json" }, { - "experiment": "claude-code-opus-5", + "experiment": "claude-code-opus-4.8", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "claude-code", "modelProvider": "anthropic", - "modelId": "claude-opus-5", + "modelId": "claude-opus-4-8", "reasoningEffort": "high" }, "eval": "build-database-001-migrate-postgres-to-supabase", @@ -347,75 +298,20 @@ ] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{ searchDocs(query: \"restore pg_dump custom format dump into Supabase migrate existing postgres database\", limit: 6) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/postgres", - "title": "Migrate from Postgres to Supabase" - }, - { - "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/vercel-postgres", - "title": "Migrate from Vercel Postgres to Supabase" - }, - { - "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/neon", - "title": "Migrate from Neon to Supabase" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform", - "title": "Restore a Platform Project to Self-Hosted" - }, - { - "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/heroku", - "title": "Migrate from Heroku to Supabase" - }, - { - "url": "https://supabase.com/docs/guides/platform/migrating-within-supabase/backup-restore", - "title": "Backup and Restore using the CLI" - } - ], - "resultChars": 78095 - }, - { - "source": "web_fetch", - "query": "What are the exact recommended commands and flags for restoring a pg_dump dump into a Supabase Postgres database? Include any notes about roles, ownership, privileges, extensions, schemas to exclude, and disabling triggers.", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/postgres.md" - } - ], - "resultChars": 1424 - }, - { - "source": "web_fetch", - "query": "List any recent entries tagged breaking-change, especially anything related to the CLI, local development, `supabase start`, `supabase db` commands, database restores/migrations, or Postgres major versions.", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 1248 - } - ] + "calls": [] }, "prompt": "I have an existing Postgres database I want to migrate to Supabase. There's a binary dump at `source.dump` in the current directory.\n\nCan you set up a local Supabase project and restore the dump into it?", "promptSourcePath": "evals/build-database-001-migrate-postgres-to-supabase/PROMPT.md", "attempts": 1, - "sourcePath": "claude-code-opus-5/build-database-001-migrate-postgres-to-supabase.json" + "sourcePath": "claude-code-opus-4.8/build-database-001-migrate-postgres-to-supabase.json" }, { - "experiment": "claude-code-opus-5", + "experiment": "claude-code-opus-4.8", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "claude-code", "modelProvider": "anthropic", - "modelId": "claude-opus-5", + "modelId": "claude-opus-4-8", "reasoningEffort": "high" }, "eval": "build-functions-004-service-role-bypass", @@ -452,12 +348,12 @@ { "name": "user A cannot force-read user B note", "passed": true, - "notes": "status=403" + "notes": "status=200" }, { "name": "user B cannot force-read user A note", "passed": true, - "notes": "status=403" + "notes": "status=200" } ], "skills": { @@ -470,49 +366,20 @@ ] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{searchDocs(query:\"Edge Function auth user JWT createClient Authorization header RLS\", limit:5){nodes{title href content}}}", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/why-is-my-service-role-key-client-getting-rls-errors-or-not-returning-data-7_1K9z", - "title": "Why is my service role key client getting RLS errors or not returning data?" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs", - "title": "Build a User Management App with Next.js" - }, - { - "url": "https://supabase.com/docs/guides/auth/choosing-a-server-package", - "title": "Which package to use" - }, - { - "url": "https://supabase.com/docs/guides/functions/error-codes", - "title": "Error codes" - } - ], - "resultChars": 65767 - } - ] + "calls": [] }, "prompt": "I built an Edge Function called `private-notes` for showing a user's saved\nprivate notes.\n\nCan you check whether there's any way one user could see another user's notes?\n\nPlease fix and deploy it if needed.", "promptSourcePath": "evals/build-functions-004-service-role-bypass/PROMPT.md", - "attempts": 2, - "sourcePath": "claude-code-opus-5/build-functions-004-service-role-bypass.json" + "attempts": 1, + "sourcePath": "claude-code-opus-4.8/build-functions-004-service-role-bypass.json" }, { - "experiment": "claude-code-opus-5", + "experiment": "claude-code-opus-4.8", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "claude-code", "modelProvider": "anthropic", - "modelId": "claude-opus-5", + "modelId": "claude-opus-4-8", "reasoningEffort": "high" }, "eval": "build-functions-005-dual-auth-user-secret", @@ -545,17 +412,17 @@ { "name": "user with JWT reads only their own rows", "passed": true, - "notes": "status 200: [{\"user_id\":\"cff75a07-aaa5-4010-95bb-ed51fa7d82a8\",\"metric\":\"steps_a_ms6x4xfn\",\"value\":111}]" + "notes": "status 200: [{\"user_id\":\"556e807d-0457-4653-a595-e38a345568f1\",\"metric\":\"steps_a_mrw5tp5f\",\"value\":111}]" }, { "name": "user cannot read another user's rows by passing user_id", "passed": true, - "notes": "status 200: [{\"user_id\":\"cff75a07-aaa5-4010-95bb-ed51fa7d82a8\",\"metric\":\"steps_a_ms6x4xfn\",\"value\":111}]" + "notes": "status 200: [{\"user_id\":\"556e807d-0457-4653-a595-e38a345568f1\",\"metric\":\"steps_a_mrw5tp5f\",\"value\":111}]" }, { "name": "service key bypasses RLS to read the target user's rows", "passed": true, - "notes": "status 200: [{\"user_id\":\"e73e078a-fec3-4f26-9532-a1e6e6f32e83\",\"metric\":\"steps_b_ms6x4xfn\",\"value\":222}]" + "notes": "status 200: [{\"user_id\":\"d0fe0442-9df4-4e41-b79b-545efebb9e3e\",\"metric\":\"steps_b_mrw5tp5f\",\"value\":222}]" }, { "name": "non-service key is not granted service access", @@ -590,18 +457,29 @@ "docs": { "calls": [ { - "source": "search_docs", - "query": "{ searchDocs(query: \"edge function verify_jwt config functions auth service role key\", limit: 6) { nodes { title href content } } }", + "source": "web_fetch", + "query": "List any breaking-change entries related to Edge Functions, API keys (publishable/secret keys), JWT verification, or environment variables available in the edge runtime.", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" - }, + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 1362 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"edge function dual authentication service role key user JWT verify_jwt environment variables\", limit: 6) { nodes { title href content } } }", + "hasContent": true, + "pages": [ { "url": "https://supabase.com/docs/guides/functions/auth-headers", "title": "Authorization headers" }, + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" + }, { "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", "title": "Self-Hosted Functions" @@ -610,63 +488,42 @@ "url": "https://supabase.com/docs/guides/functions/function-configuration", "title": "Function Configuration" }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys", - "title": "JWT Signing Keys" - }, { "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", "title": "Integrating With Supabase Auth" + }, + { + "url": "https://supabase.com/docs/guides/functions/secrets", + "title": "Environment Variables" } ], - "resultChars": 62524 + "resultChars": 40497 }, { - "source": "search_docs", - "query": "{ searchDocs(query: \"@supabase/server package secret key publishable key API keys\", limit: 6) { nodes { title href content } } }", + "source": "web_fetch", + "query": "Show the recommended patterns for an edge function that supports BOTH a signed-in user (user JWT) and a trusted backend authenticating with a secret key. How do you verify a secret key sent in the apikey header, and how do you verify a user JWT? Include code for the @supabase/server withSupabase auth modes and any manual verification approach.", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", - "title": "New API Keys and Asymmetric Authentication" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/auth/choosing-a-server-package", - "title": "Which package to use" - }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys", - "title": "JWT Signing Keys" - }, - { - "url": "https://supabase.com/docs/guides/integrations/supabase-for-platforms", - "title": "Supabase for Platforms" - }, - { - "url": "https://supabase.com/docs/guides/api/creating-routes", - "title": "Creating API Routes" + "url": "https://supabase.com/docs/guides/functions/auth.md" } ], - "resultChars": 85604 + "resultChars": 1593 } ] }, "prompt": "Build and serve a Supabase Edge Function named `user-stats` for this project,\nreachable over HTTP at `/functions/v1/user-stats`.\n\nOur product stores per-user metrics in the existing `user_stats` table.\n\nTwo very different callers need to hit this one endpoint:\n\n1. **Our mobile app**, acting for a signed-in user. It sends that user's\n Supabase access token. The endpoint should return the user's own stats.\n\n2. **Our internal billing service**, a trusted backend with no signed-in user.\n It authenticates with the project's secret (service-role) key in the `apikey`\n header, and names the target user with a `user_id` in the JSON request body.\n It needs that user's stats.\n\nReturn the matching rows as JSON. The endpoint must be secure: only ever serve\nstats to a caller that is genuinely entitled to them, and turn away callers that\nare not.", "promptSourcePath": "evals/build-functions-005-dual-auth-user-secret/PROMPT.md", "attempts": 1, - "sourcePath": "claude-code-opus-5/build-functions-005-dual-auth-user-secret.json" + "sourcePath": "claude-code-opus-4.8/build-functions-005-dual-auth-user-secret.json" }, { - "experiment": "claude-code-opus-5", + "experiment": "claude-code-opus-4.8", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "claude-code", "modelProvider": "anthropic", - "modelId": "claude-opus-5", + "modelId": "claude-opus-4-8", "reasoningEffort": "high" }, "eval": "build-storage-001-private-bucket-access", @@ -698,7 +555,7 @@ { "name": "user A lists only own files", "passed": true, - "notes": "saw: 019fb0eb-a03d-7688-b099-e5a490624b08/receipt-alpha.pdf, 019fb0eb-a03d-7688-b099-e5a490624b08/receipt-beta.pdf" + "notes": "saw: 019f8a22-3296-77da-9b3c-f7eff10e02f9/receipt-alpha.pdf, 019f8a22-3296-77da-9b3c-f7eff10e02f9/receipt-beta.pdf" }, { "name": "user B cannot read user A files", @@ -719,7 +576,7 @@ { "name": "configured private per-user storage access", "passed": true, - "judgeNotes": "Meets all criteria: private user-files bucket, authenticated owner-scoped SELECT and INSERT policies with WITH CHECK on storage.objects, RLS kept enabled, and supabase-js createSignedUrl with expiry for temporary sharing. No disallowed public bucket, permissive policies, anon/public access policies, getPublicUrl sharing, or client service-role usage." + "judgeNotes": "Meets all requirements: private user-files bucket, owner-scoped authenticated SELECT and INSERT policies with RLS enabled, and supabase-js createSignedUrl with expiry for temporary sharing." } ], "skills": { @@ -735,12 +592,12 @@ "calls": [ { "source": "search_docs", - "query": "{ searchDocs(query: \"storage RLS policies private bucket user folder auth.uid\", limit: 6) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"storage RLS policy owner folder access control signed url\", limit: 6) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/storage/schema/helper-functions", - "title": "Storage Helper Functions" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs", + "title": "Build a User Management App with Next.js" }, { "url": "https://supabase.com/docs/guides/storage/buckets/fundamentals", @@ -751,54 +608,34 @@ "title": "Storage Access Control" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3", - "title": "Configure S3 Storage" + "url": "https://supabase.com/docs/guides/storage/debugging/error-codes", + "title": "Error Codes" }, { - "url": "https://supabase.com/docs/guides/local-development/database-migrations", - "title": "Database migrations" + "url": "https://supabase.com/docs/guides/auth/third-party/clerk", + "title": "Clerk" }, { "url": "https://supabase.com/docs/guides/security/product-security", "title": "Secure configuration of Supabase products" } ], - "resultChars": 36724 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"createSignedUrl expiresIn temporary link download private bucket\", limit: 4) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsigneduploadurl" - }, - { - "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsignedurl" - }, - { - "url": "https://supabase.com/docs/reference/dart/file-buckets-createsignedurl" - }, - { - "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsignedurls" - } - ], - "resultChars": 3116 + "resultChars": 71443 } ] }, "prompt": "Our app lets signed-in users keep personal files like receipts and bank\nstatements. These files are private — a user must only ever be able to upload\nand download their own. The app uploads each file under a path that starts\nwith the owner's user id, e.g. `/receipt-march.pdf`.\n\nSet up a `user-files` bucket on our project and lock it down that way.\n\nUsers also sometimes share one of their files with someone else through a\ntemporary link that expires. Include the supabase-js code the app should use\nfor that.", "promptSourcePath": "evals/build-storage-001-private-bucket-access/PROMPT.md", "attempts": 1, - "sourcePath": "claude-code-opus-5/build-storage-001-private-bucket-access.json" + "sourcePath": "claude-code-opus-4.8/build-storage-001-private-bucket-access.json" }, { - "experiment": "claude-code-opus-5", + "experiment": "claude-code-opus-4.8", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "claude-code", "modelProvider": "anthropic", - "modelId": "claude-opus-5", + "modelId": "claude-opus-4-8", "reasoningEffort": "high" }, "eval": "build-tests-001-rls-tenant-isolation", @@ -817,17 +654,17 @@ { "name": "pgTAP test file(s) written under supabase/tests/", "passed": true, - "notes": "2 file(s): supabase/tests/002_memberships_isolation.test.sql, supabase/tests/001_tenant_isolation.test.sql" + "notes": "1 file(s): supabase/tests/tenant_isolation_test.sql" }, { "name": "pgTAP isolation tests ran and pass", "passed": true, - "notes": "7 passed, 5 failed" + "notes": "7 passed, 3 failed" }, { "name": "agent correctly identifies the posts isolation bug from test results", "passed": true, - "judgeNotes": "The agent correctly identifies `posts` as having the tenant isolation flaw: authenticated members of any org can read posts from other orgs due to a missing `m.org_id = posts.org_id` predicate. It grounds this in the pgTAP failures and direct psql reproduction, and does not blame `notes` or dismiss the tests." + "judgeNotes": "The agent correctly identifies `posts` as the broken table, explains that authenticated members can read posts from organizations they are not members of, and grounds the conclusion in pgTAP results showing `notes` passes while `posts` fails. It treats the test results as authoritative and does not blame `notes` or dismiss the failures." } ], "skills": { @@ -845,15 +682,15 @@ "prompt": "Can you audit the tenant isolation on our tables? Write some database tests covering both the happy path and the negative case, run them, and share the results.", "promptSourcePath": "evals/build-tests-001-rls-tenant-isolation/PROMPT.md", "attempts": 1, - "sourcePath": "claude-code-opus-5/build-tests-001-rls-tenant-isolation.json" + "sourcePath": "claude-code-opus-4.8/build-tests-001-rls-tenant-isolation.json" }, { - "experiment": "claude-code-opus-5", + "experiment": "claude-code-opus-4.8", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "claude-code", "modelProvider": "anthropic", - "modelId": "claude-opus-5", + "modelId": "claude-opus-4-8", "reasoningEffort": "high" }, "eval": "build-vectors-001-rag-with-permissions", @@ -873,17 +710,17 @@ { "name": "document_sections.embedding is vector(384)", "passed": true, - "notes": "vector(384)" + "notes": "extensions.vector(384)" }, { "name": "HNSW index on the embedding column", "passed": true, - "notes": "CREATE INDEX document_sections_embedding_idx ON public.document_sections USING hnsw (embedding vector_ip_ops)" + "notes": "CREATE INDEX document_sections_embedding_idx ON public.document_sections USING hnsw (embedding extensions.vector_cosine_ops)" }, { "name": "index operator class matches the search operator", "passed": true, - "notes": "function operators: <#>\nindexes: CREATE INDEX document_sections_embedding_idx ON public.document_sections USING hnsw (embedding vector_ip_ops)" + "notes": "function operators: <=>\nindexes: CREATE INDEX document_sections_embedding_idx ON public.document_sections USING hnsw (embedding extensions.vector_cosine_ops)" }, { "name": "user A search returns only own sections, best match first", @@ -915,19 +752,9 @@ "calls": [ { "source": "search_docs", - "query": "{ searchDocs(query: \"semantic search pgvector match_document_sections RLS embedding gte-small\", limit: 6) { nodes { title href content } } }", - "hasContent": true, - "pages": [] - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"semantic search pgvector match_document_sections RLS embedding gte-small edge function\", limit: 6) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"semantic search embeddings gte-small match_document_sections RLS\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ - { - "url": "https://supabase.com/docs/guides/ai/hybrid-search", - "title": "Hybrid search" - }, { "url": "https://supabase.com/docs/guides/ai/rag-with-permissions", "title": "RAG with Permissions" @@ -937,34 +764,34 @@ "title": "Semantic search" }, { - "url": "https://supabase.com/docs/guides/functions/examples/semantic-search", - "title": "Semantic Search" + "url": "https://supabase.com/docs/guides/ai/choosing-compute-addon", + "title": "Choosing your Compute Add-on" }, { - "url": "https://supabase.com/docs/guides/ai/vector-columns", - "title": "Vector columns" + "url": "https://supabase.com/docs/guides/database/extensions/pgvector", + "title": "pgvector: Embeddings and vector similarity" }, { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", - "title": "Automatic embeddings" + "url": "https://supabase.com/docs/guides/functions/examples/semantic-search", + "title": "Semantic Search" } ], - "resultChars": 84107 + "resultChars": 57915 } ] }, "prompt": "We're adding semantic search to our internal knowledge base app. I already wrote the edge functions, but search doesn't work yet. Can you set up whatever the database needs to make search work end to end?\n\nSome documents are confidential, so users should only have access to documents they own.", "promptSourcePath": "evals/build-vectors-001-rag-with-permissions/PROMPT.md", "attempts": 1, - "sourcePath": "claude-code-opus-5/build-vectors-001-rag-with-permissions.json" + "sourcePath": "claude-code-opus-4.8/build-vectors-001-rag-with-permissions.json" }, { - "experiment": "claude-code-opus-5", + "experiment": "claude-code-opus-4.8", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "claude-code", "modelProvider": "anthropic", - "modelId": "claude-opus-5", + "modelId": "claude-opus-4-8", "reasoningEffort": "high" }, "eval": "deploy-database-001-prometheus-metrics", @@ -985,12 +812,12 @@ { "name": "configured the Supabase Metrics API scrape correctly", "passed": true, - "judgeNotes": "Supabase scrape uses HTTPS, the required metrics path, basic_auth with password_file, targets the project ref on supabase.co, preserves the app job, and docker-compose mounts the secrets directory containing the password file." + "judgeNotes": "Meets requirements: app scrape preserved, Supabase HTTPS scrape uses /customer/v1/privileged/metrics with basic_auth.password_file, target is PROJECT_REF.supabase.co:443, and docker-compose mounts the secrets directory containing the password file." }, { "name": "documented live deployment and verification steps", "passed": true, - "judgeNotes": "README includes Secret API key creation, matching secret file path, Compose Prometheus restart/reload guidance, and concrete verification via curl to the endpoint, Prometheus targets API, and Grafana dashboard." + "judgeNotes": "README includes go-live steps for project ref, creating a Supabase Secret API key, writing it to the matching Prometheus secret file, restarting/reloading the Compose stack, and verifying via curl, Prometheus targets, and Grafana/dashboard guidance. Endpoint/auth and secret file path match the config." } ], "skills": { @@ -1006,7 +833,7 @@ "calls": [ { "source": "search_docs", - "query": "{ searchDocs(query: \"Prometheus metrics endpoint project observability scrape\", limit: 8) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"Prometheus metrics endpoint project observability\", limit: 6) { nodes { title href content } } }", "hasContent": true, "pages": [ { @@ -1031,21 +858,32 @@ } ], "resultChars": 32656 + }, + { + "source": "web_fetch", + "query": "What is the exact URL of the project metrics endpoint, what authentication does it require (username/password), what format does it return, and how do you configure Prometheus to scrape it? Include the scrape config example if present.", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/telemetry/metrics.md" + } + ], + "resultChars": 1067 } ] }, - "prompt": "Can you wire my Supabase project metrics into our existing observability stack and document\nin the observability README what we need to do to make the config live?", + "prompt": "Can you wire my Supabase project metrics into our existing observability stack and document\nwhat we need to do to make the config live?", "promptSourcePath": "evals/deploy-database-001-prometheus-metrics/PROMPT.md", "attempts": 1, - "sourcePath": "claude-code-opus-5/deploy-database-001-prometheus-metrics.json" + "sourcePath": "claude-code-opus-4.8/deploy-database-001-prometheus-metrics.json" }, { - "experiment": "claude-code-opus-5", + "experiment": "claude-code-opus-4.8", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "claude-code", "modelProvider": "anthropic", - "modelId": "claude-opus-5", + "modelId": "claude-opus-4-8", "reasoningEffort": "high" }, "eval": "deploy-functions-001-edge-function-secrets", @@ -1089,73 +927,20 @@ ] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{ searchDocs(query: \"Edge Function secrets environment variables manage\", limit: 6) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/secrets", - "title": "Environment Variables" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", - "title": "Self-Hosted Functions" - }, - { - "url": "https://supabase.com/docs/guides/database/secure-data", - "title": "Securing your data" - }, - { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", - "title": "Automatic embeddings" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/inspecting-edge-function-environment-variables-wg5qOQ", - "title": "Inspecting edge function environment variables" - } - ], - "resultChars": 65698 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"config.toml edge_runtime.secrets env() local development\", limit: 3) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/secrets", - "title": "Environment Variables" - }, - { - "url": "https://supabase.com/docs/guides/deployment/branching/configuration", - "title": "Configuration" - }, - { - "url": "https://supabase.com/docs/guides/local-development/managing-config", - "title": "Managing config and secrets" - } - ], - "resultChars": 18379 - } - ] + "calls": [] }, "prompt": "Our weather widget currently calls WeatherAPI straight from the browser, which\nleaks our API key. I want to move that behind a Supabase Edge Function called\n`weather` that holds the key server-side and proxies the request.\n\nThe function should read the key from an environment variable named\n`WEATHER_API_KEY`. Our key already lives in a local `.env` file at the project\nroot.\n\nDeploy the function to our project so it's live, and make sure the deployed\nfunction can actually read the key at runtime.", "promptSourcePath": "evals/deploy-functions-001-edge-function-secrets/PROMPT.md", "attempts": 1, - "sourcePath": "claude-code-opus-5/deploy-functions-001-edge-function-secrets.json" + "sourcePath": "claude-code-opus-4.8/deploy-functions-001-edge-function-secrets.json" }, { - "experiment": "claude-code-opus-5", + "experiment": "claude-code-opus-4.8", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "claude-code", "modelProvider": "anthropic", - "modelId": "claude-opus-5", + "modelId": "claude-opus-4-8", "reasoningEffort": "high" }, "eval": "deploy-self-hosting-001-docker-compose", @@ -1202,29 +987,40 @@ "calls": [ { "source": "web_fetch", - "query": "Extract the full step-by-step instructions for self-hosting Supabase with Docker: how to obtain the docker directory, the .env file, which secrets must be generated/changed (JWT secret, anon/service keys, postgres password, dashboard credentials, secret_key_base, vault enc key, etc.), how to generate them, and any securing-your-services guidance. Include exact commands and env var names verbatim.", + "query": "Extract the complete step-by-step instructions for self-hosting Supabase with Docker. Specifically: what files/directories to copy, the exact git/curl commands used to obtain the docker directory, the full list of secrets/env vars that must be set (POSTGRES_PASSWORD, JWT_SECRET, ANON_KEY, SERVICE_ROLE_KEY, DASHBOARD_USERNAME, DASHBOARD_PASSWORD, SECRET_KEY_BASE, VAULT_ENC_KEY, etc.), how to generate the API keys (anon/service_role) from the JWT secret, and any security recommendations for production. Include exact commands.", "hasContent": true, "pages": [ { "url": "https://supabase.com/docs/guides/self-hosting/docker.md" } ], - "resultChars": 4806 + "resultChars": 3964 + }, + { + "source": "web_fetch", + "query": "List any recent entries or breaking changes related to self-hosting, Docker, docker-compose, analytics/Logflare, Vector, API keys (publishable/secret keys, anon/service_role JWT keys), or environment/secret configuration for self-hosted Supabase.", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 1483 } ] }, "prompt": "I'm moving off the hosted Supabase and running the whole thing myself on a VPS I\njust spun up. Can you get a Docker setup ready for me to copy onto the box?\n\nI don't need it running here, I'll do the actual bring-up once I'm on the\nserver. I just want everything in place and the secrets set up. Put it in a `supabase-docker/`\nfolder at the repo root so I can scp the whole thing across in one go.", "promptSourcePath": "evals/deploy-self-hosting-001-docker-compose/PROMPT.md", "attempts": 1, - "sourcePath": "claude-code-opus-5/deploy-self-hosting-001-docker-compose.json" + "sourcePath": "claude-code-opus-4.8/deploy-self-hosting-001-docker-compose.json" }, { - "experiment": "claude-code-opus-5", + "experiment": "claude-code-opus-4.8", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "claude-code", "modelProvider": "anthropic", - "modelId": "claude-opus-5", + "modelId": "claude-opus-4-8", "reasoningEffort": "high" }, "eval": "investigate-auth-001-deleted-user-access", @@ -1268,4418 +1064,74 @@ }, { "name": "diagnosed and explained session revocation", + "passed": false, + "judgeNotes": "Fails because the shipped fix only deletes sessions/refresh tokens and soft-deletes the profile; it leaves the auth user/identity intact, so the user could sign in again and obtain a new session. The answer otherwise correctly explains stale JWT behavior, RLS/Data API window, and publishable vs secret keys." + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [] + }, + "prompt": "Last week support removed a user through our app's delete-account flow — the\napp calls the `delete_account` function over RPC as the signed-in user. This\nmorning that same person was back: still signed in, reading and saving their\ndata like nothing happened.\n\nFigure out why the account still works, fix the flow so a deleted account\nloses access, and tell me whether there is any window where they could still\nget in after the fix.\n\nOne more thing while you're at it: we're migrating off the legacy\nanon/service_role API keys to the new publishable and secret keys, and the\nteam isn't sure which one belongs in the frontend and what each means for\nRLS. Give us a quick rundown.", + "promptSourcePath": "evals/investigate-auth-001-deleted-user-access/PROMPT.md", + "attempts": 2, + "sourcePath": "claude-code-opus-4.8/investigate-auth-001-deleted-user-access.json" + }, + { + "experiment": "claude-code-opus-4.8", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-opus-4-8", + "reasoningEffort": "high" + }, + "eval": "investigate-realtime-001-subscribed-no-events", + "stage": "investigate", + "product": [ + "realtime", + "database" + ], + "topic": [ + "sdk" + ], + "suite": "benchmark", + "interface": "mcp", + "passed": true, + "checks": [ + { + "name": "orders table added to supabase_realtime publication", + "passed": true + }, + { + "name": "courier_locations still in supabase_realtime publication", + "passed": true + }, + { + "name": "publication still publishes INSERT events", + "passed": true + }, + { + "name": "RLS still enabled on orders", + "passed": true + }, + { + "name": "staff can still read orders through RLS", + "passed": true, + "notes": "authenticated sees 2 of 2 orders" + }, + { + "name": "diagnosed missing publication membership", "passed": true, - "judgeNotes": "Meets the rubric: identifies soft-delete-only root cause with no auth user/session revocation; implements real revocation by banning and deleting sessions/refresh tokens, plus RLS flag checks; consistently explains that stale JWTs remain valid for purely local validation until exp while its RLS fix closes the Data API path for covered tables; and correctly distinguishes publishable frontend keys with RLS from server-only secret/service_role-style keys that bypass RLS." - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" - ] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{ searchDocs(query: \"delete user account revoke sessions invalidate access token\", limit: 6) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows", - "title": "OAuth 2.1 Flows" - }, - { - "url": "https://supabase.com/docs/guides/platform/sso/multiple-providers", - "title": "Multiple SSO Providers" - }, - { - "url": "https://supabase.com/docs/guides/auth/sessions", - "title": "User sessions" - }, - { - "url": "https://supabase.com/docs/guides/platform/temporary-access", - "title": "Temporary access" - }, - { - "url": "https://supabase.com/docs/guides/auth/managing-user-data", - "title": "User Management" - }, - { - "url": "https://supabase.com/docs/guides/resources/glossary", - "title": "Glossary" - } - ], - "resultChars": 86320 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"publishable key secret key API keys migration anon service_role\", limit: 6) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys", - "title": "JWT Signing Keys" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", - "title": "New API Keys and Asymmetric Authentication" - }, - { - "url": "https://supabase.com/docs/guides/integrations/supabase-for-platforms", - "title": "Supabase for Platforms" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-redwoodjs", - "title": "Build a User Management App with RedwoodJS" - }, - { - "url": "https://supabase.com/docs/guides/auth/quickstarts/with-expo-react-native-social-auth", - "title": "Build a Social Auth App with Expo React Native" - } - ], - "resultChars": 166856 - }, - { - "source": "web_fetch", - "query": "Extract the FULL text/content of this page verbatim as much as possible. I need details on: publishable key vs secret key, which Postgres role each maps to (anon, service_role, authenticated), RLS enforcement vs bypass, whether keys are JWTs or opaque, rotation/revocation, multiple keys, disabling keys, legacy anon/service_role key deprecation timeline, apikey vs Authorization Bearer header behavior, auth.uid()/auth.jwt() implications, and any links to migration guides.", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/api/api-keys.md" - } - ], - "resultChars": 3025 - }, - { - "source": "web_fetch", - "query": "Extract the full content verbatim as much as possible. I need: step-by-step migration, deprecation timeline/dates for legacy anon and service_role keys, gotchas, clients that verify JWT locally, apikey vs Authorization Bearer header behavior, RLS behavior changes, role mapping (anon/authenticated/service_role), whether publishable keys resolve to anon and what happens after sign-in, rotation/revocation, multiple keys, disabling keys independently.", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys.md" - } - ], - "resultChars": 4100 - }, - { - "source": "web_fetch", - "query": "Extract everything about deleting users, soft delete vs hard delete, banning users (banned_until), what happens to existing JWTs/sessions after deletion, and the \"Deleting users\" section verbatim. Also any mention of auth.admin.deleteUser shouldSoftDelete.", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/auth/users.md" - } - ], - "resultChars": 593 - }, - { - "source": "web_fetch", - "query": "Extract the full reference docs for auth.admin.deleteUser: signature, parameters (including shouldSoftDelete), notes/caveats verbatim.", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/reference/javascript/auth-admin-deleteuser.md" - } - ], - "resultChars": 215 - }, - { - "source": "web_fetch", - "query": "Extract the full reference docs for auth.admin.deleteUser: signature, parameters (including shouldSoftDelete), notes/caveats verbatim.", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/reference/javascript/auth-admin-deleteuser" - } - ], - "resultChars": 706 - }, - { - "source": "web_fetch", - "query": "Extract full reference docs for auth.admin.signOut: signature, parameters, the scope options (global, local, others), and all notes verbatim.", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/reference/javascript/auth-admin-signout" - } - ], - "resultChars": 928 - }, - { - "source": "web_fetch", - "query": "Extract the full content verbatim as much as possible. Focus on: publishable key vs secret key definitions, which Postgres role each maps to (anon, authenticated, service_role), RLS enforced vs bypassed, BYPASSRLS, whether keys are JWTs or opaque, what happens to the role when a user signs in, apikey vs Authorization Bearer header, rotation/revocation, creating multiple named keys, disabling/deleting keys, legacy key deprecation timeline, and auth.uid()/auth.jwt() notes.", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/getting-started/api-keys.md" - } - ], - "resultChars": 2479 - }, - { - "source": "web_fetch", - "query": "List every changelog entry that mentions API keys, publishable keys, secret keys, sb_publishable, sb_secret, JWT signing keys, or legacy anon/service_role key deprecation. Include dates and any deprecation timeline dates verbatim.", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 1334 - }, - { - "source": "web_fetch", - "query": "Find and quote verbatim the auth config keys related to JWT expiry (jwt_expiry), refresh token rotation, refresh_token_reuse_interval, and any session timebox / inactivity timeout keys, including their default values.", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/local-development/cli/config.md" - } - ], - "resultChars": 215 - }, - { - "source": "web_fetch", - "query": "Quote verbatim the auth config keys for JWT expiry (auth.jwt_expiry), refresh token rotation (auth.enable_refresh_token_rotation), refresh_token_reuse_interval, and any session timebox/inactivity keys, with defaults.", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/cli/config.md" - } - ], - "resultChars": 215 - }, - { - "source": "web_fetch", - "query": "Quote VERBATIM, with no paraphrasing, these sections if present: (1) any table or list mapping keys to Postgres roles and RLS; (2) the section about the `anon` role and `authenticated` role; (3) the section titled \"Legacy API keys\" or similar including any deprecation dates; (4) any \"Limitations\" or \"Known limitations\" section; (5) anything about `auth.uid()`, `auth.jwt()`, claims, or JWTs; (6) anything about the apikey header and Authorization header. Output the raw markdown of those sections.", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/getting-started/api-keys.md" - } - ], - "resultChars": 1599 - }, - { - "source": "web_fetch", - "query": "Quote verbatim the config.toml auth keys: auth.jwt_expiry (with description and default), auth.enable_refresh_token_rotation, auth.refresh_token_reuse_interval. Also any session timebox/inactivity timeout keys.", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/local-development/cli/config" - } - ], - "resultChars": 899 - }, - { - "source": "web_fetch", - "query": "Quote verbatim the \"Deleting users\" section and anything about JWT remaining valid after deletion, and any mention of soft delete or signing users out.", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/auth/managing-user-data.md" - } - ], - "resultChars": 534 - }, - { - "source": "web_search", - "query": "Supabase changelog publishable secret API keys general availability legacy anon service_role deprecation 2026", - "hasContent": false, - "pages": [ - { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys | Supabase Docs" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys | Supabase Docs" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/rotating-anon-service-and-jwt-secrets-1Jq6yd", - "title": "Supabase Docs | Troubleshooting | Rotating Anon, Service, and JWT Secrets" - }, - { - "url": "https://supabase.com/changelog/29260-upcoming-changes-to-supabase-api-keys", - "title": "Upcoming changes to Supabase API Keys · Changelog" - }, - { - "url": "https://supabase.com/changelog?next=Y3Vyc29yOnYyOpK0MjAyNC0wOS0xM1QxNTowMDoyOVrOAG2JYw%3D%3D&restPage=2", - "title": "Changelog" - }, - { - "url": "https://supabase.com/changelog/42949-breaking-change-removing-access-to-openapi-spec-via-the-anon-key", - "title": "Breaking Change: Removing access to OpenAPI spec via the anon key · Changelog" - }, - { - "url": "https://supabase.com/docs/guides/database/secure-data", - "title": "Securing your data | Supabase Docs" - }, - { - "url": "https://supabase.com/docs/guides/api/creating-routes", - "title": "Creating API Routes | Supabase Docs" - } - ], - "resultChars": 2967 - }, - { - "source": "web_fetch", - "query": "List any entries (with dates) about session revocation, sign out, user deletion, JWT expiry, asymmetric JWT signing keys, or auth.sessions. Quote the entry titles and dates.", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 1335 - }, - { - "source": "web_fetch", - "query": "Quote verbatim the entire changelog entry, especially: publication date, the timeline with all dates, description of publishable and secret keys, role mapping, RLS, revocation/rotation, multiple keys, and any statement about legacy key deprecation.", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/changelog/29260-upcoming-changes-to-supabase-api-keys" - } - ], - "resultChars": 215 - }, - { - "source": "web_fetch", - "query": "Quote verbatim the list of claims in a Supabase access token (especially session_id), and anything about JWT expiry defaults, revocation, and that JWTs cannot be revoked before expiry.", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/auth/jwts.md" - } - ], - "resultChars": 911 - }, - { - "source": "web_fetch", - "query": "Quote verbatim anything about: the anon and authenticated roles, service_role bypassing RLS / BYPASSRLS, `TO authenticated` / `TO anon` policy role targeting, auth.uid(), auth.jwt(), and how API keys map to roles. Also any warnings about policies without a TO clause or about service_role.", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security.md" - } - ], - "resultChars": 1591 - }, - { - "source": "web_fetch", - "query": "Quote verbatim the entry for session_id and the exp claim, plus any note on which claims are guaranteed/required in Supabase access tokens.", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/auth/jwt-fields.md" - } - ], - "resultChars": 530 - }, - { - "source": "web_search", - "query": "site:supabase.com/changelog sb_publishable secret API keys", - "hasContent": false, - "pages": [ - { - "url": "https://supabase.com/changelog/29260-upcoming-changes-to-supabase-api-keys", - "title": "Upcoming changes to Supabase API Keys · Changelog" - }, - { - "url": "https://supabase.com/changelog?next=Y3Vyc29yOnYyOpK0MjAyNC0wOS0yM1QwNzowMzozNVrOAG46dg%3D%3D&restPage=2", - "title": "Changelog" - }, - { - "url": "https://supabase.com/changelog/42949-breaking-change-removing-access-to-openapi-spec-via-the-anon-key", - "title": "Breaking Change: Removing access to OpenAPI spec via the anon key · Changelog" - }, - { - "url": "https://supabase.com/changelog/29289-supabase-auth-asymmetric-keys-support-in-2025", - "title": "Supabase Auth: Asymmetric Keys support in 2025 · Changelog" - }, - { - "url": "https://supabase.com/changelog/46458-passkeys-for-supabase-auth-beta", - "title": "Passkeys for Supabase Auth (Beta) · Changelog" - }, - { - "url": "https://supabase.com/changelog/43465-developer-update-march-2026", - "title": "Developer Update - March 2026 · Changelog" - }, - { - "url": "https://supabase.com/changelog/18972-edge-functions-secrets-should-now-get-updated-upon-resetting-db-password-or-jwt", - "title": "Edge Functions secrets should now get updated upon resetting DB password or JWT secret · Changelog" - }, - { - "url": "https://supabase.com/changelog/45702-developer-update-may-2026", - "title": "Developer Update - May 2026 · Changelog" - }, - { - "url": "https://supabase.com/changelog/29494-supabase-platform-access-control-project-permissions-breaking-changes-on-october", - "title": "Supabase Platform Access Control: Project Permissions Breaking Changes on October 15, 2024 · Changelog" - }, - { - "url": "https://supabase.com/changelog/46346-feature-preview-temporary-token-based-database-access", - "title": "Feature Preview: Temporary token-based database access · Changelog" - } - ], - "resultChars": 2996 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"delete user account revoke sessions sign out all devices\", limit: 8) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/auth/passkeys", - "title": "Passkey authentication" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-signout" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-admin-signout" - }, - { - "url": "https://supabase.com/docs/guides/auth/managing-user-data", - "title": "User Management" - }, - { - "url": "https://supabase.com/docs/guides/auth/signout", - "title": "Signing out" - }, - { - "url": "https://supabase.com/docs/guides/auth/auth-email-passwordless", - "title": "Passwordless email logins" - }, - { - "url": "https://supabase.com/docs/reference/swift/auth-signout", - "title": "signOut()" - }, - { - "url": "https://supabase.com/docs/guides/platform/sso/multiple-providers", - "title": "Multiple SSO Providers" - } - ], - "resultChars": 55523 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"ban user banned_until block user access token\", limit: 8) { nodes { title href } } }", - "hasContent": false, - "pages": [ - { - "url": "https://supabase.com/docs/guides/platform/temporary-access", - "title": "Temporary access" - }, - { - "url": "https://supabase.com/docs/guides/auth/jwt-fields", - "title": "JWT Claims Reference" - }, - { - "url": "https://supabase.com/docs/guides/platform/sso/multiple-providers", - "title": "Multiple SSO Providers" - }, - { - "url": "https://supabase.com/docs/guides/auth/auth-anonymous", - "title": "Anonymous Sign-Ins" - }, - { - "url": "https://supabase.com/docs/guides/auth/sessions", - "title": "User sessions" - } - ], - "resultChars": 582 - }, - { - "source": "web_fetch", - "query": "Quote the entire entry verbatim including publication date, timeline dates, key descriptions, role mapping, RLS notes, rotation/revocation, and legacy key deprecation statements.", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/changelog/29260-upcoming-changes-to-supabase-api-keys" - } - ], - "resultChars": 215 - }, - { - "source": "web_fetch", - "query": "Quote verbatim the table rows / descriptions for these error codes: session_not_found, user_banned, refresh_token_not_found, refresh_token_already_used, session_expired, user_not_found, bad_jwt.", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/auth/debugging/error-codes.md" - } - ], - "resultChars": 1077 - }, - { - "source": "web_fetch", - "query": "Quote the attributes list for updateUserById, especially ban_duration - its type, description, and accepted values (e.g. 'none', '24h').", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/reference/javascript/auth-admin-updateuserbyid" - } - ], - "resultChars": 843 - }, - { - "source": "web_fetch", - "query": "Quote verbatim any warnings about getSession vs getUser vs getClaims on the server, and why getUser must be used (revalidating the token with the Auth server).", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/auth/server-side/nextjs.md" - } - ], - "resultChars": 1190 - }, - { - "source": "web_fetch", - "query": "Quote verbatim the description of getClaims: when it verifies locally (asymmetric JWT) vs calls the Auth server, and any caveats about revocation / trusting claims.", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/reference/javascript/auth-getclaims" - } - ], - "resultChars": 907 - }, - { - "source": "web_fetch", - "query": "Quote verbatim everything about the apikey header vs the Authorization header, publishable and secret keys, which Postgres role results, what happens when both a secret key in apikey and a user JWT in Authorization are sent, RLS enforcement, and verify_jwt.", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/auth-headers.md" - } - ], - "resultChars": 1519 - }, - { - "source": "web_fetch", - "query": "Quote verbatim everything about anon/publishable/secret/service_role keys, which Postgres role each results in, RLS enforcement vs bypass, and the apikey vs Authorization header.", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/api/securing-your-api.md" - } - ], - "resultChars": 1341 - }, - { - "source": "web_fetch", - "query": "Quote verbatim: any mention of (a) \"last used\" indicators for API keys; (b) what Postgres role applies when a SECRET key is sent together with a signed-in user's JWT; (c) whether a secret key can be configured to assume a role other than service_role; (d) the full \"Secret keys\" section; (e) the full \"Publishable key\" section; (f) anything about how many publishable keys can exist vs secret keys.", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/getting-started/api-keys.md" - } - ], - "resultChars": 1243 - } - ] - }, - "prompt": "Last week support removed a user through our app's delete-account flow — the\napp calls the `delete_account` function over RPC as the signed-in user. This\nmorning that same person was back: still signed in, reading and saving their\ndata like nothing happened.\n\nFigure out why the account still works, fix the flow so a deleted account\nloses access, and tell me whether there is any window where they could still\nget in after the fix.\n\nOne more thing while you're at it: we're migrating off the legacy\nanon/service_role API keys to the new publishable and secret keys, and the\nteam isn't sure which one belongs in the frontend and what each means for\nRLS. Give us a quick rundown.", - "promptSourcePath": "evals/investigate-auth-001-deleted-user-access/PROMPT.md", - "attempts": 2, - "sourcePath": "claude-code-opus-5/investigate-auth-001-deleted-user-access.json" - }, - { - "experiment": "claude-code-opus-5", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-opus-5", - "reasoningEffort": "high" - }, - "eval": "investigate-realtime-001-subscribed-no-events", - "stage": "investigate", - "product": [ - "realtime", - "database" - ], - "topic": [ - "sdk" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "orders table added to supabase_realtime publication", - "passed": true - }, - { - "name": "courier_locations still in supabase_realtime publication", - "passed": true - }, - { - "name": "publication still publishes INSERT events", - "passed": true - }, - { - "name": "RLS still enabled on orders", - "passed": true - }, - { - "name": "staff can still read orders through RLS", - "passed": true, - "notes": "authenticated sees 2 of 2 orders" - }, - { - "name": "diagnosed missing publication membership", - "passed": true, - "judgeNotes": "Diagnosed the issue as orders missing from the supabase_realtime publication despite SUBSCRIBED status, applied ALTER PUBLICATION supabase_realtime ADD TABLE public.orders, and preserved RLS/policies and courier_locations. Did not blame or fix via client code/RLS/networking." - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" - ] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"enable postgres changes realtime publication table\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/realtime/subscribing-to-database-changes", - "title": "Subscribing to Database Changes" - }, - { - "url": "https://supabase.com/docs/guides/realtime/postgres-changes", - "title": "Postgres Changes" - }, - { - "url": "https://supabase.com/docs/guides/realtime/benchmarks", - "title": "Benchmarks" - }, - { - "url": "https://supabase.com/docs/guides/realtime/authorization", - "title": "Realtime Authorization" - }, - { - "url": "https://supabase.com/docs/guides/realtime/protocol", - "title": "Realtime Protocol" - } - ], - "resultChars": 112153 - } - ] - }, - "prompt": "Our dispatch dashboard shows incoming orders as they happen. The courier\nlocation feed on the same page updates live without problems, but new orders\nonly show up after a page refresh.\n\nThe dashboard uses supabase-js to subscribe to INSERT events on the `orders`\ntable through postgres_changes, the same way it subscribes to courier\nlocations. The channel's status callback logs SUBSCRIBED and there are no\nerrors in the browser console.\n\nFigure out why no order events ever arrive and fix it.", - "promptSourcePath": "evals/investigate-realtime-001-subscribed-no-events/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-opus-5/investigate-realtime-001-subscribed-no-events.json" - }, - { - "experiment": "claude-code-opus-5", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-opus-5", - "reasoningEffort": "high" - }, - "eval": "investigate-reliability-003-edge-function-5xx-correlation", - "stage": "investigate", - "product": [ - "edge-functions" - ], - "topic": [ - "observability" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "identified image-transform and the recurring 503 pattern", - "passed": true, - "judgeNotes": "Identified image-transform as the affected function and described the recurring HTTP 503 pattern across the morning of 2026-04-28, covering the 8 gateway failures from 07:00Z through 12:00Z. Also correctly avoided treating the older billing-webhook 503s as the main issue." - }, - { - "name": "attributed recurring 503s to gateway/platform layer, not function code", - "passed": true, - "judgeNotes": "Attributes the recurring image-transform 503s to the gateway/platform layer, explicitly stating they appear only in gateway/API logs and that failed requests never reached the function runtime. Grounds this in valid observations: no corresponding edge-function log rows for 503s, nearby successful invocations with 200s and normal runtimes, and distinguishes gateway 503s from avatar-upload's function-level 500." - }, - { - "name": "recommended a concrete next step", - "passed": true, - "judgeNotes": "The assistant recommended several concrete next steps: find half-hourly triggers, obtain volume data for :00/:30 windows from Logs Explorer, flatten spikes with jitter/queueing, add retry with backoff, and triage correlated avatar-upload errors." - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" - ] - }, - "docs": { - "calls": [] - }, - "prompt": "Users have been reporting that image uploads are intermittently failing with server errors throughout the morning of `2026-04-28`.\n\nCan you investigate the project logs and tell me what is going on and what we\nshould do next?", - "promptSourcePath": "evals/investigate-reliability-003-edge-function-5xx-correlation/PROMPT.md", - "attempts": 2, - "sourcePath": "claude-code-opus-5/investigate-reliability-003-edge-function-5xx-correlation.json" - }, - { - "experiment": "claude-code-opus-5", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-opus-5", - "reasoningEffort": "high" - }, - "eval": "resolve-dataapi-001-empty-results", - "stage": "resolve", - "product": [ - "data-api", - "database", - "auth" - ], - "topic": [ - "rls", - "sdk" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "RLS still enabled on bookmarks", - "passed": true - }, - { - "name": "user A reads own bookmarks", - "passed": true - }, - { - "name": "user B cannot read user A bookmarks", - "passed": true - }, - { - "name": "anon reads no bookmarks", - "passed": true - }, - { - "name": "user A can save a new bookmark", - "passed": true - }, - { - "name": "user B cannot insert a bookmark as user A", - "passed": true - }, - { - "name": "diagnosed RLS and added owner-scoped policies", - "passed": true, - "judgeNotes": "Diagnosed RLS enabled with zero policies as deny-all for Data API/authenticated, kept RLS enabled, and created authenticated SELECT and INSERT policies scoped to user_id = auth.uid() with WITH CHECK for inserts. Did not use permissive/anon policies or disable RLS." - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" - ] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{ searchDocs(query: \"row level security policy select insert auth.uid user_id private table\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/api/securing-your-api", - "title": "Securing your API" - }, - { - "url": "https://supabase.com/docs/guides/realtime/getting_started", - "title": "Getting Started with Realtime" - }, - { - "url": "https://supabase.com/docs/guides/local-development/testing/pgtap-extended", - "title": "Advanced pgTAP Testing" - }, - { - "url": "https://supabase.com/docs/guides/realtime/authorization", - "title": "Realtime Authorization" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/column-level-security", - "title": "Column Level Security" - } - ], - "resultChars": 81499 - } - ] - }, - "prompt": "Our app lets signed-in users save bookmarks and view them on their dashboard. Bookmarks are stored in the `bookmarks` table and are private — a user must only ever see their own. \nUsers also need to be able to save new bookmarks from the app.\n\nI can see the rows when I query the table directly, but the dashboard shows an empty list for every user.\n\nFind out why the Data API returns nothing and fix it.", - "promptSourcePath": "evals/resolve-dataapi-001-empty-results/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-opus-5/resolve-dataapi-001-empty-results.json" - }, - { - "experiment": "claude-code-opus-5", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-opus-5", - "reasoningEffort": "high" - }, - "eval": "resolve-database-001-migration-history-mismatch", - "stage": "resolve", - "product": [ - "database" - ], - "topic": [ - "migrations" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "the avatar_url column is applied on the hosted profiles table", - "passed": true - }, - { - "name": "migration 20240220000000 is recorded in the remote history", - "passed": true - }, - { - "name": "remote migration history matches local migration files", - "passed": true - }, - { - "name": "local migrations are a valid reconciled sequence", - "passed": true - }, - { - "name": "production profile data is intact (not reset)", - "passed": true - }, - { - "name": "the avatar migration and history reconciliation were done via the Supabase CLI", - "passed": true, - "judgeNotes": "Avatar migration was applied through `supabase db push` in #22, with output `Applying migration 20240220000000_add_avatar_url.sql...` and `Finished supabase db push.` History was reconciled by adding the missing local migration file `supabase/migrations/20240115000000_add_profile_bio.sql` in #20, after which `supabase migration list` (#21/#24) showed local and remote aligned. Only read-only `psql` inspections were used; no disallowed direct SQL mutation or prepared-statement workaround was seen." - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" - ] - }, - "docs": { - "calls": [] - }, - "prompt": "I'm trying to ship a migration to our hosted project and it's not working. Can you figure out what's wrong and get it deployed?", - "promptSourcePath": "evals/resolve-database-001-migration-history-mismatch/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-opus-5/resolve-database-001-migration-history-mismatch.json" - }, - { - "experiment": "claude-code-opus-5", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-opus-5", - "reasoningEffort": "high" - }, - "eval": "resolve-performance-001-slow-query-cpu-spike", - "stage": "resolve", - "product": [ - "database" - ], - "topic": [ - "observability", - "sql" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "inspected pg_stat_statements for query performance", - "passed": true - }, - { - "name": "ran EXPLAIN on the expensive query", - "passed": true - }, - { - "name": "created index covering user_id and created_at", - "passed": true - }, - { - "name": "query plan uses an index and avoids sequential scan", - "passed": true, - "notes": "Limit (cost=79.74..79.86 rows=50 width=58)\n -> Sort (cost=79.74..79.99 rows=100 width=58)\n Sort Key: created_at DESC\n -> Bitmap Heap Scan on events (cost=5.06..76.42 rows=100 width=58)\n Recheck Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)\n -> Bitmap Index Scan on events_user_id_created_at_desc_idx (cost=0.00..5.03 rows=100 width=0)\n Index Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)" - }, - { - "name": "inserts still work", - "passed": true - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" - ] - }, - "docs": { - "calls": [] - }, - "prompt": "My database CPU keeps spiking and the app gets slow when loading recent events for a user. Can you figure out what query is causing it and make the database change needed to fix it?\n\nEnd your turn with a short summary of what you changed and why.", - "promptSourcePath": "evals/resolve-performance-001-slow-query-cpu-spike/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-opus-5/resolve-performance-001-slow-query-cpu-spike.json" - }, - { - "experiment": "claude-code-opus-5", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-opus-5", - "reasoningEffort": "high" - }, - "eval": "resolve-security-002-rls-cross-tenant-leak", - "stage": "resolve", - "product": [ - "database", - "auth" - ], - "topic": [ - "rls", - "security" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "RLS enabled on notes", - "passed": true - }, - { - "name": "tenant A sees only org A notes", - "passed": true - }, - { - "name": "tenant B cannot read org A notes", - "passed": true - }, - { - "name": "tenant A author can update own note", - "passed": true - }, - { - "name": "tenant B cannot update org A note", - "passed": true - }, - { - "name": "tenant B author can delete own note", - "passed": true - }, - { - "name": "tenant B cannot delete org A note", - "passed": true - }, - { - "name": "tenant A can insert note in own org", - "passed": true - }, - { - "name": "tenant B cannot insert into org A", - "passed": true - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] - }, - "docs": { - "calls": [] - }, - "prompt": "A customer reported that notes showed up in the wrong workspace.\n\nCan you investigate what is going on and fix it?", - "promptSourcePath": "evals/resolve-security-002-rls-cross-tenant-leak/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-opus-5/resolve-security-002-rls-cross-tenant-leak.json" - }, - { - "experiment": "claude-code-opus-5-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-opus-5", - "reasoningEffort": "high" - }, - "eval": "build-cli-001-bootstrap-app", - "stage": "build", - "product": [ - "database", - "data-api" - ], - "topic": [ - "migrations", - "rls" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "supabase project initialised (supabase/config.toml exists)", - "passed": true - }, - { - "name": "todos table is created by a migration file", - "passed": true - }, - { - "name": "todos table exists with at least 2 seeded rows", - "passed": true, - "notes": "found 2 rows" - }, - { - "name": "row level security is enabled on todos", - "passed": true - }, - { - "name": "a SELECT policy targets the authenticated role", - "passed": true - }, - { - "name": "REST API returns no todos to anonymous requests", - "passed": true, - "notes": "0 rows" - }, - { - "name": "REST API returns the todos to authenticated requests", - "passed": true, - "notes": "2 rows" - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "We're kicking off a todos app and I want the Supabase side ready for the team\nto build on. Set it up the way we'd run it in development, with schema changes\ntracked as migrations so they can be reviewed and replayed.\n\nFor the first slice we just need a `todos` table. Todos aren't public: anyone\nsigned in can read all of them, but nothing should be writable through the API\nfor now. Add a couple of sample todos so there's something to look at.\n\nBefore you hand it back, make sure the running API actually behaves that way —\nsigned-in users get the todos, signed-out requests get nothing.", - "promptSourcePath": "evals/build-cli-001-bootstrap-app/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-opus-5-no-skills/build-cli-001-bootstrap-app.json" - }, - { - "experiment": "claude-code-opus-5-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-opus-5", - "reasoningEffort": "high" - }, - "eval": "build-cli-002-declarative-schema", - "stage": "build", - "product": [ - "database" - ], - "topic": [ - "declarative-schema", - "migrations" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "supabase db diff used to generate the migration", - "passed": true - }, - { - "name": "schema file updated to include description column", - "passed": true - }, - { - "name": "a new migration was generated for the change", - "passed": true - }, - { - "name": "description column exists in the live database", - "passed": true - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{searchDocs(query:\"declarative database schemas db diff generate migration\", limit:4){nodes{title href content}}}", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/local-development/declarative-database-schemas", - "title": "Declarative database schemas" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows", - "title": "Local development workflow" - }, - { - "url": "https://supabase.com/docs/guides/deployment/database-migrations", - "title": "Database Migrations" - }, - { - "url": "https://supabase.com/docs/guides/deployment/managing-environments", - "title": "Managing Environments" - } - ], - "resultChars": 66910 - } - ] - }, - "prompt": "Add a description text column to the `products` table in my local Supabase stack", - "promptSourcePath": "evals/build-cli-002-declarative-schema/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-opus-5-no-skills/build-cli-002-declarative-schema.json" - }, - { - "experiment": "claude-code-opus-5-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-opus-5", - "reasoningEffort": "high" - }, - "eval": "build-cli-003-pg-cron-queue-workflow", - "stage": "build", - "product": [ - "database", - "edge-functions", - "cron", - "queues" - ], - "topic": [ - "sql", - "sdk" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "pg_cron job 'enqueue-tasks' scheduled to run every minute", - "passed": true, - "notes": "schedule='* * * * *', active=true" - }, - { - "name": "cron command enqueues to the 'tasks' queue", - "passed": true, - "notes": "queue depth 0 -> 1" - }, - { - "name": "process-tasks function drains the queue", - "passed": true, - "notes": "function removed the seeded message (id 36) from the queue" - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{ searchDocs(query: \"queues cron schedule pgmq edge function drain queue\", limit: 8) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", - "title": "Automatic embeddings" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net", - "title": "pg_net: Async Networking" - }, - { - "url": "https://supabase.com/docs/guides/functions/schedule-functions", - "title": "Scheduling Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/cron/quickstart", - "title": "Quickstart" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/pgcron-debugging-guide-n1KTaz", - "title": "pg_cron debugging guide" - } - ], - "resultChars": 63907 - } - ] - }, - "prompt": "I want to set up a recurring background workflow on my local Supabase stack.\n\nCan you set up a cron job called `enqueue-tasks` to run every minute and push a task into a queue called `tasks`? Then add a `process-tasks` edge function that reads messages off the `tasks` queue and removes them, so a scheduled worker can keep the backlog drained.", - "promptSourcePath": "evals/build-cli-003-pg-cron-queue-workflow/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-opus-5-no-skills/build-cli-003-pg-cron-queue-workflow.json" - }, - { - "experiment": "claude-code-opus-5-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-opus-5", - "reasoningEffort": "high" - }, - "eval": "build-database-001-migrate-postgres-to-supabase", - "stage": "build", - "product": [ - "database" - ], - "topic": [ - "migrations" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "all 3 tables exist (teams, members, tasks)", - "passed": true - }, - { - "name": "row counts match (teams=5, members=10, tasks=13)", - "passed": true - }, - { - "name": "foreign key constraints survived the restore", - "passed": true - }, - { - "name": "tasks_team_status_idx index survived the restore", - "passed": true - }, - { - "name": "sequences synced (next insert won't conflict with existing IDs)", - "passed": true - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "I have an existing Postgres database I want to migrate to Supabase. There's a binary dump at `source.dump` in the current directory.\n\nCan you set up a local Supabase project and restore the dump into it?", - "promptSourcePath": "evals/build-database-001-migrate-postgres-to-supabase/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-opus-5-no-skills/build-database-001-migrate-postgres-to-supabase.json" - }, - { - "experiment": "claude-code-opus-5-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-opus-5", - "reasoningEffort": "high" - }, - "eval": "build-functions-004-service-role-bypass", - "stage": "build", - "product": [ - "edge-functions", - "auth", - "database" - ], - "topic": [ - "rls", - "security", - "sdk" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "rejects missing auth", - "passed": true, - "notes": "status=401" - }, - { - "name": "user A reads own note", - "passed": true, - "notes": "status=200" - }, - { - "name": "reads only with the caller's JWT", - "passed": true, - "notes": "bearer_tokens=2, all_match=true" - }, - { - "name": "user A cannot force-read user B note", - "passed": true, - "notes": "status=200" - }, - { - "name": "user B cannot force-read user A note", - "passed": true, - "notes": "status=200" - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "I built an Edge Function called `private-notes` for showing a user's saved\nprivate notes.\n\nCan you check whether there's any way one user could see another user's notes?\n\nPlease fix and deploy it if needed.", - "promptSourcePath": "evals/build-functions-004-service-role-bypass/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-opus-5-no-skills/build-functions-004-service-role-bypass.json" - }, - { - "experiment": "claude-code-opus-5-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-opus-5", - "reasoningEffort": "high" - }, - "eval": "build-functions-005-dual-auth-user-secret", - "stage": "build", - "product": [ - "edge-functions", - "auth", - "database" - ], - "topic": [ - "sdk", - "rls", - "security" - ], - "suite": "benchmark", - "interface": "cli", - "cliVersion": "2.109.1", - "passed": true, - "checks": [ - { - "name": "seed rows present", - "passed": true, - "notes": "found 2/2 seeded rows" - }, - { - "name": "rejects request with no credentials", - "passed": true, - "notes": "status 401: {\"error\":\"Unauthorized\"}" - }, - { - "name": "user with JWT reads only their own rows", - "passed": true, - "notes": "status 200: [{\"user_id\":\"370cd0f2-cda6-44ce-afad-754f7b5fd3fb\",\"metric\":\"steps_a_ms6xp2i0\",\"value\":111}]" - }, - { - "name": "user cannot read another user's rows by passing user_id", - "passed": true, - "notes": "status 200: [{\"user_id\":\"370cd0f2-cda6-44ce-afad-754f7b5fd3fb\",\"metric\":\"steps_a_ms6xp2i0\",\"value\":111}]" - }, - { - "name": "service key bypasses RLS to read the target user's rows", - "passed": true, - "notes": "status 200: [{\"user_id\":\"2b64c372-40da-4ea6-90cd-c4ac620fa7e7\",\"metric\":\"steps_b_ms6xp2i0\",\"value\":222}]" - }, - { - "name": "non-service key is not granted service access", - "passed": true, - "notes": "status 401: {\"error\":\"Unauthorized\"}" - }, - { - "name": "rejects an unverified (forged) user token", - "passed": true, - "notes": "status 401: {\"error\":\"Unauthorized\"}" - }, - { - "name": "a user token in the apikey slot is not treated as the service key", - "passed": true, - "notes": "status 401: {\"error\":\"Unauthorized\"}" - }, - { - "name": "implementation uses @supabase/server", - "passed": true, - "notes": "imports @supabase/server / withSupabase" - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{ searchDocs(query: \"@supabase/server package edge function api keys secret key publishable key\", limit: 6) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/functions/secrets", - "title": "Environment Variables" - }, - { - "url": "https://supabase.com/docs/guides/auth/choosing-a-server-package", - "title": "Which package to use" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" - }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys", - "title": "JWT Signing Keys" - } - ], - "resultChars": 76864 - } - ] - }, - "prompt": "Build and serve a Supabase Edge Function named `user-stats` for this project,\nreachable over HTTP at `/functions/v1/user-stats`.\n\nOur product stores per-user metrics in the existing `user_stats` table.\n\nTwo very different callers need to hit this one endpoint:\n\n1. **Our mobile app**, acting for a signed-in user. It sends that user's\n Supabase access token. The endpoint should return the user's own stats.\n\n2. **Our internal billing service**, a trusted backend with no signed-in user.\n It authenticates with the project's secret (service-role) key in the `apikey`\n header, and names the target user with a `user_id` in the JSON request body.\n It needs that user's stats.\n\nReturn the matching rows as JSON. The endpoint must be secure: only ever serve\nstats to a caller that is genuinely entitled to them, and turn away callers that\nare not.", - "promptSourcePath": "evals/build-functions-005-dual-auth-user-secret/PROMPT.md", - "attempts": 2, - "sourcePath": "claude-code-opus-5-no-skills/build-functions-005-dual-auth-user-secret.json" - }, - { - "experiment": "claude-code-opus-5-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-opus-5", - "reasoningEffort": "high" - }, - "eval": "build-storage-001-private-bucket-access", - "stage": "build", - "product": [ - "storage", - "database" - ], - "topic": [ - "rls", - "sdk" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "bucket user-files exists", - "passed": true - }, - { - "name": "bucket user-files is private", - "passed": true - }, - { - "name": "RLS still enabled on storage.objects", - "passed": true - }, - { - "name": "user A lists only own files", - "passed": true, - "notes": "saw: 019fb0e9-cc49-7119-a6b5-8129b1bef946/receipt-alpha.pdf, 019fb0e9-cc49-7119-a6b5-8129b1bef946/receipt-beta.pdf" - }, - { - "name": "user B cannot read user A files", - "passed": true - }, - { - "name": "anon reads no files", - "passed": true - }, - { - "name": "user A can upload into own folder", - "passed": true - }, - { - "name": "user B cannot upload into user A folder", - "passed": true - }, - { - "name": "configured private per-user storage access", - "passed": true, - "judgeNotes": "Meets requirements: private user-files bucket, authenticated owner-scoped SELECT and INSERT storage.objects policies with WITH CHECK, no RLS disabling or public access, and supabase-js createSignedUrl/createSignedUrls examples with expiries." - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{\n searchDocs(query: \"storage RLS policy user id folder path createSignedUrl private bucket\", limit: 8) {\n nodes {\n title\n href\n content\n }\n }\n}", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/storage/buckets/fundamentals", - "title": "Storage Buckets" - }, - { - "url": "https://supabase.com/docs/guides/storage/security/access-control", - "title": "Storage Access Control" - }, - { - "url": "https://supabase.com/docs/guides/security/product-security", - "title": "Secure configuration of Supabase products" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3", - "title": "Configure S3 Storage" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/why-cant-i-uploadlistetc-my-public-bucket-Z6CmGt", - "title": "Why can't I upload/list/etc my public bucket?" - }, - { - "url": "https://supabase.com/docs/guides/storage/schema/helper-functions", - "title": "Storage Helper Functions" - }, - { - "url": "https://supabase.com/docs/guides/storage/serving/downloads", - "title": "Serving assets from Storage" - }, - { - "url": "https://supabase.com/docs/guides/auth/third-party/clerk", - "title": "Clerk" - } - ], - "resultChars": 35640 - } - ] - }, - "prompt": "Our app lets signed-in users keep personal files like receipts and bank\nstatements. These files are private — a user must only ever be able to upload\nand download their own. The app uploads each file under a path that starts\nwith the owner's user id, e.g. `/receipt-march.pdf`.\n\nSet up a `user-files` bucket on our project and lock it down that way.\n\nUsers also sometimes share one of their files with someone else through a\ntemporary link that expires. Include the supabase-js code the app should use\nfor that.", - "promptSourcePath": "evals/build-storage-001-private-bucket-access/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-opus-5-no-skills/build-storage-001-private-bucket-access.json" - }, - { - "experiment": "claude-code-opus-5-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-opus-5", - "reasoningEffort": "high" - }, - "eval": "build-tests-001-rls-tenant-isolation", - "stage": "build", - "product": [ - "database" - ], - "topic": [ - "tests", - "rls" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "pgTAP test file(s) written under supabase/tests/", - "passed": true, - "notes": "1 file(s): supabase/tests/tenant_isolation_test.sql" - }, - { - "name": "pgTAP isolation tests ran and pass", - "passed": true, - "notes": "10 passed, 3 failed" - }, - { - "name": "agent correctly identifies the posts isolation bug from test results", - "passed": true, - "judgeNotes": "The agent correctly identifies `posts` as the table with the broken tenant isolation policy, grounds it in pgTAP failures showing cross-tenant reads, and distinguishes that `notes` passes. It treats the test results as authoritative and identifies the missing `m.org_id = posts.org_id` correlation as the root cause." - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "Can you audit the tenant isolation on our tables? Write some database tests covering both the happy path and the negative case, run them, and share the results.", - "promptSourcePath": "evals/build-tests-001-rls-tenant-isolation/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-opus-5-no-skills/build-tests-001-rls-tenant-isolation.json" - }, - { - "experiment": "claude-code-opus-5-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-opus-5", - "reasoningEffort": "high" - }, - "eval": "build-vectors-001-rag-with-permissions", - "stage": "build", - "product": [ - "database", - "vectors" - ], - "topic": [ - "sql", - "rls" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "document_sections.embedding is vector(384)", - "passed": true, - "notes": "vector(384)" - }, - { - "name": "HNSW index on the embedding column", - "passed": true, - "notes": "CREATE INDEX document_sections_embedding_idx ON public.document_sections USING hnsw (embedding vector_ip_ops)" - }, - { - "name": "index operator class matches the search operator", - "passed": true, - "notes": "function operators: <#>\nindexes: CREATE INDEX document_sections_embedding_idx ON public.document_sections USING hnsw (embedding vector_ip_ops)" - }, - { - "name": "user A search returns only own sections, best match first", - "passed": true - }, - { - "name": "user B search returns only own sections, best match first", - "passed": true - }, - { - "name": "user A reads only own sections through the API", - "passed": true - }, - { - "name": "user A reads only own documents through the API", - "passed": true - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{ searchDocs(query: \"gte-small embedding dimensions Supabase.ai Session edge function\", limit: 4) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/examples/semantic-search", - "title": "Semantic Search" - }, - { - "url": "https://supabase.com/docs/guides/ai/semantic-search", - "title": "Semantic search" - }, - { - "url": "https://supabase.com/docs/guides/functions/ai-models", - "title": "Running AI Models" - }, - { - "url": "https://supabase.com/docs/guides/ai/quickstarts/generate-text-embeddings", - "title": "Generate Embeddings" - } - ], - "resultChars": 42731 - } - ] - }, - "prompt": "We're adding semantic search to our internal knowledge base app. I already wrote the edge functions, but search doesn't work yet. Can you set up whatever the database needs to make search work end to end?\n\nSome documents are confidential, so users should only have access to documents they own.", - "promptSourcePath": "evals/build-vectors-001-rag-with-permissions/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-opus-5-no-skills/build-vectors-001-rag-with-permissions.json" - }, - { - "experiment": "claude-code-opus-5-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-opus-5", - "reasoningEffort": "high" - }, - "eval": "deploy-database-001-prometheus-metrics", - "stage": "deploy", - "product": [ - "database" - ], - "topic": [ - "observability" - ], - "suite": "benchmark", - "passed": true, - "checks": [ - { - "name": "preserved existing app scrape job", - "passed": true - }, - { - "name": "configured the Supabase Metrics API scrape correctly", - "passed": true, - "judgeNotes": "Meets requirements: preserves app scrape, adds HTTPS Supabase Metrics API target with correct path and project host, uses basic_auth password_file, and docker-compose mounts the secrets directory containing that password file." - }, - { - "name": "documented live deployment and verification steps", - "passed": true, - "judgeNotes": "README includes creating a Supabase Secret API key, writing it to the mounted secret file, reloading Prometheus/Compose stack, and verifying via Prometheus targets and Grafana." - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{ searchDocs(query: \"Prometheus metrics endpoint project observability\", limit: 6) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/telemetry/metrics/grafana-self-hosted", - "title": "Metrics API with Prometheus & Grafana (self-hosted)" - }, - { - "url": "https://supabase.com/docs/guides/telemetry/metrics", - "title": "Metrics API" - }, - { - "url": "https://supabase.com/docs/guides/telemetry/metrics/vendor-agnostic", - "title": "Vendor-agnostic Metrics API setup" - }, - { - "url": "https://supabase.com/docs/guides/telemetry/metrics/grafana-cloud", - "title": "Metrics API with Grafana Cloud" - }, - { - "url": "https://supabase.com/docs/guides/platform/read-replicas", - "title": "Read Replicas" - } - ], - "resultChars": 32656 - } - ] - }, - "prompt": "Can you wire my Supabase project metrics into our existing observability stack and document\nin the observability README what we need to do to make the config live?", - "promptSourcePath": "evals/deploy-database-001-prometheus-metrics/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-opus-5-no-skills/deploy-database-001-prometheus-metrics.json" - }, - { - "experiment": "claude-code-opus-5-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-opus-5", - "reasoningEffort": "high" - }, - "eval": "deploy-functions-001-edge-function-secrets", - "stage": "deploy", - "product": [ - "edge-functions" - ], - "topic": [ - "security" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "WEATHER_API_KEY is set as a Function secret on the project", - "passed": true - }, - { - "name": "the weather function is deployed to the project", - "passed": true, - "notes": "status ACTIVE" - }, - { - "name": "the weather function reads WEATHER_API_KEY from the environment", - "passed": true, - "judgeNotes": "The function reads WEATHER_API_KEY from the runtime environment using Deno.env.get(\"WEATHER_API_KEY\")." - }, - { - "name": "WEATHER_API_KEY value is not committed to the repo", - "passed": true - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{ searchDocs(query: \"edge function secrets environment variables set env-file deploy\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/secrets", - "title": "Environment Variables" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/inspecting-edge-function-environment-variables-wg5qOQ", - "title": "Inspecting edge function environment variables" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech", - "title": "Transcription Telegram Bot" - }, - { - "url": "https://supabase.com/docs/guides/deployment/branching/configuration", - "title": "Configuration" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream", - "title": "Streaming Speech with ElevenLabs" - } - ], - "resultChars": 38347 - } - ] - }, - "prompt": "Our weather widget currently calls WeatherAPI straight from the browser, which\nleaks our API key. I want to move that behind a Supabase Edge Function called\n`weather` that holds the key server-side and proxies the request.\n\nThe function should read the key from an environment variable named\n`WEATHER_API_KEY`. Our key already lives in a local `.env` file at the project\nroot.\n\nDeploy the function to our project so it's live, and make sure the deployed\nfunction can actually read the key at runtime.", - "promptSourcePath": "evals/deploy-functions-001-edge-function-secrets/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-opus-5-no-skills/deploy-functions-001-edge-function-secrets.json" - }, - { - "experiment": "claude-code-opus-5-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-opus-5", - "reasoningEffort": "high" - }, - "eval": "deploy-self-hosting-001-docker-compose", - "stage": "deploy", - "product": [ - "database", - "auth", - "storage" - ], - "topic": [ - "self-hosting" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "cloned the self-host stack (docker-compose.yml + volumes/db)", - "passed": true - }, - { - "name": "didn't conflate with the CLI (no supabase/config.toml in the stack)", - "passed": true - }, - { - "name": "secrets rotated off the shipped defaults", - "passed": true - }, - { - "name": "ANON_KEY and SERVICE_ROLE_KEY are HS256 JWTs signed by JWT_SECRET", - "passed": true - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "I'm moving off the hosted Supabase and running the whole thing myself on a VPS I\njust spun up. Can you get a Docker setup ready for me to copy onto the box?\n\nI don't need it running here, I'll do the actual bring-up once I'm on the\nserver. I just want everything in place and the secrets set up. Put it in a `supabase-docker/`\nfolder at the repo root so I can scp the whole thing across in one go.", - "promptSourcePath": "evals/deploy-self-hosting-001-docker-compose/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-opus-5-no-skills/deploy-self-hosting-001-docker-compose.json" - }, - { - "experiment": "claude-code-opus-5-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-opus-5", - "reasoningEffort": "high" - }, - "eval": "investigate-auth-001-deleted-user-access", - "stage": "investigate", - "product": [ - "auth" - ], - "topic": [ - "security", - "sdk" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": false, - "checks": [ - { - "name": "victim session active before delete-account", - "passed": true - }, - { - "name": "delete_account flow ran for the victim", - "passed": true - }, - { - "name": "delete-account revokes the user's sessions", - "passed": true, - "notes": "sessions left: 0" - }, - { - "name": "deleted user's refresh token is rejected", - "passed": true - }, - { - "name": "deleted user cannot sign back in", - "passed": false, - "notes": "deleted account can still sign in" - }, - { - "name": "other users keep their sessions and access", - "passed": true - }, - { - "name": "diagnosed and explained session revocation", - "passed": true, - "judgeNotes": "Meets the rubric: diagnoses soft-delete-only flow, implements real auth/session/refresh-token revocation plus RLS gating, explains Data API has no post-deletion window under its implemented RLS fix while unexpired JWTs can still matter for non-policy/local-ish surfaces, and correctly distinguishes publishable frontend/RLS-enforced keys from server-only secret keys that bypass RLS." - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{ searchDocs(query: \"publishable key secret key legacy anon service_role API keys RLS\", limit: 6) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/auth/signing-keys", - "title": "JWT Signing Keys" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" - }, - { - "url": "https://supabase.com/docs/guides/auth/server-side/creating-a-client", - "title": "Creating a Supabase client for SSR" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/functions/secrets", - "title": "Environment Variables" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-redwoodjs", - "title": "Build a User Management App with RedwoodJS" - } - ], - "resultChars": 136765 - }, - { - "source": "web_fetch", - "query": "Report verbatim any statements about: (a) how many secret keys or publishable keys a project can have (limits/maximums), (b) whether secret keys can be viewed again after creation or are shown only once, (c) whether publishable/secret keys expire or have an expiry, (d) whether keys are JWTs, (e) key format prefixes. Quote exact sentences.", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/getting-started/api-keys" - } - ], - "resultChars": 964 - }, - { - "source": "web_fetch", - "query": "Report verbatim any statements about deprecation timeline for legacy anon/service_role keys, limits on number of keys, and whether legacy and new keys can coexist. Also note the page's stated last-updated date if visible.", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys" - } - ], - "resultChars": 1009 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"how many secret keys can I create limit reveal secret key dashboard\", limit: 2) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" - }, - { - "url": "https://supabase.com/docs/guides/functions/limits", - "title": "Limits" - } - ], - "resultChars": 15838 - } - ] - }, - "prompt": "Last week support removed a user through our app's delete-account flow — the\napp calls the `delete_account` function over RPC as the signed-in user. This\nmorning that same person was back: still signed in, reading and saving their\ndata like nothing happened.\n\nFigure out why the account still works, fix the flow so a deleted account\nloses access, and tell me whether there is any window where they could still\nget in after the fix.\n\nOne more thing while you're at it: we're migrating off the legacy\nanon/service_role API keys to the new publishable and secret keys, and the\nteam isn't sure which one belongs in the frontend and what each means for\nRLS. Give us a quick rundown.", - "promptSourcePath": "evals/investigate-auth-001-deleted-user-access/PROMPT.md", - "attempts": 2, - "sourcePath": "claude-code-opus-5-no-skills/investigate-auth-001-deleted-user-access.json" - }, - { - "experiment": "claude-code-opus-5-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-opus-5", - "reasoningEffort": "high" - }, - "eval": "investigate-realtime-001-subscribed-no-events", - "stage": "investigate", - "product": [ - "realtime", - "database" - ], - "topic": [ - "sdk" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "orders table added to supabase_realtime publication", - "passed": true - }, - { - "name": "courier_locations still in supabase_realtime publication", - "passed": true - }, - { - "name": "publication still publishes INSERT events", - "passed": true - }, - { - "name": "RLS still enabled on orders", - "passed": true - }, - { - "name": "staff can still read orders through RLS", - "passed": true, - "notes": "authenticated sees 2 of 2 orders" - }, - { - "name": "diagnosed missing publication membership", - "passed": true, - "judgeNotes": "The assistant correctly identified the root cause as public.orders missing from the supabase_realtime publication despite the channel reaching SUBSCRIBED, applied an idempotent ALTER PUBLICATION supabase_realtime ADD TABLE public.orders migration, preserved courier_locations and existing RLS/policies, and did not blame or alter RLS, grants, client code, networking, or recreate the publication." - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "Our dispatch dashboard shows incoming orders as they happen. The courier\nlocation feed on the same page updates live without problems, but new orders\nonly show up after a page refresh.\n\nThe dashboard uses supabase-js to subscribe to INSERT events on the `orders`\ntable through postgres_changes, the same way it subscribes to courier\nlocations. The channel's status callback logs SUBSCRIBED and there are no\nerrors in the browser console.\n\nFigure out why no order events ever arrive and fix it.", - "promptSourcePath": "evals/investigate-realtime-001-subscribed-no-events/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-opus-5-no-skills/investigate-realtime-001-subscribed-no-events.json" - }, - { - "experiment": "claude-code-opus-5-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-opus-5", - "reasoningEffort": "high" - }, - "eval": "investigate-reliability-003-edge-function-5xx-correlation", - "stage": "investigate", - "product": [ - "edge-functions" - ], - "topic": [ - "observability" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "identified image-transform and the recurring 503 pattern", - "passed": true, - "judgeNotes": "The assistant explicitly identified `image-transform` as returning 503 eight times on 2026-04-28 between 07:00 and 12:00 UTC, and described the recurring pattern across the morning with the specific gateway failures. It did not incorrectly center the old billing-webhook issue." - }, - { - "name": "attributed recurring 503s to gateway/platform layer, not function code", - "passed": true, - "judgeNotes": "The assistant clearly attributes the recurring 503s to the gateway/platform layer, not image-transform code, and grounds this in valid observations: 503s appear only in API/gateway logs, lack function execution metadata, have no corresponding edge-function log entries, while actual function executions returned 200. It also distinguishes the avatar-upload 500 as a function-level error and recommends escalation to Supabase/platform support rather than redeploying or fixing image-transform code." - }, - { - "name": "recommended a concrete next step", - "passed": true, - "judgeNotes": "The assistant recommended concrete next steps, including escalating gateway 503s to Supabase support with request IDs, identifying the scheduled caller, obtaining a real failing user request with timestamp/request ID, and pulling scoped logs from Logs Explorer/log drain." - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "Users have been reporting that image uploads are intermittently failing with server errors throughout the morning of `2026-04-28`.\n\nCan you investigate the project logs and tell me what is going on and what we\nshould do next?", - "promptSourcePath": "evals/investigate-reliability-003-edge-function-5xx-correlation/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-opus-5-no-skills/investigate-reliability-003-edge-function-5xx-correlation.json" - }, - { - "experiment": "claude-code-opus-5-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-opus-5", - "reasoningEffort": "high" - }, - "eval": "resolve-dataapi-001-empty-results", - "stage": "resolve", - "product": [ - "data-api", - "database", - "auth" - ], - "topic": [ - "rls", - "sdk" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "RLS still enabled on bookmarks", - "passed": true - }, - { - "name": "user A reads own bookmarks", - "passed": true - }, - { - "name": "user B cannot read user A bookmarks", - "passed": true - }, - { - "name": "anon reads no bookmarks", - "passed": true - }, - { - "name": "user A can save a new bookmark", - "passed": true - }, - { - "name": "user B cannot insert a bookmark as user A", - "passed": true - }, - { - "name": "diagnosed RLS and added owner-scoped policies", - "passed": true, - "judgeNotes": "Diagnosed deny-all RLS due to no policies, kept RLS enabled, and added authenticated SELECT and INSERT policies scoped to user_id = auth.uid() with WITH CHECK for inserts." - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "Our app lets signed-in users save bookmarks and view them on their dashboard. Bookmarks are stored in the `bookmarks` table and are private — a user must only ever see their own. \nUsers also need to be able to save new bookmarks from the app.\n\nI can see the rows when I query the table directly, but the dashboard shows an empty list for every user.\n\nFind out why the Data API returns nothing and fix it.", - "promptSourcePath": "evals/resolve-dataapi-001-empty-results/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-opus-5-no-skills/resolve-dataapi-001-empty-results.json" - }, - { - "experiment": "claude-code-opus-5-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-opus-5", - "reasoningEffort": "high" - }, - "eval": "resolve-database-001-migration-history-mismatch", - "stage": "resolve", - "product": [ - "database" - ], - "topic": [ - "migrations" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "the avatar_url column is applied on the hosted profiles table", - "passed": true - }, - { - "name": "migration 20240220000000 is recorded in the remote history", - "passed": true - }, - { - "name": "remote migration history matches local migration files", - "passed": true - }, - { - "name": "local migrations are a valid reconciled sequence", - "passed": true - }, - { - "name": "production profile data is intact (not reset)", - "passed": true - }, - { - "name": "the avatar migration and history reconciliation were done via the Supabase CLI", - "passed": true, - "judgeNotes": "Applied pending avatar_url via `supabase db push` in #25, with output `Applying migration 20240220000000_add_avatar_url.sql...` and `Finished supabase db push.` Reconciled orphan bio migration by adding the missing local file `20240115000000_add_profile_bio.sql` in #22, after which `supabase migration list` aligned and `db push` proceeded. No disallowed workaround or direct remote mutation observed." - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "I'm trying to ship a migration to our hosted project and it's not working. Can you figure out what's wrong and get it deployed?", - "promptSourcePath": "evals/resolve-database-001-migration-history-mismatch/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-opus-5-no-skills/resolve-database-001-migration-history-mismatch.json" - }, - { - "experiment": "claude-code-opus-5-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-opus-5", - "reasoningEffort": "high" - }, - "eval": "resolve-performance-001-slow-query-cpu-spike", - "stage": "resolve", - "product": [ - "database" - ], - "topic": [ - "observability", - "sql" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "inspected pg_stat_statements for query performance", - "passed": true - }, - { - "name": "ran EXPLAIN on the expensive query", - "passed": true - }, - { - "name": "created index covering user_id and created_at", - "passed": true - }, - { - "name": "query plan uses an index and avoids sequential scan", - "passed": true, - "notes": "Limit (cost=79.74..79.86 rows=50 width=58)\n -> Sort (cost=79.74..79.99 rows=100 width=58)\n Sort Key: created_at DESC\n -> Bitmap Heap Scan on events (cost=5.06..76.42 rows=100 width=58)\n Recheck Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)\n -> Bitmap Index Scan on events_user_id_created_at_desc_idx (cost=0.00..5.03 rows=100 width=0)\n Index Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)" - }, - { - "name": "inserts still work", - "passed": true - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "My database CPU keeps spiking and the app gets slow when loading recent events for a user. Can you figure out what query is causing it and make the database change needed to fix it?\n\nEnd your turn with a short summary of what you changed and why.", - "promptSourcePath": "evals/resolve-performance-001-slow-query-cpu-spike/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-opus-5-no-skills/resolve-performance-001-slow-query-cpu-spike.json" - }, - { - "experiment": "claude-code-opus-5-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-opus-5", - "reasoningEffort": "high" - }, - "eval": "resolve-security-002-rls-cross-tenant-leak", - "stage": "resolve", - "product": [ - "database", - "auth" - ], - "topic": [ - "rls", - "security" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "RLS enabled on notes", - "passed": true - }, - { - "name": "tenant A sees only org A notes", - "passed": true - }, - { - "name": "tenant B cannot read org A notes", - "passed": true - }, - { - "name": "tenant A author can update own note", - "passed": true - }, - { - "name": "tenant B cannot update org A note", - "passed": true - }, - { - "name": "tenant B author can delete own note", - "passed": true - }, - { - "name": "tenant B cannot delete org A note", - "passed": true - }, - { - "name": "tenant A can insert note in own org", - "passed": true - }, - { - "name": "tenant B cannot insert into org A", - "passed": true - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "A customer reported that notes showed up in the wrong workspace.\n\nCan you investigate what is going on and fix it?", - "promptSourcePath": "evals/resolve-security-002-rls-cross-tenant-leak/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-opus-5-no-skills/resolve-security-002-rls-cross-tenant-leak.json" - }, - { - "experiment": "claude-code-sonnet-5", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-sonnet-5", - "reasoningEffort": "high" - }, - "eval": "build-cli-001-bootstrap-app", - "stage": "build", - "product": [ - "database", - "data-api" - ], - "topic": [ - "migrations", - "rls" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "supabase project initialised (supabase/config.toml exists)", - "passed": true - }, - { - "name": "todos table is created by a migration file", - "passed": true - }, - { - "name": "todos table exists with at least 2 seeded rows", - "passed": true, - "notes": "found 2 rows" - }, - { - "name": "row level security is enabled on todos", - "passed": true - }, - { - "name": "a SELECT policy targets the authenticated role", - "passed": true - }, - { - "name": "REST API returns no todos to anonymous requests", - "passed": true, - "notes": "0 rows" - }, - { - "name": "REST API returns the todos to authenticated requests", - "passed": true, - "notes": "2 rows" - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" - ] - }, - "docs": { - "calls": [] - }, - "prompt": "We're kicking off a todos app and I want the Supabase side ready for the team\nto build on. Set it up the way we'd run it in development, with schema changes\ntracked as migrations so they can be reviewed and replayed.\n\nFor the first slice we just need a `todos` table. Todos aren't public: anyone\nsigned in can read all of them, but nothing should be writable through the API\nfor now. Add a couple of sample todos so there's something to look at.\n\nBefore you hand it back, make sure the running API actually behaves that way —\nsigned-in users get the todos, signed-out requests get nothing.", - "promptSourcePath": "evals/build-cli-001-bootstrap-app/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-sonnet-5/build-cli-001-bootstrap-app.json" - }, - { - "experiment": "claude-code-sonnet-5", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-sonnet-5", - "reasoningEffort": "high" - }, - "eval": "build-cli-002-declarative-schema", - "stage": "build", - "product": [ - "database" - ], - "topic": [ - "declarative-schema", - "migrations" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "supabase db diff used to generate the migration", - "passed": true - }, - { - "name": "schema file updated to include description column", - "passed": true - }, - { - "name": "a new migration was generated for the change", - "passed": true - }, - { - "name": "description column exists in the live database", - "passed": true - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" - ] - }, - "docs": { - "calls": [] - }, - "prompt": "Add a description text column to the `products` table in my local Supabase stack", - "promptSourcePath": "evals/build-cli-002-declarative-schema/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-sonnet-5/build-cli-002-declarative-schema.json" - }, - { - "experiment": "claude-code-sonnet-5", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-sonnet-5", - "reasoningEffort": "high" - }, - "eval": "build-cli-003-pg-cron-queue-workflow", - "stage": "build", - "product": [ - "database", - "edge-functions", - "cron", - "queues" - ], - "topic": [ - "sql", - "sdk" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "pg_cron job 'enqueue-tasks' scheduled to run every minute", - "passed": true, - "notes": "schedule='* * * * *', active=true" - }, - { - "name": "cron command enqueues to the 'tasks' queue", - "passed": true, - "notes": "queue depth 0 -> 1" - }, - { - "name": "process-tasks function drains the queue", - "passed": true, - "notes": "function removed the seeded message (id 3) from the queue" - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" - ] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{ searchDocs(query: \"pg_cron schedule pgmq queue send message example\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/queues/pgmq", - "title": "PGMQ Extension" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net", - "title": "pg_net: Async Networking" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/pgcron-debugging-guide-n1KTaz", - "title": "pg_cron debugging guide" - }, - { - "url": "https://supabase.com/docs/guides/queues/quickstart", - "title": "Quickstart" - }, - { - "url": "https://supabase.com/docs/guides/functions/schedule-functions", - "title": "Scheduling Edge Functions" - } - ], - "resultChars": 55834 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"edge function supabase-js service role client import example Deno.serve\", limit: 3) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions", - "title": "Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/database/connecting-to-postgres/serverless-drivers", - "title": "Serverless Drivers" - }, - { - "url": "https://supabase.com/docs/guides/functions/unit-test", - "title": "Testing your Edge Functions" - } - ], - "resultChars": 26350 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"Edge Functions default secrets SUPABASE_URL SUPABASE_SERVICE_ROLE_KEY automatically populated local\", limit: 3) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/secrets", - "title": "Environment Variables" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", - "title": "Self-Hosted Functions" - } - ], - "resultChars": 28165 - } - ] - }, - "prompt": "I want to set up a recurring background workflow on my local Supabase stack.\n\nCan you set up a cron job called `enqueue-tasks` to run every minute and push a task into a queue called `tasks`? Then add a `process-tasks` edge function that reads messages off the `tasks` queue and removes them, so a scheduled worker can keep the backlog drained.", - "promptSourcePath": "evals/build-cli-003-pg-cron-queue-workflow/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-sonnet-5/build-cli-003-pg-cron-queue-workflow.json" - }, - { - "experiment": "claude-code-sonnet-5", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-sonnet-5", - "reasoningEffort": "high" - }, - "eval": "build-database-001-migrate-postgres-to-supabase", - "stage": "build", - "product": [ - "database" - ], - "topic": [ - "migrations" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "all 3 tables exist (teams, members, tasks)", - "passed": true - }, - { - "name": "row counts match (teams=5, members=10, tasks=13)", - "passed": true - }, - { - "name": "foreign key constraints survived the restore", - "passed": true - }, - { - "name": "tasks_team_status_idx index survived the restore", - "passed": true - }, - { - "name": "sequences synced (next insert won't conflict with existing IDs)", - "passed": true - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" - ] - }, - "docs": { - "calls": [] - }, - "prompt": "I have an existing Postgres database I want to migrate to Supabase. There's a binary dump at `source.dump` in the current directory.\n\nCan you set up a local Supabase project and restore the dump into it?", - "promptSourcePath": "evals/build-database-001-migrate-postgres-to-supabase/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-sonnet-5/build-database-001-migrate-postgres-to-supabase.json" - }, - { - "experiment": "claude-code-sonnet-5", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-sonnet-5", - "reasoningEffort": "high" - }, - "eval": "build-functions-004-service-role-bypass", - "stage": "build", - "product": [ - "edge-functions", - "auth", - "database" - ], - "topic": [ - "rls", - "security", - "sdk" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "rejects missing auth", - "passed": true, - "notes": "status=401" - }, - { - "name": "user A reads own note", - "passed": true, - "notes": "status=200" - }, - { - "name": "reads only with the caller's JWT", - "passed": true, - "notes": "bearer_tokens=2, all_match=true" - }, - { - "name": "user A cannot force-read user B note", - "passed": true, - "notes": "status=200" - }, - { - "name": "user B cannot force-read user A note", - "passed": true, - "notes": "status=200" - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" - ] - }, - "docs": { - "calls": [] - }, - "prompt": "I built an Edge Function called `private-notes` for showing a user's saved\nprivate notes.\n\nCan you check whether there's any way one user could see another user's notes?\n\nPlease fix and deploy it if needed.", - "promptSourcePath": "evals/build-functions-004-service-role-bypass/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-sonnet-5/build-functions-004-service-role-bypass.json" - }, - { - "experiment": "claude-code-sonnet-5", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-sonnet-5", - "reasoningEffort": "high" - }, - "eval": "build-functions-005-dual-auth-user-secret", - "stage": "build", - "product": [ - "edge-functions", - "auth", - "database" - ], - "topic": [ - "sdk", - "rls", - "security" - ], - "suite": "benchmark", - "interface": "cli", - "cliVersion": "2.109.1", - "passed": true, - "checks": [ - { - "name": "seed rows present", - "passed": true, - "notes": "found 2/2 seeded rows" - }, - { - "name": "rejects request with no credentials", - "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" - }, - { - "name": "user with JWT reads only their own rows", - "passed": true, - "notes": "status 200: {\"data\":[{\"user_id\":\"b0cee6b2-6005-4375-acfc-e42243e7b9fe\",\"metric\":\"steps_a_ms6x1j4o\",\"value\":111}]}" - }, - { - "name": "user cannot read another user's rows by passing user_id", - "passed": true, - "notes": "status 200: {\"data\":[{\"user_id\":\"b0cee6b2-6005-4375-acfc-e42243e7b9fe\",\"metric\":\"steps_a_ms6x1j4o\",\"value\":111}]}" - }, - { - "name": "service key bypasses RLS to read the target user's rows", - "passed": true, - "notes": "status 200: {\"data\":[{\"user_id\":\"ab402c7a-0f99-490c-87f2-8214ddbef994\",\"metric\":\"steps_b_ms6x1j4o\",\"value\":222}]}" - }, - { - "name": "non-service key is not granted service access", - "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" - }, - { - "name": "rejects an unverified (forged) user token", - "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" - }, - { - "name": "a user token in the apikey slot is not treated as the service key", - "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" - }, - { - "name": "implementation uses @supabase/server", - "passed": true, - "notes": "imports @supabase/server / withSupabase" - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" - ] - }, - "docs": { - "calls": [ - { - "source": "web_fetch", - "query": "List any entries about: new API key system (sb_publishable_/sb_secret_), Edge Functions env vars, edge runtime, JWT signing keys, getClaims, or breaking changes to auth/apikey handling. Include dates and links.", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 1355 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"edge functions environment variables SUPABASE_URL SUPABASE_ANON_KEY SUPABASE_SERVICE_ROLE_KEY default secrets\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/secrets", - "title": "Environment Variables" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", - "title": "Self-Hosted Functions" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" - } - ], - "resultChars": 41771 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"new API keys publishable secret key sb_secret sb_publishable service_role apikey header\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", - "title": "New API Keys and Asymmetric Authentication" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", - "title": "Envoy API Gateway" - }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys", - "title": "JWT Signing Keys" - } - ], - "resultChars": 98570 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"edge function call with service role bypass RLS forward Authorization header user JWT getUser\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/troubleshooting/why-is-my-service-role-key-client-getting-rls-errors-or-not-returning-data-7_1K9z", - "title": "Why is my service role key client getting RLS errors or not returning data?" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth-headers", - "title": "Authorization headers" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" - }, - { - "url": "https://supabase.com/docs/guides/functions/error-codes", - "title": "Error codes" - } - ], - "resultChars": 28724 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"@supabase/server withSupabase edge function auth\") { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/auth/choosing-a-server-package", - "title": "Which package to use" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/resumable-websockets", - "title": "Resumable WebSockets with Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/functions", - "title": "Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/ai-tools/byo-mcp", - "title": "Deploy MCP servers" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net", - "title": "pg_net: Async Networking" - }, - { - "url": "https://supabase.com/docs/guides/auth", - "title": "Auth" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/roles", - "title": "Postgres Roles" - }, - { - "url": "https://supabase.com/docs/guides/functions/connect-to-postgres", - "title": "Integrating with Supabase Database (Postgres)" - }, - { - "url": "https://supabase.com/docs/guides/functions/cors", - "title": "CORS (Cross-Origin Resource Sharing) support for Invoking from the browser" - }, - { - "url": "https://supabase.com/docs/guides/functions/quickstart-dashboard", - "title": "Getting Started with Edge Functions (Dashboard)" - }, - { - "url": "https://supabase.com/docs/guides/auth/architecture", - "title": "Auth architecture" - }, - { - "url": "https://supabase.com/docs/guides/database/secure-data", - "title": "Securing your data" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs", - "title": "Build a User Management App with Next.js" - }, - { - "url": "https://supabase.com/docs/guides/functions/websockets", - "title": "Handling WebSockets" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/edge-function-401-error-response", - "title": "Edge Function 401 error response" - }, - { - "url": "https://supabase.com/docs/guides/database/connecting-to-postgres/serverless-drivers", - "title": "Serverless Drivers" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth-headers", - "title": "Authorization headers" - }, - { - "url": "https://supabase.com/docs/guides/functions/architecture", - "title": "Edge Functions Architecture" - }, - { - "url": "https://supabase.com/docs/guides/functions/quickstart", - "title": "Getting Started with Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/auth-send-email-hook-react-email-resend", - "title": "Custom Auth Emails with React Email and Resend" - }, - { - "url": "https://supabase.com/docs/guides/auth/social-login/auth-apple", - "title": "Login with Apple" - }, - { - "url": "https://supabase.com/docs/guides/auth/enterprise-sso", - "title": "Enterprise Single Sign-On" - }, - { - "url": "https://supabase.com/docs/guides/functions/recursive-functions", - "title": "Recursive / Nested Function Calls" - }, - { - "url": "https://supabase.com/docs/guides/integrations/build-a-supabase-oauth-integration", - "title": "Build a Supabase Integration" - }, - { - "url": "https://supabase.com/docs/guides/auth/sessions", - "title": "User sessions" - }, - { - "url": "https://supabase.com/docs/guides/auth/quickstarts/astrojs", - "title": "Use Supabase Auth with Astro" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", - "title": "Self-Hosted Functions" - } - ], - "resultChars": 12 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"edge function verify_jwt secret key apikey header\") { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth-headers", - "title": "Authorization headers" - }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys", - "title": "JWT Signing Keys" - }, - { - "url": "https://supabase.com/docs/guides/functions/error-codes", - "title": "Error codes" - }, - { - "url": "https://supabase.com/docs/guides/ai/examples/openai", - "title": "Generating OpenAI GPT3 completions" - }, - { - "url": "https://supabase.com/docs/guides/auth/auth-hooks/send-email-hook", - "title": "Send Email Hook" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", - "title": "Envoy API Gateway" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", - "title": "New API Keys and Asymmetric Authentication" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin", - "title": "Build a Product Management Android App with Jetpack Compose" - }, - { - "url": "https://supabase.com/docs/guides/auth/jwts", - "title": "JSON Web Token (JWT)" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", - "title": "Self-Hosted Functions" - }, - { - "url": "https://supabase.com/docs/guides/functions/function-configuration", - "title": "Function Configuration" - }, - { - "url": "https://supabase.com/docs/guides/functions/status-codes", - "title": "Status codes" - }, - { - "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows", - "title": "OAuth 2.1 Flows" - }, - { - "url": "https://supabase.com/docs/guides/auth/auth-mfa", - "title": "Multi-Factor Authentication" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" - }, - { - "url": "https://supabase.com/docs/guides/functions/websockets", - "title": "Handling WebSockets" - }, - { - "url": "https://supabase.com/docs/guides/integrations/partner-integration-guide", - "title": "Supabase Partner Integration Guide" - }, - { - "url": "https://supabase.com/docs/guides/resources/glossary", - "title": "Glossary" - }, - { - "url": "https://supabase.com/docs/guides/database/secure-data", - "title": "Securing your data" - } - ], - "resultChars": 335959 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"SUPABASE_SECRET_KEYS SUPABASE_PUBLISHABLE_KEYS environment variables edge functions\") { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/functions/secrets", - "title": "Environment Variables" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", - "title": "Self-Hosted Functions" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", - "title": "New API Keys and Asymmetric Authentication" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker", - "title": "Self-Hosting with Docker" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", - "title": "Envoy API Gateway" - }, - { - "url": "https://supabase.com/docs/guides/auth/quickstarts/with-expo-react-native-social-auth", - "title": "Build a Social Auth App with Expo React Native" - }, - { - "url": "https://supabase.com/docs/guides/functions/quickstart-dashboard", - "title": "Getting Started with Edge Functions (Dashboard)" - }, - { - "url": "https://supabase.com/docs/guides/ai/examples/nextjs-vector-search", - "title": "Vector search with Next.js and OpenAI" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/inspecting-edge-function-environment-variables-wg5qOQ", - "title": "Inspecting edge function environment variables" - }, - { - "url": "https://supabase.com/docs/guides/database/secure-data", - "title": "Securing your data" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream", - "title": "Streaming Speech with ElevenLabs" - }, - { - "url": "https://supabase.com/docs/guides/functions/architecture", - "title": "Edge Functions Architecture" - }, - { - "url": "https://supabase.com/docs/guides/integrations/vercel-marketplace", - "title": "Vercel Marketplace" - }, - { - "url": "https://supabase.com/docs/guides/functions/unit-test", - "title": "Testing your Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/functions", - "title": "Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys", - "title": "JWT Signing Keys" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech", - "title": "Transcription Telegram Bot" - }, - { - "url": "https://supabase.com/docs/guides/auth/server-side/creating-a-client", - "title": "Creating a Supabase client for SSR" - }, - { - "url": "https://supabase.com/docs/guides/functions/connect-to-postgres", - "title": "Integrating with Supabase Database (Postgres)" - }, - { - "url": "https://supabase.com/docs/guides/integrations/build-a-supabase-oauth-integration", - "title": "Build a Supabase Integration" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/cloudflare-turnstile", - "title": "CAPTCHA support with Cloudflare Turnstile" - }, - { - "url": "https://supabase.com/docs/guides/integrations/supabase-for-platforms", - "title": "Supabase for Platforms" - }, - { - "url": "https://supabase.com/docs/guides/functions/deploy", - "title": "Deploy to Production" - }, - { - "url": "https://supabase.com/docs/guides/telemetry/reports", - "title": "Reports" - }, - { - "url": "https://supabase.com/docs/guides/functions/schedule-functions", - "title": "Scheduling Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/functions/recursive-functions", - "title": "Recursive / Nested Function Calls" - } - ], - "resultChars": 436561 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"new API keys publishable secret migration edge functions\") { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys", - "title": "JWT Signing Keys" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", - "title": "New API Keys and Asymmetric Authentication" - }, - { - "url": "https://supabase.com/docs/guides/integrations/supabase-for-platforms", - "title": "Supabase for Platforms" - }, - { - "url": "https://supabase.com/docs/guides/ai-tools/mcp", - "title": "Supabase MCP Server" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs", - "title": "Build a User Management App with Next.js" - }, - { - "url": "https://supabase.com/docs/guides/functions/development-tips", - "title": "Development tips" - }, - { - "url": "https://supabase.com/docs/guides/platform/migrating-within-supabase/backup-restore", - "title": "Backup and Restore using the CLI" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker", - "title": "Self-Hosting with Docker" - }, - { - "url": "https://supabase.com/docs/guides/database/secure-data", - "title": "Securing your data" - }, - { - "url": "https://supabase.com/docs/guides/ai/examples/nextjs-vector-search", - "title": "Vector search with Next.js and OpenAI" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli/getting-started", - "title": "Supabase CLI" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features", - "title": "Features" - }, - { - "url": "https://supabase.com/docs/guides/functions/quickstart-dashboard", - "title": "Getting Started with Edge Functions (Dashboard)" - }, - { - "url": "https://supabase.com/docs/guides/functions/secrets", - "title": "Environment Variables" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", - "title": "Envoy API Gateway" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin", - "title": "Build a Product Management Android App with Jetpack Compose" - }, - { - "url": "https://supabase.com/docs/guides/platform/sso/multiple-providers", - "title": "Multiple SSO Providers" - }, - { - "url": "https://supabase.com/docs/guides/auth/quickstarts/with-expo-react-native-social-auth", - "title": "Build a Social Auth App with Expo React Native" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-expo-react-native", - "title": "Build a User Management App with Expo React Native" - }, - { - "url": "https://supabase.com/docs/guides/platform/delete-project", - "title": "Deleting Your Project" - }, - { - "url": "https://supabase.com/docs/guides/functions", - "title": "Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/functions/recursive-functions", - "title": "Recursive / Nested Function Calls" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-ionic-react", - "title": "Build a User Management App with Ionic React" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-ionic-angular", - "title": "Build a User Management App with Ionic Angular" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-angular", - "title": "Build a User Management App with Angular" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", - "title": "Self-Hosted Functions" - } - ], - "resultChars": 611775 - } - ] - }, - "prompt": "Build and serve a Supabase Edge Function named `user-stats` for this project,\nreachable over HTTP at `/functions/v1/user-stats`.\n\nOur product stores per-user metrics in the existing `user_stats` table.\n\nTwo very different callers need to hit this one endpoint:\n\n1. **Our mobile app**, acting for a signed-in user. It sends that user's\n Supabase access token. The endpoint should return the user's own stats.\n\n2. **Our internal billing service**, a trusted backend with no signed-in user.\n It authenticates with the project's secret (service-role) key in the `apikey`\n header, and names the target user with a `user_id` in the JSON request body.\n It needs that user's stats.\n\nReturn the matching rows as JSON. The endpoint must be secure: only ever serve\nstats to a caller that is genuinely entitled to them, and turn away callers that\nare not.", - "promptSourcePath": "evals/build-functions-005-dual-auth-user-secret/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-sonnet-5/build-functions-005-dual-auth-user-secret.json" - }, - { - "experiment": "claude-code-sonnet-5", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-sonnet-5", - "reasoningEffort": "high" - }, - "eval": "build-storage-001-private-bucket-access", - "stage": "build", - "product": [ - "storage", - "database" - ], - "topic": [ - "rls", - "sdk" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "bucket user-files exists", - "passed": true - }, - { - "name": "bucket user-files is private", - "passed": true - }, - { - "name": "RLS still enabled on storage.objects", - "passed": true - }, - { - "name": "user A lists only own files", - "passed": true, - "notes": "saw: 019fb0e6-fae1-72ae-a0ef-528df766bb3f/receipt-alpha.pdf, 019fb0e6-fae1-72ae-a0ef-528df766bb3f/receipt-beta.pdf" - }, - { - "name": "user B cannot read user A files", - "passed": true - }, - { - "name": "anon reads no files", - "passed": true - }, - { - "name": "user A can upload into own folder", - "passed": true - }, - { - "name": "user B cannot upload into user A folder", - "passed": true - }, - { - "name": "configured private per-user storage access", - "passed": true, - "judgeNotes": "Meets all criteria: private user-files bucket, authenticated owner-scoped SELECT and INSERT policies with WITH CHECK, no public/anon access or permissive policies, RLS not disabled, and supabase-js createSignedUrl with expiry for temporary sharing." - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" - ] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{ searchDocs(query: \"storage RLS policy private user folder signed URL\", limit: 5) { nodes { title href ... on Guide { content } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/storage/buckets/fundamentals", - "title": "Storage Buckets" - }, - { - "url": "https://supabase.com/docs/guides/storage/schema/helper-functions", - "title": "Storage Helper Functions" - }, - { - "url": "https://supabase.com/docs/guides/auth/third-party/clerk", - "title": "Clerk" - }, - { - "url": "https://supabase.com/docs/guides/storage/security/access-control", - "title": "Storage Access Control" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3", - "title": "Configure S3 Storage" - } - ], - "resultChars": 28558 - } - ] - }, - "prompt": "Our app lets signed-in users keep personal files like receipts and bank\nstatements. These files are private — a user must only ever be able to upload\nand download their own. The app uploads each file under a path that starts\nwith the owner's user id, e.g. `/receipt-march.pdf`.\n\nSet up a `user-files` bucket on our project and lock it down that way.\n\nUsers also sometimes share one of their files with someone else through a\ntemporary link that expires. Include the supabase-js code the app should use\nfor that.", - "promptSourcePath": "evals/build-storage-001-private-bucket-access/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-sonnet-5/build-storage-001-private-bucket-access.json" - }, - { - "experiment": "claude-code-sonnet-5", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-sonnet-5", - "reasoningEffort": "high" - }, - "eval": "build-tests-001-rls-tenant-isolation", - "stage": "build", - "product": [ - "database" - ], - "topic": [ - "tests", - "rls" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "pgTAP test file(s) written under supabase/tests/", - "passed": true, - "notes": "1 file(s): supabase/tests/tenant_isolation_test.sql" - }, - { - "name": "pgTAP isolation tests ran and pass", - "passed": true, - "notes": "4 passed, 4 failed" - }, - { - "name": "agent correctly identifies the posts isolation bug from test results", - "passed": true, - "judgeNotes": "The agent correctly identifies posts as having the tenant isolation flaw, specifically that authenticated members can read posts from orgs they do not belong to, and grounds this in pgTAP failures. It also notes notes is correctly isolated. Extra discussion of memberships does not undermine the required conclusion." - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" - ] - }, - "docs": { - "calls": [] - }, - "prompt": "Can you audit the tenant isolation on our tables? Write some database tests covering both the happy path and the negative case, run them, and share the results.", - "promptSourcePath": "evals/build-tests-001-rls-tenant-isolation/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-sonnet-5/build-tests-001-rls-tenant-isolation.json" - }, - { - "experiment": "claude-code-sonnet-5", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-sonnet-5", - "reasoningEffort": "high" - }, - "eval": "build-vectors-001-rag-with-permissions", - "stage": "build", - "product": [ - "database", - "vectors" - ], - "topic": [ - "sql", - "rls" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "document_sections.embedding is vector(384)", - "passed": true, - "notes": "vector(384)" - }, - { - "name": "HNSW index on the embedding column", - "passed": true, - "notes": "CREATE INDEX document_sections_embedding_idx ON public.document_sections USING hnsw (embedding vector_cosine_ops)" - }, - { - "name": "index operator class matches the search operator", - "passed": true, - "notes": "function operators: <=>\nindexes: CREATE INDEX document_sections_embedding_idx ON public.document_sections USING hnsw (embedding vector_cosine_ops)" - }, - { - "name": "user A search returns only own sections, best match first", - "passed": true - }, - { - "name": "user B search returns only own sections, best match first", - "passed": true - }, - { - "name": "user A reads only own sections through the API", - "passed": true - }, - { - "name": "user A reads only own documents through the API", - "passed": true - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" - ] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{ searchDocs(query: \"gte-small embedding dimensions Supabase.ai Session\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/ai/choosing-compute-addon", - "title": "Choosing your Compute Add-on" - }, - { - "url": "https://supabase.com/docs/guides/functions/ai-models", - "title": "Running AI Models" - }, - { - "url": "https://supabase.com/docs/guides/ai/semantic-search", - "title": "Semantic search" - }, - { - "url": "https://supabase.com/docs/guides/ai/concepts", - "title": "Concepts" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/semantic-search", - "title": "Semantic Search" - } - ], - "resultChars": 62210 - } - ] - }, - "prompt": "We're adding semantic search to our internal knowledge base app. I already wrote the edge functions, but search doesn't work yet. Can you set up whatever the database needs to make search work end to end?\n\nSome documents are confidential, so users should only have access to documents they own.", - "promptSourcePath": "evals/build-vectors-001-rag-with-permissions/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-sonnet-5/build-vectors-001-rag-with-permissions.json" - }, - { - "experiment": "claude-code-sonnet-5", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-sonnet-5", - "reasoningEffort": "high" - }, - "eval": "deploy-database-001-prometheus-metrics", - "stage": "deploy", - "product": [ - "database" - ], - "topic": [ - "observability" - ], - "suite": "benchmark", - "passed": true, - "checks": [ - { - "name": "preserved existing app scrape job", - "passed": true - }, - { - "name": "configured the Supabase Metrics API scrape correctly", - "passed": true, - "judgeNotes": "Supabase scrape is deployable: HTTPS, correct metrics path, Basic Auth with password_file, project target on supabase.co, app job preserved, and docker-compose mounts the password file at the referenced path." - }, - { - "name": "documented live deployment and verification steps", - "passed": true, - "judgeNotes": "README includes Secret API key creation, matching secret file placement, Compose start/reload instructions, and concrete verification via Prometheus targets." - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" - ] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{ searchDocs(query: \"project metrics endpoint prometheus\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/telemetry/metrics/grafana-self-hosted", - "title": "Metrics API with Prometheus & Grafana (self-hosted)" - }, - { - "url": "https://supabase.com/docs/guides/telemetry/metrics", - "title": "Metrics API" - }, - { - "url": "https://supabase.com/docs/guides/telemetry/metrics/vendor-agnostic", - "title": "Vendor-agnostic Metrics API setup" - }, - { - "url": "https://supabase.com/docs/guides/telemetry/metrics/grafana-cloud", - "title": "Metrics API with Grafana Cloud" - }, - { - "url": "https://supabase.com/docs/guides/database/replication/manual-replication-monitoring", - "title": "Manual replication monitoring" - } - ], - "resultChars": 27147 - } - ] - }, - "prompt": "Can you wire my Supabase project metrics into our existing observability stack and document\nin the observability README what we need to do to make the config live?", - "promptSourcePath": "evals/deploy-database-001-prometheus-metrics/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-sonnet-5/deploy-database-001-prometheus-metrics.json" - }, - { - "experiment": "claude-code-sonnet-5", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-sonnet-5", - "reasoningEffort": "high" - }, - "eval": "deploy-functions-001-edge-function-secrets", - "stage": "deploy", - "product": [ - "edge-functions" - ], - "topic": [ - "security" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "WEATHER_API_KEY is set as a Function secret on the project", - "passed": true - }, - { - "name": "the weather function is deployed to the project", - "passed": true, - "notes": "status ACTIVE" - }, - { - "name": "the weather function reads WEATHER_API_KEY from the environment", - "passed": true, - "judgeNotes": "Reads WEATHER_API_KEY from the runtime environment using Deno.env.get(\"WEATHER_API_KEY\")." - }, - { - "name": "WEATHER_API_KEY value is not committed to the repo", - "passed": true - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" - ] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{ searchDocs(query: \"edge functions environment variables secrets set\", limit: 5) { nodes { title href ... on Guide { content } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/troubleshooting/inspecting-edge-function-environment-variables-wg5qOQ", - "title": "Inspecting edge function environment variables" - }, - { - "url": "https://supabase.com/docs/guides/functions/secrets", - "title": "Environment Variables" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech", - "title": "Transcription Telegram Bot" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", - "title": "Self-Hosted Functions" - } - ], - "resultChars": 39587 - } - ] - }, - "prompt": "Our weather widget currently calls WeatherAPI straight from the browser, which\nleaks our API key. I want to move that behind a Supabase Edge Function called\n`weather` that holds the key server-side and proxies the request.\n\nThe function should read the key from an environment variable named\n`WEATHER_API_KEY`. Our key already lives in a local `.env` file at the project\nroot.\n\nDeploy the function to our project so it's live, and make sure the deployed\nfunction can actually read the key at runtime.", - "promptSourcePath": "evals/deploy-functions-001-edge-function-secrets/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-sonnet-5/deploy-functions-001-edge-function-secrets.json" - }, - { - "experiment": "claude-code-sonnet-5", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-sonnet-5", - "reasoningEffort": "high" - }, - "eval": "deploy-self-hosting-001-docker-compose", - "stage": "deploy", - "product": [ - "database", - "auth", - "storage" - ], - "topic": [ - "self-hosting" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "cloned the self-host stack (docker-compose.yml + volumes/db)", - "passed": true - }, - { - "name": "didn't conflate with the CLI (no supabase/config.toml in the stack)", - "passed": true - }, - { - "name": "secrets rotated off the shipped defaults", - "passed": true - }, - { - "name": "ANON_KEY and SERVICE_ROLE_KEY are HS256 JWTs signed by JWT_SECRET", - "passed": true - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" - ] - }, - "docs": { - "calls": [] - }, - "prompt": "I'm moving off the hosted Supabase and running the whole thing myself on a VPS I\njust spun up. Can you get a Docker setup ready for me to copy onto the box?\n\nI don't need it running here, I'll do the actual bring-up once I'm on the\nserver. I just want everything in place and the secrets set up. Put it in a `supabase-docker/`\nfolder at the repo root so I can scp the whole thing across in one go.", - "promptSourcePath": "evals/deploy-self-hosting-001-docker-compose/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-sonnet-5/deploy-self-hosting-001-docker-compose.json" - }, - { - "experiment": "claude-code-sonnet-5", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-sonnet-5", - "reasoningEffort": "high" - }, - "eval": "investigate-auth-001-deleted-user-access", - "stage": "investigate", - "product": [ - "auth" - ], - "topic": [ - "security", - "sdk" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": false, - "checks": [ - { - "name": "victim session active before delete-account", - "passed": true - }, - { - "name": "delete_account flow ran for the victim", - "passed": true - }, - { - "name": "delete-account revokes the user's sessions", - "passed": true, - "notes": "sessions left: 0" - }, - { - "name": "deleted user's refresh token is rejected", - "passed": true - }, - { - "name": "deleted user cannot sign back in", - "passed": false, - "notes": "deleted account can still sign in" - }, - { - "name": "other users keep their sessions and access", - "passed": true - }, - { - "name": "diagnosed and explained session revocation", - "passed": true, - "judgeNotes": "The answer correctly identifies the soft-delete-only bug, implements real revocation via session deletion/refresh-token cascade plus sign-in blocking, adds RLS checks to close stale-token data access for the covered tables, and explains the remaining stateless JWT caveat consistently. It also correctly distinguishes publishable frontend keys from server-only secret keys that bypass RLS." - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" - ] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{ searchDocs(query: \"publishable key secret key vs anon service_role migration\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [] - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"publishable key secret key vs anon service_role migration\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys", - "title": "JWT Signing Keys" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/rotating-anon-service-and-jwt-secrets-1Jq6yd", - "title": "Rotating Anon, Service, and JWT Secrets" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" - } - ], - "resultChars": 61183 - } - ] - }, - "prompt": "Last week support removed a user through our app's delete-account flow — the\napp calls the `delete_account` function over RPC as the signed-in user. This\nmorning that same person was back: still signed in, reading and saving their\ndata like nothing happened.\n\nFigure out why the account still works, fix the flow so a deleted account\nloses access, and tell me whether there is any window where they could still\nget in after the fix.\n\nOne more thing while you're at it: we're migrating off the legacy\nanon/service_role API keys to the new publishable and secret keys, and the\nteam isn't sure which one belongs in the frontend and what each means for\nRLS. Give us a quick rundown.", - "promptSourcePath": "evals/investigate-auth-001-deleted-user-access/PROMPT.md", - "attempts": 2, - "sourcePath": "claude-code-sonnet-5/investigate-auth-001-deleted-user-access.json" - }, - { - "experiment": "claude-code-sonnet-5", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-sonnet-5", - "reasoningEffort": "high" - }, - "eval": "investigate-realtime-001-subscribed-no-events", - "stage": "investigate", - "product": [ - "realtime", - "database" - ], - "topic": [ - "sdk" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "orders table added to supabase_realtime publication", - "passed": true - }, - { - "name": "courier_locations still in supabase_realtime publication", - "passed": true - }, - { - "name": "publication still publishes INSERT events", - "passed": true - }, - { - "name": "RLS still enabled on orders", - "passed": true - }, - { - "name": "staff can still read orders through RLS", - "passed": true, - "notes": "authenticated sees 2 of 2 orders" - }, - { - "name": "diagnosed missing publication membership", - "passed": true, - "judgeNotes": "Diagnosed missing orders table from supabase_realtime publication, applied ALTER PUBLICATION ... ADD TABLE public.orders, preserved courier_locations and RLS/policies." - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" - ] - }, - "docs": { - "calls": [] - }, - "prompt": "Our dispatch dashboard shows incoming orders as they happen. The courier\nlocation feed on the same page updates live without problems, but new orders\nonly show up after a page refresh.\n\nThe dashboard uses supabase-js to subscribe to INSERT events on the `orders`\ntable through postgres_changes, the same way it subscribes to courier\nlocations. The channel's status callback logs SUBSCRIBED and there are no\nerrors in the browser console.\n\nFigure out why no order events ever arrive and fix it.", - "promptSourcePath": "evals/investigate-realtime-001-subscribed-no-events/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-sonnet-5/investigate-realtime-001-subscribed-no-events.json" - }, - { - "experiment": "claude-code-sonnet-5", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-sonnet-5", - "reasoningEffort": "high" - }, - "eval": "investigate-reliability-003-edge-function-5xx-correlation", - "stage": "investigate", - "product": [ - "edge-functions" - ], - "topic": [ - "observability" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "identified image-transform and the recurring 503 pattern", - "passed": true, - "judgeNotes": "Identified image-transform as the affected function and described recurring 503s throughout the morning of 2026-04-28, covering the gateway failure pattern and distinguishing it from unrelated billing-webhook errors." - }, - { - "name": "attributed recurring 503s to gateway/platform layer, not function code", - "passed": true, - "judgeNotes": "Attributes recurring 503s to gateway/infrastructure before function execution, not function code, and grounds it in valid observations: gateway-only 503s with zero corresponding invocation 503s, nearby successful invocations on unchanged deployment/version, and distinction from avatar-upload's function-level 500." - }, - { - "name": "recommended a concrete next step", - "passed": true, - "judgeNotes": "The assistant provided concrete actionable next steps, including checking Edge Function concurrency/instance limits and cold-start behavior for the specific time window, investigating periodic traffic spikes, adding retry-with-backoff, and digging into specific function logs." - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "Users have been reporting that image uploads are intermittently failing with server errors throughout the morning of `2026-04-28`.\n\nCan you investigate the project logs and tell me what is going on and what we\nshould do next?", - "promptSourcePath": "evals/investigate-reliability-003-edge-function-5xx-correlation/PROMPT.md", - "attempts": 2, - "sourcePath": "claude-code-sonnet-5/investigate-reliability-003-edge-function-5xx-correlation.json" - }, - { - "experiment": "claude-code-sonnet-5", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-sonnet-5", - "reasoningEffort": "high" - }, - "eval": "resolve-dataapi-001-empty-results", - "stage": "resolve", - "product": [ - "data-api", - "database", - "auth" - ], - "topic": [ - "rls", - "sdk" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "RLS still enabled on bookmarks", - "passed": true - }, - { - "name": "user A reads own bookmarks", - "passed": true - }, - { - "name": "user B cannot read user A bookmarks", - "passed": true - }, - { - "name": "anon reads no bookmarks", - "passed": true - }, - { - "name": "user A can save a new bookmark", - "passed": true - }, - { - "name": "user B cannot insert a bookmark as user A", - "passed": true - }, - { - "name": "diagnosed RLS and added owner-scoped policies", - "passed": true, - "judgeNotes": "Diagnosed RLS enabled with no policies causing deny-all Data API results, kept RLS enabled, and created authenticated SELECT and INSERT policies scoped to user_id = auth.uid() with USING and WITH CHECK respectively." - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" - ] - }, - "docs": { - "calls": [] - }, - "prompt": "Our app lets signed-in users save bookmarks and view them on their dashboard. Bookmarks are stored in the `bookmarks` table and are private — a user must only ever see their own. \nUsers also need to be able to save new bookmarks from the app.\n\nI can see the rows when I query the table directly, but the dashboard shows an empty list for every user.\n\nFind out why the Data API returns nothing and fix it.", - "promptSourcePath": "evals/resolve-dataapi-001-empty-results/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-sonnet-5/resolve-dataapi-001-empty-results.json" - }, - { - "experiment": "claude-code-sonnet-5", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-sonnet-5", - "reasoningEffort": "high" - }, - "eval": "resolve-database-001-migration-history-mismatch", - "stage": "resolve", - "product": [ - "database" - ], - "topic": [ - "migrations" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "the avatar_url column is applied on the hosted profiles table", - "passed": true - }, - { - "name": "migration 20240220000000 is recorded in the remote history", - "passed": true - }, - { - "name": "remote migration history matches local migration files", - "passed": true - }, - { - "name": "local migrations are a valid reconciled sequence", - "passed": true - }, - { - "name": "production profile data is intact (not reset)", - "passed": true - }, - { - "name": "the avatar migration and history reconciliation were done via the Supabase CLI", - "passed": true, - "judgeNotes": "Applied pending avatar_url migration with `supabase db push` (#17), which output `Applying migration 20240220000000_add_avatar_url.sql...` and `Finished supabase db push.` Reconciled the orphan bio migration by adding the missing local migration file `20240115000000_add_bio.sql` (#15), after which `supabase migration list` showed local and remote aligned (#16). Read-only psql inspection was used; no disallowed workaround or direct mutation was seen." - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" - ] - }, - "docs": { - "calls": [] - }, - "prompt": "I'm trying to ship a migration to our hosted project and it's not working. Can you figure out what's wrong and get it deployed?", - "promptSourcePath": "evals/resolve-database-001-migration-history-mismatch/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-sonnet-5/resolve-database-001-migration-history-mismatch.json" - }, - { - "experiment": "claude-code-sonnet-5", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-sonnet-5", - "reasoningEffort": "high" - }, - "eval": "resolve-performance-001-slow-query-cpu-spike", - "stage": "resolve", - "product": [ - "database" - ], - "topic": [ - "observability", - "sql" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "inspected pg_stat_statements for query performance", - "passed": true - }, - { - "name": "ran EXPLAIN on the expensive query", - "passed": true - }, - { - "name": "created index covering user_id and created_at", - "passed": true - }, - { - "name": "query plan uses an index and avoids sequential scan", - "passed": true, - "notes": "Limit (cost=55.55..55.61 rows=25 width=88)\n -> Sort (cost=55.55..55.61 rows=25 width=88)\n Sort Key: created_at DESC\n -> Bitmap Heap Scan on events (cost=4.48..54.97 rows=25 width=88)\n Recheck Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)\n -> Bitmap Index Scan on events_user_id_created_at_idx (cost=0.00..4.47 rows=25 width=0)\n Index Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)" - }, - { - "name": "inserts still work", - "passed": true - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" - ] - }, - "docs": { - "calls": [] - }, - "prompt": "My database CPU keeps spiking and the app gets slow when loading recent events for a user. Can you figure out what query is causing it and make the database change needed to fix it?\n\nEnd your turn with a short summary of what you changed and why.", - "promptSourcePath": "evals/resolve-performance-001-slow-query-cpu-spike/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-sonnet-5/resolve-performance-001-slow-query-cpu-spike.json" - }, - { - "experiment": "claude-code-sonnet-5", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-sonnet-5", - "reasoningEffort": "high" - }, - "eval": "resolve-security-002-rls-cross-tenant-leak", - "stage": "resolve", - "product": [ - "database", - "auth" - ], - "topic": [ - "rls", - "security" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "RLS enabled on notes", - "passed": true - }, - { - "name": "tenant A sees only org A notes", - "passed": true - }, - { - "name": "tenant B cannot read org A notes", - "passed": true - }, - { - "name": "tenant A author can update own note", - "passed": true - }, - { - "name": "tenant B cannot update org A note", - "passed": true - }, - { - "name": "tenant B author can delete own note", - "passed": true - }, - { - "name": "tenant B cannot delete org A note", - "passed": true - }, - { - "name": "tenant A can insert note in own org", - "passed": true - }, - { - "name": "tenant B cannot insert into org A", - "passed": true + "judgeNotes": "The assistant correctly identified the root cause as orders missing from the supabase_realtime publication, noted SUBSCRIBED can still occur without INSERT events, applied ALTER PUBLICATION supabase_realtime ADD TABLE public.orders, verified membership, and did not weaken RLS/policies or disrupt courier_locations." } ], "skills": { @@ -5694,193 +1146,147 @@ "docs": { "calls": [] }, - "prompt": "A customer reported that notes showed up in the wrong workspace.\n\nCan you investigate what is going on and fix it?", - "promptSourcePath": "evals/resolve-security-002-rls-cross-tenant-leak/PROMPT.md", + "prompt": "Our dispatch dashboard shows incoming orders as they happen. The courier\nlocation feed on the same page updates live without problems, but new orders\nonly show up after a page refresh.\n\nThe dashboard uses supabase-js to subscribe to INSERT events on the `orders`\ntable through postgres_changes, the same way it subscribes to courier\nlocations. The channel's status callback logs SUBSCRIBED and there are no\nerrors in the browser console.\n\nFigure out why no order events ever arrive and fix it.", + "promptSourcePath": "evals/investigate-realtime-001-subscribed-no-events/PROMPT.md", "attempts": 1, - "sourcePath": "claude-code-sonnet-5/resolve-security-002-rls-cross-tenant-leak.json" + "sourcePath": "claude-code-opus-4.8/investigate-realtime-001-subscribed-no-events.json" }, { - "experiment": "claude-code-sonnet-5-no-skills", - "experimentSuite": "no-skills", + "experiment": "claude-code-opus-4.8", + "experimentSuite": "benchmark", "experimentDisplay": { "agent": "claude-code", "modelProvider": "anthropic", - "modelId": "claude-sonnet-5", + "modelId": "claude-opus-4-8", "reasoningEffort": "high" }, - "eval": "build-cli-001-bootstrap-app", - "stage": "build", + "eval": "investigate-reliability-003-edge-function-5xx-correlation", + "stage": "investigate", "product": [ - "database", - "data-api" + "edge-functions" ], "topic": [ - "migrations", - "rls" + "observability" ], "suite": "benchmark", - "interface": "cli", + "interface": "mcp", "passed": true, "checks": [ { - "name": "supabase project initialised (supabase/config.toml exists)", - "passed": true - }, - { - "name": "todos table is created by a migration file", - "passed": true - }, - { - "name": "todos table exists with at least 2 seeded rows", + "name": "identified image-transform and the recurring 503 pattern", "passed": true, - "notes": "found 2 rows" - }, - { - "name": "row level security is enabled on todos", - "passed": true - }, - { - "name": "a SELECT policy targets the authenticated role", - "passed": true + "judgeNotes": "The assistant correctly identified image-transform as the affected function and described the recurring pattern of eight HTTP 503 gateway failures throughout the morning of 2026-04-28 from 07:00Z to 12:00Z, while distinguishing unrelated billing-webhook errors." }, { - "name": "REST API returns no todos to anonymous requests", + "name": "attributed recurring 503s to gateway/platform layer, not function code", "passed": true, - "notes": "0 rows" + "judgeNotes": "The assistant clearly attributes the recurring image-transform 503s to the gateway/worker boot/platform layer, not application code. It grounds this in valid observations: 503s appear only in API/gateway logs with no matching runtime logs, successful nearby invocations have runtime logs and complete quickly, deployment v42 did not change, and it distinguishes these gateway 503s from avatar-upload's function-level 500." }, { - "name": "REST API returns the todos to authenticated requests", + "name": "recommended a concrete next step", "passed": true, - "notes": "2 rows" + "judgeNotes": "Recommended concrete next steps: check Edge Function memory/CPU limits and cold-start behavior, consider raising memory/keeping warm instance, review dependency version, rerun under load while watching for WORKER_LIMIT/OOM errors, and inspect configured memory/limits." } ], "skills": { - "available": [], - "loaded": [] + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] }, "docs": { "calls": [] }, - "prompt": "We're kicking off a todos app and I want the Supabase side ready for the team\nto build on. Set it up the way we'd run it in development, with schema changes\ntracked as migrations so they can be reviewed and replayed.\n\nFor the first slice we just need a `todos` table. Todos aren't public: anyone\nsigned in can read all of them, but nothing should be writable through the API\nfor now. Add a couple of sample todos so there's something to look at.\n\nBefore you hand it back, make sure the running API actually behaves that way —\nsigned-in users get the todos, signed-out requests get nothing.", - "promptSourcePath": "evals/build-cli-001-bootstrap-app/PROMPT.md", + "prompt": "Users have been reporting that image uploads are intermittently failing with server errors throughout the morning of `2026-04-28`.\n\nCan you investigate the project logs and tell me what is going on and what we\nshould do next?", + "promptSourcePath": "evals/investigate-reliability-003-edge-function-5xx-correlation/PROMPT.md", "attempts": 1, - "sourcePath": "claude-code-sonnet-5-no-skills/build-cli-001-bootstrap-app.json" + "sourcePath": "claude-code-opus-4.8/investigate-reliability-003-edge-function-5xx-correlation.json" }, { - "experiment": "claude-code-sonnet-5-no-skills", - "experimentSuite": "no-skills", + "experiment": "claude-code-opus-4.8", + "experimentSuite": "benchmark", "experimentDisplay": { "agent": "claude-code", "modelProvider": "anthropic", - "modelId": "claude-sonnet-5", + "modelId": "claude-opus-4-8", "reasoningEffort": "high" }, - "eval": "build-cli-002-declarative-schema", - "stage": "build", + "eval": "resolve-dataapi-001-empty-results", + "stage": "resolve", "product": [ - "database" + "data-api", + "database", + "auth" ], "topic": [ - "declarative-schema", - "migrations" + "rls", + "sdk" ], "suite": "benchmark", - "interface": "cli", + "interface": "mcp", "passed": true, "checks": [ { - "name": "supabase db diff used to generate the migration", + "name": "RLS still enabled on bookmarks", "passed": true }, { - "name": "schema file updated to include description column", + "name": "user A reads own bookmarks", "passed": true }, { - "name": "a new migration was generated for the change", + "name": "user B cannot read user A bookmarks", "passed": true }, { - "name": "description column exists in the live database", + "name": "anon reads no bookmarks", "passed": true - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "Add a description text column to the `products` table in my local Supabase stack", - "promptSourcePath": "evals/build-cli-002-declarative-schema/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-sonnet-5-no-skills/build-cli-002-declarative-schema.json" - }, - { - "experiment": "claude-code-sonnet-5-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-sonnet-5", - "reasoningEffort": "high" - }, - "eval": "build-cli-003-pg-cron-queue-workflow", - "stage": "build", - "product": [ - "database", - "edge-functions", - "cron", - "queues" - ], - "topic": [ - "sql", - "sdk" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ + }, { - "name": "pg_cron job 'enqueue-tasks' scheduled to run every minute", - "passed": true, - "notes": "schedule='* * * * *', active=true" + "name": "user A can save a new bookmark", + "passed": true }, { - "name": "cron command enqueues to the 'tasks' queue", - "passed": true, - "notes": "queue depth 0 -> 1" + "name": "user B cannot insert a bookmark as user A", + "passed": true }, { - "name": "process-tasks function drains the queue", + "name": "diagnosed RLS and added owner-scoped policies", "passed": true, - "notes": "function removed the seeded message (id 3) from the queue" + "judgeNotes": "Diagnosed RLS enabled with no policies as deny-all for Data API, kept RLS enabled, and created authenticated SELECT/INSERT policies scoped to user_id = auth.uid() with WITH CHECK for inserts." } ], "skills": { - "available": [], - "loaded": [] + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] }, "docs": { "calls": [] }, - "prompt": "I want to set up a recurring background workflow on my local Supabase stack.\n\nCan you set up a cron job called `enqueue-tasks` to run every minute and push a task into a queue called `tasks`? Then add a `process-tasks` edge function that reads messages off the `tasks` queue and removes them, so a scheduled worker can keep the backlog drained.", - "promptSourcePath": "evals/build-cli-003-pg-cron-queue-workflow/PROMPT.md", + "prompt": "Our app lets signed-in users save bookmarks and view them on their dashboard. Bookmarks are stored in the `bookmarks` table and are private — a user must only ever see their own. \nUsers also need to be able to save new bookmarks from the app.\n\nI can see the rows when I query the table directly, but the dashboard shows an empty list for every user.\n\nFind out why the Data API returns nothing and fix it.", + "promptSourcePath": "evals/resolve-dataapi-001-empty-results/PROMPT.md", "attempts": 1, - "sourcePath": "claude-code-sonnet-5-no-skills/build-cli-003-pg-cron-queue-workflow.json" + "sourcePath": "claude-code-opus-4.8/resolve-dataapi-001-empty-results.json" }, { - "experiment": "claude-code-sonnet-5-no-skills", - "experimentSuite": "no-skills", + "experiment": "claude-code-opus-4.8", + "experimentSuite": "benchmark", "experimentDisplay": { "agent": "claude-code", "modelProvider": "anthropic", - "modelId": "claude-sonnet-5", + "modelId": "claude-opus-4-8", "reasoningEffort": "high" }, - "eval": "build-database-001-migrate-postgres-to-supabase", - "stage": "build", + "eval": "resolve-database-001-migration-history-mismatch", + "stage": "resolve", "product": [ "database" ], @@ -5892,275 +1298,203 @@ "passed": true, "checks": [ { - "name": "all 3 tables exist (teams, members, tasks)", - "passed": true - }, - { - "name": "row counts match (teams=5, members=10, tasks=13)", - "passed": true - }, - { - "name": "foreign key constraints survived the restore", + "name": "the avatar_url column is applied on the hosted profiles table", "passed": true }, { - "name": "tasks_team_status_idx index survived the restore", + "name": "migration 20240220000000 is recorded in the remote history", "passed": true }, { - "name": "sequences synced (next insert won't conflict with existing IDs)", + "name": "remote migration history matches local migration files", "passed": true - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "I have an existing Postgres database I want to migrate to Supabase. There's a binary dump at `source.dump` in the current directory.\n\nCan you set up a local Supabase project and restore the dump into it?", - "promptSourcePath": "evals/build-database-001-migrate-postgres-to-supabase/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-sonnet-5-no-skills/build-database-001-migrate-postgres-to-supabase.json" - }, - { - "experiment": "claude-code-sonnet-5-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-sonnet-5", - "reasoningEffort": "high" - }, - "eval": "build-functions-004-service-role-bypass", - "stage": "build", - "product": [ - "edge-functions", - "auth", - "database" - ], - "topic": [ - "rls", - "security", - "sdk" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": false, - "checks": [ - { - "name": "rejects missing auth", - "passed": true, - "notes": "status=401" }, - { - "name": "user A reads own note", - "passed": true, - "notes": "status=200" - }, - { - "name": "reads only with the caller's JWT", - "passed": false, - "notes": "bearer_tokens=2, all_match=false" + { + "name": "local migrations are a valid reconciled sequence", + "passed": true }, { - "name": "user A cannot force-read user B note", - "passed": true, - "notes": "status=200" + "name": "production profile data is intact (not reset)", + "passed": true }, { - "name": "user B cannot force-read user A note", + "name": "the avatar migration and history reconciliation were done via the Supabase CLI", "passed": true, - "notes": "status=200" + "judgeNotes": "Avatar migration was applied through `supabase db push` in action #14, with output `Applying migration 20240220000000_add_avatar_url.sql...` and `Finished supabase db push.` The orphan bio migration history was reconciled by adding the missing local migration file `20240115000000_add_bio.sql` in action #12, after which `supabase migration list` showed local and remote aligned. No disallowed workaround or direct SQL mutation was used." } ], "skills": { - "available": [], - "loaded": [] + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] }, "docs": { "calls": [] }, - "prompt": "I built an Edge Function called `private-notes` for showing a user's saved\nprivate notes.\n\nCan you check whether there's any way one user could see another user's notes?\n\nPlease fix and deploy it if needed.", - "promptSourcePath": "evals/build-functions-004-service-role-bypass/PROMPT.md", - "attempts": 2, - "sourcePath": "claude-code-sonnet-5-no-skills/build-functions-004-service-role-bypass.json" + "prompt": "I'm trying to ship a migration to our hosted project and it's not working. Can you figure out what's wrong and get it deployed?", + "promptSourcePath": "evals/resolve-database-001-migration-history-mismatch/PROMPT.md", + "attempts": 1, + "sourcePath": "claude-code-opus-4.8/resolve-database-001-migration-history-mismatch.json" }, { - "experiment": "claude-code-sonnet-5-no-skills", - "experimentSuite": "no-skills", + "experiment": "claude-code-opus-4.8", + "experimentSuite": "benchmark", "experimentDisplay": { "agent": "claude-code", "modelProvider": "anthropic", - "modelId": "claude-sonnet-5", + "modelId": "claude-opus-4-8", "reasoningEffort": "high" }, - "eval": "build-functions-005-dual-auth-user-secret", - "stage": "build", + "eval": "resolve-performance-001-slow-query-cpu-spike", + "stage": "resolve", "product": [ - "edge-functions", - "auth", "database" ], "topic": [ - "sdk", - "rls", - "security" + "observability", + "sql" ], "suite": "benchmark", - "interface": "cli", - "cliVersion": "2.109.1", - "passed": false, + "interface": "mcp", + "passed": true, "checks": [ { - "name": "seed rows present", - "passed": true, - "notes": "found 2/2 seeded rows" - }, - { - "name": "rejects request with no credentials", - "passed": true, - "notes": "status 401: {\"error\":\"Missing bearer token\"}" - }, - { - "name": "user with JWT reads only their own rows", - "passed": true, - "notes": "status 200: {\"data\":[{\"user_id\":\"53dfc849-cc40-4ef2-80d0-bf282c98b6c7\",\"metric\":\"steps_a_ms6xiocg\",\"value\":111}]}" - }, - { - "name": "user cannot read another user's rows by passing user_id", - "passed": true, - "notes": "status 200: {\"data\":[{\"user_id\":\"53dfc849-cc40-4ef2-80d0-bf282c98b6c7\",\"metric\":\"steps_a_ms6xiocg\",\"value\":111}]}" - }, - { - "name": "service key bypasses RLS to read the target user's rows", - "passed": true, - "notes": "status 200: {\"data\":[{\"user_id\":\"e51e9bc6-ff44-46b5-83e3-08763a07bd87\",\"metric\":\"steps_b_ms6xiocg\",\"value\":222}]}" + "name": "inspected pg_stat_statements for query performance", + "passed": true }, { - "name": "non-service key is not granted service access", - "passed": true, - "notes": "status 401: {\"error\":\"Missing bearer token\"}" + "name": "ran EXPLAIN on the expensive query", + "passed": true }, { - "name": "rejects an unverified (forged) user token", - "passed": true, - "notes": "status 401: {\"error\":\"Invalid or expired access token\"}" + "name": "created index covering user_id and created_at", + "passed": true }, { - "name": "a user token in the apikey slot is not treated as the service key", + "name": "query plan uses an index and avoids sequential scan", "passed": true, - "notes": "status 401: {\"error\":\"Missing bearer token\"}" + "notes": "Limit (cost=55.55..55.61 rows=25 width=88)\n -> Sort (cost=55.55..55.61 rows=25 width=88)\n Sort Key: created_at DESC\n -> Bitmap Heap Scan on events (cost=4.48..54.97 rows=25 width=88)\n Recheck Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)\n -> Bitmap Index Scan on events_user_id_created_at_idx (cost=0.00..4.47 rows=25 width=0)\n Index Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)" }, { - "name": "implementation uses @supabase/server", - "passed": false, - "notes": "hand-rolled (raw supabase-js or other) — this eval requires @supabase/server" + "name": "inserts still work", + "passed": true } ], "skills": { - "available": [], - "loaded": [] + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] }, "docs": { "calls": [] }, - "prompt": "Build and serve a Supabase Edge Function named `user-stats` for this project,\nreachable over HTTP at `/functions/v1/user-stats`.\n\nOur product stores per-user metrics in the existing `user_stats` table.\n\nTwo very different callers need to hit this one endpoint:\n\n1. **Our mobile app**, acting for a signed-in user. It sends that user's\n Supabase access token. The endpoint should return the user's own stats.\n\n2. **Our internal billing service**, a trusted backend with no signed-in user.\n It authenticates with the project's secret (service-role) key in the `apikey`\n header, and names the target user with a `user_id` in the JSON request body.\n It needs that user's stats.\n\nReturn the matching rows as JSON. The endpoint must be secure: only ever serve\nstats to a caller that is genuinely entitled to them, and turn away callers that\nare not.", - "promptSourcePath": "evals/build-functions-005-dual-auth-user-secret/PROMPT.md", - "attempts": 2, - "sourcePath": "claude-code-sonnet-5-no-skills/build-functions-005-dual-auth-user-secret.json" + "prompt": "My database CPU keeps spiking and the app gets slow when loading recent events for a user. Can you figure out what query is causing it and make the database change needed to fix it?\n\nEnd your turn with a short summary of what you changed and why.", + "promptSourcePath": "evals/resolve-performance-001-slow-query-cpu-spike/PROMPT.md", + "attempts": 1, + "sourcePath": "claude-code-opus-4.8/resolve-performance-001-slow-query-cpu-spike.json" }, { - "experiment": "claude-code-sonnet-5-no-skills", - "experimentSuite": "no-skills", + "experiment": "claude-code-opus-4.8", + "experimentSuite": "benchmark", "experimentDisplay": { "agent": "claude-code", "modelProvider": "anthropic", - "modelId": "claude-sonnet-5", + "modelId": "claude-opus-4-8", "reasoningEffort": "high" }, - "eval": "build-storage-001-private-bucket-access", - "stage": "build", + "eval": "resolve-security-002-rls-cross-tenant-leak", + "stage": "resolve", "product": [ - "storage", - "database" + "database", + "auth" ], "topic": [ "rls", - "sdk" + "security" ], "suite": "benchmark", "interface": "mcp", "passed": true, "checks": [ { - "name": "bucket user-files exists", + "name": "RLS enabled on notes", "passed": true }, { - "name": "bucket user-files is private", + "name": "tenant A sees only org A notes", "passed": true }, { - "name": "RLS still enabled on storage.objects", + "name": "tenant B cannot read org A notes", "passed": true }, { - "name": "user A lists only own files", - "passed": true, - "notes": "saw: 019fb0e7-0613-730a-ae13-0efd0c195137/receipt-alpha.pdf, 019fb0e7-0613-730a-ae13-0efd0c195137/receipt-beta.pdf" + "name": "tenant A author can update own note", + "passed": true }, { - "name": "user B cannot read user A files", + "name": "tenant B cannot update org A note", "passed": true }, { - "name": "anon reads no files", + "name": "tenant B author can delete own note", "passed": true }, { - "name": "user A can upload into own folder", + "name": "tenant B cannot delete org A note", "passed": true }, { - "name": "user B cannot upload into user A folder", + "name": "tenant A can insert note in own org", "passed": true }, { - "name": "configured private per-user storage access", - "passed": true, - "judgeNotes": "Meets requirements: private user-files bucket, authenticated owner-scoped SELECT and INSERT policies with WITH CHECK on storage.objects, no RLS disabling or public access, and supabase-js createSignedUrl with expiry for sharing." + "name": "tenant B cannot insert into org A", + "passed": true } ], "skills": { - "available": [], - "loaded": [] + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] }, "docs": { "calls": [] }, - "prompt": "Our app lets signed-in users keep personal files like receipts and bank\nstatements. These files are private — a user must only ever be able to upload\nand download their own. The app uploads each file under a path that starts\nwith the owner's user id, e.g. `/receipt-march.pdf`.\n\nSet up a `user-files` bucket on our project and lock it down that way.\n\nUsers also sometimes share one of their files with someone else through a\ntemporary link that expires. Include the supabase-js code the app should use\nfor that.", - "promptSourcePath": "evals/build-storage-001-private-bucket-access/PROMPT.md", + "prompt": "A customer reported that notes showed up in the wrong workspace.\n\nCan you investigate what is going on and fix it?", + "promptSourcePath": "evals/resolve-security-002-rls-cross-tenant-leak/PROMPT.md", "attempts": 1, - "sourcePath": "claude-code-sonnet-5-no-skills/build-storage-001-private-bucket-access.json" + "sourcePath": "claude-code-opus-4.8/resolve-security-002-rls-cross-tenant-leak.json" }, { - "experiment": "claude-code-sonnet-5-no-skills", + "experiment": "claude-code-opus-4.8-no-skills", "experimentSuite": "no-skills", "experimentDisplay": { "agent": "claude-code", "modelProvider": "anthropic", - "modelId": "claude-sonnet-5", + "modelId": "claude-opus-4-8", "reasoningEffort": "high" }, - "eval": "build-tests-001-rls-tenant-isolation", + "eval": "build-cli-001-bootstrap-app", "stage": "build", "product": [ - "database" + "database", + "data-api" ], "topic": [ - "tests", + "migrations", "rls" ], "suite": "benchmark", @@ -6168,19 +1502,35 @@ "passed": true, "checks": [ { - "name": "pgTAP test file(s) written under supabase/tests/", + "name": "supabase project initialised (supabase/config.toml exists)", + "passed": true + }, + { + "name": "todos table is created by a migration file", + "passed": true + }, + { + "name": "todos table exists with at least 2 seeded rows", "passed": true, - "notes": "1 file(s): supabase/tests/database/tenant_isolation_test.sql" + "notes": "found 2 rows" }, { - "name": "pgTAP isolation tests ran and pass", + "name": "row level security is enabled on todos", + "passed": true + }, + { + "name": "a SELECT policy targets the authenticated role", + "passed": true + }, + { + "name": "REST API returns no todos to anonymous requests", "passed": true, - "notes": "6 passed, 2 failed" + "notes": "0 rows" }, { - "name": "agent correctly identifies the posts isolation bug from test results", + "name": "REST API returns the todos to authenticated requests", "passed": true, - "judgeNotes": "The agent correctly identifies `posts` as the table with the broken tenant isolation policy, explains that authenticated members of any org can read other organizations’ posts, and grounds the conclusion in pgTAP failures. It distinguishes `notes` as correctly isolated for reads and treats the test results as authoritative." + "notes": "2 rows" } ], "skills": { @@ -6190,63 +1540,47 @@ "docs": { "calls": [] }, - "prompt": "Can you audit the tenant isolation on our tables? Write some database tests covering both the happy path and the negative case, run them, and share the results.", - "promptSourcePath": "evals/build-tests-001-rls-tenant-isolation/PROMPT.md", + "prompt": "We're kicking off a todos app and I want the Supabase side ready for the team\nto build on. Set it up the way we'd run it in development, with schema changes\ntracked as migrations so they can be reviewed and replayed.\n\nFor the first slice we just need a `todos` table. Todos aren't public: anyone\nsigned in can read all of them, but nothing should be writable through the API\nfor now. Add a couple of sample todos so there's something to look at.\n\nBefore you hand it back, make sure the running API actually behaves that way —\nsigned-in users get the todos, signed-out requests get nothing.", + "promptSourcePath": "evals/build-cli-001-bootstrap-app/PROMPT.md", "attempts": 1, - "sourcePath": "claude-code-sonnet-5-no-skills/build-tests-001-rls-tenant-isolation.json" + "sourcePath": "claude-code-opus-4.8-no-skills/build-cli-001-bootstrap-app.json" }, { - "experiment": "claude-code-sonnet-5-no-skills", + "experiment": "claude-code-opus-4.8-no-skills", "experimentSuite": "no-skills", "experimentDisplay": { "agent": "claude-code", "modelProvider": "anthropic", - "modelId": "claude-sonnet-5", + "modelId": "claude-opus-4-8", "reasoningEffort": "high" }, - "eval": "build-vectors-001-rag-with-permissions", + "eval": "build-cli-002-declarative-schema", "stage": "build", "product": [ - "database", - "vectors" + "database" ], "topic": [ - "sql", - "rls" + "declarative-schema", + "migrations" ], "suite": "benchmark", - "interface": "mcp", + "interface": "cli", "passed": true, "checks": [ { - "name": "document_sections.embedding is vector(384)", - "passed": true, - "notes": "vector(384)" - }, - { - "name": "HNSW index on the embedding column", - "passed": true, - "notes": "CREATE INDEX document_sections_embedding_idx ON public.document_sections USING hnsw (embedding vector_cosine_ops)" - }, - { - "name": "index operator class matches the search operator", - "passed": true, - "notes": "function operators: <=>\nindexes: CREATE INDEX document_sections_embedding_idx ON public.document_sections USING hnsw (embedding vector_cosine_ops)" - }, - { - "name": "user A search returns only own sections, best match first", + "name": "supabase db diff used to generate the migration", "passed": true }, { - "name": "user B search returns only own sections, best match first", + "name": "schema file updated to include description column", "passed": true }, { - "name": "user A reads only own sections through the API", + "name": "a new migration was generated for the change", "passed": true }, { - "name": "user A reads only own documents through the API", + "name": "description column exists in the live database", "passed": true } ], @@ -6255,75 +1589,52 @@ "loaded": [] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{ searchDocs(query: \"match_document_sections pgvector edge function embeddings\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/examples/semantic-search", - "title": "Semantic Search" - }, - { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", - "title": "Automatic embeddings" - }, - { - "url": "https://supabase.com/docs/guides/ai/semantic-search", - "title": "Semantic search" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pgvector", - "title": "pgvector: Embeddings and vector similarity" - }, - { - "url": "https://supabase.com/docs/guides/ai/hybrid-search", - "title": "Hybrid search" - } - ], - "resultChars": 68270 - } - ] + "calls": [] }, - "prompt": "We're adding semantic search to our internal knowledge base app. I already wrote the edge functions, but search doesn't work yet. Can you set up whatever the database needs to make search work end to end?\n\nSome documents are confidential, so users should only have access to documents they own.", - "promptSourcePath": "evals/build-vectors-001-rag-with-permissions/PROMPT.md", + "prompt": "Add a description text column to the `products` table in my local Supabase stack", + "promptSourcePath": "evals/build-cli-002-declarative-schema/PROMPT.md", "attempts": 1, - "sourcePath": "claude-code-sonnet-5-no-skills/build-vectors-001-rag-with-permissions.json" + "sourcePath": "claude-code-opus-4.8-no-skills/build-cli-002-declarative-schema.json" }, { - "experiment": "claude-code-sonnet-5-no-skills", + "experiment": "claude-code-opus-4.8-no-skills", "experimentSuite": "no-skills", "experimentDisplay": { "agent": "claude-code", "modelProvider": "anthropic", - "modelId": "claude-sonnet-5", + "modelId": "claude-opus-4-8", "reasoningEffort": "high" }, - "eval": "deploy-database-001-prometheus-metrics", - "stage": "deploy", + "eval": "build-cli-003-pg-cron-queue-workflow", + "stage": "build", "product": [ - "database" + "database", + "edge-functions", + "cron", + "queues" ], "topic": [ - "observability" + "sql", + "sdk" ], "suite": "benchmark", - "passed": false, + "interface": "cli", + "passed": true, "checks": [ { - "name": "preserved existing app scrape job", - "passed": true + "name": "pg_cron job 'enqueue-tasks' scheduled to run every minute", + "passed": true, + "notes": "schedule='* * * * *', active=true" }, { - "name": "configured the Supabase Metrics API scrape correctly", - "passed": false, - "judgeNotes": "Fails: Supabase scrape is not deployable against .supabase.co/red, uses http and host.docker.internal, uses inline basic_auth password instead of password_file, and docker-compose.yml does not mount a password_file via volume or secret." + "name": "cron command enqueues to the 'tasks' queue", + "passed": true, + "notes": "queue depth 1 -> 2" }, { - "name": "documented live deployment and verification steps", - "passed": false, - "judgeNotes": "README includes Secret API key creation, reload/restart, and Prometheus target verification, but it does not provide required steps to place a matching secret file; the config uses an inline placeholder instead of a concrete password_file/secret-file setup." + "name": "process-tasks function drains the queue", + "passed": true, + "notes": "function removed the seeded message (id 47) from the queue" } ], "skills": { @@ -6331,75 +1642,52 @@ "loaded": [] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{ searchDocs(query: \"self-hosted metrics endpoint prometheus\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/telemetry/metrics/grafana-self-hosted", - "title": "Metrics API with Prometheus & Grafana (self-hosted)" - }, - { - "url": "https://supabase.com/docs/guides/telemetry/metrics", - "title": "Metrics API" - }, - { - "url": "https://supabase.com/docs/guides/telemetry/metrics/vendor-agnostic", - "title": "Vendor-agnostic Metrics API setup" - }, - { - "url": "https://supabase.com/docs/guides/telemetry/metrics/grafana-cloud", - "title": "Metrics API with Grafana Cloud" - } - ], - "resultChars": 23542 - } - ] + "calls": [] }, - "prompt": "Can you wire my Supabase project metrics into our existing observability stack and document\nin the observability README what we need to do to make the config live?", - "promptSourcePath": "evals/deploy-database-001-prometheus-metrics/PROMPT.md", - "attempts": 2, - "sourcePath": "claude-code-sonnet-5-no-skills/deploy-database-001-prometheus-metrics.json" + "prompt": "I want to set up a recurring background workflow on my local Supabase stack.\n\nCan you set up a cron job called `enqueue-tasks` to run every minute and push a task into a queue called `tasks`? Then add a `process-tasks` edge function that reads messages off the `tasks` queue and removes them, so a scheduled worker can keep the backlog drained.", + "promptSourcePath": "evals/build-cli-003-pg-cron-queue-workflow/PROMPT.md", + "attempts": 1, + "sourcePath": "claude-code-opus-4.8-no-skills/build-cli-003-pg-cron-queue-workflow.json" }, { - "experiment": "claude-code-sonnet-5-no-skills", + "experiment": "claude-code-opus-4.8-no-skills", "experimentSuite": "no-skills", "experimentDisplay": { "agent": "claude-code", "modelProvider": "anthropic", - "modelId": "claude-sonnet-5", + "modelId": "claude-opus-4-8", "reasoningEffort": "high" }, - "eval": "deploy-functions-001-edge-function-secrets", - "stage": "deploy", + "eval": "build-database-001-migrate-postgres-to-supabase", + "stage": "build", "product": [ - "edge-functions" + "database" ], "topic": [ - "security" + "migrations" ], "suite": "benchmark", "interface": "cli", "passed": true, "checks": [ { - "name": "WEATHER_API_KEY is set as a Function secret on the project", + "name": "all 3 tables exist (teams, members, tasks)", "passed": true }, { - "name": "the weather function is deployed to the project", - "passed": true, - "notes": "status ACTIVE" + "name": "row counts match (teams=5, members=10, tasks=13)", + "passed": true }, { - "name": "the weather function reads WEATHER_API_KEY from the environment", - "passed": true, - "judgeNotes": "The function reads WEATHER_API_KEY from the runtime environment using Deno.env.get(\"WEATHER_API_KEY\")." + "name": "foreign key constraints survived the restore", + "passed": true }, { - "name": "WEATHER_API_KEY value is not committed to the repo", + "name": "tasks_team_status_idx index survived the restore", + "passed": true + }, + { + "name": "sequences synced (next insert won't conflict with existing IDs)", "passed": true } ], @@ -6410,49 +1698,60 @@ "docs": { "calls": [] }, - "prompt": "Our weather widget currently calls WeatherAPI straight from the browser, which\nleaks our API key. I want to move that behind a Supabase Edge Function called\n`weather` that holds the key server-side and proxies the request.\n\nThe function should read the key from an environment variable named\n`WEATHER_API_KEY`. Our key already lives in a local `.env` file at the project\nroot.\n\nDeploy the function to our project so it's live, and make sure the deployed\nfunction can actually read the key at runtime.", - "promptSourcePath": "evals/deploy-functions-001-edge-function-secrets/PROMPT.md", + "prompt": "I have an existing Postgres database I want to migrate to Supabase. There's a binary dump at `source.dump` in the current directory.\n\nCan you set up a local Supabase project and restore the dump into it?", + "promptSourcePath": "evals/build-database-001-migrate-postgres-to-supabase/PROMPT.md", "attempts": 1, - "sourcePath": "claude-code-sonnet-5-no-skills/deploy-functions-001-edge-function-secrets.json" + "sourcePath": "claude-code-opus-4.8-no-skills/build-database-001-migrate-postgres-to-supabase.json" }, { - "experiment": "claude-code-sonnet-5-no-skills", + "experiment": "claude-code-opus-4.8-no-skills", "experimentSuite": "no-skills", "experimentDisplay": { "agent": "claude-code", "modelProvider": "anthropic", - "modelId": "claude-sonnet-5", + "modelId": "claude-opus-4-8", "reasoningEffort": "high" }, - "eval": "deploy-self-hosting-001-docker-compose", - "stage": "deploy", + "eval": "build-functions-004-service-role-bypass", + "stage": "build", "product": [ - "database", + "edge-functions", "auth", - "storage" + "database" ], "topic": [ - "self-hosting" + "rls", + "security", + "sdk" ], "suite": "benchmark", - "interface": "cli", + "interface": "mcp", "passed": true, "checks": [ { - "name": "cloned the self-host stack (docker-compose.yml + volumes/db)", - "passed": true + "name": "rejects missing auth", + "passed": true, + "notes": "status=401" }, { - "name": "didn't conflate with the CLI (no supabase/config.toml in the stack)", - "passed": true + "name": "user A reads own note", + "passed": true, + "notes": "status=200" }, { - "name": "secrets rotated off the shipped defaults", - "passed": true + "name": "reads only with the caller's JWT", + "passed": true, + "notes": "bearer_tokens=2, all_match=true" }, { - "name": "ANON_KEY and SERVICE_ROLE_KEY are HS256 JWTs signed by JWT_SECRET", - "passed": true + "name": "user A cannot force-read user B note", + "passed": true, + "notes": "status=200" + }, + { + "name": "user B cannot force-read user A note", + "passed": true, + "notes": "status=200" } ], "skills": { @@ -6462,63 +1761,81 @@ "docs": { "calls": [] }, - "prompt": "I'm moving off the hosted Supabase and running the whole thing myself on a VPS I\njust spun up. Can you get a Docker setup ready for me to copy onto the box?\n\nI don't need it running here, I'll do the actual bring-up once I'm on the\nserver. I just want everything in place and the secrets set up. Put it in a `supabase-docker/`\nfolder at the repo root so I can scp the whole thing across in one go.", - "promptSourcePath": "evals/deploy-self-hosting-001-docker-compose/PROMPT.md", + "prompt": "I built an Edge Function called `private-notes` for showing a user's saved\nprivate notes.\n\nCan you check whether there's any way one user could see another user's notes?\n\nPlease fix and deploy it if needed.", + "promptSourcePath": "evals/build-functions-004-service-role-bypass/PROMPT.md", "attempts": 1, - "sourcePath": "claude-code-sonnet-5-no-skills/deploy-self-hosting-001-docker-compose.json" + "sourcePath": "claude-code-opus-4.8-no-skills/build-functions-004-service-role-bypass.json" }, { - "experiment": "claude-code-sonnet-5-no-skills", + "experiment": "claude-code-opus-4.8-no-skills", "experimentSuite": "no-skills", "experimentDisplay": { "agent": "claude-code", "modelProvider": "anthropic", - "modelId": "claude-sonnet-5", + "modelId": "claude-opus-4-8", "reasoningEffort": "high" }, - "eval": "investigate-auth-001-deleted-user-access", - "stage": "investigate", + "eval": "build-functions-005-dual-auth-user-secret", + "stage": "build", "product": [ - "auth" + "edge-functions", + "auth", + "database" ], "topic": [ - "security", - "sdk" + "sdk", + "rls", + "security" ], "suite": "benchmark", - "interface": "mcp", + "interface": "cli", + "cliVersion": "2.109.1", "passed": false, "checks": [ { - "name": "victim session active before delete-account", - "passed": true + "name": "seed rows present", + "passed": true, + "notes": "found 2/2 seeded rows" }, { - "name": "delete_account flow ran for the victim", - "passed": true + "name": "rejects request with no credentials", + "passed": true, + "notes": "status 401: {\"error\":\"Unauthorized\"}" }, { - "name": "delete-account revokes the user's sessions", + "name": "user with JWT reads only their own rows", "passed": true, - "notes": "sessions left: 0" + "notes": "status 200: [{\"user_id\":\"de8bd691-9818-49f1-bb4b-6c48aed1910b\",\"metric\":\"steps_a_mrw6buc0\",\"value\":111}]" }, { - "name": "deleted user's refresh token is rejected", - "passed": true + "name": "user cannot read another user's rows by passing user_id", + "passed": true, + "notes": "status 200: [{\"user_id\":\"de8bd691-9818-49f1-bb4b-6c48aed1910b\",\"metric\":\"steps_a_mrw6buc0\",\"value\":111}]" }, { - "name": "deleted user cannot sign back in", - "passed": false, - "notes": "deleted account can still sign in" + "name": "service key bypasses RLS to read the target user's rows", + "passed": true, + "notes": "status 200: [{\"user_id\":\"2d2fc3f8-6aee-4ebb-bf2d-2fdbbe2ff4c6\",\"metric\":\"steps_b_mrw6buc0\",\"value\":222}]" }, { - "name": "other users keep their sessions and access", - "passed": true + "name": "non-service key is not granted service access", + "passed": true, + "notes": "status 401: {\"error\":\"Unauthorized\"}" }, { - "name": "diagnosed and explained session revocation", + "name": "rejects an unverified (forged) user token", + "passed": true, + "notes": "status 401: {\"error\":\"Unauthorized\"}" + }, + { + "name": "a user token in the apikey slot is not treated as the service key", "passed": true, - "judgeNotes": "Meets rubric: identifies soft-delete-only cause, implements auth/session revocation plus RLS checks, accurately explains JWT expiry/local validation caveat consistent with its RLS fix, and correctly distinguishes publishable vs secret keys and RLS behavior." + "notes": "status 401: {\"error\":\"Unauthorized\"}" + }, + { + "name": "implementation uses @supabase/server", + "passed": false, + "notes": "hand-rolled (raw supabase-js or other) — this eval requires @supabase/server" } ], "skills": { @@ -6529,40 +1846,48 @@ "calls": [ { "source": "search_docs", - "query": "{ searchDocs(query: \"publishable key secret key anon service_role migration RLS\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [] - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"publishable key secret key anon service_role migration RLS\", limit: 5) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"functions serve failed to determine entrypoint main worker boot error\", limit: 5) { nodes { title href content } } }", "hasContent": true, - "pages": [], - "resultChars": 149036 + "pages": [ + { + "url": "https://supabase.com/docs/guides/functions/error-codes", + "title": "Error codes" + }, + { + "url": "https://supabase.com/docs/guides/functions/status-codes", + "title": "Status codes" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", + "title": "Self-Hosted Functions" + } + ], + "resultChars": 23247 } ] }, - "prompt": "Last week support removed a user through our app's delete-account flow — the\napp calls the `delete_account` function over RPC as the signed-in user. This\nmorning that same person was back: still signed in, reading and saving their\ndata like nothing happened.\n\nFigure out why the account still works, fix the flow so a deleted account\nloses access, and tell me whether there is any window where they could still\nget in after the fix.\n\nOne more thing while you're at it: we're migrating off the legacy\nanon/service_role API keys to the new publishable and secret keys, and the\nteam isn't sure which one belongs in the frontend and what each means for\nRLS. Give us a quick rundown.", - "promptSourcePath": "evals/investigate-auth-001-deleted-user-access/PROMPT.md", + "prompt": "Build and serve a Supabase Edge Function named `user-stats` for this project,\nreachable over HTTP at `/functions/v1/user-stats`.\n\nOur product stores per-user metrics in the existing `user_stats` table.\n\nTwo very different callers need to hit this one endpoint:\n\n1. **Our mobile app**, acting for a signed-in user. It sends that user's\n Supabase access token. The endpoint should return the user's own stats.\n\n2. **Our internal billing service**, a trusted backend with no signed-in user.\n It authenticates with the project's secret (service-role) key in the `apikey`\n header, and names the target user with a `user_id` in the JSON request body.\n It needs that user's stats.\n\nReturn the matching rows as JSON. The endpoint must be secure: only ever serve\nstats to a caller that is genuinely entitled to them, and turn away callers that\nare not.", + "promptSourcePath": "evals/build-functions-005-dual-auth-user-secret/PROMPT.md", "attempts": 2, - "sourcePath": "claude-code-sonnet-5-no-skills/investigate-auth-001-deleted-user-access.json" + "sourcePath": "claude-code-opus-4.8-no-skills/build-functions-005-dual-auth-user-secret.json" }, { - "experiment": "claude-code-sonnet-5-no-skills", + "experiment": "claude-code-opus-4.8-no-skills", "experimentSuite": "no-skills", "experimentDisplay": { "agent": "claude-code", "modelProvider": "anthropic", - "modelId": "claude-sonnet-5", + "modelId": "claude-opus-4-8", "reasoningEffort": "high" }, - "eval": "investigate-realtime-001-subscribed-no-events", - "stage": "investigate", + "eval": "build-storage-001-private-bucket-access", + "stage": "build", "product": [ - "realtime", + "storage", "database" ], "topic": [ + "rls", "sdk" ], "suite": "benchmark", @@ -6570,30 +1895,42 @@ "passed": true, "checks": [ { - "name": "orders table added to supabase_realtime publication", + "name": "bucket user-files exists", "passed": true }, { - "name": "courier_locations still in supabase_realtime publication", + "name": "bucket user-files is private", "passed": true }, { - "name": "publication still publishes INSERT events", + "name": "RLS still enabled on storage.objects", "passed": true }, { - "name": "RLS still enabled on orders", + "name": "user A lists only own files", + "passed": true, + "notes": "saw: 019f8a21-ebbd-753c-a6c4-491819928807/receipt-alpha.pdf, 019f8a21-ebbd-753c-a6c4-491819928807/receipt-beta.pdf" + }, + { + "name": "user B cannot read user A files", + "passed": true + }, + { + "name": "anon reads no files", + "passed": true + }, + { + "name": "user A can upload into own folder", "passed": true }, { - "name": "staff can still read orders through RLS", - "passed": true, - "notes": "authenticated sees 2 of 2 orders" + "name": "user B cannot upload into user A folder", + "passed": true }, { - "name": "diagnosed missing publication membership", + "name": "configured private per-user storage access", "passed": true, - "judgeNotes": "The assistant correctly identified the root cause as orders missing from the supabase_realtime publication, explained why SUBSCRIBED can occur without INSERT events, added public.orders to the existing publication with ALTER PUBLICATION, and preserved RLS/policies and the courier_locations feed. It did not blame client code, networking, grants, or weaken security." + "judgeNotes": "Meets all requirements: private bucket, authenticated owner-scoped SELECT/INSERT policies with WITH CHECK, no RLS disabling or public access, and signed URL sharing via createSignedUrl with expiry." } ], "skills": { @@ -6601,48 +1938,78 @@ "loaded": [] }, "docs": { - "calls": [] + "calls": [ + { + "source": "search_docs", + "query": "{ searchDocs(query: \"storage bucket RLS policy restrict users own folder user id\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/storage/security/access-control", + "title": "Storage Access Control" + }, + { + "url": "https://supabase.com/docs/guides/storage/schema/helper-functions", + "title": "Storage Helper Functions" + }, + { + "url": "https://supabase.com/docs/guides/storage/buckets/fundamentals", + "title": "Storage Buckets" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3", + "title": "Configure S3 Storage" + }, + { + "url": "https://supabase.com/docs/guides/security/product-security", + "title": "Secure configuration of Supabase products" + } + ], + "resultChars": 23289 + } + ] }, - "prompt": "Our dispatch dashboard shows incoming orders as they happen. The courier\nlocation feed on the same page updates live without problems, but new orders\nonly show up after a page refresh.\n\nThe dashboard uses supabase-js to subscribe to INSERT events on the `orders`\ntable through postgres_changes, the same way it subscribes to courier\nlocations. The channel's status callback logs SUBSCRIBED and there are no\nerrors in the browser console.\n\nFigure out why no order events ever arrive and fix it.", - "promptSourcePath": "evals/investigate-realtime-001-subscribed-no-events/PROMPT.md", + "prompt": "Our app lets signed-in users keep personal files like receipts and bank\nstatements. These files are private — a user must only ever be able to upload\nand download their own. The app uploads each file under a path that starts\nwith the owner's user id, e.g. `/receipt-march.pdf`.\n\nSet up a `user-files` bucket on our project and lock it down that way.\n\nUsers also sometimes share one of their files with someone else through a\ntemporary link that expires. Include the supabase-js code the app should use\nfor that.", + "promptSourcePath": "evals/build-storage-001-private-bucket-access/PROMPT.md", "attempts": 1, - "sourcePath": "claude-code-sonnet-5-no-skills/investigate-realtime-001-subscribed-no-events.json" + "sourcePath": "claude-code-opus-4.8-no-skills/build-storage-001-private-bucket-access.json" }, { - "experiment": "claude-code-sonnet-5-no-skills", + "experiment": "claude-code-opus-4.8-no-skills", "experimentSuite": "no-skills", "experimentDisplay": { "agent": "claude-code", "modelProvider": "anthropic", - "modelId": "claude-sonnet-5", + "modelId": "claude-opus-4-8", "reasoningEffort": "high" }, - "eval": "investigate-reliability-003-edge-function-5xx-correlation", - "stage": "investigate", + "eval": "build-tests-001-rls-tenant-isolation", + "stage": "build", "product": [ - "edge-functions" + "database" ], "topic": [ - "observability" + "tests", + "rls" ], "suite": "benchmark", - "interface": "mcp", + "interface": "cli", "passed": true, "checks": [ { - "name": "identified image-transform and the recurring 503 pattern", + "name": "pgTAP test file(s) written under supabase/tests/", "passed": true, - "judgeNotes": "Identified `image-transform` as the affected function and described the recurring HTTP 503 gateway pattern across the morning of 2026-04-28, covering most of the failures in the 07:00Z–12:00Z window. Also correctly treated old billing-webhook 503s as unrelated." + "notes": "1 file(s): supabase/tests/tenant_isolation_test.sql" }, { - "name": "attributed recurring 503s to gateway/platform layer, not function code", + "name": "pgTAP isolation tests ran and pass", "passed": true, - "judgeNotes": "Attributes the recurring image-transform 503s to the gateway/platform layer before function execution, grounded in gateway-only 503s with no matching invocation/runtime rows, unchanged version/deployment, and contrast with avatar-upload's true in-function 500." + "notes": "6 passed, 2 failed" }, { - "name": "recommended a concrete next step", + "name": "agent correctly identifies the posts isolation bug from test results", "passed": true, - "judgeNotes": "The assistant recommended concrete actionable next steps, including checking resource/concurrency limits, investigating a memory-heavy dependency, adding retry/backoff, and separately improving error logging for avatar-upload." + "judgeNotes": "The agent correctly identifies `posts` as the broken table, explains that authenticated members can read posts from other organizations due to missing `m.org_id = posts.org_id`, and grounds the conclusion in the pgTAP failures (tests 4 and 8). It treats the test results as authoritative and does not blame `notes` or dismiss the failures." } ], "skills": { @@ -6652,63 +2019,64 @@ "docs": { "calls": [] }, - "prompt": "Users have been reporting that image uploads are intermittently failing with server errors throughout the morning of `2026-04-28`.\n\nCan you investigate the project logs and tell me what is going on and what we\nshould do next?", - "promptSourcePath": "evals/investigate-reliability-003-edge-function-5xx-correlation/PROMPT.md", + "prompt": "Can you audit the tenant isolation on our tables? Write some database tests covering both the happy path and the negative case, run them, and share the results.", + "promptSourcePath": "evals/build-tests-001-rls-tenant-isolation/PROMPT.md", "attempts": 1, - "sourcePath": "claude-code-sonnet-5-no-skills/investigate-reliability-003-edge-function-5xx-correlation.json" + "sourcePath": "claude-code-opus-4.8-no-skills/build-tests-001-rls-tenant-isolation.json" }, { - "experiment": "claude-code-sonnet-5-no-skills", + "experiment": "claude-code-opus-4.8-no-skills", "experimentSuite": "no-skills", "experimentDisplay": { "agent": "claude-code", "modelProvider": "anthropic", - "modelId": "claude-sonnet-5", + "modelId": "claude-opus-4-8", "reasoningEffort": "high" }, - "eval": "resolve-dataapi-001-empty-results", - "stage": "resolve", + "eval": "build-vectors-001-rag-with-permissions", + "stage": "build", "product": [ - "data-api", "database", - "auth" + "vectors" ], "topic": [ - "rls", - "sdk" + "sql", + "rls" ], "suite": "benchmark", "interface": "mcp", "passed": true, "checks": [ { - "name": "RLS still enabled on bookmarks", - "passed": true + "name": "document_sections.embedding is vector(384)", + "passed": true, + "notes": "vector(384)" }, { - "name": "user A reads own bookmarks", - "passed": true + "name": "HNSW index on the embedding column", + "passed": true, + "notes": "CREATE INDEX document_sections_embedding_idx ON public.document_sections USING hnsw (embedding vector_cosine_ops)" }, { - "name": "user B cannot read user A bookmarks", - "passed": true + "name": "index operator class matches the search operator", + "passed": true, + "notes": "function operators: <=>\nindexes: CREATE INDEX document_sections_embedding_idx ON public.document_sections USING hnsw (embedding vector_cosine_ops)" }, { - "name": "anon reads no bookmarks", + "name": "user A search returns only own sections, best match first", "passed": true }, { - "name": "user A can save a new bookmark", + "name": "user B search returns only own sections, best match first", "passed": true }, { - "name": "user B cannot insert a bookmark as user A", + "name": "user A reads only own sections through the API", "passed": true }, { - "name": "diagnosed RLS and added owner-scoped policies", - "passed": true, - "judgeNotes": "Diagnoses RLS enabled with no policies causing deny-all/empty Data API results; keeps RLS enabled; creates authenticated SELECT policy with auth.uid() = user_id and INSERT policy with WITH CHECK enforcing auth.uid() = user_id." + "name": "user A reads only own documents through the API", + "passed": true } ], "skills": { @@ -6718,56 +2086,44 @@ "docs": { "calls": [] }, - "prompt": "Our app lets signed-in users save bookmarks and view them on their dashboard. Bookmarks are stored in the `bookmarks` table and are private — a user must only ever see their own. \nUsers also need to be able to save new bookmarks from the app.\n\nI can see the rows when I query the table directly, but the dashboard shows an empty list for every user.\n\nFind out why the Data API returns nothing and fix it.", - "promptSourcePath": "evals/resolve-dataapi-001-empty-results/PROMPT.md", + "prompt": "We're adding semantic search to our internal knowledge base app. I already wrote the edge functions, but search doesn't work yet. Can you set up whatever the database needs to make search work end to end?\n\nSome documents are confidential, so users should only have access to documents they own.", + "promptSourcePath": "evals/build-vectors-001-rag-with-permissions/PROMPT.md", "attempts": 1, - "sourcePath": "claude-code-sonnet-5-no-skills/resolve-dataapi-001-empty-results.json" + "sourcePath": "claude-code-opus-4.8-no-skills/build-vectors-001-rag-with-permissions.json" }, { - "experiment": "claude-code-sonnet-5-no-skills", + "experiment": "claude-code-opus-4.8-no-skills", "experimentSuite": "no-skills", "experimentDisplay": { "agent": "claude-code", "modelProvider": "anthropic", - "modelId": "claude-sonnet-5", + "modelId": "claude-opus-4-8", "reasoningEffort": "high" }, - "eval": "resolve-database-001-migration-history-mismatch", - "stage": "resolve", + "eval": "deploy-database-001-prometheus-metrics", + "stage": "deploy", "product": [ "database" ], "topic": [ - "migrations" + "observability" ], "suite": "benchmark", - "interface": "cli", "passed": true, "checks": [ { - "name": "the avatar_url column is applied on the hosted profiles table", - "passed": true - }, - { - "name": "migration 20240220000000 is recorded in the remote history", - "passed": true - }, - { - "name": "remote migration history matches local migration files", - "passed": true - }, - { - "name": "local migrations are a valid reconciled sequence", + "name": "preserved existing app scrape job", "passed": true }, { - "name": "production profile data is intact (not reset)", - "passed": true + "name": "configured the Supabase Metrics API scrape correctly", + "passed": true, + "judgeNotes": "Adds Supabase scrape over HTTPS to /customer/v1/privileged/metrics with Basic Auth password_file, preserves app scrape, and docker-compose mounts the secrets directory containing the password file." }, { - "name": "the avatar migration and history reconciliation were done via the Supabase CLI", + "name": "documented live deployment and verification steps", "passed": true, - "judgeNotes": "Applied via `supabase db push` (#12), which shows `Applying migration 20240220000000_add_avatar_url.sql...` and `Finished supabase db push.` Reconciled the orphan bio migration by adding the missing local migration file `supabase/migrations/20240115000000_add_profile_bio.sql` (#10), after which `supabase migration list` (#11) showed local and remote aligned. Only read-only psql inspection was used; no disallowed workaround seen." + "judgeNotes": "README includes Secret API key creation, matching secret file placement, Compose start/reload steps, and concrete verification via Prometheus targets; endpoint/auth and secret mounting are consistent." } ], "skills": { @@ -6775,54 +2131,79 @@ "loaded": [] }, "docs": { - "calls": [] + "calls": [ + { + "source": "search_docs", + "query": "{ searchDocs(query: \"Prometheus metrics endpoint project observability scrape\", limit: 6) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/telemetry/metrics/grafana-self-hosted", + "title": "Metrics API with Prometheus & Grafana (self-hosted)" + }, + { + "url": "https://supabase.com/docs/guides/telemetry/metrics", + "title": "Metrics API" + }, + { + "url": "https://supabase.com/docs/guides/telemetry/metrics/vendor-agnostic", + "title": "Vendor-agnostic Metrics API setup" + }, + { + "url": "https://supabase.com/docs/guides/telemetry/metrics/grafana-cloud", + "title": "Metrics API with Grafana Cloud" + }, + { + "url": "https://supabase.com/docs/guides/platform/read-replicas", + "title": "Read Replicas" + } + ], + "resultChars": 32656 + } + ] }, - "prompt": "I'm trying to ship a migration to our hosted project and it's not working. Can you figure out what's wrong and get it deployed?", - "promptSourcePath": "evals/resolve-database-001-migration-history-mismatch/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-sonnet-5-no-skills/resolve-database-001-migration-history-mismatch.json" + "prompt": "Can you wire my Supabase project metrics into our existing observability stack and document\nwhat we need to do to make the config live?", + "promptSourcePath": "evals/deploy-database-001-prometheus-metrics/PROMPT.md", + "attempts": 2, + "sourcePath": "claude-code-opus-4.8-no-skills/deploy-database-001-prometheus-metrics.json" }, { - "experiment": "claude-code-sonnet-5-no-skills", + "experiment": "claude-code-opus-4.8-no-skills", "experimentSuite": "no-skills", "experimentDisplay": { "agent": "claude-code", "modelProvider": "anthropic", - "modelId": "claude-sonnet-5", + "modelId": "claude-opus-4-8", "reasoningEffort": "high" }, - "eval": "resolve-performance-001-slow-query-cpu-spike", - "stage": "resolve", + "eval": "deploy-functions-001-edge-function-secrets", + "stage": "deploy", "product": [ - "database" + "edge-functions" ], "topic": [ - "observability", - "sql" + "security" ], "suite": "benchmark", - "interface": "mcp", + "interface": "cli", "passed": true, "checks": [ { - "name": "inspected pg_stat_statements for query performance", - "passed": true - }, - { - "name": "ran EXPLAIN on the expensive query", + "name": "WEATHER_API_KEY is set as a Function secret on the project", "passed": true }, { - "name": "created index covering user_id and created_at", - "passed": true + "name": "the weather function is deployed to the project", + "passed": true, + "notes": "status ACTIVE" }, { - "name": "query plan uses an index and avoids sequential scan", + "name": "the weather function reads WEATHER_API_KEY from the environment", "passed": true, - "notes": "Limit (cost=55.55..55.61 rows=25 width=88)\n -> Sort (cost=55.55..55.61 rows=25 width=88)\n Sort Key: created_at DESC\n -> Bitmap Heap Scan on events (cost=4.48..54.97 rows=25 width=88)\n Recheck Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)\n -> Bitmap Index Scan on events_user_id_created_at_idx (cost=0.00..4.47 rows=25 width=0)\n Index Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)" + "judgeNotes": "The function reads WEATHER_API_KEY from the runtime environment using Deno.env.get(\"WEATHER_API_KEY\")." }, { - "name": "inserts still work", + "name": "WEATHER_API_KEY value is not committed to the repo", "passed": true } ], @@ -6833,68 +2214,48 @@ "docs": { "calls": [] }, - "prompt": "My database CPU keeps spiking and the app gets slow when loading recent events for a user. Can you figure out what query is causing it and make the database change needed to fix it?\n\nEnd your turn with a short summary of what you changed and why.", - "promptSourcePath": "evals/resolve-performance-001-slow-query-cpu-spike/PROMPT.md", + "prompt": "Our weather widget currently calls WeatherAPI straight from the browser, which\nleaks our API key. I want to move that behind a Supabase Edge Function called\n`weather` that holds the key server-side and proxies the request.\n\nThe function should read the key from an environment variable named\n`WEATHER_API_KEY`. Our key already lives in a local `.env` file at the project\nroot.\n\nDeploy the function to our project so it's live, and make sure the deployed\nfunction can actually read the key at runtime.", + "promptSourcePath": "evals/deploy-functions-001-edge-function-secrets/PROMPT.md", "attempts": 1, - "sourcePath": "claude-code-sonnet-5-no-skills/resolve-performance-001-slow-query-cpu-spike.json" + "sourcePath": "claude-code-opus-4.8-no-skills/deploy-functions-001-edge-function-secrets.json" }, { - "experiment": "claude-code-sonnet-5-no-skills", + "experiment": "claude-code-opus-4.8-no-skills", "experimentSuite": "no-skills", "experimentDisplay": { "agent": "claude-code", "modelProvider": "anthropic", - "modelId": "claude-sonnet-5", + "modelId": "claude-opus-4-8", "reasoningEffort": "high" }, - "eval": "resolve-security-002-rls-cross-tenant-leak", - "stage": "resolve", + "eval": "deploy-self-hosting-001-docker-compose", + "stage": "deploy", "product": [ "database", - "auth" + "auth", + "storage" ], "topic": [ - "rls", - "security" + "self-hosting" ], "suite": "benchmark", - "interface": "mcp", + "interface": "cli", "passed": true, "checks": [ { - "name": "RLS enabled on notes", - "passed": true - }, - { - "name": "tenant A sees only org A notes", - "passed": true - }, - { - "name": "tenant B cannot read org A notes", - "passed": true - }, - { - "name": "tenant A author can update own note", - "passed": true - }, - { - "name": "tenant B cannot update org A note", - "passed": true - }, - { - "name": "tenant B author can delete own note", + "name": "cloned the self-host stack (docker-compose.yml + volumes/db)", "passed": true }, { - "name": "tenant B cannot delete org A note", + "name": "didn't conflate with the CLI (no supabase/config.toml in the stack)", "passed": true }, { - "name": "tenant A can insert note in own org", + "name": "secrets rotated off the shipped defaults", "passed": true }, { - "name": "tenant B cannot insert into org A", + "name": "ANON_KEY and SERVICE_ROLE_KEY are HS256 JWTs signed by JWT_SECRET", "passed": true } ], @@ -6905,322 +2266,263 @@ "docs": { "calls": [] }, - "prompt": "A customer reported that notes showed up in the wrong workspace.\n\nCan you investigate what is going on and fix it?", - "promptSourcePath": "evals/resolve-security-002-rls-cross-tenant-leak/PROMPT.md", + "prompt": "I'm moving off the hosted Supabase and running the whole thing myself on a VPS I\njust spun up. Can you get a Docker setup ready for me to copy onto the box?\n\nI don't need it running here, I'll do the actual bring-up once I'm on the\nserver. I just want everything in place and the secrets set up. Put it in a `supabase-docker/`\nfolder at the repo root so I can scp the whole thing across in one go.", + "promptSourcePath": "evals/deploy-self-hosting-001-docker-compose/PROMPT.md", "attempts": 1, - "sourcePath": "claude-code-sonnet-5-no-skills/resolve-security-002-rls-cross-tenant-leak.json" + "sourcePath": "claude-code-opus-4.8-no-skills/deploy-self-hosting-001-docker-compose.json" }, { - "experiment": "codex-gpt-5.4-mini", - "experimentSuite": "benchmark", + "experiment": "claude-code-opus-4.8-no-skills", + "experimentSuite": "no-skills", "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.4-mini", - "reasoningEffort": "medium" + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-opus-4-8", + "reasoningEffort": "high" }, - "eval": "build-cli-001-bootstrap-app", - "stage": "build", + "eval": "investigate-auth-001-deleted-user-access", + "stage": "investigate", "product": [ - "database", - "data-api" + "auth" ], "topic": [ - "migrations", - "rls" + "security", + "sdk" ], "suite": "benchmark", - "interface": "cli", + "interface": "mcp", "passed": true, "checks": [ { - "name": "supabase project initialised (supabase/config.toml exists)", + "name": "victim session active before delete-account", "passed": true }, { - "name": "todos table is created by a migration file", + "name": "delete_account flow ran for the victim", "passed": true }, { - "name": "todos table exists with at least 2 seeded rows", + "name": "delete-account revokes the user's sessions", "passed": true, - "notes": "found 2 rows" + "notes": "sessions left: 0" }, { - "name": "row level security is enabled on todos", + "name": "deleted user's refresh token is rejected", "passed": true }, { - "name": "a SELECT policy targets the authenticated role", + "name": "deleted user cannot sign back in", "passed": true }, { - "name": "REST API returns no todos to anonymous requests", - "passed": true, - "notes": "0 rows" + "name": "other users keep their sessions and access", + "passed": true }, { - "name": "REST API returns the todos to authenticated requests", + "name": "diagnosed and explained session revocation", "passed": true, - "notes": "2 rows" + "judgeNotes": "Meets all rubric requirements: diagnoses soft-delete-only flow, implements auth user deletion/session and refresh-token revocation plus RLS guard, explains stateless JWT expiry window consistently with that fix, and correctly distinguishes publishable vs secret keys and RLS behavior." } ], "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" - ] + "available": [], + "loaded": [] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"RLS expose table anon authenticated policies select insert update supabase local development migrations\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows", - "title": "Local development workflow" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", - "title": "Row Level Security" - }, - { - "url": "https://supabase.com/docs/guides/api/securing-your-api", - "title": "Securing your API" - }, - { - "url": "https://supabase.com/docs/guides/auth/quickstarts/with-expo-react-native-social-auth", - "title": "Build a Social Auth App with Expo React Native" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-redwoodjs", - "title": "Build a User Management App with RedwoodJS" - } - ], - "resultChars": 163038 - } - ] + "calls": [] }, - "prompt": "We're kicking off a todos app and I want the Supabase side ready for the team\nto build on. Set it up the way we'd run it in development, with schema changes\ntracked as migrations so they can be reviewed and replayed.\n\nFor the first slice we just need a `todos` table. Todos aren't public: anyone\nsigned in can read all of them, but nothing should be writable through the API\nfor now. Add a couple of sample todos so there's something to look at.\n\nBefore you hand it back, make sure the running API actually behaves that way —\nsigned-in users get the todos, signed-out requests get nothing.", - "promptSourcePath": "evals/build-cli-001-bootstrap-app/PROMPT.md", + "prompt": "Last week support removed a user through our app's delete-account flow — the\napp calls the `delete_account` function over RPC as the signed-in user. This\nmorning that same person was back: still signed in, reading and saving their\ndata like nothing happened.\n\nFigure out why the account still works, fix the flow so a deleted account\nloses access, and tell me whether there is any window where they could still\nget in after the fix.\n\nOne more thing while you're at it: we're migrating off the legacy\nanon/service_role API keys to the new publishable and secret keys, and the\nteam isn't sure which one belongs in the frontend and what each means for\nRLS. Give us a quick rundown.", + "promptSourcePath": "evals/investigate-auth-001-deleted-user-access/PROMPT.md", "attempts": 1, - "sourcePath": "codex-gpt-5.4-mini/build-cli-001-bootstrap-app.json" + "sourcePath": "claude-code-opus-4.8-no-skills/investigate-auth-001-deleted-user-access.json" }, { - "experiment": "codex-gpt-5.4-mini", - "experimentSuite": "benchmark", + "experiment": "claude-code-opus-4.8-no-skills", + "experimentSuite": "no-skills", "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.4-mini", - "reasoningEffort": "medium" + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-opus-4-8", + "reasoningEffort": "high" }, - "eval": "build-cli-002-declarative-schema", - "stage": "build", + "eval": "investigate-realtime-001-subscribed-no-events", + "stage": "investigate", "product": [ + "realtime", "database" ], "topic": [ - "declarative-schema", - "migrations" + "sdk" ], "suite": "benchmark", - "interface": "cli", + "interface": "mcp", "passed": true, "checks": [ { - "name": "supabase db diff used to generate the migration", + "name": "orders table added to supabase_realtime publication", "passed": true }, { - "name": "schema file updated to include description column", + "name": "courier_locations still in supabase_realtime publication", "passed": true }, { - "name": "a new migration was generated for the change", + "name": "publication still publishes INSERT events", "passed": true }, { - "name": "description column exists in the live database", + "name": "RLS still enabled on orders", "passed": true + }, + { + "name": "staff can still read orders through RLS", + "passed": true, + "notes": "authenticated sees 2 of 2 orders" + }, + { + "name": "diagnosed missing publication membership", + "passed": true, + "judgeNotes": "The answer identifies the root cause as orders missing from the supabase_realtime publication despite SUBSCRIBED, applies ALTER PUBLICATION supabase_realtime ADD TABLE public.orders, preserves courier_locations and existing RLS/policies, and does not blame or change RLS/client/networking." } ], "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" - ] + "available": [], + "loaded": [] }, "docs": { "calls": [] }, - "prompt": "Add a description text column to the `products` table in my local Supabase stack", - "promptSourcePath": "evals/build-cli-002-declarative-schema/PROMPT.md", + "prompt": "Our dispatch dashboard shows incoming orders as they happen. The courier\nlocation feed on the same page updates live without problems, but new orders\nonly show up after a page refresh.\n\nThe dashboard uses supabase-js to subscribe to INSERT events on the `orders`\ntable through postgres_changes, the same way it subscribes to courier\nlocations. The channel's status callback logs SUBSCRIBED and there are no\nerrors in the browser console.\n\nFigure out why no order events ever arrive and fix it.", + "promptSourcePath": "evals/investigate-realtime-001-subscribed-no-events/PROMPT.md", "attempts": 1, - "sourcePath": "codex-gpt-5.4-mini/build-cli-002-declarative-schema.json" + "sourcePath": "claude-code-opus-4.8-no-skills/investigate-realtime-001-subscribed-no-events.json" }, { - "experiment": "codex-gpt-5.4-mini", - "experimentSuite": "benchmark", + "experiment": "claude-code-opus-4.8-no-skills", + "experimentSuite": "no-skills", "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.4-mini", - "reasoningEffort": "medium" + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-opus-4-8", + "reasoningEffort": "high" }, - "eval": "build-cli-003-pg-cron-queue-workflow", - "stage": "build", + "eval": "investigate-reliability-003-edge-function-5xx-correlation", + "stage": "investigate", "product": [ - "database", - "edge-functions", - "cron", - "queues" + "edge-functions" ], "topic": [ - "sql", - "sdk" + "observability" ], "suite": "benchmark", - "interface": "cli", + "interface": "mcp", "passed": true, "checks": [ { - "name": "pg_cron job 'enqueue-tasks' scheduled to run every minute", + "name": "identified image-transform and the recurring 503 pattern", "passed": true, - "notes": "schedule='* * * * *', active=true" + "judgeNotes": "Identified image-transform as the main affected function and described the recurring 503 pattern across the morning of 2026-04-28, including most/all eight gateway failures from 07:00Z to 12:00Z. Also correctly distinguished older billing-webhook noise." }, { - "name": "cron command enqueues to the 'tasks' queue", + "name": "attributed recurring 503s to gateway/platform layer, not function code", "passed": true, - "notes": "queue depth 1 -> 2" + "judgeNotes": "The assistant clearly attributes the recurring image-transform 503s to the gateway/platform layer, not function code. It grounds this in valid observations: 503s appear only in gateway/API logs with no runtime rows, successful nearby invocations reached runtime, and the avatar-upload 500 is distinguished as a separate function-level error." }, { - "name": "process-tasks function drains the queue", + "name": "recommended a concrete next step", "passed": true, - "notes": "function removed the seeded message (id 4) from the queue" + "judgeNotes": "The assistant provided concrete actionable next steps: confirm platform/worker availability with Supabase, check memory/CPU/concurrency limits, identify the scheduled traffic source, add retry/backoff, and triage the separate avatar-upload 500." } ], "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" - ] + "available": [], + "loaded": [] }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"pg_cron schedule cron jobs Supabase local development\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/cron", - "title": "Cron" - }, - { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", - "title": "Automatic embeddings" - }, - { - "url": "https://supabase.com/docs/guides/auth/auth-hooks/send-email-hook", - "title": "Send Email Hook" - }, - { - "url": "https://supabase.com/docs/guides/cron/install", - "title": "Install" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net", - "title": "pg_net: Async Networking" - } - ], - "resultChars": 94119 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase Queues pgmq pop send delete read edge functions local development\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/queues", - "title": "Supabase Queues" - }, - { - "url": "https://supabase.com/docs/guides/queues/consuming-messages-with-edge-functions", - "title": "Consuming Supabase Queue Messages with Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/queues/api", - "title": "API" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli/testing-and-linting", - "title": "Testing and linting" - }, - { - "url": "https://supabase.com/docs/guides/functions/architecture", - "title": "Edge Functions Architecture" - } - ], - "resultChars": 18872 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase edge functions local development deploy serve deno\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/quickstart", - "title": "Getting Started with Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/functions/quickstart-dashboard", - "title": "Getting Started with Edge Functions (Dashboard)" - }, - { - "url": "https://supabase.com/docs/guides/functions", - "title": "Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/functions/development-environment", - "title": "Development Environment" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/mcp-server-mcp-lite", - "title": "Building an MCP Server with mcp-lite" - } - ], - "resultChars": 47492 - } - ] + "docs": { + "calls": [] }, - "prompt": "I want to set up a recurring background workflow on my local Supabase stack.\n\nCan you set up a cron job called `enqueue-tasks` to run every minute and push a task into a queue called `tasks`? Then add a `process-tasks` edge function that reads messages off the `tasks` queue and removes them, so a scheduled worker can keep the backlog drained.", - "promptSourcePath": "evals/build-cli-003-pg-cron-queue-workflow/PROMPT.md", + "prompt": "Users have been reporting that image uploads are intermittently failing with server errors throughout the morning of `2026-04-28`.\n\nCan you investigate the project logs and tell me what is going on and what we\nshould do next?", + "promptSourcePath": "evals/investigate-reliability-003-edge-function-5xx-correlation/PROMPT.md", "attempts": 1, - "sourcePath": "codex-gpt-5.4-mini/build-cli-003-pg-cron-queue-workflow.json" + "sourcePath": "claude-code-opus-4.8-no-skills/investigate-reliability-003-edge-function-5xx-correlation.json" }, { - "experiment": "codex-gpt-5.4-mini", - "experimentSuite": "benchmark", + "experiment": "claude-code-opus-4.8-no-skills", + "experimentSuite": "no-skills", "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.4-mini", - "reasoningEffort": "medium" + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-opus-4-8", + "reasoningEffort": "high" }, - "eval": "build-database-001-migrate-postgres-to-supabase", - "stage": "build", + "eval": "resolve-dataapi-001-empty-results", + "stage": "resolve", + "product": [ + "data-api", + "database", + "auth" + ], + "topic": [ + "rls", + "sdk" + ], + "suite": "benchmark", + "interface": "mcp", + "passed": true, + "checks": [ + { + "name": "RLS still enabled on bookmarks", + "passed": true + }, + { + "name": "user A reads own bookmarks", + "passed": true + }, + { + "name": "user B cannot read user A bookmarks", + "passed": true + }, + { + "name": "anon reads no bookmarks", + "passed": true + }, + { + "name": "user A can save a new bookmark", + "passed": true + }, + { + "name": "user B cannot insert a bookmark as user A", + "passed": true + }, + { + "name": "diagnosed RLS and added owner-scoped policies", + "passed": true, + "judgeNotes": "Diagnosed RLS enabled with no policies causing deny-all Data API results, kept RLS enabled, and created authenticated SELECT and INSERT policies scoped to user_id = auth.uid() with WITH CHECK for inserts." + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] + }, + "prompt": "Our app lets signed-in users save bookmarks and view them on their dashboard. Bookmarks are stored in the `bookmarks` table and are private — a user must only ever see their own. \nUsers also need to be able to save new bookmarks from the app.\n\nI can see the rows when I query the table directly, but the dashboard shows an empty list for every user.\n\nFind out why the Data API returns nothing and fix it.", + "promptSourcePath": "evals/resolve-dataapi-001-empty-results/PROMPT.md", + "attempts": 1, + "sourcePath": "claude-code-opus-4.8-no-skills/resolve-dataapi-001-empty-results.json" + }, + { + "experiment": "claude-code-opus-4.8-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-opus-4-8", + "reasoningEffort": "high" + }, + "eval": "resolve-database-001-migration-history-mismatch", + "stage": "resolve", "product": [ "database" ], @@ -7232,397 +2534,224 @@ "passed": true, "checks": [ { - "name": "all 3 tables exist (teams, members, tasks)", + "name": "the avatar_url column is applied on the hosted profiles table", "passed": true }, { - "name": "row counts match (teams=5, members=10, tasks=13)", + "name": "migration 20240220000000 is recorded in the remote history", "passed": true }, { - "name": "foreign key constraints survived the restore", + "name": "remote migration history matches local migration files", "passed": true }, { - "name": "tasks_team_status_idx index survived the restore", + "name": "local migrations are a valid reconciled sequence", "passed": true }, { - "name": "sequences synced (next insert won't conflict with existing IDs)", + "name": "production profile data is intact (not reset)", "passed": true + }, + { + "name": "the avatar migration and history reconciliation were done via the Supabase CLI", + "passed": true, + "judgeNotes": "Avatar migration was applied through `supabase db push` in #17, with output `Applying migration 20240220000000_add_avatar_url.sql...` and `Finished supabase db push.` The orphan bio migration was reconciled by adding the missing local file `20240115000000_add_profile_bio.sql` (#15), after which `supabase migration list` aligned and `supabase db push` proceeded. No disallowed workaround or direct SQL mutation was used; psql was only used for read-only verification." } ], "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" - ] + "available": [], + "loaded": [] }, "docs": { "calls": [] }, - "prompt": "I have an existing Postgres database I want to migrate to Supabase. There's a binary dump at `source.dump` in the current directory.\n\nCan you set up a local Supabase project and restore the dump into it?", - "promptSourcePath": "evals/build-database-001-migrate-postgres-to-supabase/PROMPT.md", + "prompt": "I'm trying to ship a migration to our hosted project and it's not working. Can you figure out what's wrong and get it deployed?", + "promptSourcePath": "evals/resolve-database-001-migration-history-mismatch/PROMPT.md", "attempts": 1, - "sourcePath": "codex-gpt-5.4-mini/build-database-001-migrate-postgres-to-supabase.json" + "sourcePath": "claude-code-opus-4.8-no-skills/resolve-database-001-migration-history-mismatch.json" }, { - "experiment": "codex-gpt-5.4-mini", - "experimentSuite": "benchmark", + "experiment": "claude-code-opus-4.8-no-skills", + "experimentSuite": "no-skills", "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.4-mini", - "reasoningEffort": "medium" + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-opus-4-8", + "reasoningEffort": "high" }, - "eval": "build-functions-004-service-role-bypass", - "stage": "build", + "eval": "resolve-performance-001-slow-query-cpu-spike", + "stage": "resolve", "product": [ - "edge-functions", - "auth", "database" ], "topic": [ - "rls", - "security", - "sdk" + "observability", + "sql" ], "suite": "benchmark", "interface": "mcp", "passed": true, "checks": [ { - "name": "rejects missing auth", - "passed": true, - "notes": "status=401" + "name": "inspected pg_stat_statements for query performance", + "passed": true }, { - "name": "user A reads own note", - "passed": true, - "notes": "status=200" + "name": "ran EXPLAIN on the expensive query", + "passed": true }, { - "name": "reads only with the caller's JWT", - "passed": true, - "notes": "bearer_tokens=2, all_match=true" + "name": "created index covering user_id and created_at", + "passed": true }, { - "name": "user A cannot force-read user B note", + "name": "query plan uses an index and avoids sequential scan", "passed": true, - "notes": "status=200" + "notes": "Limit (cost=55.55..55.61 rows=25 width=88)\n -> Sort (cost=55.55..55.61 rows=25 width=88)\n Sort Key: created_at DESC\n -> Bitmap Heap Scan on events (cost=4.48..54.97 rows=25 width=88)\n Recheck Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)\n -> Bitmap Index Scan on events_user_id_created_at_idx (cost=0.00..4.47 rows=25 width=0)\n Index Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)" }, { - "name": "user B cannot force-read user A note", - "passed": true, - "notes": "status=200" + "name": "inserts still work", + "passed": true } ], "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" - ] + "available": [], + "loaded": [] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query {\n searchDocs(query: \"Supabase Edge Functions verify JWT Authorization header createClient service role key anon key auth.getUser\", limit: 5) {\n nodes {\n title\n href\n content\n }\n }\n}", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", - "title": "New API Keys and Asymmetric Authentication" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/why-is-my-service-role-key-client-getting-rls-errors-or-not-returning-data-7_1K9z", - "title": "Why is my service role key client getting RLS errors or not returning data?" - } - ], - "resultChars": 45333 - }, - { - "source": "web_search", - "query": "https://supabase.com/changelog.md", - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ] - }, - { - "source": "search_docs", - "query": "query {\n searchDocs(query: \"Supabase Edge Functions environment variables SUPABASE_ANON_KEY SUPABASE_PUBLISHABLE_KEY\", limit: 10) {\n nodes {\n title\n href\n content\n }\n }\n}", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/functions/secrets", - "title": "Environment Variables" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", - "title": "Self-Hosted Functions" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", - "title": "New API Keys and Asymmetric Authentication" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/auth/quickstarts/with-expo-react-native-social-auth", - "title": "Build a Social Auth App with Expo React Native" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream", - "title": "Streaming Speech with ElevenLabs" - }, - { - "url": "https://supabase.com/docs/guides/integrations/vercel-marketplace", - "title": "Vercel Marketplace" - }, - { - "url": "https://supabase.com/docs/guides/auth/server-side/creating-a-client", - "title": "Creating a Supabase client for SSR" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" - } - ], - "resultChars": 189134 - } - ] + "calls": [] }, - "prompt": "I built an Edge Function called `private-notes` for showing a user's saved\nprivate notes.\n\nCan you check whether there's any way one user could see another user's notes?\n\nPlease fix and deploy it if needed.", - "promptSourcePath": "evals/build-functions-004-service-role-bypass/PROMPT.md", + "prompt": "My database CPU keeps spiking and the app gets slow when loading recent events for a user. Can you figure out what query is causing it and make the database change needed to fix it?\n\nEnd your turn with a short summary of what you changed and why.", + "promptSourcePath": "evals/resolve-performance-001-slow-query-cpu-spike/PROMPT.md", "attempts": 1, - "sourcePath": "codex-gpt-5.4-mini/build-functions-004-service-role-bypass.json" + "sourcePath": "claude-code-opus-4.8-no-skills/resolve-performance-001-slow-query-cpu-spike.json" }, { - "experiment": "codex-gpt-5.4-mini", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.4-mini", - "reasoningEffort": "medium" + "experiment": "claude-code-opus-4.8-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-opus-4-8", + "reasoningEffort": "high" }, - "eval": "build-functions-005-dual-auth-user-secret", - "stage": "build", + "eval": "resolve-security-002-rls-cross-tenant-leak", + "stage": "resolve", "product": [ - "edge-functions", - "auth", - "database" + "database", + "auth" ], "topic": [ - "sdk", "rls", "security" ], "suite": "benchmark", - "interface": "cli", - "cliVersion": "2.109.1", - "passed": false, + "interface": "mcp", + "passed": true, "checks": [ { - "name": "read stack config from `supabase status`", - "passed": false, - "notes": "missing API_URL/SECRET_KEY/PUBLISHABLE_KEY — new API keys are required for @supabase/server; is the stack running on a new-enough CLI? got keys: ANON_KEY, DB_URL, JWT_SECRET, PUBLISHABLE_KEY, SECRET_KEY, SERVICE_ROLE_KEY" + "name": "RLS enabled on notes", + "passed": true + }, + { + "name": "tenant A sees only org A notes", + "passed": true + }, + { + "name": "tenant B cannot read org A notes", + "passed": true + }, + { + "name": "tenant A author can update own note", + "passed": true + }, + { + "name": "tenant B cannot update org A note", + "passed": true + }, + { + "name": "tenant B author can delete own note", + "passed": true + }, + { + "name": "tenant B cannot delete org A note", + "passed": true + }, + { + "name": "tenant A can insert note in own org", + "passed": true + }, + { + "name": "tenant B cannot insert into org A", + "passed": true } ], "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] + "available": [], + "loaded": [] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Edge Functions verify JWT service role apikey header auth.getUser Deno.serve Supabase\", limit: 5) { nodes { __typename title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" - }, - { - "url": "https://supabase.com/docs/guides/functions/websockets", - "title": "Handling WebSockets" - }, - { - "url": "https://supabase.com/docs/guides/functions", - "title": "Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/resumable-websockets", - "title": "Resumable WebSockets with Edge Functions" - } - ], - "resultChars": 49160 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"functions verify_jwt false config.toml edge function\", limit: 5) { nodes { __typename title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/function-configuration", - "title": "Function Configuration" - }, - { - "url": "https://supabase.com/docs/guides/functions/deploy", - "title": "Deploy to Production" - }, - { - "url": "https://supabase.com/docs/guides/functions/development-tips", - "title": "Development tips" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/github-actions", - "title": "GitHub Actions" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth-headers", - "title": "Authorization headers" - } - ], - "resultChars": 20522 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"supabase functions serve entrypoint config.toml no-verify-jwt import_map deno.json\", limit: 5) { nodes { __typename title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/development-tips", - "title": "Development tips" - }, - { - "url": "https://supabase.com/docs/guides/functions/dependencies", - "title": "Managing dependencies" - }, - { - "url": "https://supabase.com/docs/guides/functions/function-configuration", - "title": "Function Configuration" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/mcp-server-mcp-lite", - "title": "Building an MCP Server with mcp-lite" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", - "title": "Self-Hosted Functions" - } - ], - "resultChars": 32932 - }, - { - "source": "web_search", - "query": "\"failed to determine entrypoint\" \"No .npmrc file found\" Supabase functions serve", - "pages": [] - } - ] + "calls": [] }, - "prompt": "Build and serve a Supabase Edge Function named `user-stats` for this project,\nreachable over HTTP at `/functions/v1/user-stats`.\n\nOur product stores per-user metrics in the existing `user_stats` table.\n\nTwo very different callers need to hit this one endpoint:\n\n1. **Our mobile app**, acting for a signed-in user. It sends that user's\n Supabase access token. The endpoint should return the user's own stats.\n\n2. **Our internal billing service**, a trusted backend with no signed-in user.\n It authenticates with the project's secret (service-role) key in the `apikey`\n header, and names the target user with a `user_id` in the JSON request body.\n It needs that user's stats.\n\nReturn the matching rows as JSON. The endpoint must be secure: only ever serve\nstats to a caller that is genuinely entitled to them, and turn away callers that\nare not.", - "promptSourcePath": "evals/build-functions-005-dual-auth-user-secret/PROMPT.md", - "attempts": 2, - "sourcePath": "codex-gpt-5.4-mini/build-functions-005-dual-auth-user-secret.json" + "prompt": "A customer reported that notes showed up in the wrong workspace.\n\nCan you investigate what is going on and fix it?", + "promptSourcePath": "evals/resolve-security-002-rls-cross-tenant-leak/PROMPT.md", + "attempts": 1, + "sourcePath": "claude-code-opus-4.8-no-skills/resolve-security-002-rls-cross-tenant-leak.json" }, { - "experiment": "codex-gpt-5.4-mini", + "experiment": "claude-code-sonnet-5", "experimentSuite": "benchmark", "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.4-mini", - "reasoningEffort": "medium" + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" }, - "eval": "build-storage-001-private-bucket-access", + "eval": "build-cli-001-bootstrap-app", "stage": "build", "product": [ - "storage", - "database" + "database", + "data-api" ], "topic": [ - "rls", - "sdk" + "migrations", + "rls" ], "suite": "benchmark", - "interface": "mcp", + "interface": "cli", "passed": true, "checks": [ { - "name": "bucket user-files exists", - "passed": true - }, - { - "name": "bucket user-files is private", + "name": "supabase project initialised (supabase/config.toml exists)", "passed": true }, { - "name": "RLS still enabled on storage.objects", + "name": "todos table is created by a migration file", "passed": true }, { - "name": "user A lists only own files", + "name": "todos table exists with at least 2 seeded rows", "passed": true, - "notes": "saw: 019fb0e7-3951-71a0-8d70-ee9bba881248/receipt-alpha.pdf, 019fb0e7-3951-71a0-8d70-ee9bba881248/receipt-beta.pdf" - }, - { - "name": "user B cannot read user A files", - "passed": true + "notes": "found 3 rows" }, { - "name": "anon reads no files", + "name": "row level security is enabled on todos", "passed": true }, { - "name": "user A can upload into own folder", + "name": "a SELECT policy targets the authenticated role", "passed": true }, { - "name": "user B cannot upload into user A folder", - "passed": true + "name": "REST API returns no todos to anonymous requests", + "passed": true, + "notes": "0 rows" }, { - "name": "configured private per-user storage access", + "name": "REST API returns the todos to authenticated requests", "passed": true, - "judgeNotes": "Creates private user-files bucket, authenticated owner-scoped SELECT and INSERT policies with WITH CHECK using first path segment = auth.uid(), keeps RLS enabled, and provides createSignedUrl code with expiry. No disallowed public bucket/getPublicUrl/service-role usage." + "notes": "3 rows" } ], "skills": { @@ -7635,98 +2764,50 @@ ] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"storage policies signed urls user owns files bucket create policies\", limit: 5) { nodes { __typename ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/storage/buckets/fundamentals", - "title": "Storage Buckets" - }, - { - "url": "https://supabase.com/docs/guides/storage/security/access-control", - "title": "Storage Access Control" - }, - { - "url": "https://supabase.com/docs/guides/storage/schema/custom-roles", - "title": "Custom Roles" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3", - "title": "Configure S3 Storage" - } - ], - "resultChars": 20163 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"createSignedUrl storage objects select policy signed url private bucket\", limit: 5) { nodes { __typename ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/storage/buckets/fundamentals", - "title": "Storage Buckets" - }, - { - "url": "https://supabase.com/docs/guides/storage/serving/downloads", - "title": "Serving assets from Storage" - }, - { - "url": "https://supabase.com/docs/guides/storage/cdn/smart-cdn", - "title": "Smart CDN" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3", - "title": "Configure S3 Storage" - } - ], - "resultChars": 21896 - } - ] + "calls": [] }, - "prompt": "Our app lets signed-in users keep personal files like receipts and bank\nstatements. These files are private — a user must only ever be able to upload\nand download their own. The app uploads each file under a path that starts\nwith the owner's user id, e.g. `/receipt-march.pdf`.\n\nSet up a `user-files` bucket on our project and lock it down that way.\n\nUsers also sometimes share one of their files with someone else through a\ntemporary link that expires. Include the supabase-js code the app should use\nfor that.", - "promptSourcePath": "evals/build-storage-001-private-bucket-access/PROMPT.md", + "prompt": "We're kicking off a todos app and I want the Supabase side ready for the team\nto build on. Set it up the way we'd run it in development, with schema changes\ntracked as migrations so they can be reviewed and replayed.\n\nFor the first slice we just need a `todos` table. Todos aren't public: anyone\nsigned in can read all of them, but nothing should be writable through the API\nfor now. Add a couple of sample todos so there's something to look at.\n\nBefore you hand it back, make sure the running API actually behaves that way —\nsigned-in users get the todos, signed-out requests get nothing.", + "promptSourcePath": "evals/build-cli-001-bootstrap-app/PROMPT.md", "attempts": 1, - "sourcePath": "codex-gpt-5.4-mini/build-storage-001-private-bucket-access.json" + "sourcePath": "claude-code-sonnet-5/build-cli-001-bootstrap-app.json" }, { - "experiment": "codex-gpt-5.4-mini", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.4-mini", - "reasoningEffort": "medium" + "experiment": "claude-code-sonnet-5", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" }, - "eval": "build-tests-001-rls-tenant-isolation", + "eval": "build-cli-002-declarative-schema", "stage": "build", "product": [ "database" ], "topic": [ - "tests", - "rls" + "declarative-schema", + "migrations" ], "suite": "benchmark", "interface": "cli", "passed": true, "checks": [ { - "name": "pgTAP test file(s) written under supabase/tests/", - "passed": true, - "notes": "1 file(s): supabase/tests/database/tenant_isolation.test.sql" + "name": "supabase db diff used to generate the migration", + "passed": true }, { - "name": "pgTAP isolation tests ran and pass", - "passed": true, - "notes": "4 passed, 0 failed" + "name": "schema file updated to include description column", + "passed": true }, { - "name": "agent correctly identifies the posts isolation bug from test results", - "passed": true, - "judgeNotes": "The agent correctly identifies `posts` as the table with the tenant isolation flaw, states that `notes` was already scoped correctly, explains that authenticated members could read posts from other orgs, and cites the pgTAP verification results after fixing the policy." + "name": "a new migration was generated for the change", + "passed": true + }, + { + "name": "description column exists in the live database", + "passed": true } ], "skills": { @@ -7739,87 +2820,52 @@ ] }, "docs": { - "calls": [ - { - "source": "web_search", - "query": "site:supabase.com/changelog.md Supabase changelog breaking-change row level security testing", - "pages": [] - }, - { - "source": "web_search", - "query": "site:supabase.com/docs \"RLS policies testing\" Supabase pgTAP", - "pages": [] - }, - { - "source": "web_search", - "query": "site:supabase.com/docs \"RLS policies testing\" pgTAP auth.uid() set_config request.jwt.claim.sub", - "pages": [] - }, - { - "source": "web_search", - "query": "site:supabase.com/docs \"RLS policies testing\" \"set_config\" Supabase", - "pages": [] - } - ] + "calls": [] }, - "prompt": "Can you audit the tenant isolation on our tables? Write some database tests covering both the happy path and the negative case, run them, and share the results.", - "promptSourcePath": "evals/build-tests-001-rls-tenant-isolation/PROMPT.md", + "prompt": "Add a description text column to the `products` table in my local Supabase stack", + "promptSourcePath": "evals/build-cli-002-declarative-schema/PROMPT.md", "attempts": 1, - "sourcePath": "codex-gpt-5.4-mini/build-tests-001-rls-tenant-isolation.json" + "sourcePath": "claude-code-sonnet-5/build-cli-002-declarative-schema.json" }, { - "experiment": "codex-gpt-5.4-mini", + "experiment": "claude-code-sonnet-5", "experimentSuite": "benchmark", "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.4-mini", - "reasoningEffort": "medium" + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" }, - "eval": "build-vectors-001-rag-with-permissions", + "eval": "build-cli-003-pg-cron-queue-workflow", "stage": "build", "product": [ "database", - "vectors" + "edge-functions", + "cron", + "queues" ], "topic": [ "sql", - "rls" + "sdk" ], "suite": "benchmark", - "interface": "mcp", + "interface": "cli", "passed": true, "checks": [ { - "name": "document_sections.embedding is vector(384)", + "name": "pg_cron job 'enqueue-tasks' scheduled to run every minute", "passed": true, - "notes": "vector(384)" + "notes": "schedule='* * * * *', active=true" }, { - "name": "HNSW index on the embedding column", + "name": "cron command enqueues to the 'tasks' queue", "passed": true, - "notes": "CREATE INDEX document_sections_embedding_idx ON public.document_sections USING hnsw (embedding vector_cosine_ops)" + "notes": "queue depth 1 -> 2" }, { - "name": "index operator class matches the search operator", + "name": "process-tasks function drains the queue", "passed": true, - "notes": "function operators: <=>\nindexes: CREATE INDEX document_sections_embedding_idx ON public.document_sections USING hnsw (embedding vector_cosine_ops)" - }, - { - "name": "user A search returns only own sections, best match first", - "passed": true - }, - { - "name": "user B search returns only own sections, best match first", - "passed": true - }, - { - "name": "user A reads only own sections through the API", - "passed": true - }, - { - "name": "user A reads only own documents through the API", - "passed": true + "notes": "function removed the seeded message (id 5) from the queue" } ], "skills": { @@ -7835,214 +2881,147 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"pgvector semantic search embeddings RLS Supabase\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/ai/rag-with-permissions", - "title": "RAG with Permissions" - }, - { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", - "title": "Automatic embeddings" - }, - { - "url": "https://supabase.com/docs/guides/ai/hybrid-search", - "title": "Hybrid search" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pgvector", - "title": "pgvector: Embeddings and vector similarity" - }, - { - "url": "https://supabase.com/docs/guides/ai/semantic-search", - "title": "Semantic search" - } - ], - "resultChars": 76895 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"pgvector vector type without dimension Supabase\", limit: 5) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"pg_cron schedule job queues pgmq send\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/ai/vector-columns", - "title": "Vector columns" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pgvector", - "title": "pgvector: Embeddings and vector similarity" - }, - { - "url": "https://supabase.com/docs/guides/ai/vector-indexes/hnsw-indexes", - "title": "HNSW indexes" - }, - { - "url": "https://supabase.com/docs/guides/ai/semantic-search", - "title": "Semantic search" - }, - { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", - "title": "Automatic embeddings" - } - ], - "resultChars": 65961 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase RLS documents owner_id authenticated policy\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/ai/rag-with-permissions", - "title": "RAG with Permissions" + "url": "https://supabase.com/docs/guides/queues/pgmq", + "title": "PGMQ Extension" }, { - "url": "https://supabase.com/docs/guides/storage/schema/helper-functions", - "title": "Storage Helper Functions" + "url": "https://supabase.com/docs/guides/database/extensions/pg_net", + "title": "pg_net: Async Networking" }, { - "url": "https://supabase.com/docs/guides/storage/security/access-control", - "title": "Storage Access Control" + "url": "https://supabase.com/docs/guides/troubleshooting/pgcron-debugging-guide-n1KTaz", + "title": "pg_cron debugging guide" }, { - "url": "https://supabase.com/docs/guides/auth/users", - "title": "Users" + "url": "https://supabase.com/docs/guides/cron", + "title": "Cron" }, { - "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security", - "title": "Token Security and Row Level Security" + "url": "https://supabase.com/docs/guides/cron/install", + "title": "Install" } ], - "resultChars": 46468 + "resultChars": 45486 } ] }, - "prompt": "We're adding semantic search to our internal knowledge base app. I already wrote the edge functions, but search doesn't work yet. Can you set up whatever the database needs to make search work end to end?\n\nSome documents are confidential, so users should only have access to documents they own.", - "promptSourcePath": "evals/build-vectors-001-rag-with-permissions/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.4-mini/build-vectors-001-rag-with-permissions.json" + "prompt": "I want to set up a recurring background workflow on my local Supabase stack.\n\nCan you set up a cron job called `enqueue-tasks` to run every minute and push a task into a queue called `tasks`? Then add a `process-tasks` edge function that reads messages off the `tasks` queue and removes them, so a scheduled worker can keep the backlog drained.", + "promptSourcePath": "evals/build-cli-003-pg-cron-queue-workflow/PROMPT.md", + "attempts": 2, + "sourcePath": "claude-code-sonnet-5/build-cli-003-pg-cron-queue-workflow.json" }, { - "experiment": "codex-gpt-5.4-mini", + "experiment": "claude-code-sonnet-5", "experimentSuite": "benchmark", "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.4-mini", - "reasoningEffort": "medium" + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" }, - "eval": "deploy-database-001-prometheus-metrics", - "stage": "deploy", + "eval": "build-database-001-migrate-postgres-to-supabase", + "stage": "build", "product": [ "database" ], "topic": [ - "observability" + "migrations" ], "suite": "benchmark", - "passed": false, + "interface": "cli", + "passed": true, "checks": [ { - "name": "preserved existing app scrape job", + "name": "all 3 tables exist (teams, members, tasks)", "passed": true }, { - "name": "configured the Supabase Metrics API scrape correctly", - "passed": false, - "judgeNotes": "Fails because Prometheus uses basic_auth.password with an environment variable instead of basic_auth.password_file, and docker-compose.yml does not mount that password_file via a volume or Compose secret. The app scrape is preserved and the endpoint/path/scheme are otherwise correct." + "name": "row counts match (teams=5, members=10, tasks=13)", + "passed": true }, { - "name": "documented live deployment and verification steps", - "passed": false, - "judgeNotes": "README explains creating a Supabase Secret API key, placing it in observability/.env, and restarting the Compose stack, but it lacks concrete verification steps such as checking Prometheus targets or running a PromQL/Grafana query to confirm the Supabase scrape is live." + "name": "foreign key constraints survived the restore", + "passed": true + }, + { + "name": "tasks_team_status_idx index survived the restore", + "passed": true + }, + { + "name": "sequences synced (next insert won't conflict with existing IDs)", + "passed": true } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" - ] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query {\n searchDocs(query: \"project metrics observability prometheus metrics endpoint\", limit: 5) {\n nodes {\n title\n href\n content\n }\n }\n}", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/telemetry/metrics", - "title": "Metrics API" - }, - { - "url": "https://supabase.com/docs/guides/telemetry/metrics/grafana-self-hosted", - "title": "Metrics API with Prometheus & Grafana (self-hosted)" - }, - { - "url": "https://supabase.com/docs/guides/telemetry/metrics/vendor-agnostic", - "title": "Vendor-agnostic Metrics API setup" - }, - { - "url": "https://supabase.com/docs/guides/telemetry/metrics/grafana-cloud", - "title": "Metrics API with Grafana Cloud" - }, - { - "url": "https://supabase.com/docs/guides/platform/read-replicas", - "title": "Read Replicas" - } - ], - "resultChars": 32694 - } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" ] }, - "prompt": "Can you wire my Supabase project metrics into our existing observability stack and document\nin the observability README what we need to do to make the config live?", - "promptSourcePath": "evals/deploy-database-001-prometheus-metrics/PROMPT.md", - "attempts": 2, - "sourcePath": "codex-gpt-5.4-mini/deploy-database-001-prometheus-metrics.json" + "docs": { + "calls": [] + }, + "prompt": "I have an existing Postgres database I want to migrate to Supabase. There's a binary dump at `source.dump` in the current directory.\n\nCan you set up a local Supabase project and restore the dump into it?", + "promptSourcePath": "evals/build-database-001-migrate-postgres-to-supabase/PROMPT.md", + "attempts": 1, + "sourcePath": "claude-code-sonnet-5/build-database-001-migrate-postgres-to-supabase.json" }, { - "experiment": "codex-gpt-5.4-mini", + "experiment": "claude-code-sonnet-5", "experimentSuite": "benchmark", "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.4-mini", - "reasoningEffort": "medium" + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" }, - "eval": "deploy-functions-001-edge-function-secrets", - "stage": "deploy", + "eval": "build-functions-004-service-role-bypass", + "stage": "build", "product": [ - "edge-functions" + "edge-functions", + "auth", + "database" ], "topic": [ - "security" + "rls", + "security", + "sdk" ], "suite": "benchmark", - "interface": "cli", + "interface": "mcp", "passed": true, "checks": [ { - "name": "WEATHER_API_KEY is set as a Function secret on the project", - "passed": true + "name": "rejects missing auth", + "passed": true, + "notes": "status=401" }, { - "name": "the weather function is deployed to the project", + "name": "user A reads own note", "passed": true, - "notes": "status ACTIVE" + "notes": "status=200" }, { - "name": "the weather function reads WEATHER_API_KEY from the environment", + "name": "reads only with the caller's JWT", "passed": true, - "judgeNotes": "Reads WEATHER_API_KEY from the runtime environment via Deno.env.get(\"WEATHER_API_KEY\")." + "notes": "bearer_tokens=2, all_match=true" }, { - "name": "WEATHER_API_KEY value is not committed to the repo", - "passed": true + "name": "user A cannot force-read user B note", + "passed": true, + "notes": "status=200" + }, + { + "name": "user B cannot force-read user A note", + "passed": true, + "notes": "status=200" } ], "skills": { @@ -8055,164 +3034,83 @@ ] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query:\"create edge function deno serve request supabase edge functions weather proxy fetch headers\", limit: 5) { nodes { ... on Guide { title href content } ... on CLICommandReference { title href content } ... on TroubleshootingGuide { title href content } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions", - "title": "Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/database/connecting-to-postgres/serverless-drivers", - "title": "Serverless Drivers" - }, - { - "url": "https://supabase.com/docs/guides/functions/http-methods", - "title": "Routing" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli/testing-and-linting", - "title": "Testing and linting" - }, - { - "url": "https://supabase.com/docs/guides/functions/cors", - "title": "CORS (Cross-Origin Resource Sharing) support for Invoking from the browser" - } - ], - "resultChars": 32587 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query:\"Edge Functions secrets environment variables deploy functions supabase secrets set edge function runtime environment variable\", limit: 5) { nodes { ... on Guide { title href content } ... on CLICommandReference { title href content } ... on ManagementApiReference { title href content } ... on TroubleshootingGuide { title href content } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", - "title": "Automatic embeddings" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream", - "title": "Streaming Speech with ElevenLabs" - }, - { - "url": "https://supabase.com/docs/guides/functions/secrets", - "title": "Environment Variables" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech", - "title": "Transcription Telegram Bot" - } - ], - "resultChars": 70876 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query:\"Edge Functions invoke curl apikey Authorization header no verify jwt public function invoke\", limit: 5) { nodes { ... on Guide { title href content } ... on CLICommandReference { title href content } ... on TroubleshootingGuide { title href content } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/error-codes", - "title": "Error codes" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth-headers", - "title": "Authorization headers" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" - }, - { - "url": "https://supabase.com/docs/guides/functions/status-codes", - "title": "Status codes" - }, - { - "url": "https://supabase.com/docs/guides/functions/function-configuration", - "title": "Function Configuration" - } - ], - "resultChars": 26210 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query:\"Authorization headers Edge Functions apikey anon key invoke browser\", limit: 5) { nodes { ... on Guide { title href content } ... on TroubleshootingGuide { title href content } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/cors", - "title": "CORS (Cross-Origin Resource Sharing) support for Invoking from the browser" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features", - "title": "Features" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth-headers", - "title": "Authorization headers" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" - }, - { - "url": "https://supabase.com/docs/guides/functions/error-codes", - "title": "Error codes" - } - ], - "resultChars": 42697 - } - ] + "calls": [] }, - "prompt": "Our weather widget currently calls WeatherAPI straight from the browser, which\nleaks our API key. I want to move that behind a Supabase Edge Function called\n`weather` that holds the key server-side and proxies the request.\n\nThe function should read the key from an environment variable named\n`WEATHER_API_KEY`. Our key already lives in a local `.env` file at the project\nroot.\n\nDeploy the function to our project so it's live, and make sure the deployed\nfunction can actually read the key at runtime.", - "promptSourcePath": "evals/deploy-functions-001-edge-function-secrets/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.4-mini/deploy-functions-001-edge-function-secrets.json" + "prompt": "I built an Edge Function called `private-notes` for showing a user's saved\nprivate notes.\n\nCan you check whether there's any way one user could see another user's notes?\n\nPlease fix and deploy it if needed.", + "promptSourcePath": "evals/build-functions-004-service-role-bypass/PROMPT.md", + "attempts": 2, + "sourcePath": "claude-code-sonnet-5/build-functions-004-service-role-bypass.json" }, { - "experiment": "codex-gpt-5.4-mini", + "experiment": "claude-code-sonnet-5", "experimentSuite": "benchmark", "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.4-mini", - "reasoningEffort": "medium" + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" }, - "eval": "deploy-self-hosting-001-docker-compose", - "stage": "deploy", + "eval": "build-functions-005-dual-auth-user-secret", + "stage": "build", "product": [ - "database", + "edge-functions", "auth", - "storage" + "database" ], "topic": [ - "self-hosting" + "sdk", + "rls", + "security" ], "suite": "benchmark", "interface": "cli", + "cliVersion": "2.109.1", "passed": true, "checks": [ { - "name": "cloned the self-host stack (docker-compose.yml + volumes/db)", - "passed": true + "name": "seed rows present", + "passed": true, + "notes": "found 2/2 seeded rows" }, { - "name": "didn't conflate with the CLI (no supabase/config.toml in the stack)", - "passed": true + "name": "rejects request with no credentials", + "passed": true, + "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" }, { - "name": "secrets rotated off the shipped defaults", - "passed": true + "name": "user with JWT reads only their own rows", + "passed": true, + "notes": "status 200: {\"data\":[{\"user_id\":\"79bf56d6-9c00-405b-9509-b940c5ed77a8\",\"metric\":\"steps_a_mrw5onyl\",\"value\":111}]}" }, { - "name": "ANON_KEY and SERVICE_ROLE_KEY are HS256 JWTs signed by JWT_SECRET", - "passed": true + "name": "user cannot read another user's rows by passing user_id", + "passed": true, + "notes": "status 200: {\"data\":[{\"user_id\":\"79bf56d6-9c00-405b-9509-b940c5ed77a8\",\"metric\":\"steps_a_mrw5onyl\",\"value\":111}]}" + }, + { + "name": "service key bypasses RLS to read the target user's rows", + "passed": true, + "notes": "status 200: {\"data\":[{\"user_id\":\"7cb8dbff-4d10-4c1b-82eb-cd09dd8c6df6\",\"metric\":\"steps_b_mrw5onyl\",\"value\":222}]}" + }, + { + "name": "non-service key is not granted service access", + "passed": true, + "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" + }, + { + "name": "rejects an unverified (forged) user token", + "passed": true, + "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" + }, + { + "name": "a user token in the apikey slot is not treated as the service key", + "passed": true, + "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" + }, + { + "name": "implementation uses @supabase/server", + "passed": true, + "notes": "imports @supabase/server / withSupabase" } ], "skills": { @@ -8228,83 +3126,140 @@ "calls": [ { "source": "search_docs", - "query": "query {\n searchDocs(query: \"self-hosted Supabase docker compose API_EXTERNAL_URL auth/v1 envoy kong\", limit: 5) {\n nodes {\n ... on Guide {\n title\n href\n content\n }\n ... on TroubleshootingGuide {\n title\n href\n content\n }\n ... on CLICommandReference {\n title\n href\n content\n }\n }\n totalCount\n }\n}", + "query": "{ searchDocs(query: \"Edge Functions custom headers apikey secret key authentication verify_jwt\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/auth/signing-keys", + "title": "JWT Signing Keys" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth-headers", + "title": "Authorization headers" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth", + "title": "Securing Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", + "title": "Integrating With Supabase Auth" + }, + { + "url": "https://supabase.com/docs/guides/functions/error-codes", + "title": "Error codes" + } + ], + "resultChars": 59656 + }, + { + "source": "search_docs", + "query": "{ a: searchDocs(query: \"Edge Functions Deno.serve authorization header user JWT createClient auth.getUser\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", - "title": "Envoy API Gateway" + "url": "https://supabase.com/docs/guides/functions/websockets", + "title": "Handling WebSockets" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-proxy-https", - "title": "Configure Reverse Proxy and HTTPS" + "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", + "title": "Integrating With Supabase Auth" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", - "title": "New API Keys and Asymmetric Authentication" + "url": "https://supabase.com/docs/guides/functions/auth-headers", + "title": "Authorization headers" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth", - "title": "Configure Social Login (OAuth) Providers" + "url": "https://supabase.com/docs/guides/functions", + "title": "Edge Functions" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-saml-sso", - "title": "Configure SAML SSO" + "url": "https://supabase.com/docs/guides/functions/examples/resumable-websockets", + "title": "Resumable WebSockets with Edge Functions" } ], - "resultChars": 97845 + "resultChars": 39376 }, { "source": "search_docs", - "query": "query {\n searchDocs(query: \"Self-Hosting with Docker docker-compose.yml self-hosted Supabase\", limit: 5) {\n nodes {\n ... on Guide {\n title\n href\n content\n }\n }\n totalCount\n }\n}", + "query": "{ a: searchDocs(query: \"verify_jwt config.toml edge function no-verify-jwt service role\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/self-hosting/docker", - "title": "Self-Hosting with Docker" + "url": "https://supabase.com/docs/guides/functions/function-configuration", + "title": "Function Configuration" }, { - "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform", - "title": "Restore a Platform Project to Self-Hosted" + "url": "https://supabase.com/docs/guides/functions/deploy", + "title": "Deploy to Production" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-proxy-https", - "title": "Configure Reverse Proxy and HTTPS" + "url": "https://supabase.com/docs/guides/functions/development-tips", + "title": "Development tips" }, { - "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17", - "title": "Upgrade to Postgres 17" + "url": "https://supabase.com/docs/guides/functions/auth-headers", + "title": "Authorization headers" }, { - "url": "https://supabase.com/docs/guides/queues/expose-self-hosted-queues", - "title": "Expose Queues for local and self-hosted Supabase" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", + "title": "Self-Hosted Functions" + } + ], + "resultChars": 28137 + }, + { + "source": "search_docs", + "query": "{ a: searchDocs(query: \"Securing Edge Functions verifyCredentials createAdminClient createContextClient @supabase/server service role trusted backend\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" + }, + { + "url": "https://supabase.com/docs/guides/functions/examples/resumable-websockets", + "title": "Resumable WebSockets with Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", + "title": "Integrating With Supabase Auth" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth", + "title": "Securing Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth-headers", + "title": "Authorization headers" } ], - "resultChars": 69788 + "resultChars": 37362 } ] }, - "prompt": "I'm moving off the hosted Supabase and running the whole thing myself on a VPS I\njust spun up. Can you get a Docker setup ready for me to copy onto the box?\n\nI don't need it running here, I'll do the actual bring-up once I'm on the\nserver. I just want everything in place and the secrets set up. Put it in a `supabase-docker/`\nfolder at the repo root so I can scp the whole thing across in one go.", - "promptSourcePath": "evals/deploy-self-hosting-001-docker-compose/PROMPT.md", + "prompt": "Build and serve a Supabase Edge Function named `user-stats` for this project,\nreachable over HTTP at `/functions/v1/user-stats`.\n\nOur product stores per-user metrics in the existing `user_stats` table.\n\nTwo very different callers need to hit this one endpoint:\n\n1. **Our mobile app**, acting for a signed-in user. It sends that user's\n Supabase access token. The endpoint should return the user's own stats.\n\n2. **Our internal billing service**, a trusted backend with no signed-in user.\n It authenticates with the project's secret (service-role) key in the `apikey`\n header, and names the target user with a `user_id` in the JSON request body.\n It needs that user's stats.\n\nReturn the matching rows as JSON. The endpoint must be secure: only ever serve\nstats to a caller that is genuinely entitled to them, and turn away callers that\nare not.", + "promptSourcePath": "evals/build-functions-005-dual-auth-user-secret/PROMPT.md", "attempts": 1, - "sourcePath": "codex-gpt-5.4-mini/deploy-self-hosting-001-docker-compose.json" + "sourcePath": "claude-code-sonnet-5/build-functions-005-dual-auth-user-secret.json" }, { - "experiment": "codex-gpt-5.4-mini", + "experiment": "claude-code-sonnet-5", "experimentSuite": "benchmark", "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.4-mini", - "reasoningEffort": "medium" + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" }, - "eval": "investigate-auth-001-deleted-user-access", - "stage": "investigate", + "eval": "build-storage-001-private-bucket-access", + "stage": "build", "product": [ - "auth" + "storage", + "database" ], "topic": [ - "security", + "rls", "sdk" ], "suite": "benchmark", @@ -8312,34 +3267,42 @@ "passed": true, "checks": [ { - "name": "victim session active before delete-account", + "name": "bucket user-files exists", "passed": true }, { - "name": "delete_account flow ran for the victim", + "name": "bucket user-files is private", "passed": true }, { - "name": "delete-account revokes the user's sessions", + "name": "RLS still enabled on storage.objects", + "passed": true + }, + { + "name": "user A lists only own files", "passed": true, - "notes": "sessions left: 0" + "notes": "saw: 019f8a22-0300-72bb-840a-66265cec2922/receipt-alpha.pdf, 019f8a22-0300-72bb-840a-66265cec2922/receipt-beta.pdf" }, { - "name": "deleted user's refresh token is rejected", + "name": "user B cannot read user A files", "passed": true }, { - "name": "deleted user cannot sign back in", + "name": "anon reads no files", "passed": true }, { - "name": "other users keep their sessions and access", + "name": "user A can upload into own folder", "passed": true }, { - "name": "diagnosed and explained session revocation", + "name": "user B cannot upload into user A folder", + "passed": true + }, + { + "name": "configured private per-user storage access", "passed": true, - "judgeNotes": "Diagnosed soft-delete-only flow, implemented auth user deletion plus RLS that closes the Data API stale-JWT path, explained JWT expiry caveat sufficiently, and correctly distinguished publishable/anon from secret/service_role keys." + "judgeNotes": "Creates a private user-files bucket, defines authenticated SELECT and INSERT policies on storage.objects scoped to bucket and owner via first path segment auth.uid(), keeps RLS enabled implicitly, and uses createSignedUrl with expiry for temporary sharing. No public bucket, public URL, permissive policy, or client service-role misuse." } ], "skills": { @@ -8355,276 +3318,424 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"publishable secret keys anon service_role frontend RLS\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/database/secure-data", - "title": "Securing your data" - }, - { - "url": "https://supabase.com/docs/guides/auth/jwt-fields", - "title": "JWT Claims Reference" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-redwoodjs", - "title": "Build a User Management App with RedwoodJS" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" - }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys", - "title": "JWT Signing Keys" - } - ], - "resultChars": 91173 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"delete user revoke sessions auth.admin.deleteUser delete_account sessions invalidation\", limit: 5) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"storage RLS policy path user id folder structure private bucket\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/auth/managing-user-data", - "title": "User Management" + "url": "https://supabase.com/docs/guides/storage/buckets/fundamentals", + "title": "Storage Buckets" }, { - "url": "https://supabase.com/docs/reference/javascript/auth-admin-deleteuser" + "url": "https://supabase.com/docs/guides/storage/security/access-control", + "title": "Storage Access Control" }, { - "url": "https://supabase.com/docs/reference/dart/auth-admin-deleteuser" + "url": "https://supabase.com/docs/guides/security/product-security", + "title": "Secure configuration of Supabase products" }, { - "url": "https://supabase.com/docs/guides/platform/sso/multiple-providers", - "title": "Multiple SSO Providers" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3", + "title": "Configure S3 Storage" }, { - "url": "https://supabase.com/docs/reference/swift/auth-admin-deleteuser", - "title": "deleteUser()" + "url": "https://supabase.com/docs/guides/storage/schema/design", + "title": "The Storage Schema" } ], - "resultChars": 27499 + "resultChars": 21275 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"row level security auth.uid deleted flag policy\", limit: 5) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"createSignedUrl temporary link expiring share file\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows", - "title": "Local development workflow" + "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsignedurl" }, { - "url": "https://supabase.com/docs/guides/storage/management/delete-objects", - "title": "Delete Objects" + "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsignedurls" }, { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", - "title": "Row Level Security" + "url": "https://supabase.com/docs/reference/dart/file-buckets-createsignedurl" }, { - "url": "https://supabase.com/docs/guides/auth/auth-mfa", - "title": "Multi-Factor Authentication" + "url": "https://supabase.com/docs/reference/csharp/storage-from-createsignedurl", + "title": "From().CreateSignedUrl()" }, { - "url": "https://supabase.com/docs/guides/database/database-advisors?queryGroups=lint&lint=0003_auth_rls_initplan", - "title": "Database Advisor: Lint 0003_auth_rls_initplan" + "url": "https://supabase.com/docs/reference/python/storage-from-createsignedurl", + "title": "from_.create_signed_url()" } ], - "resultChars": 74757 - }, + "resultChars": 4221 + } + ] + }, + "prompt": "Our app lets signed-in users keep personal files like receipts and bank\nstatements. These files are private — a user must only ever be able to upload\nand download their own. The app uploads each file under a path that starts\nwith the owner's user id, e.g. `/receipt-march.pdf`.\n\nSet up a `user-files` bucket on our project and lock it down that way.\n\nUsers also sometimes share one of their files with someone else through a\ntemporary link that expires. Include the supabase-js code the app should use\nfor that.", + "promptSourcePath": "evals/build-storage-001-private-bucket-access/PROMPT.md", + "attempts": 1, + "sourcePath": "claude-code-sonnet-5/build-storage-001-private-bucket-access.json" + }, + { + "experiment": "claude-code-sonnet-5", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" + }, + "eval": "build-tests-001-rls-tenant-isolation", + "stage": "build", + "product": [ + "database" + ], + "topic": [ + "tests", + "rls" + ], + "suite": "benchmark", + "interface": "cli", + "passed": true, + "checks": [ + { + "name": "pgTAP test file(s) written under supabase/tests/", + "passed": true, + "notes": "1 file(s): supabase/tests/database/tenant_isolation_test.sql" + }, + { + "name": "pgTAP isolation tests ran and pass", + "passed": true, + "notes": "5 passed, 4 failed" + }, + { + "name": "agent correctly identifies the posts isolation bug from test results", + "passed": true, + "judgeNotes": "The agent correctly identifies `posts` as having a broken tenant isolation policy, specifically that authenticated members can read posts from other organizations, and grounds this in pgTAP test failures. It also correctly distinguishes `notes` as isolated and treats the test results as authoritative." + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [] + }, + "prompt": "Can you audit the tenant isolation on our tables? Write some database tests covering both the happy path and the negative case, run them, and share the results.", + "promptSourcePath": "evals/build-tests-001-rls-tenant-isolation/PROMPT.md", + "attempts": 1, + "sourcePath": "claude-code-sonnet-5/build-tests-001-rls-tenant-isolation.json" + }, + { + "experiment": "claude-code-sonnet-5", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" + }, + "eval": "build-vectors-001-rag-with-permissions", + "stage": "build", + "product": [ + "database", + "vectors" + ], + "topic": [ + "sql", + "rls" + ], + "suite": "benchmark", + "interface": "mcp", + "passed": true, + "checks": [ + { + "name": "document_sections.embedding is vector(384)", + "passed": true, + "notes": "vector(384)" + }, + { + "name": "HNSW index on the embedding column", + "passed": true, + "notes": "CREATE INDEX document_sections_embedding_idx ON public.document_sections USING hnsw (embedding vector_cosine_ops)" + }, + { + "name": "index operator class matches the search operator", + "passed": true, + "notes": "function operators: <=>\nindexes: CREATE INDEX document_sections_embedding_idx ON public.document_sections USING hnsw (embedding vector_cosine_ops)" + }, + { + "name": "user A search returns only own sections, best match first", + "passed": true + }, + { + "name": "user B search returns only own sections, best match first", + "passed": true + }, + { + "name": "user A reads only own sections through the API", + "passed": true + }, + { + "name": "user A reads only own documents through the API", + "passed": true + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"auth.admin signOut deleteUser revoke refresh token\", limit: 10) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"gte-small embedding dimensions Supabase.ai edge function\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/reference/javascript/auth-signout" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-admin-deleteuser" - }, - { - "url": "https://supabase.com/docs/guides/auth/managing-user-data", - "title": "User Management" - }, - { - "url": "https://supabase.com/docs/reference/dart/auth-admin-deleteuser" - }, - { - "url": "https://supabase.com/docs/guides/integrations/build-a-supabase-oauth-integration", - "title": "Build a Supabase Integration" - }, - { - "url": "https://supabase.com/docs/reference/api/v1-revoke-token", - "title": "[Beta] Revoke oauth app authorization and it's corresponding tokens" + "url": "https://supabase.com/docs/guides/ai/semantic-search", + "title": "Semantic search" }, { - "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows", - "title": "OAuth 2.1 Flows" + "url": "https://supabase.com/docs/guides/functions/examples/semantic-search", + "title": "Semantic Search" }, { - "url": "https://supabase.com/docs/reference/javascript/oauth-admin", - "title": "OAuth Admin" + "url": "https://supabase.com/docs/guides/ai/concepts", + "title": "Concepts" }, { - "url": "https://supabase.com/docs/reference/swift/auth-admin-deleteuser", - "title": "deleteUser()" + "url": "https://supabase.com/docs/guides/ai/choosing-compute-addon", + "title": "Choosing your Compute Add-on" }, { - "url": "https://supabase.com/docs/reference/javascript/auth-admin-signout" + "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", + "title": "Automatic embeddings" } ], - "resultChars": 56854 + "resultChars": 74041 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"revoke sessions delete user auth.sessions revoke refresh tokens\", limit: 10) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"vector similarity search match function RLS pgvector\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/auth/sessions", - "title": "User sessions" - }, - { - "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows", - "title": "OAuth 2.1 Flows" - }, - { - "url": "https://supabase.com/docs/guides/resources/glossary", - "title": "Glossary" - }, - { - "url": "https://supabase.com/docs/reference/javascript/oauth-server-revokegrant" - }, - { - "url": "https://supabase.com/docs/guides/auth/managing-user-data", - "title": "User Management" - }, - { - "url": "https://supabase.com/docs/reference/api/v1-revoke-token", - "title": "[Beta] Revoke oauth app authorization and it's corresponding tokens" + "url": "https://supabase.com/docs/guides/ai/rag-with-permissions", + "title": "RAG with Permissions" }, { - "url": "https://supabase.com/docs/guides/auth/signing-keys", - "title": "JWT Signing Keys" + "url": "https://supabase.com/docs/guides/ai/hybrid-search", + "title": "Hybrid search" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/should-i-set-a-shorter-max-age-parameter-on-the-cookies-8sbF4V", - "title": "Should I set a shorter Max-Age parameter on the cookies?" + "url": "https://supabase.com/docs/guides/ai/vector-columns", + "title": "Vector columns" }, { - "url": "https://supabase.com/docs/guides/auth/server-side/creating-a-client", - "title": "Creating a Supabase client for SSR" + "url": "https://supabase.com/docs/guides/storage/vector/querying-vectors", + "title": "Querying Vectors" }, { - "url": "https://supabase.com/docs/guides/auth/auth-anonymous", - "title": "Anonymous Sign-Ins" + "url": "https://supabase.com/docs/guides/database/extensions/pgvector", + "title": "pgvector: Embeddings and vector similarity" } ], - "resultChars": 160501 - }, + "resultChars": 62992 + } + ] + }, + "prompt": "We're adding semantic search to our internal knowledge base app. I already wrote the edge functions, but search doesn't work yet. Can you set up whatever the database needs to make search work end to end?\n\nSome documents are confidential, so users should only have access to documents they own.", + "promptSourcePath": "evals/build-vectors-001-rag-with-permissions/PROMPT.md", + "attempts": 1, + "sourcePath": "claude-code-sonnet-5/build-vectors-001-rag-with-permissions.json" + }, + { + "experiment": "claude-code-sonnet-5", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" + }, + "eval": "deploy-database-001-prometheus-metrics", + "stage": "deploy", + "product": [ + "database" + ], + "topic": [ + "observability" + ], + "suite": "benchmark", + "passed": true, + "checks": [ + { + "name": "preserved existing app scrape job", + "passed": true + }, + { + "name": "configured the Supabase Metrics API scrape correctly", + "passed": true, + "judgeNotes": "Prometheus preserves the app scrape and adds a Supabase scrape using HTTPS, the required metrics path, Basic Auth with password_file, and a .supabase.co target. docker-compose mounts the secrets directory containing the password_file into Prometheus." + }, + { + "name": "documented live deployment and verification steps", + "passed": true, + "judgeNotes": "README includes Secret API key creation, matching password_file secret placement, Prometheus reload, and concrete verification via Prometheus targets/Grafana." + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"sign out all sessions delete user auth admin revoke\", limit: 10) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"project metrics endpoint Prometheus\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/reference/javascript/auth-signout" - }, - { - "url": "https://supabase.com/docs/guides/auth/passkeys", - "title": "Passkey authentication" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", - "title": "Row Level Security" - }, - { - "url": "https://supabase.com/docs/guides/auth/signout", - "title": "Signing out" - }, - { - "url": "https://supabase.com/docs/guides/auth/managing-user-data", - "title": "User Management" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-admin-signout" + "url": "https://supabase.com/docs/guides/telemetry/metrics/grafana-self-hosted", + "title": "Metrics API with Prometheus & Grafana (self-hosted)" }, { - "url": "https://supabase.com/docs/guides/auth/sessions", - "title": "User sessions" + "url": "https://supabase.com/docs/guides/telemetry/metrics", + "title": "Metrics API" }, { - "url": "https://supabase.com/docs/guides/resources/glossary", - "title": "Glossary" + "url": "https://supabase.com/docs/guides/telemetry/metrics/grafana-cloud", + "title": "Metrics API with Grafana Cloud" }, { - "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows", - "title": "OAuth 2.1 Flows" + "url": "https://supabase.com/docs/guides/telemetry/metrics/vendor-agnostic", + "title": "Vendor-agnostic Metrics API setup" }, { - "url": "https://supabase.com/docs/reference/javascript/auth-admin-deletefactor" + "url": "https://supabase.com/docs/guides/database/replication/manual-replication-monitoring", + "title": "Manual replication monitoring" } ], - "resultChars": 108752 + "resultChars": 27147 } ] }, - "prompt": "Last week support removed a user through our app's delete-account flow — the\napp calls the `delete_account` function over RPC as the signed-in user. This\nmorning that same person was back: still signed in, reading and saving their\ndata like nothing happened.\n\nFigure out why the account still works, fix the flow so a deleted account\nloses access, and tell me whether there is any window where they could still\nget in after the fix.\n\nOne more thing while you're at it: we're migrating off the legacy\nanon/service_role API keys to the new publishable and secret keys, and the\nteam isn't sure which one belongs in the frontend and what each means for\nRLS. Give us a quick rundown.", - "promptSourcePath": "evals/investigate-auth-001-deleted-user-access/PROMPT.md", + "prompt": "Can you wire my Supabase project metrics into our existing observability stack and document\nwhat we need to do to make the config live?", + "promptSourcePath": "evals/deploy-database-001-prometheus-metrics/PROMPT.md", + "attempts": 1, + "sourcePath": "claude-code-sonnet-5/deploy-database-001-prometheus-metrics.json" + }, + { + "experiment": "claude-code-sonnet-5", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" + }, + "eval": "deploy-functions-001-edge-function-secrets", + "stage": "deploy", + "product": [ + "edge-functions" + ], + "topic": [ + "security" + ], + "suite": "benchmark", + "interface": "cli", + "passed": true, + "checks": [ + { + "name": "WEATHER_API_KEY is set as a Function secret on the project", + "passed": true + }, + { + "name": "the weather function is deployed to the project", + "passed": true, + "notes": "status ACTIVE" + }, + { + "name": "the weather function reads WEATHER_API_KEY from the environment", + "passed": true, + "judgeNotes": "Reads WEATHER_API_KEY from the runtime environment using Deno.env.get(\"WEATHER_API_KEY\")." + }, + { + "name": "WEATHER_API_KEY value is not committed to the repo", + "passed": true + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [] + }, + "prompt": "Our weather widget currently calls WeatherAPI straight from the browser, which\nleaks our API key. I want to move that behind a Supabase Edge Function called\n`weather` that holds the key server-side and proxies the request.\n\nThe function should read the key from an environment variable named\n`WEATHER_API_KEY`. Our key already lives in a local `.env` file at the project\nroot.\n\nDeploy the function to our project so it's live, and make sure the deployed\nfunction can actually read the key at runtime.", + "promptSourcePath": "evals/deploy-functions-001-edge-function-secrets/PROMPT.md", "attempts": 1, - "sourcePath": "codex-gpt-5.4-mini/investigate-auth-001-deleted-user-access.json" + "sourcePath": "claude-code-sonnet-5/deploy-functions-001-edge-function-secrets.json" }, { - "experiment": "codex-gpt-5.4-mini", + "experiment": "claude-code-sonnet-5", "experimentSuite": "benchmark", "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.4-mini", - "reasoningEffort": "medium" + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" }, - "eval": "investigate-realtime-001-subscribed-no-events", - "stage": "investigate", + "eval": "deploy-self-hosting-001-docker-compose", + "stage": "deploy", "product": [ - "realtime", - "database" + "database", + "auth", + "storage" ], "topic": [ - "sdk" + "self-hosting" ], "suite": "benchmark", - "interface": "mcp", + "interface": "cli", "passed": true, "checks": [ { - "name": "orders table added to supabase_realtime publication", + "name": "cloned the self-host stack (docker-compose.yml + volumes/db)", "passed": true }, { - "name": "courier_locations still in supabase_realtime publication", + "name": "didn't conflate with the CLI (no supabase/config.toml in the stack)", "passed": true }, { - "name": "publication still publishes INSERT events", + "name": "secrets rotated off the shipped defaults", "passed": true }, { - "name": "RLS still enabled on orders", + "name": "ANON_KEY and SERVICE_ROLE_KEY are HS256 JWTs signed by JWT_SECRET", "passed": true - }, - { - "name": "staff can still read orders through RLS", - "passed": true, - "notes": "authenticated sees 2 of 2 orders" - }, - { - "name": "diagnosed missing publication membership", - "passed": true, - "judgeNotes": "Identified missing orders table in supabase_realtime as root cause, added only public.orders to existing publication, verified courier_locations remained, and did not weaken RLS/policies." } ], "skills": { @@ -8637,77 +3748,64 @@ ] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"postgres_changes realtime publication insert table not receiving events\", limit: 5) { nodes { __typename title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/realtime/subscribing-to-database-changes", - "title": "Subscribing to Database Changes" - }, - { - "url": "https://supabase.com/docs/guides/realtime/postgres-changes", - "title": "Postgres Changes" - }, - { - "url": "https://supabase.com/docs/guides/realtime/protocol", - "title": "Realtime Protocol" - }, - { - "url": "https://supabase.com/docs/guides/realtime/benchmarks", - "title": "Benchmarks" - }, - { - "url": "https://supabase.com/docs/guides/database/replication/pipelines-faq", - "title": "Pipelines FAQ" - } - ], - "resultChars": 124600 - } - ] + "calls": [] }, - "prompt": "Our dispatch dashboard shows incoming orders as they happen. The courier\nlocation feed on the same page updates live without problems, but new orders\nonly show up after a page refresh.\n\nThe dashboard uses supabase-js to subscribe to INSERT events on the `orders`\ntable through postgres_changes, the same way it subscribes to courier\nlocations. The channel's status callback logs SUBSCRIBED and there are no\nerrors in the browser console.\n\nFigure out why no order events ever arrive and fix it.", - "promptSourcePath": "evals/investigate-realtime-001-subscribed-no-events/PROMPT.md", + "prompt": "I'm moving off the hosted Supabase and running the whole thing myself on a VPS I\njust spun up. Can you get a Docker setup ready for me to copy onto the box?\n\nI don't need it running here, I'll do the actual bring-up once I'm on the\nserver. I just want everything in place and the secrets set up. Put it in a `supabase-docker/`\nfolder at the repo root so I can scp the whole thing across in one go.", + "promptSourcePath": "evals/deploy-self-hosting-001-docker-compose/PROMPT.md", "attempts": 1, - "sourcePath": "codex-gpt-5.4-mini/investigate-realtime-001-subscribed-no-events.json" + "sourcePath": "claude-code-sonnet-5/deploy-self-hosting-001-docker-compose.json" }, { - "experiment": "codex-gpt-5.4-mini", + "experiment": "claude-code-sonnet-5", "experimentSuite": "benchmark", "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.4-mini", - "reasoningEffort": "medium" + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" }, - "eval": "investigate-reliability-003-edge-function-5xx-correlation", + "eval": "investigate-auth-001-deleted-user-access", "stage": "investigate", "product": [ - "edge-functions" + "auth" ], "topic": [ - "observability" + "security", + "sdk" ], "suite": "benchmark", "interface": "mcp", - "passed": false, + "passed": true, "checks": [ { - "name": "identified image-transform and the recurring 503 pattern", + "name": "victim session active before delete-account", + "passed": true + }, + { + "name": "delete_account flow ran for the victim", + "passed": true + }, + { + "name": "delete-account revokes the user's sessions", "passed": true, - "judgeNotes": "The assistant explicitly identified `image-transform` as the affected function and described the recurring HTTP 503 pattern across the morning of 2026-04-28, covering all 8 failures from 07:00Z through 12:00Z." + "notes": "sessions left: 0" }, { - "name": "attributed recurring 503s to gateway/platform layer, not function code", - "passed": false, - "judgeNotes": "The assistant does not attribute the recurring image-transform 503s to the gateway/platform layer. It instead frames them as function-specific failures likely inside external @acme packages or an upstream service and recommends inspecting function implementations/logging, despite noting unchanged deployment and intermittent successes." + "name": "deleted user's refresh token is rejected", + "passed": true }, { - "name": "recommended a concrete next step", + "name": "deleted user cannot sign back in", + "passed": true + }, + { + "name": "other users keep their sessions and access", + "passed": true + }, + { + "name": "diagnosed and explained session revocation", "passed": true, - "judgeNotes": "The assistant provided concrete actionable next steps, including inspecting specific external package implementations/release history, adding structured error logging, creating/verifying Storage buckets and policies, and re-testing uploads with correlation factors." + "judgeNotes": "Meets rubric: diagnoses soft-delete-only flow, implements auth user deletion with session/refresh-token revocation via cascades, gives a consistent remaining JWT access-token window and mitigation, and correctly distinguishes publishable/anon frontend keys from secret/service_role server-only RLS-bypassing keys." } ], "skills": { @@ -8722,29 +3820,27 @@ "docs": { "calls": [] }, - "prompt": "Users have been reporting that image uploads are intermittently failing with server errors throughout the morning of `2026-04-28`.\n\nCan you investigate the project logs and tell me what is going on and what we\nshould do next?", - "promptSourcePath": "evals/investigate-reliability-003-edge-function-5xx-correlation/PROMPT.md", - "attempts": 2, - "sourcePath": "codex-gpt-5.4-mini/investigate-reliability-003-edge-function-5xx-correlation.json" + "prompt": "Last week support removed a user through our app's delete-account flow — the\napp calls the `delete_account` function over RPC as the signed-in user. This\nmorning that same person was back: still signed in, reading and saving their\ndata like nothing happened.\n\nFigure out why the account still works, fix the flow so a deleted account\nloses access, and tell me whether there is any window where they could still\nget in after the fix.\n\nOne more thing while you're at it: we're migrating off the legacy\nanon/service_role API keys to the new publishable and secret keys, and the\nteam isn't sure which one belongs in the frontend and what each means for\nRLS. Give us a quick rundown.", + "promptSourcePath": "evals/investigate-auth-001-deleted-user-access/PROMPT.md", + "attempts": 1, + "sourcePath": "claude-code-sonnet-5/investigate-auth-001-deleted-user-access.json" }, { - "experiment": "codex-gpt-5.4-mini", + "experiment": "claude-code-sonnet-5", "experimentSuite": "benchmark", "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.4-mini", - "reasoningEffort": "medium" + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" }, - "eval": "resolve-dataapi-001-empty-results", - "stage": "resolve", + "eval": "investigate-realtime-001-subscribed-no-events", + "stage": "investigate", "product": [ - "data-api", - "database", - "auth" + "realtime", + "database" ], "topic": [ - "rls", "sdk" ], "suite": "benchmark", @@ -8752,33 +3848,82 @@ "passed": true, "checks": [ { - "name": "RLS still enabled on bookmarks", + "name": "orders table added to supabase_realtime publication", "passed": true }, { - "name": "user A reads own bookmarks", + "name": "courier_locations still in supabase_realtime publication", "passed": true }, { - "name": "user B cannot read user A bookmarks", + "name": "publication still publishes INSERT events", "passed": true }, { - "name": "anon reads no bookmarks", + "name": "RLS still enabled on orders", "passed": true }, { - "name": "user A can save a new bookmark", - "passed": true + "name": "staff can still read orders through RLS", + "passed": true, + "notes": "authenticated sees 2 of 2 orders" }, { - "name": "user B cannot insert a bookmark as user A", - "passed": true + "name": "diagnosed missing publication membership", + "passed": true, + "judgeNotes": "The answer correctly identifies orders missing from the supabase_realtime publication as the root cause despite SUBSCRIBED status, applies ALTER PUBLICATION supabase_realtime ADD TABLE public.orders, preserves courier_locations and existing RLS/policies, and does not blame or alter client code/RLS/grants/networking." + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [] + }, + "docs": { + "calls": [] + }, + "prompt": "Our dispatch dashboard shows incoming orders as they happen. The courier\nlocation feed on the same page updates live without problems, but new orders\nonly show up after a page refresh.\n\nThe dashboard uses supabase-js to subscribe to INSERT events on the `orders`\ntable through postgres_changes, the same way it subscribes to courier\nlocations. The channel's status callback logs SUBSCRIBED and there are no\nerrors in the browser console.\n\nFigure out why no order events ever arrive and fix it.", + "promptSourcePath": "evals/investigate-realtime-001-subscribed-no-events/PROMPT.md", + "attempts": 1, + "sourcePath": "claude-code-sonnet-5/investigate-realtime-001-subscribed-no-events.json" + }, + { + "experiment": "claude-code-sonnet-5", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" + }, + "eval": "investigate-reliability-003-edge-function-5xx-correlation", + "stage": "investigate", + "product": [ + "edge-functions" + ], + "topic": [ + "observability" + ], + "suite": "benchmark", + "interface": "mcp", + "passed": true, + "checks": [ + { + "name": "identified image-transform and the recurring 503 pattern", + "passed": true, + "judgeNotes": "The assistant correctly identified image-transform as the affected function and described the recurring HTTP 503 pattern across the morning of 2026-04-28, covering the failures from roughly 07:00Z through 12:00Z. It also distinguished the older billing-webhook 503s as unrelated." }, { - "name": "diagnosed RLS and added owner-scoped policies", + "name": "attributed recurring 503s to gateway/platform layer, not function code", + "passed": true, + "judgeNotes": "Attributes the recurring image-transform 503s to gateway/platform layer, not application code. Grounds this in valid observations: 503s lack execution_time_ms/deployment_id and thus did not reach function instances, nearby 200 invocations succeeded, and distinguishes avatar-upload's 500 as a separate function-level error." + }, + { + "name": "recommended a concrete next step", "passed": true, - "judgeNotes": "The assistant correctly diagnosed RLS enabled with no policies causing deny-all Data API results, kept RLS enabled, and created authenticated SELECT and INSERT owner-scoped policies using auth.uid() = user_id with WITH CHECK for insert." + "judgeNotes": "The assistant recommended specific actionable next steps, including correlating client-side request volume, adding retries, reviewing Edge Function concurrency/memory limits, and improving logging/log retention." } ], "skills": { @@ -8791,104 +3936,65 @@ ] }, "docs": { - "calls": [ - { - "source": "web_search", - "query": "https://supabase.com/changelog.md", - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ] - }, - { - "source": "web_search", - "query": "site:supabase.com changelog.md supabase", - "pages": [] - }, - { - "source": "search_docs", - "query": "query {\n searchDocs(query: \"row level security auth.uid select insert policy authenticated user_id bookmarks\", limit: 5) {\n nodes {\n title\n href\n ... on Guide { content }\n }\n }\n}", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", - "title": "Row Level Security" - }, - { - "url": "https://supabase.com/docs/guides/storage/management/copy-move-objects", - "title": "Copy Objects" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features", - "title": "Features" - }, - { - "url": "https://supabase.com/docs/guides/realtime/getting_started", - "title": "Getting Started with Realtime" - }, - { - "url": "https://supabase.com/docs/guides/auth/auth-mfa", - "title": "Multi-Factor Authentication" - } - ], - "resultChars": 81216 - } - ] + "calls": [] }, - "prompt": "Our app lets signed-in users save bookmarks and view them on their dashboard. Bookmarks are stored in the `bookmarks` table and are private — a user must only ever see their own. \nUsers also need to be able to save new bookmarks from the app.\n\nI can see the rows when I query the table directly, but the dashboard shows an empty list for every user.\n\nFind out why the Data API returns nothing and fix it.", - "promptSourcePath": "evals/resolve-dataapi-001-empty-results/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.4-mini/resolve-dataapi-001-empty-results.json" + "prompt": "Users have been reporting that image uploads are intermittently failing with server errors throughout the morning of `2026-04-28`.\n\nCan you investigate the project logs and tell me what is going on and what we\nshould do next?", + "promptSourcePath": "evals/investigate-reliability-003-edge-function-5xx-correlation/PROMPT.md", + "attempts": 2, + "sourcePath": "claude-code-sonnet-5/investigate-reliability-003-edge-function-5xx-correlation.json" }, { - "experiment": "codex-gpt-5.4-mini", + "experiment": "claude-code-sonnet-5", "experimentSuite": "benchmark", "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.4-mini", - "reasoningEffort": "medium" + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" }, - "eval": "resolve-database-001-migration-history-mismatch", + "eval": "resolve-dataapi-001-empty-results", "stage": "resolve", "product": [ - "database" + "data-api", + "database", + "auth" ], "topic": [ - "migrations" + "rls", + "sdk" ], "suite": "benchmark", - "interface": "cli", - "passed": false, + "interface": "mcp", + "passed": true, "checks": [ { - "name": "the avatar_url column is applied on the hosted profiles table", - "passed": false, - "notes": "avatar_url not found on public.profiles" + "name": "RLS still enabled on bookmarks", + "passed": true }, { - "name": "migration 20240220000000 is recorded in the remote history", - "passed": false, - "notes": "remote history versions: [\"20240101000000\",\"20240115000000\"]" + "name": "user A reads own bookmarks", + "passed": true }, { - "name": "remote migration history matches local migration files", + "name": "user B cannot read user A bookmarks", "passed": true }, { - "name": "local migrations are a valid reconciled sequence", - "passed": false, - "notes": "expected 20240101000000_create_profiles → → 20240220000000_add_avatar_url, strictly ascending; got [20240101000000_create_profiles.sql, 20240115000000_add_profile_bio.sql]" + "name": "anon reads no bookmarks", + "passed": true }, { - "name": "production profile data is intact (not reset)", + "name": "user A can save a new bookmark", "passed": true }, { - "name": "the avatar migration and history reconciliation were done via the Supabase CLI", - "passed": false, - "judgeNotes": "No real remote `supabase db push` succeeded; only dry-runs/errors and local `db reset` occurred. The avatar_url migration was not applied to the hosted remote via CLI. The orphan bio migration was only inspected via Management API and apparently added locally (`20240115000000_add_profile_bio.sql`), but no CLI repair/pull/push reconciled remote history. No prohibited mutation workaround observed; Management API uses shown were read-only SELECTs." + "name": "user B cannot insert a bookmark as user A", + "passed": true + }, + { + "name": "diagnosed RLS and added owner-scoped policies", + "passed": true, + "judgeNotes": "Diagnosed deny-all RLS with no policies and added authenticated-only SELECT and INSERT policies scoped to user_id = auth.uid(), with INSERT enforced via WITH CHECK, while keeping RLS enabled." } ], "skills": { @@ -8901,180 +4007,58 @@ ] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"management api database connection string project ref password db host\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/reference/api/v1-update-database-password", - "title": "Updates the database password" - }, - { - "url": "https://supabase.com/docs/guides/platform/temporary-access", - "title": "Temporary access" - }, - { - "url": "https://supabase.com/docs/reference/api/v1-enable-database-webhook", - "title": "[Beta] Enables Database Webhooks on the project" - }, - { - "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/postgres", - "title": "Migrate from Postgres to Supabase" - }, - { - "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/amazon-rds", - "title": "Migrate from Amazon RDS to Supabase" - } - ], - "resultChars": 35186 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"management api database host connection string project details pooler host\", limit: 10) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/self-hosting/docker", - "title": "Self-Hosting with Docker" - }, - { - "url": "https://supabase.com/docs/guides/platform/ipv4-address", - "title": "Dedicated IPv4 Address for Ingress" - }, - { - "url": "https://supabase.com/docs/guides/ai/integrations/roboflow", - "title": "Roboflow" - }, - { - "url": "https://supabase.com/docs/reference/api/v1-update-pooler-config", - "title": "Updates project's supavisor config" - }, - { - "url": "https://supabase.com/docs/guides/platform/read-replicas", - "title": "Read Replicas" - }, - { - "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/postgres", - "title": "Migrate from Postgres to Supabase" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/quickstarts/laravel", - "title": "Use Supabase with Laravel" - }, - { - "url": "https://supabase.com/docs/reference/api/v1-get-pooler-config", - "title": "Gets project's supavisor config" - }, - { - "url": "https://supabase.com/docs/guides/database/connecting-to-postgres", - "title": "Connect to your database" - }, - { - "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/heroku", - "title": "Migrate from Heroku to Supabase" - } - ], - "resultChars": 109113 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"project supavisor config management api host session pooler\", limit: 10) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/self-hosting/docker", - "title": "Self-Hosting with Docker" - }, - { - "url": "https://supabase.com/docs/guides/database/connecting-to-postgres", - "title": "Connect to your database" - }, - { - "url": "https://supabase.com/docs/reference/api/v1-update-pooler-config", - "title": "Updates project's supavisor config" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/architecture", - "title": "Architecture" - }, - { - "url": "https://supabase.com/docs/guides/telemetry/reports", - "title": "Reports" - }, - { - "url": "https://supabase.com/docs/guides/platform/read-replicas", - "title": "Read Replicas" - }, - { - "url": "https://supabase.com/docs/reference/api/v1-get-pooler-config", - "title": "Gets project's supavisor config" - }, - { - "url": "https://supabase.com/docs/guides/database/connection-management", - "title": "Connection management" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/supavisor-faq-YyP5tI", - "title": "Supavisor FAQ" - }, - { - "url": "https://supabase.com/docs/guides/platform/ipv4-address", - "title": "Dedicated IPv4 Address for Ingress" - } - ], - "resultChars": 128627 - } - ] + "calls": [] }, - "prompt": "I'm trying to ship a migration to our hosted project and it's not working. Can you figure out what's wrong and get it deployed?", - "promptSourcePath": "evals/resolve-database-001-migration-history-mismatch/PROMPT.md", - "attempts": 2, - "sourcePath": "codex-gpt-5.4-mini/resolve-database-001-migration-history-mismatch.json" + "prompt": "Our app lets signed-in users save bookmarks and view them on their dashboard. Bookmarks are stored in the `bookmarks` table and are private — a user must only ever see their own. \nUsers also need to be able to save new bookmarks from the app.\n\nI can see the rows when I query the table directly, but the dashboard shows an empty list for every user.\n\nFind out why the Data API returns nothing and fix it.", + "promptSourcePath": "evals/resolve-dataapi-001-empty-results/PROMPT.md", + "attempts": 1, + "sourcePath": "claude-code-sonnet-5/resolve-dataapi-001-empty-results.json" }, { - "experiment": "codex-gpt-5.4-mini", + "experiment": "claude-code-sonnet-5", "experimentSuite": "benchmark", "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.4-mini", - "reasoningEffort": "medium" + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" }, - "eval": "resolve-performance-001-slow-query-cpu-spike", + "eval": "resolve-database-001-migration-history-mismatch", "stage": "resolve", "product": [ "database" ], "topic": [ - "observability", - "sql" + "migrations" ], "suite": "benchmark", - "interface": "mcp", + "interface": "cli", "passed": true, "checks": [ { - "name": "inspected pg_stat_statements for query performance", + "name": "the avatar_url column is applied on the hosted profiles table", "passed": true }, { - "name": "ran EXPLAIN on the expensive query", + "name": "migration 20240220000000 is recorded in the remote history", "passed": true }, { - "name": "created index covering user_id and created_at", + "name": "remote migration history matches local migration files", "passed": true }, { - "name": "query plan uses an index and avoids sequential scan", - "passed": true, - "notes": "Limit (cost=79.74..79.86 rows=50 width=58)\n -> Sort (cost=79.74..79.99 rows=100 width=58)\n Sort Key: created_at DESC\n -> Bitmap Heap Scan on events (cost=5.06..76.42 rows=100 width=58)\n Recheck Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)\n -> Bitmap Index Scan on events_user_id_created_at_idx (cost=0.00..5.03 rows=100 width=0)\n Index Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)" + "name": "local migrations are a valid reconciled sequence", + "passed": true }, { - "name": "inserts still work", + "name": "production profile data is intact (not reset)", "passed": true + }, + { + "name": "the avatar migration and history reconciliation were done via the Supabase CLI", + "passed": true, + "judgeNotes": "Avatar migration was applied through `supabase db push` (#23), which showed `Applying migration 20240220000000_add_avatar_url.sql...` and `Finished supabase db push.` History was reconciled through Supabase CLI migration repair: `supabase migration repair --status reverted 20240115000000` (#16), then after adding the local bio migration file, `supabase migration repair --status applied 20240115000000` (#21). Read-only `psql` inspection was used, but no prohibited direct SQL mutation or prepared-statement workaround was seen." } ], "skills": { @@ -9087,99 +4071,54 @@ ] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Postgres index recent rows user_id order by desc limit\", limit: 5) { nodes { __typename title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/storage/analytics/query-with-postgres", - "title": "Query with Postgres" - }, - { - "url": "https://supabase.com/docs/guides/storage/debugging/logs", - "title": "Logs" - }, - { - "url": "https://supabase.com/docs/guides/database/orioledb", - "title": "OrioleDB Overview" - }, - { - "url": "https://supabase.com/docs/guides/database/inspect", - "title": "Debugging and monitoring" - }, - { - "url": "https://supabase.com/docs/guides/telemetry/advanced-log-filtering", - "title": "Advanced Log Querying and Filtering" - } - ], - "resultChars": 37224 - } - ] + "calls": [] }, - "prompt": "My database CPU keeps spiking and the app gets slow when loading recent events for a user. Can you figure out what query is causing it and make the database change needed to fix it?\n\nEnd your turn with a short summary of what you changed and why.", - "promptSourcePath": "evals/resolve-performance-001-slow-query-cpu-spike/PROMPT.md", + "prompt": "I'm trying to ship a migration to our hosted project and it's not working. Can you figure out what's wrong and get it deployed?", + "promptSourcePath": "evals/resolve-database-001-migration-history-mismatch/PROMPT.md", "attempts": 1, - "sourcePath": "codex-gpt-5.4-mini/resolve-performance-001-slow-query-cpu-spike.json" + "sourcePath": "claude-code-sonnet-5/resolve-database-001-migration-history-mismatch.json" }, { - "experiment": "codex-gpt-5.4-mini", + "experiment": "claude-code-sonnet-5", "experimentSuite": "benchmark", "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.4-mini", - "reasoningEffort": "medium" + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" }, - "eval": "resolve-security-002-rls-cross-tenant-leak", + "eval": "resolve-performance-001-slow-query-cpu-spike", "stage": "resolve", "product": [ - "database", - "auth" + "database" ], "topic": [ - "rls", - "security" + "observability", + "sql" ], "suite": "benchmark", "interface": "mcp", "passed": true, "checks": [ { - "name": "RLS enabled on notes", - "passed": true - }, - { - "name": "tenant A sees only org A notes", - "passed": true - }, - { - "name": "tenant B cannot read org A notes", - "passed": true - }, - { - "name": "tenant A author can update own note", - "passed": true - }, - { - "name": "tenant B cannot update org A note", + "name": "inspected pg_stat_statements for query performance", "passed": true }, { - "name": "tenant B author can delete own note", + "name": "ran EXPLAIN on the expensive query", "passed": true }, { - "name": "tenant B cannot delete org A note", + "name": "created index covering user_id and created_at", "passed": true }, { - "name": "tenant A can insert note in own org", - "passed": true + "name": "query plan uses an index and avoids sequential scan", + "passed": true, + "notes": "Limit (cost=55.55..55.61 rows=25 width=88)\n -> Sort (cost=55.55..55.61 rows=25 width=88)\n Sort Key: created_at DESC\n -> Bitmap Heap Scan on events (cost=4.48..54.97 rows=25 width=88)\n Recheck Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)\n -> Bitmap Index Scan on events_user_id_created_at_idx (cost=0.00..4.47 rows=25 width=0)\n Index Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)" }, { - "name": "tenant B cannot insert into org A", + "name": "inserts still work", "passed": true } ], @@ -9193,322 +4132,96 @@ ] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase RLS policy EXISTS membership org_id auth.uid select policy workspace\", limit: 5) { nodes { title href content ... on Guide { subsections { nodes { title href content } } } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/auth/third-party/clerk", - "title": "Clerk" - }, - { - "url": "https://supabase.com/docs/guides/auth/third-party/clerk#example-check-user-has-passed-second-factor-verification", - "title": "Example: Check user has passed second factor verification" - }, - { - "url": "https://supabase.com/docs/guides/auth/third-party/clerk#deprecated-integration-with-jwt-templates", - "title": "Deprecated integration with JWT templates" - }, - { - "url": "https://supabase.com/docs/guides/auth/third-party/clerk#getting-started", - "title": "Getting started" - }, - { - "url": "https://supabase.com/docs/guides/auth/third-party/clerk#configure-for-local-development-or-self-hosting", - "title": "Configure for local development or self-hosting" - }, - { - "url": "https://supabase.com/docs/guides/auth/third-party/clerk#manually-configuring-your-clerk-instance", - "title": "Manually configuring your Clerk instance" - }, - { - "url": "https://supabase.com/docs/guides/auth/third-party/clerk#setup-the-supabase-client-library", - "title": "Setup the Supabase client library" - }, - { - "url": "https://supabase.com/docs/guides/auth/third-party/clerk#using-rls-policies", - "title": "Using RLS policies" - }, - { - "url": "https://supabase.com/docs/guides/auth/third-party/clerk#example-check-user-organization-role", - "title": "Example: Check user organization role" - }, - { - "url": "https://supabase.com/docs/guides/storage/schema/helper-functions", - "title": "Storage Helper Functions" - }, - { - "url": "https://supabase.com/docs/guides/storage/schema/helper-functions#storageallow_any_operation", - "title": "storage.allow_any_operation()" - }, - { - "url": "https://supabase.com/docs/guides/storage/schema/helper-functions#storageallow_only_operation", - "title": "storage.allow_only_operation()" - }, - { - "url": "https://supabase.com/docs/guides/storage/schema/helper-functions#storageextension", - "title": "storage.extension()" - }, - { - "url": "https://supabase.com/docs/guides/storage/schema/helper-functions#storagefoldername", - "title": "storage.foldername()" - }, - { - "url": "https://supabase.com/docs/guides/storage/schema/helper-functions#storagefilename", - "title": "storage.filename()" - }, - { - "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security", - "title": "Token Security and Row Level Security" - }, - { - "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security#extracting-oauth-claims-in-rls", - "title": "Extracting OAuth claims in RLS" - }, - { - "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security#common-rls-patterns-for-oauth", - "title": "Common RLS patterns for OAuth" - }, - { - "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security#pattern-1-grant-specific-client-full-access", - "title": "Pattern 1: Grant specific client full access" - }, - { - "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security#pattern-2-grant-multiple-clients-read-only-access", - "title": "Pattern 2: Grant multiple clients read-only access" - }, - { - "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security#pattern-3-restrict-sensitive-data-from-oauth-clients", - "title": "Pattern 3: Restrict sensitive data from OAuth clients" - }, - { - "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security#pattern-4-client-specific-data-access", - "title": "Pattern 4: Client-specific data access" - }, - { - "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security#real-world-examples", - "title": "Real-world examples" - }, - { - "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security#example-1-multi-platform-application", - "title": "Example 1: Multi-platform application" - }, - { - "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security#custom-access-token-hooks", - "title": "Custom access token hooks" - }, - { - "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security#customizing-the-audience-claim", - "title": "Customizing the audience claim" - }, - { - "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security#adding-client-specific-claims", - "title": "Adding client-specific claims" - }, - { - "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security#security-best-practices", - "title": "Security best practices" - }, - { - "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security#1-principle-of-least-privilege", - "title": "1. Principle of least privilege" - }, - { - "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security#2-separate-policies-for-oauth-clients", - "title": "2. Separate policies for OAuth clients" - }, - { - "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security#3-regularly-audit-oauth-clients", - "title": "3. Regularly audit OAuth clients" - }, - { - "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security#testing-your-policies", - "title": "Testing your policies" - }, - { - "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security#troubleshooting", - "title": "Troubleshooting" - }, - { - "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security#policy-not-working-for-oauth-client", - "title": "Policy not working for OAuth client" - }, - { - "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security#policy-too-permissive", - "title": "Policy too permissive" - }, - { - "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security#cant-differentiate-between-users-and-oauth-clients", - "title": "Can't differentiate between users and OAuth clients" - }, - { - "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security#next-steps", - "title": "Next steps" - }, - { - "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security#how-oauth-tokens-work-with-rls", - "title": "How OAuth tokens work with RLS" - }, - { - "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security#token-structure", - "title": "Token structure" - }, - { - "url": "https://supabase.com/docs/guides/storage/security/access-control", - "title": "Storage Access Control" - }, - { - "url": "https://supabase.com/docs/guides/storage/security/access-control#bypassing-access-controls", - "title": "Bypassing access controls" - }, - { - "url": "https://supabase.com/docs/guides/storage/security/access-control#policy-examples", - "title": "Policy examples" - }, - { - "url": "https://supabase.com/docs/guides/storage/security/access-control#access-policies", - "title": "Access policies" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", - "title": "Row Level Security" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#minimize-joins", - "title": "Minimize joins" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#benchmarks-3", - "title": "Benchmarks" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#benchmarks-4", - "title": "Benchmarks" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#row-level-security-in-supabase", - "title": "Row Level Security in Supabase" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#policies", - "title": "Policies" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#enabling-row-level-security", - "title": "Enabling Row Level Security" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#auto-enable-rls-for-new-tables", - "title": "Auto-enable RLS for new tables" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#authenticated-and-unauthenticated-roles", - "title": "Authenticated and unauthenticated roles" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#creating-policies", - "title": "Creating policies" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#select-policies", - "title": "SELECT policies" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#insert-policies", - "title": "INSERT policies" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#update-policies", - "title": "UPDATE policies" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#delete-policies", - "title": "DELETE policies" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#views", - "title": "Views" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#helper-functions", - "title": "Helper functions" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#authuid", - "title": "auth.uid()" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#authjwt", - "title": "auth.jwt()" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#mfa", - "title": "MFA" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#bypassing-row-level-security", - "title": "Bypassing Row Level Security" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#rls-performance-recommendations", - "title": "RLS performance recommendations" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#add-indexes", - "title": "Add indexes" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#benchmarks", - "title": "Benchmarks" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#more-resources", - "title": "More resources" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#call-functions-with-select", - "title": "Call functions with select" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#specify-roles-in-your-policies", - "title": "Specify roles in your policies" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#benchmarks-1", - "title": "Benchmarks" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#add-filters-to-every-query", - "title": "Add filters to every query" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#benchmarks-2", - "title": "Benchmarks" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#use-security-definer-functions", - "title": "Use security definer functions" - } - ], - "resultChars": 166639 - } - ] + "calls": [] + }, + "prompt": "My database CPU keeps spiking and the app gets slow when loading recent events for a user. Can you figure out what query is causing it and make the database change needed to fix it?\n\nEnd your turn with a short summary of what you changed and why.", + "promptSourcePath": "evals/resolve-performance-001-slow-query-cpu-spike/PROMPT.md", + "attempts": 1, + "sourcePath": "claude-code-sonnet-5/resolve-performance-001-slow-query-cpu-spike.json" + }, + { + "experiment": "claude-code-sonnet-5", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" + }, + "eval": "resolve-security-002-rls-cross-tenant-leak", + "stage": "resolve", + "product": [ + "database", + "auth" + ], + "topic": [ + "rls", + "security" + ], + "suite": "benchmark", + "interface": "mcp", + "passed": true, + "checks": [ + { + "name": "RLS enabled on notes", + "passed": true + }, + { + "name": "tenant A sees only org A notes", + "passed": true + }, + { + "name": "tenant B cannot read org A notes", + "passed": true + }, + { + "name": "tenant A author can update own note", + "passed": true + }, + { + "name": "tenant B cannot update org A note", + "passed": true + }, + { + "name": "tenant B author can delete own note", + "passed": true + }, + { + "name": "tenant B cannot delete org A note", + "passed": true + }, + { + "name": "tenant A can insert note in own org", + "passed": true + }, + { + "name": "tenant B cannot insert into org A", + "passed": true + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [] + }, + "docs": { + "calls": [] }, "prompt": "A customer reported that notes showed up in the wrong workspace.\n\nCan you investigate what is going on and fix it?", "promptSourcePath": "evals/resolve-security-002-rls-cross-tenant-leak/PROMPT.md", "attempts": 1, - "sourcePath": "codex-gpt-5.4-mini/resolve-security-002-rls-cross-tenant-leak.json" + "sourcePath": "claude-code-sonnet-5/resolve-security-002-rls-cross-tenant-leak.json" }, { - "experiment": "codex-gpt-5.4-mini-no-skills", + "experiment": "claude-code-sonnet-5-no-skills", "experimentSuite": "no-skills", "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.4-mini", - "reasoningEffort": "medium" + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" }, "eval": "build-cli-001-bootstrap-app", "stage": "build", @@ -9535,7 +4248,7 @@ { "name": "todos table exists with at least 2 seeded rows", "passed": true, - "notes": "found 2 rows" + "notes": "found 3 rows" }, { "name": "row level security is enabled on todos", @@ -9553,7 +4266,7 @@ { "name": "REST API returns the todos to authenticated requests", "passed": true, - "notes": "2 rows" + "notes": "3 rows" } ], "skills": { @@ -9566,16 +4279,16 @@ "prompt": "We're kicking off a todos app and I want the Supabase side ready for the team\nto build on. Set it up the way we'd run it in development, with schema changes\ntracked as migrations so they can be reviewed and replayed.\n\nFor the first slice we just need a `todos` table. Todos aren't public: anyone\nsigned in can read all of them, but nothing should be writable through the API\nfor now. Add a couple of sample todos so there's something to look at.\n\nBefore you hand it back, make sure the running API actually behaves that way —\nsigned-in users get the todos, signed-out requests get nothing.", "promptSourcePath": "evals/build-cli-001-bootstrap-app/PROMPT.md", "attempts": 1, - "sourcePath": "codex-gpt-5.4-mini-no-skills/build-cli-001-bootstrap-app.json" + "sourcePath": "claude-code-sonnet-5-no-skills/build-cli-001-bootstrap-app.json" }, { - "experiment": "codex-gpt-5.4-mini-no-skills", + "experiment": "claude-code-sonnet-5-no-skills", "experimentSuite": "no-skills", "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.4-mini", - "reasoningEffort": "medium" + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" }, "eval": "build-cli-002-declarative-schema", "stage": "build", @@ -9588,11 +4301,11 @@ ], "suite": "benchmark", "interface": "cli", - "passed": false, + "passed": true, "checks": [ { "name": "supabase db diff used to generate the migration", - "passed": false + "passed": true }, { "name": "schema file updated to include description column", @@ -9600,12 +4313,11 @@ }, { "name": "a new migration was generated for the change", - "passed": false, - "notes": "found 1 migration file(s)" + "passed": true }, { "name": "description column exists in the live database", - "passed": false + "passed": true } ], "skills": { @@ -9617,17 +4329,17 @@ }, "prompt": "Add a description text column to the `products` table in my local Supabase stack", "promptSourcePath": "evals/build-cli-002-declarative-schema/PROMPT.md", - "attempts": 2, - "sourcePath": "codex-gpt-5.4-mini-no-skills/build-cli-002-declarative-schema.json" + "attempts": 1, + "sourcePath": "claude-code-sonnet-5-no-skills/build-cli-002-declarative-schema.json" }, { - "experiment": "codex-gpt-5.4-mini-no-skills", + "experiment": "claude-code-sonnet-5-no-skills", "experimentSuite": "no-skills", "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.4-mini", - "reasoningEffort": "medium" + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" }, "eval": "build-cli-003-pg-cron-queue-workflow", "stage": "build", @@ -9658,7 +4370,7 @@ { "name": "process-tasks function drains the queue", "passed": true, - "notes": "function removed the seeded message (id 2) from the queue" + "notes": "function removed the seeded message (id 38) from the queue" } ], "skills": { @@ -9666,56 +4378,21 @@ "loaded": [] }, "docs": { - "calls": [ - { - "source": "web_search", - "query": "site:supabase.com/docs pg_cron cron.schedule Supabase queue pgmq read delete edge function service role", - "pages": [] - }, - { - "source": "web_search", - "query": "https://supabase.com/docs/guides/queues/consuming-messages-with-edge-functions", - "pages": [ - { - "url": "https://supabase.com/docs/guides/queues/consuming-messages-with-edge-functions" - } - ] - }, - { - "source": "web_search", - "query": "site:supabase.com/docs pgmq create queue send auto create queue", - "pages": [] - }, - { - "source": "web_search", - "query": "site:supabase.com/docs invoke edge function from pg_cron service role key apikey header verify_jwt false", - "pages": [] - }, - { - "source": "web_search", - "query": "site:supabase.com/docs pgmq list queues list_queues", - "pages": [] - }, - { - "source": "web_search", - "query": "site:supabase.com/docs edge function config.toml verify_jwt false", - "pages": [] - } - ] + "calls": [] }, "prompt": "I want to set up a recurring background workflow on my local Supabase stack.\n\nCan you set up a cron job called `enqueue-tasks` to run every minute and push a task into a queue called `tasks`? Then add a `process-tasks` edge function that reads messages off the `tasks` queue and removes them, so a scheduled worker can keep the backlog drained.", "promptSourcePath": "evals/build-cli-003-pg-cron-queue-workflow/PROMPT.md", "attempts": 1, - "sourcePath": "codex-gpt-5.4-mini-no-skills/build-cli-003-pg-cron-queue-workflow.json" + "sourcePath": "claude-code-sonnet-5-no-skills/build-cli-003-pg-cron-queue-workflow.json" }, { - "experiment": "codex-gpt-5.4-mini-no-skills", + "experiment": "claude-code-sonnet-5-no-skills", "experimentSuite": "no-skills", "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.4-mini", - "reasoningEffort": "medium" + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" }, "eval": "build-database-001-migrate-postgres-to-supabase", "stage": "build", @@ -9760,16 +4437,16 @@ "prompt": "I have an existing Postgres database I want to migrate to Supabase. There's a binary dump at `source.dump` in the current directory.\n\nCan you set up a local Supabase project and restore the dump into it?", "promptSourcePath": "evals/build-database-001-migrate-postgres-to-supabase/PROMPT.md", "attempts": 1, - "sourcePath": "codex-gpt-5.4-mini-no-skills/build-database-001-migrate-postgres-to-supabase.json" + "sourcePath": "claude-code-sonnet-5-no-skills/build-database-001-migrate-postgres-to-supabase.json" }, { - "experiment": "codex-gpt-5.4-mini-no-skills", + "experiment": "claude-code-sonnet-5-no-skills", "experimentSuite": "no-skills", "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.4-mini", - "reasoningEffort": "medium" + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" }, "eval": "build-functions-004-service-role-bypass", "stage": "build", @@ -9800,17 +4477,17 @@ { "name": "reads only with the caller's JWT", "passed": true, - "notes": "bearer_tokens=2, all_match=true" + "notes": "bearer_tokens=1, all_match=true" }, { "name": "user A cannot force-read user B note", "passed": true, - "notes": "status=403" + "notes": "status=200" }, { "name": "user B cannot force-read user A note", "passed": true, - "notes": "status=403" + "notes": "status=200" } ], "skills": { @@ -9823,158 +4500,78 @@ "prompt": "I built an Edge Function called `private-notes` for showing a user's saved\nprivate notes.\n\nCan you check whether there's any way one user could see another user's notes?\n\nPlease fix and deploy it if needed.", "promptSourcePath": "evals/build-functions-004-service-role-bypass/PROMPT.md", "attempts": 1, - "sourcePath": "codex-gpt-5.4-mini-no-skills/build-functions-004-service-role-bypass.json" - }, - { - "experiment": "codex-gpt-5.4-mini-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.4-mini", - "reasoningEffort": "medium" - }, - "eval": "build-functions-005-dual-auth-user-secret", - "stage": "build", - "product": [ - "edge-functions", - "auth", - "database" - ], - "topic": [ - "sdk", - "rls", - "security" - ], - "suite": "benchmark", - "interface": "cli", - "cliVersion": "2.109.1", - "passed": false, - "checks": [ - { - "name": "seed rows present", - "passed": true, - "notes": "found 2/2 seeded rows" - }, - { - "name": "rejects request with no credentials", - "passed": true, - "notes": "status 401: {\"error\":\"Missing or invalid apikey\"}" - }, - { - "name": "user with JWT reads only their own rows", - "passed": false, - "notes": "status 401: {\"error\":\"Missing or invalid apikey\"}" - }, - { - "name": "user cannot read another user's rows by passing user_id", - "passed": false, - "notes": "status 401: {\"error\":\"Missing or invalid apikey\"}" - }, - { - "name": "service key bypasses RLS to read the target user's rows", - "passed": false, - "notes": "status 401: {\"error\":\"Missing or invalid apikey\"}" - }, - { - "name": "non-service key is not granted service access", - "passed": true, - "notes": "status 401: {\"error\":\"Missing or invalid apikey\"}" - }, - { - "name": "rejects an unverified (forged) user token", - "passed": true, - "notes": "status 401: {\"error\":\"Missing or invalid apikey\"}" - }, - { - "name": "a user token in the apikey slot is not treated as the service key", - "passed": true, - "notes": "status 401: {\"error\":\"Missing or invalid apikey\"}" - }, - { - "name": "implementation uses @supabase/server", - "passed": false, - "notes": "hand-rolled (raw supabase-js or other) — this eval requires @supabase/server" - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [ - { - "source": "web_search", - "query": "site:supabase.com docs supabase functions serve entrypoint failed to determine entrypoint verify_jwt config.toml user-stats", - "pages": [] - } - ] - }, - "prompt": "Build and serve a Supabase Edge Function named `user-stats` for this project,\nreachable over HTTP at `/functions/v1/user-stats`.\n\nOur product stores per-user metrics in the existing `user_stats` table.\n\nTwo very different callers need to hit this one endpoint:\n\n1. **Our mobile app**, acting for a signed-in user. It sends that user's\n Supabase access token. The endpoint should return the user's own stats.\n\n2. **Our internal billing service**, a trusted backend with no signed-in user.\n It authenticates with the project's secret (service-role) key in the `apikey`\n header, and names the target user with a `user_id` in the JSON request body.\n It needs that user's stats.\n\nReturn the matching rows as JSON. The endpoint must be secure: only ever serve\nstats to a caller that is genuinely entitled to them, and turn away callers that\nare not.", - "promptSourcePath": "evals/build-functions-005-dual-auth-user-secret/PROMPT.md", - "attempts": 2, - "sourcePath": "codex-gpt-5.4-mini-no-skills/build-functions-005-dual-auth-user-secret.json" + "sourcePath": "claude-code-sonnet-5-no-skills/build-functions-004-service-role-bypass.json" }, { - "experiment": "codex-gpt-5.4-mini-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.4-mini", - "reasoningEffort": "medium" + "experiment": "claude-code-sonnet-5-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" }, - "eval": "build-storage-001-private-bucket-access", + "eval": "build-functions-005-dual-auth-user-secret", "stage": "build", "product": [ - "storage", + "edge-functions", + "auth", "database" ], "topic": [ + "sdk", "rls", - "sdk" + "security" ], "suite": "benchmark", - "interface": "mcp", + "interface": "cli", + "cliVersion": "2.109.1", "passed": true, "checks": [ { - "name": "bucket user-files exists", - "passed": true + "name": "seed rows present", + "passed": true, + "notes": "found 2/2 seeded rows" }, { - "name": "bucket user-files is private", - "passed": true + "name": "rejects request with no credentials", + "passed": true, + "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" }, { - "name": "RLS still enabled on storage.objects", - "passed": true + "name": "user with JWT reads only their own rows", + "passed": true, + "notes": "status 200: {\"data\":[{\"user_id\":\"6ca63d6b-9ced-44cf-8362-35ea3786d005\",\"metric\":\"steps_a_mrw66yrs\",\"value\":111}]}" }, { - "name": "user A lists only own files", + "name": "user cannot read another user's rows by passing user_id", "passed": true, - "notes": "saw: 019fb0e7-4308-75bd-a52e-bb64802d730c/receipt-alpha.pdf, 019fb0e7-4308-75bd-a52e-bb64802d730c/receipt-beta.pdf" + "notes": "status 200: {\"data\":[{\"user_id\":\"6ca63d6b-9ced-44cf-8362-35ea3786d005\",\"metric\":\"steps_a_mrw66yrs\",\"value\":111}]}" }, { - "name": "user B cannot read user A files", - "passed": true + "name": "service key bypasses RLS to read the target user's rows", + "passed": true, + "notes": "status 200: {\"data\":[{\"user_id\":\"c97e718b-9b4e-4d71-8c8e-694fe9b1469a\",\"metric\":\"steps_b_mrw66yrs\",\"value\":222}]}" }, { - "name": "anon reads no files", - "passed": true + "name": "non-service key is not granted service access", + "passed": true, + "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" }, { - "name": "user A can upload into own folder", - "passed": true + "name": "rejects an unverified (forged) user token", + "passed": true, + "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" }, { - "name": "user B cannot upload into user A folder", - "passed": true + "name": "a user token in the apikey slot is not treated as the service key", + "passed": true, + "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" }, { - "name": "configured private per-user storage access", + "name": "implementation uses @supabase/server", "passed": true, - "judgeNotes": "Configured a private user-files bucket (public=false), kept storage.objects RLS enabled, added authenticated SELECT and INSERT policies scoped to bucket and owner via first folder segment = auth.uid(), and provided supabase-js createSignedUrl with a 15-minute expiry. No public bucket/getPublicUrl/service-role client misuse." + "notes": "imports @supabase/server / withSupabase" } ], "skills": { @@ -9985,129 +4582,211 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"storage bucket RLS policy auth.uid prefix storage.objects signed url\", limit: 5) { nodes { __typename title href content } } }", + "query": "{ searchDocs(query: \"Edge Functions authorization header service role key user JWT example\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/storage/security/access-control", - "title": "Storage Access Control" + "url": "https://supabase.com/docs/guides/auth/jwt-fields", + "title": "JWT Claims Reference" }, { - "url": "https://supabase.com/docs/guides/storage/buckets/fundamentals", - "title": "Storage Buckets" + "url": "https://supabase.com/docs/guides/auth/signing-keys", + "title": "JWT Signing Keys" }, { - "url": "https://supabase.com/docs/guides/security/product-security", - "title": "Secure configuration of Supabase products" + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3", - "title": "Configure S3 Storage" + "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", + "title": "Integrating With Supabase Auth" }, { - "url": "https://supabase.com/docs/guides/storage/schema/design", - "title": "The Storage Schema" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", + "title": "Self-Hosted Functions" } ], - "resultChars": 21438 + "resultChars": 71763 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"supabase-js createSignedUrl storage download temporary link\", limit: 5) { nodes { __typename title href content } } }", + "query": "{ searchDocs(query: \"Securing Edge Functions auth modes user secret publishable multiple\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/storage/serving/downloads", - "title": "Serving assets from Storage" + "url": "https://supabase.com/docs/guides/self-hosting/docker", + "title": "Self-Hosting with Docker" }, { - "url": "https://supabase.com/docs/guides/storage/uploads/resumable-uploads", - "title": "Resumable Uploads" + "url": "https://supabase.com/docs/guides/functions/auth-headers", + "title": "Authorization headers" }, { - "url": "https://supabase.com/docs/reference/dart/file-buckets-createsignedurl" + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" }, { - "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsignedurl" + "url": "https://supabase.com/docs/guides/getting-started/api-keys", + "title": "Understanding API keys" }, { - "url": "https://supabase.com/docs/guides/storage/management/download-objects", - "title": "Download Objects" + "url": "https://supabase.com/docs/guides/database/secure-data", + "title": "Securing your data" } ], - "resultChars": 25058 + "resultChars": 66818 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"storage.buckets policy create bucket RLS\", limit: 5) { nodes { __typename title href content } } }", + "query": "{ searchDocs(query: \"Securing Edge Functions guide auth user secret combined same function accept either\", limit: 3) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/storage/security/access-control", - "title": "Storage Access Control" - }, - { - "url": "https://supabase.com/docs/guides/platform/migrating-within-supabase/backup-restore", - "title": "Backup and Restore using the CLI" - }, - { - "url": "https://supabase.com/docs/guides/storage/buckets/fundamentals", - "title": "Storage Buckets" + "url": "https://supabase.com/docs/guides/functions/auth-headers", + "title": "Authorization headers" }, { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-ionic-vue", - "title": "Build a User Management App with Ionic Vue" + "url": "https://supabase.com/docs/guides/database/secure-data", + "title": "Securing your data" }, { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-svelte", - "title": "Build a User Management App with Svelte" + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" } ], - "resultChars": 96990 + "resultChars": 19847 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"storage.createBucket RLS policy required bucket table\", limit: 5) { nodes { __typename title href content } } }", + "query": "{ searchDocs(query: \"Securing Edge Functions auth guide every auth mode use case withSupabase array optional\", limit: 3) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/storage/security/access-control", - "title": "Storage Access Control" + "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", + "title": "Integrating With Supabase Auth" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3", - "title": "Configure S3 Storage" + "url": "https://supabase.com/docs/guides/functions/architecture", + "title": "Edge Functions Architecture" }, { - "url": "https://supabase.com/docs/guides/storage/buckets/fundamentals", - "title": "Storage Buckets" - }, + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" + } + ], + "resultChars": 25569 + }, + { + "source": "web_fetch", + "query": "Does the withSupabase function or auth option support accepting multiple auth modes for a single function (e.g., allowing either a signed-in user JWT OR a secret key to call the same endpoint)? Show any code examples of combining 'user' and 'secret' auth, and list all valid values/shapes for the `auth` option.", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/guides/local-development/database-migrations", - "title": "Database migrations" - }, + "url": "https://supabase.com/docs/guides/functions/auth" + } + ], + "resultChars": 1274 + }, + { + "source": "web_fetch", + "query": "Give the full documentation content about the ctx object passed to the withSupabase handler: what properties does ctx have (ctx.supabase, ctx.supabaseAdmin, ctx.userClaims, ctx.authMode, ctx.apiKeyName, etc)? What fields does ctx.userClaims contain (does it have 'sub' for user id)? Also show the full example combining 'user' and 'secret' auth modes with the actual database query. Quote exact code samples verbatim.", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/guides/storage/schema/design", - "title": "The Storage Schema" + "url": "https://supabase.com/docs/guides/functions/auth" } ], - "resultChars": 32878 + "resultChars": 1407 } ] }, + "prompt": "Build and serve a Supabase Edge Function named `user-stats` for this project,\nreachable over HTTP at `/functions/v1/user-stats`.\n\nOur product stores per-user metrics in the existing `user_stats` table.\n\nTwo very different callers need to hit this one endpoint:\n\n1. **Our mobile app**, acting for a signed-in user. It sends that user's\n Supabase access token. The endpoint should return the user's own stats.\n\n2. **Our internal billing service**, a trusted backend with no signed-in user.\n It authenticates with the project's secret (service-role) key in the `apikey`\n header, and names the target user with a `user_id` in the JSON request body.\n It needs that user's stats.\n\nReturn the matching rows as JSON. The endpoint must be secure: only ever serve\nstats to a caller that is genuinely entitled to them, and turn away callers that\nare not.", + "promptSourcePath": "evals/build-functions-005-dual-auth-user-secret/PROMPT.md", + "attempts": 2, + "sourcePath": "claude-code-sonnet-5-no-skills/build-functions-005-dual-auth-user-secret.json" + }, + { + "experiment": "claude-code-sonnet-5-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" + }, + "eval": "build-storage-001-private-bucket-access", + "stage": "build", + "product": [ + "storage", + "database" + ], + "topic": [ + "rls", + "sdk" + ], + "suite": "benchmark", + "interface": "mcp", + "passed": true, + "checks": [ + { + "name": "bucket user-files exists", + "passed": true + }, + { + "name": "bucket user-files is private", + "passed": true + }, + { + "name": "RLS still enabled on storage.objects", + "passed": true + }, + { + "name": "user A lists only own files", + "passed": true, + "notes": "saw: 019f8a21-9dc2-72bb-9b82-2098d830d7f3/receipt-alpha.pdf, 019f8a21-9dc2-72bb-9b82-2098d830d7f3/receipt-beta.pdf" + }, + { + "name": "user B cannot read user A files", + "passed": true + }, + { + "name": "anon reads no files", + "passed": true + }, + { + "name": "user A can upload into own folder", + "passed": true + }, + { + "name": "user B cannot upload into user A folder", + "passed": true + }, + { + "name": "configured private per-user storage access", + "passed": true, + "judgeNotes": "Meets all criteria: private bucket, authenticated owner-scoped SELECT and INSERT policies on storage.objects, no RLS disabling or public access, and uses createSignedUrl with an expiry for sharing." + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] + }, "prompt": "Our app lets signed-in users keep personal files like receipts and bank\nstatements. These files are private — a user must only ever be able to upload\nand download their own. The app uploads each file under a path that starts\nwith the owner's user id, e.g. `/receipt-march.pdf`.\n\nSet up a `user-files` bucket on our project and lock it down that way.\n\nUsers also sometimes share one of their files with someone else through a\ntemporary link that expires. Include the supabase-js code the app should use\nfor that.", "promptSourcePath": "evals/build-storage-001-private-bucket-access/PROMPT.md", "attempts": 1, - "sourcePath": "codex-gpt-5.4-mini-no-skills/build-storage-001-private-bucket-access.json" + "sourcePath": "claude-code-sonnet-5-no-skills/build-storage-001-private-bucket-access.json" }, { - "experiment": "codex-gpt-5.4-mini-no-skills", + "experiment": "claude-code-sonnet-5-no-skills", "experimentSuite": "no-skills", "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.4-mini", - "reasoningEffort": "medium" + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" }, "eval": "build-tests-001-rls-tenant-isolation", "stage": "build", @@ -10125,17 +4804,17 @@ { "name": "pgTAP test file(s) written under supabase/tests/", "passed": true, - "notes": "1 file(s): supabase/tests/tenant_isolation.sql" + "notes": "3 file(s): supabase/tests/database/02_posts_isolation.test.sql, supabase/tests/database/01_notes_isolation.test.sql, supabase/tests/database/00_memberships_isolation.test.sql" }, { "name": "pgTAP isolation tests ran and pass", "passed": true, - "notes": "6 passed, 0 failed" + "notes": "1 passed, 1 failed" }, { "name": "agent correctly identifies the posts isolation bug from test results", "passed": true, - "judgeNotes": "The agent correctly identifies `posts` as having the tenant isolation flaw: the policy checks membership but not the row's `org_id`, allowing authenticated members to read posts from other orgs. It does not blame `notes` or dismiss pgTAP; it adds/runs tests and reports passing after fixing the policy." + "judgeNotes": "The agent correctly identifies `posts` as having a broken SELECT policy that lacks an `org_id` check, causing authenticated members of one org to read posts from another org. It grounds this in pgTAP results (`have: 1, want: 0`) and correctly distinguishes `notes` as isolated." } ], "skills": { @@ -10148,16 +4827,16 @@ "prompt": "Can you audit the tenant isolation on our tables? Write some database tests covering both the happy path and the negative case, run them, and share the results.", "promptSourcePath": "evals/build-tests-001-rls-tenant-isolation/PROMPT.md", "attempts": 1, - "sourcePath": "codex-gpt-5.4-mini-no-skills/build-tests-001-rls-tenant-isolation.json" + "sourcePath": "claude-code-sonnet-5-no-skills/build-tests-001-rls-tenant-isolation.json" }, { - "experiment": "codex-gpt-5.4-mini-no-skills", + "experiment": "claude-code-sonnet-5-no-skills", "experimentSuite": "no-skills", "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.4-mini", - "reasoningEffort": "medium" + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" }, "eval": "build-vectors-001-rag-with-permissions", "stage": "build", @@ -10181,12 +4860,12 @@ { "name": "HNSW index on the embedding column", "passed": true, - "notes": "CREATE INDEX document_sections_embedding_hnsw_idx ON public.document_sections USING hnsw (embedding vector_cosine_ops) WHERE (embedding IS NOT NULL)" + "notes": "CREATE INDEX document_sections_embedding_idx ON public.document_sections USING hnsw (embedding vector_cosine_ops)" }, { "name": "index operator class matches the search operator", "passed": true, - "notes": "function operators: <=>\nindexes: CREATE INDEX document_sections_embedding_hnsw_idx ON public.document_sections USING hnsw (embedding vector_cosine_ops) WHERE (embedding IS NOT NULL)" + "notes": "function operators: <=>\nindexes: CREATE INDEX document_sections_embedding_idx ON public.document_sections USING hnsw (embedding vector_cosine_ops)" }, { "name": "user A search returns only own sections, best match first", @@ -10215,16 +4894,16 @@ "prompt": "We're adding semantic search to our internal knowledge base app. I already wrote the edge functions, but search doesn't work yet. Can you set up whatever the database needs to make search work end to end?\n\nSome documents are confidential, so users should only have access to documents they own.", "promptSourcePath": "evals/build-vectors-001-rag-with-permissions/PROMPT.md", "attempts": 1, - "sourcePath": "codex-gpt-5.4-mini-no-skills/build-vectors-001-rag-with-permissions.json" + "sourcePath": "claude-code-sonnet-5-no-skills/build-vectors-001-rag-with-permissions.json" }, { - "experiment": "codex-gpt-5.4-mini-no-skills", + "experiment": "claude-code-sonnet-5-no-skills", "experimentSuite": "no-skills", "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.4-mini", - "reasoningEffort": "medium" + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" }, "eval": "deploy-database-001-prometheus-metrics", "stage": "deploy", @@ -10235,7 +4914,7 @@ "observability" ], "suite": "benchmark", - "passed": false, + "passed": true, "checks": [ { "name": "preserved existing app scrape job", @@ -10243,13 +4922,13 @@ }, { "name": "configured the Supabase Metrics API scrape correctly", - "passed": false, - "judgeNotes": "Fails: Supabase scrape uses basic_auth.password instead of password_file, and docker-compose.yml does not mount the password file via a volume or Compose secret. README also instructs hardcoding the Secret API key in prometheus.yml." + "passed": true, + "judgeNotes": "Adds Supabase scrape over HTTPS with correct metrics path, Basic Auth using password_file, preserves app scrape, and mounts the secrets directory into Prometheus via docker-compose volume." }, { "name": "documented live deployment and verification steps", - "passed": false, - "judgeNotes": "README instructs replacing a placeholder directly in prometheus.yml rather than placing a matching secret file, and the Compose setup does not mount/use a secret file. It also only says restart Prometheus or reload, not restart/reload the Compose stack. Verification is present but secret setup does not meet the rubric." + "passed": true, + "judgeNotes": "README includes correct Supabase Metrics endpoint/auth, Secret API key creation, matching password_file setup, placeholder replacement, Prometheus reload, and concrete verification via Prometheus targets." } ], "skills": { @@ -10259,30 +4938,48 @@ "docs": { "calls": [ { - "source": "web_search", - "query": "site:supabase.com/docs Supabase Prometheus metrics endpoint observability", - "pages": [] - }, - { - "source": "web_search", - "query": "site:supabase.com/docs/guides/telemetry/metrics/grafana-self-hosted Supabase Metrics API self-hosted Prometheus Grafana official", - "pages": [] + "source": "search_docs", + "query": "{ searchDocs(query: \"prometheus metrics endpoint project observability\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/telemetry/metrics/grafana-self-hosted", + "title": "Metrics API with Prometheus & Grafana (self-hosted)" + }, + { + "url": "https://supabase.com/docs/guides/telemetry/metrics", + "title": "Metrics API" + }, + { + "url": "https://supabase.com/docs/guides/telemetry/metrics/vendor-agnostic", + "title": "Vendor-agnostic Metrics API setup" + }, + { + "url": "https://supabase.com/docs/guides/telemetry/metrics/grafana-cloud", + "title": "Metrics API with Grafana Cloud" + }, + { + "url": "https://supabase.com/docs/guides/platform/read-replicas", + "title": "Read Replicas" + } + ], + "resultChars": 32656 } ] }, - "prompt": "Can you wire my Supabase project metrics into our existing observability stack and document\nin the observability README what we need to do to make the config live?", + "prompt": "Can you wire my Supabase project metrics into our existing observability stack and document\nwhat we need to do to make the config live?", "promptSourcePath": "evals/deploy-database-001-prometheus-metrics/PROMPT.md", - "attempts": 2, - "sourcePath": "codex-gpt-5.4-mini-no-skills/deploy-database-001-prometheus-metrics.json" + "attempts": 1, + "sourcePath": "claude-code-sonnet-5-no-skills/deploy-database-001-prometheus-metrics.json" }, { - "experiment": "codex-gpt-5.4-mini-no-skills", + "experiment": "claude-code-sonnet-5-no-skills", "experimentSuite": "no-skills", "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.4-mini", - "reasoningEffort": "medium" + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" }, "eval": "deploy-functions-001-edge-function-secrets", "stage": "deploy", @@ -10320,100 +5017,21 @@ "loaded": [] }, "docs": { - "calls": [ - { - "source": "web_search", - "query": "site:supabase.com/docs Supabase Edge Functions Project not specified functions.supabase.co", - "pages": [] - }, - { - "source": "web_search", - "query": "https://supabase.com/docs/guides/functions/deploy", - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/deploy" - } - ] - }, - { - "source": "web_search", - "query": "site:supabase.com/docs management api invoke edge function supabase", - "pages": [] - }, - { - "source": "web_search", - "query": "\"Project not specified\" supabase function", - "pages": [] - }, - { - "source": "web_search", - "query": "site:supabase.com/docs \"Edge Function\" invoke management API", - "pages": [] - }, - { - "source": "web_search", - "query": "site:supabase.com/docs \"functions/v1\" \"project not specified\"", - "pages": [] - }, - { - "source": "web_search", - "query": "site:supabase.com/docs \"functions.supabase.co\" \"apikey\" \"project\"", - "pages": [] - }, - { - "source": "web_search", - "query": "'functions.supabase.co'", - "pages": [] - }, - { - "source": "web_search", - "query": "site:supabase.com/docs/reference/api/v1 functions invoke edge function", - "pages": [] - }, - { - "source": "web_search", - "query": "site:supabase.com/docs/reference/api v1 projects functions logs supabase", - "pages": [] - }, - { - "source": "web_search", - "query": "site:supabase.com/docs/reference/api publishable keys project api keys supabase management api", - "pages": [] - }, - { - "source": "web_search", - "query": "https://supabase.com/docs/reference/api/introduction", - "pages": [ - { - "url": "https://supabase.com/docs/reference/api/introduction" - } - ] - }, - { - "source": "web_search", - "query": "site:supabase.com/docs \"publishable key\" \"Management API\" Supabase project", - "pages": [] - }, - { - "source": "web_search", - "query": "site:supabase.com/docs/reference/api \"publishable key\" \"projects/{ref}\" supabase", - "pages": [] - } - ] + "calls": [] }, "prompt": "Our weather widget currently calls WeatherAPI straight from the browser, which\nleaks our API key. I want to move that behind a Supabase Edge Function called\n`weather` that holds the key server-side and proxies the request.\n\nThe function should read the key from an environment variable named\n`WEATHER_API_KEY`. Our key already lives in a local `.env` file at the project\nroot.\n\nDeploy the function to our project so it's live, and make sure the deployed\nfunction can actually read the key at runtime.", "promptSourcePath": "evals/deploy-functions-001-edge-function-secrets/PROMPT.md", "attempts": 1, - "sourcePath": "codex-gpt-5.4-mini-no-skills/deploy-functions-001-edge-function-secrets.json" + "sourcePath": "claude-code-sonnet-5-no-skills/deploy-functions-001-edge-function-secrets.json" }, { - "experiment": "codex-gpt-5.4-mini-no-skills", + "experiment": "claude-code-sonnet-5-no-skills", "experimentSuite": "no-skills", "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.4-mini", - "reasoningEffort": "medium" + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" }, "eval": "deploy-self-hosting-001-docker-compose", "stage": "deploy", @@ -10451,78 +5069,21 @@ "loaded": [] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase self-host Docker compose environment variables anon key service_role JWT secrets\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", - "title": "New API Keys and Asymmetric Authentication" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", - "title": "Self-Hosted Functions" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", - "title": "Envoy API Gateway" - }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys", - "title": "JWT Signing Keys" - }, - { - "url": "https://supabase.com/docs/guides/functions/secrets", - "title": "Environment Variables" - } - ], - "resultChars": 88894 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Self-hosting with Docker quick start Linux generate-keys.sh docker-compose.yml .env.example\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", - "title": "New API Keys and Asymmetric Authentication" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker", - "title": "Self-Hosting with Docker" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth", - "title": "Configure Social Login (OAuth) Providers" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/custom-email-templates", - "title": "Custom Email Templates" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", - "title": "Envoy API Gateway" - } - ], - "resultChars": 103382 - } - ] + "calls": [] }, "prompt": "I'm moving off the hosted Supabase and running the whole thing myself on a VPS I\njust spun up. Can you get a Docker setup ready for me to copy onto the box?\n\nI don't need it running here, I'll do the actual bring-up once I'm on the\nserver. I just want everything in place and the secrets set up. Put it in a `supabase-docker/`\nfolder at the repo root so I can scp the whole thing across in one go.", "promptSourcePath": "evals/deploy-self-hosting-001-docker-compose/PROMPT.md", "attempts": 1, - "sourcePath": "codex-gpt-5.4-mini-no-skills/deploy-self-hosting-001-docker-compose.json" + "sourcePath": "claude-code-sonnet-5-no-skills/deploy-self-hosting-001-docker-compose.json" }, { - "experiment": "codex-gpt-5.4-mini-no-skills", + "experiment": "claude-code-sonnet-5-no-skills", "experimentSuite": "no-skills", "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.4-mini", - "reasoningEffort": "medium" + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" }, "eval": "investigate-auth-001-deleted-user-access", "stage": "investigate", @@ -10565,8 +5126,8 @@ }, { "name": "diagnosed and explained session revocation", - "passed": false, - "judgeNotes": "The answer correctly diagnoses the soft-delete-only bug, revokes auth sessions by deleting auth.sessions (cascading refresh tokens), adds RLS checks against deleted profiles, and correctly distinguishes publishable vs secret keys. However, it says 'deleting a user does not automatically sign them out' even though its implemented fix does not delete the auth user, only sessions, and the rubric requires fixing the flow so the auth user is deleted or equivalently identity and sessions are removed. The implemented fix leaves identities/auth user intact, so future sign-in may still be possible; it does not fully satisfy real account deletion/removal of identity. It also overstates 'no database-access window' based on a deleted profile RLS check, which is acceptable for the shown data path, but the missing identity/user deletion is a failing issue." + "passed": true, + "judgeNotes": "Meets the rubric: correctly diagnoses soft-delete-only flow, implements auth/session revocation plus RLS enforcement, explains JWT expiry/local-validation caveat consistently with the RLS fix, and correctly distinguishes publishable vs secret keys and RLS behavior." } ], "skills": { @@ -10574,32 +5135,21 @@ "loaded": [] }, "docs": { - "calls": [ - { - "source": "web_search", - "query": "site:supabase.com/docs publishable key secret key anon service_role RLS Supabase", - "pages": [] - }, - { - "source": "web_search", - "query": "site:supabase.com/docs migrating to new API keys publishable secret keys Supabase docs", - "pages": [] - } - ] + "calls": [] }, "prompt": "Last week support removed a user through our app's delete-account flow — the\napp calls the `delete_account` function over RPC as the signed-in user. This\nmorning that same person was back: still signed in, reading and saving their\ndata like nothing happened.\n\nFigure out why the account still works, fix the flow so a deleted account\nloses access, and tell me whether there is any window where they could still\nget in after the fix.\n\nOne more thing while you're at it: we're migrating off the legacy\nanon/service_role API keys to the new publishable and secret keys, and the\nteam isn't sure which one belongs in the frontend and what each means for\nRLS. Give us a quick rundown.", "promptSourcePath": "evals/investigate-auth-001-deleted-user-access/PROMPT.md", "attempts": 2, - "sourcePath": "codex-gpt-5.4-mini-no-skills/investigate-auth-001-deleted-user-access.json" + "sourcePath": "claude-code-sonnet-5-no-skills/investigate-auth-001-deleted-user-access.json" }, { - "experiment": "codex-gpt-5.4-mini-no-skills", + "experiment": "claude-code-sonnet-5-no-skills", "experimentSuite": "no-skills", "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.4-mini", - "reasoningEffort": "medium" + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" }, "eval": "investigate-realtime-001-subscribed-no-events", "stage": "investigate", @@ -10638,7 +5188,7 @@ { "name": "diagnosed missing publication membership", "passed": true, - "judgeNotes": "Identified orders missing from supabase_realtime publication, added only public.orders with ALTER PUBLICATION, verified courier_locations remained, and did not weaken RLS/policies or blame client/RLS as root cause." + "judgeNotes": "The assistant correctly identified that orders was missing from the supabase_realtime publication despite SUBSCRIBED status, added only public.orders to the existing publication, preserved courier_locations/RLS/policies, and did not blame or alter RLS/client/networking." } ], "skills": { @@ -10651,16 +5201,16 @@ "prompt": "Our dispatch dashboard shows incoming orders as they happen. The courier\nlocation feed on the same page updates live without problems, but new orders\nonly show up after a page refresh.\n\nThe dashboard uses supabase-js to subscribe to INSERT events on the `orders`\ntable through postgres_changes, the same way it subscribes to courier\nlocations. The channel's status callback logs SUBSCRIBED and there are no\nerrors in the browser console.\n\nFigure out why no order events ever arrive and fix it.", "promptSourcePath": "evals/investigate-realtime-001-subscribed-no-events/PROMPT.md", "attempts": 1, - "sourcePath": "codex-gpt-5.4-mini-no-skills/investigate-realtime-001-subscribed-no-events.json" + "sourcePath": "claude-code-sonnet-5-no-skills/investigate-realtime-001-subscribed-no-events.json" }, { - "experiment": "codex-gpt-5.4-mini-no-skills", + "experiment": "claude-code-sonnet-5-no-skills", "experimentSuite": "no-skills", "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.4-mini", - "reasoningEffort": "medium" + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" }, "eval": "investigate-reliability-003-edge-function-5xx-correlation", "stage": "investigate", @@ -10677,17 +5227,17 @@ { "name": "identified image-transform and the recurring 503 pattern", "passed": true, - "judgeNotes": "Identified image-transform as the affected function and described recurring HTTP 503 gateway failures throughout the morning of 2026-04-28, covering the relevant 07:00Z-12:00Z window and most/all failures." + "judgeNotes": "Assistant identified image-transform as affected and described the recurring pattern of ~8 HTTP 503 failures across 07:00Z–12:00Z on 2026-04-28, while distinguishing unrelated billing-webhook 503s." }, { "name": "attributed recurring 503s to gateway/platform layer, not function code", "passed": true, - "judgeNotes": "Attributes image-transform 503s to gateway/platform layer, supported by unchanged deployment/version with intermittent successes and distinction from avatar-upload's function-level 500." + "judgeNotes": "The assistant attributes the recurring image-transform 503s to the gateway/platform layer rather than function code, and grounds this in valid observations: the 503s appear only in gateway/API logs with no corresponding edge function invocation logs while nearby 200 invocations succeeded, and the deployment/version was unchanged for successful invocations. It also distinguishes these gateway 503s from a separate function-level 500." }, { "name": "recommended a concrete next step", "passed": true, - "judgeNotes": "The assistant recommended concrete next steps: checking Supabase platform/edge runtime incident health, correlating failing requests with payloads/request IDs, adding retries for 503s, and opening a Supabase support case with timestamps, function slugs, deployment IDs, and failing request IDs." + "judgeNotes": "The assistant recommended multiple concrete next steps, including checking Supabase status history for the time window, reviewing package/deployment changes, adding explicit logging, client retries, and checking concurrency limits." } ], "skills": { @@ -10699,17 +5249,17 @@ }, "prompt": "Users have been reporting that image uploads are intermittently failing with server errors throughout the morning of `2026-04-28`.\n\nCan you investigate the project logs and tell me what is going on and what we\nshould do next?", "promptSourcePath": "evals/investigate-reliability-003-edge-function-5xx-correlation/PROMPT.md", - "attempts": 2, - "sourcePath": "codex-gpt-5.4-mini-no-skills/investigate-reliability-003-edge-function-5xx-correlation.json" + "attempts": 1, + "sourcePath": "claude-code-sonnet-5-no-skills/investigate-reliability-003-edge-function-5xx-correlation.json" }, { - "experiment": "codex-gpt-5.4-mini-no-skills", + "experiment": "claude-code-sonnet-5-no-skills", "experimentSuite": "no-skills", "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.4-mini", - "reasoningEffort": "medium" + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" }, "eval": "resolve-dataapi-001-empty-results", "stage": "resolve", @@ -10753,7 +5303,7 @@ { "name": "diagnosed RLS and added owner-scoped policies", "passed": true, - "judgeNotes": "Diagnosed RLS enabled with no policies as deny-all, kept RLS enabled, and created authenticated owner-scoped SELECT and INSERT policies using user_id = auth.uid() with WITH CHECK for INSERT. Extra UPDATE/DELETE owner policies do not violate the rubric." + "judgeNotes": "Diagnoses RLS enabled with no policies as deny-all causing empty Data API results, keeps RLS enabled, and creates authenticated SELECT and INSERT policies scoped to auth.uid() = user_id using USING and WITH CHECK." } ], "skills": { @@ -10766,16 +5316,16 @@ "prompt": "Our app lets signed-in users save bookmarks and view them on their dashboard. Bookmarks are stored in the `bookmarks` table and are private — a user must only ever see their own. \nUsers also need to be able to save new bookmarks from the app.\n\nI can see the rows when I query the table directly, but the dashboard shows an empty list for every user.\n\nFind out why the Data API returns nothing and fix it.", "promptSourcePath": "evals/resolve-dataapi-001-empty-results/PROMPT.md", "attempts": 1, - "sourcePath": "codex-gpt-5.4-mini-no-skills/resolve-dataapi-001-empty-results.json" + "sourcePath": "claude-code-sonnet-5-no-skills/resolve-dataapi-001-empty-results.json" }, { - "experiment": "codex-gpt-5.4-mini-no-skills", + "experiment": "claude-code-sonnet-5-no-skills", "experimentSuite": "no-skills", "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.4-mini", - "reasoningEffort": "medium" + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" }, "eval": "resolve-database-001-migration-history-mismatch", "stage": "resolve", @@ -10787,7 +5337,7 @@ ], "suite": "benchmark", "interface": "cli", - "passed": false, + "passed": true, "checks": [ { "name": "the avatar_url column is applied on the hosted profiles table", @@ -10795,18 +5345,15 @@ }, { "name": "migration 20240220000000 is recorded in the remote history", - "passed": false, - "notes": "remote history versions: [\"20240101000000\",\"20240115000000\"]" + "passed": true }, { "name": "remote migration history matches local migration files", - "passed": false, - "notes": "remote-only versions still unreconciled: [\"20240115000000\"] (remote: [\"20240101000000\",\"20240115000000\"], local: [\"20240101000000\",\"20240220000000\"])" + "passed": true }, { "name": "local migrations are a valid reconciled sequence", - "passed": false, - "notes": "expected 20240101000000_create_profiles → → 20240220000000_add_avatar_url, strictly ascending; got [20240101000000_create_profiles.sql, 20240220000000_add_avatar_url.sql]" + "passed": true }, { "name": "production profile data is intact (not reset)", @@ -10814,8 +5361,8 @@ }, { "name": "the avatar migration and history reconciliation were done via the Supabase CLI", - "passed": false, - "judgeNotes": "No successful `supabase db push` occurred; all push/list attempts failed with IPv6/DNS errors. The avatar_url change was applied via a non-CLI Management API call to `/v1/projects/{ref}/database/migrations` in command #58, not via `supabase db push`. No CLI reconciliation command such as `supabase migration repair` or `supabase db pull` succeeded." + "passed": true, + "judgeNotes": "Avatar_url was applied through the Supabase CLI with `supabase db push` in step #17, which shows `Applying migration 20240220000000_add_avatar_url.sql...` and `Finished supabase db push.` The orphan bio migration was reconciled by adding the local file `supabase/migrations/20240115000000_add_bio.sql` in step #15, after which `supabase migration list` showed it matched remote. No prohibited workaround was used." } ], "skills": { @@ -10827,17 +5374,17 @@ }, "prompt": "I'm trying to ship a migration to our hosted project and it's not working. Can you figure out what's wrong and get it deployed?", "promptSourcePath": "evals/resolve-database-001-migration-history-mismatch/PROMPT.md", - "attempts": 2, - "sourcePath": "codex-gpt-5.4-mini-no-skills/resolve-database-001-migration-history-mismatch.json" + "attempts": 1, + "sourcePath": "claude-code-sonnet-5-no-skills/resolve-database-001-migration-history-mismatch.json" }, { - "experiment": "codex-gpt-5.4-mini-no-skills", + "experiment": "claude-code-sonnet-5-no-skills", "experimentSuite": "no-skills", "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.4-mini", - "reasoningEffort": "medium" + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" }, "eval": "resolve-performance-001-slow-query-cpu-spike", "stage": "resolve", @@ -10867,7 +5414,7 @@ { "name": "query plan uses an index and avoids sequential scan", "passed": true, - "notes": "Limit (cost=55.55..55.61 rows=25 width=88)\n -> Sort (cost=55.55..55.61 rows=25 width=88)\n Sort Key: created_at DESC\n -> Bitmap Heap Scan on events (cost=4.48..54.97 rows=25 width=88)\n Recheck Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)\n -> Bitmap Index Scan on events_user_id_created_at_desc_idx (cost=0.00..4.47 rows=25 width=0)\n Index Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)" + "notes": "Limit (cost=55.55..55.61 rows=25 width=88)\n -> Sort (cost=55.55..55.61 rows=25 width=88)\n Sort Key: created_at DESC\n -> Bitmap Heap Scan on events (cost=4.48..54.97 rows=25 width=88)\n Recheck Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)\n -> Bitmap Index Scan on events_user_id_created_at_idx (cost=0.00..4.47 rows=25 width=0)\n Index Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)" }, { "name": "inserts still work", @@ -10884,16 +5431,16 @@ "prompt": "My database CPU keeps spiking and the app gets slow when loading recent events for a user. Can you figure out what query is causing it and make the database change needed to fix it?\n\nEnd your turn with a short summary of what you changed and why.", "promptSourcePath": "evals/resolve-performance-001-slow-query-cpu-spike/PROMPT.md", "attempts": 1, - "sourcePath": "codex-gpt-5.4-mini-no-skills/resolve-performance-001-slow-query-cpu-spike.json" + "sourcePath": "claude-code-sonnet-5-no-skills/resolve-performance-001-slow-query-cpu-spike.json" }, { - "experiment": "codex-gpt-5.4-mini-no-skills", + "experiment": "claude-code-sonnet-5-no-skills", "experimentSuite": "no-skills", "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.4-mini", - "reasoningEffort": "medium" + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" }, "eval": "resolve-security-002-rls-cross-tenant-leak", "stage": "resolve", @@ -10956,15 +5503,15 @@ "prompt": "A customer reported that notes showed up in the wrong workspace.\n\nCan you investigate what is going on and fix it?", "promptSourcePath": "evals/resolve-security-002-rls-cross-tenant-leak/PROMPT.md", "attempts": 1, - "sourcePath": "codex-gpt-5.4-mini-no-skills/resolve-security-002-rls-cross-tenant-leak.json" + "sourcePath": "claude-code-sonnet-5-no-skills/resolve-security-002-rls-cross-tenant-leak.json" }, { - "experiment": "codex-gpt-5.6", + "experiment": "codex-gpt-5.4-mini", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.6-sol", + "modelId": "gpt-5.4-mini", "reasoningEffort": "medium" }, "eval": "build-cli-001-bootstrap-app", @@ -11026,46 +5573,94 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"Row Level Security policy authenticated role Data API grants select anon authenticated local development migrations seed\", limit: 5) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"local development migrations RLS exposed schema Data API anon authenticated\", limit: 5) { nodes { __typename ... on Guide { title href content subsections { totalCount } } ... on CLICommandReference { title href content } ... on TroubleshootingGuide { title href content } ... on ManagementApiReference { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } } totalCount } }", "hasContent": true, "pages": [ + { + "url": "https://supabase.com/docs/guides/local-development/cli-workflows", + "title": "Local development workflow" + }, { "url": "https://supabase.com/docs/guides/api/securing-your-api", "title": "Securing your API" }, { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows", - "title": "Local development workflow" + "url": "https://supabase.com/docs/guides/api/using-custom-schemas", + "title": "Using Custom Schemas" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-refine", + "title": "Build a User Management App with Refine" + }, + { + "url": "https://supabase.com/docs/guides/database/connecting-to-postgres", + "title": "Connect to your database" + } + ], + "resultChars": 103807 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Exposing a Table to the Data API grant authenticated anon RLS public schema\", limit: 10) { nodes { __typename ... on Guide { title href content } } totalCount } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/api/securing-your-api", + "title": "Securing your API" }, { "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", "title": "Row Level Security" }, { - "url": "https://supabase.com/docs/guides/auth/auth-anonymous", - "title": "Anonymous Sign-Ins" + "url": "https://supabase.com/docs/guides/realtime/postgres-changes", + "title": "Postgres Changes" }, { - "url": "https://supabase.com/docs/guides/getting-started/features", - "title": "Features" + "url": "https://supabase.com/docs/guides/database/database-advisors?queryGroups=lint&lint=0026_pg_graphql_anon_table_exposed", + "title": "Database Advisor: Lint 0026_pg_graphql_anon_table_exposed" + }, + { + "url": "https://supabase.com/docs/guides/database/database-advisors?queryGroups=lint&lint=0027_pg_graphql_authenticated_table_exposed", + "title": "Database Advisor: Lint 0027_pg_graphql_authenticated_table_exposed" + }, + { + "url": "https://supabase.com/docs/guides/database/database-advisors?queryGroups=lint&lint=0017_foreign_table_in_api", + "title": "Database Advisor: Lint 0017_foreign_table_in_api" + }, + { + "url": "https://supabase.com/docs/guides/database/database-advisors?queryGroups=lint&lint=0023_sensitive_columns_exposed", + "title": "Database Advisor: Lint 0023_sensitive_columns_exposed" + }, + { + "url": "https://supabase.com/docs/guides/database/database-advisors?queryGroups=lint&lint=0013_rls_disabled_in_public", + "title": "Database Advisor: Lint 0013_rls_disabled_in_public" + }, + { + "url": "https://supabase.com/docs/guides/api/using-custom-schemas", + "title": "Using Custom Schemas" + }, + { + "url": "https://supabase.com/docs/guides/database/secure-data", + "title": "Securing your data" } ], - "resultChars": 100768 + "resultChars": 132936 } ] }, "prompt": "We're kicking off a todos app and I want the Supabase side ready for the team\nto build on. Set it up the way we'd run it in development, with schema changes\ntracked as migrations so they can be reviewed and replayed.\n\nFor the first slice we just need a `todos` table. Todos aren't public: anyone\nsigned in can read all of them, but nothing should be writable through the API\nfor now. Add a couple of sample todos so there's something to look at.\n\nBefore you hand it back, make sure the running API actually behaves that way —\nsigned-in users get the todos, signed-out requests get nothing.", "promptSourcePath": "evals/build-cli-001-bootstrap-app/PROMPT.md", "attempts": 1, - "sourcePath": "codex-gpt-5.6/build-cli-001-bootstrap-app.json" + "sourcePath": "codex-gpt-5.4-mini/build-cli-001-bootstrap-app.json" }, { - "experiment": "codex-gpt-5.6", + "experiment": "codex-gpt-5.4-mini", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.6-sol", + "modelId": "gpt-5.4-mini", "reasoningEffort": "medium" }, "eval": "build-cli-002-declarative-schema", @@ -11111,46 +5706,46 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"declarative database schemas schema_paths db diff local migration\", limit: 5) { nodes { ... on Guide { title href content } ... on CLICommandReference { title href content } } } }", + "query": "query { searchDocs(query: \"declarative database schemas local development supabase schema_paths migrations\", limit: 5) { nodes { __typename ... on Guide { title href content } ... on TroubleshootingGuide { title href content } ... on CLICommandReference { title href content } ... on ClientLibraryFunctionReference { title href content } } totalCount } }", "hasContent": true, "pages": [ { "url": "https://supabase.com/docs/guides/local-development/declarative-database-schemas", "title": "Declarative database schemas" }, - { - "url": "https://supabase.com/docs/guides/deployment/database-migrations", - "title": "Database Migrations" - }, { "url": "https://supabase.com/docs/guides/local-development/cli-workflows", "title": "Local development workflow" }, { - "url": "https://supabase.com/docs/reference/cli/supabase-db-diff", - "title": "Diffs the local database for schema changes" + "url": "https://supabase.com/docs/guides/deployment/branching/working-with-branches", + "title": "Working with branches" + }, + { + "url": "https://supabase.com/docs/guides/local-development/database-migrations", + "title": "Database migrations" }, { "url": "https://supabase.com/docs/guides/deployment/managing-environments", "title": "Managing Environments" } ], - "resultChars": 72112 + "resultChars": 76347 } ] }, "prompt": "Add a description text column to the `products` table in my local Supabase stack", "promptSourcePath": "evals/build-cli-002-declarative-schema/PROMPT.md", "attempts": 1, - "sourcePath": "codex-gpt-5.6/build-cli-002-declarative-schema.json" + "sourcePath": "codex-gpt-5.4-mini/build-cli-002-declarative-schema.json" }, { - "experiment": "codex-gpt-5.6", + "experiment": "codex-gpt-5.4-mini", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.6-sol", + "modelId": "gpt-5.4-mini", "reasoningEffort": "medium" }, "eval": "build-cli-003-pg-cron-queue-workflow", @@ -11177,7 +5772,7 @@ { "name": "cron command enqueues to the 'tasks' queue", "passed": true, - "notes": "queue depth 0 -> 1" + "notes": "queue depth 1 -> 2" }, { "name": "process-tasks function drains the queue", @@ -11198,585 +5793,110 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase Queues pgmq create queue send read pop delete messages cron schedule every minute Edge Function local scheduled worker\", limit: 8) { nodes { __typename title href content ... on Guide { subsections { nodes { title href content } } } } } }", + "query": "query { searchDocs(query: \"pg_cron create schedule every minute supabase cron schedule SQL\", limit: 5) { nodes { __typename ... on Guide { title href content } ... on CLICommandReference { title href content } ... on ManagementApiReference { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } ... on TroubleshootingGuide { title href content } } } }", "hasContent": true, "pages": [ - { - "url": "https://supabase.com/docs/guides/queues", - "title": "Supabase Queues" - }, - { - "url": "https://supabase.com/docs/guides/queues#features", - "title": "Features" - }, - { - "url": "https://supabase.com/docs/guides/queues#resources", - "title": "Resources" - }, - { - "url": "https://supabase.com/docs/guides/cron", - "title": "Cron" - }, - { - "url": "https://supabase.com/docs/guides/cron#how-does-cron-work", - "title": "How does Cron work?" - }, - { - "url": "https://supabase.com/docs/guides/cron#resources", - "title": "Resources" - }, - { - "url": "https://supabase.com/docs/guides/functions/schedule-functions", - "title": "Scheduling Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/functions/schedule-functions#examples", - "title": "Examples" - }, - { - "url": "https://supabase.com/docs/guides/functions/schedule-functions#resources", - "title": "Resources" - }, - { - "url": "https://supabase.com/docs/guides/functions/schedule-functions#invoke-an-edge-function-every-minute", - "title": "Invoke an Edge Function every minute" - }, { "url": "https://supabase.com/docs/guides/database/extensions/pg_net", - "title": "pg_net: Async Networking" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#http_delete", - "title": "http_delete" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#debugging-requests", - "title": "Debugging requests" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#analyzing-responses", - "title": "Analyzing responses" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#delete-usage", - "title": "Usage " - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#post-usage", - "title": "Usage " - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#post-signature", - "title": "Signature " - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#http_post", - "title": "http_post" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#get-usage", - "title": "Usage " - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#get-signature", - "title": "Signature " - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#http_get", - "title": "http_get" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#enable-the-extension", - "title": "Enable the extension" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#resources", - "title": "Resources" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#limitations", - "title": "Limitations" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#send-multiple-table-rows-in-one-request", - "title": "Send multiple table rows in one request" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#execute-pg_net-in-a-trigger", - "title": "Execute pg_net in a trigger" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#call-an-endpoint-every-minute-with-pg_cron", - "title": "Call an endpoint every minute with pg_cron" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#invoke-a-supabase-edge-function", - "title": "Invoke a Supabase Edge Function" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#examples", - "title": "Examples" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#alter-settings", - "title": "Alter settings" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#get-current-settings", - "title": "Get current settings" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#configuration", - "title": "Configuration" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#inspecting-failed-requests", - "title": "Inspecting failed requests" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#inspecting-request-data", - "title": "Inspecting request data" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#post-signature-1", - "title": "Signature " - }, - { - "url": "https://supabase.com/docs/guides/queues/consuming-messages-with-edge-functions", - "title": "Consuming Supabase Queue Messages with Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/queues/consuming-messages-with-edge-functions#consuming-messages-in-an-edge-function", - "title": "Consuming messages in an Edge Function" - }, - { - "url": "https://supabase.com/docs/guides/queues/consuming-messages-with-edge-functions#concepts", - "title": "Concepts" - }, - { - "url": "https://supabase.com/docs/guides/queues/quickstart", - "title": "Quickstart" - }, - { - "url": "https://supabase.com/docs/guides/queues/quickstart#pull-based-queue", - "title": "Pull-Based Queue" - }, - { - "url": "https://supabase.com/docs/guides/queues/quickstart#create-queues", - "title": "Create Queues" - }, - { - "url": "https://supabase.com/docs/guides/queues/quickstart#queue-types", - "title": "Queue types" - }, - { - "url": "https://supabase.com/docs/guides/queues/quickstart#message", - "title": "Message" - }, - { - "url": "https://supabase.com/docs/guides/queues/quickstart#concepts", - "title": "Concepts" - }, - { - "url": "https://supabase.com/docs/guides/queues/quickstart#enqueueing-and-dequeueing-messages", - "title": "Enqueueing and dequeueing messages" - }, - { - "url": "https://supabase.com/docs/guides/queues/quickstart#grant-permissions-to-pgmq_public-database-functions", - "title": "Grant permissions to pgmq_public database functions" - }, - { - "url": "https://supabase.com/docs/guides/queues/quickstart#enable-rls-on-your-tables-in-pgmq-schema", - "title": "Enable RLS on your tables in pgmq schema" - }, - { - "url": "https://supabase.com/docs/guides/queues/quickstart#expose-queues-to-client-side-consumers", - "title": "Expose Queues to client-side consumers" - }, - { - "url": "https://supabase.com/docs/guides/queues/quickstart#what-happens-when-you-create-a-queue", - "title": "What happens when you create a queue?" - }, - { - "url": "https://supabase.com/docs/guides/cron/quickstart", - "title": "Quickstart" - }, - { - "url": "https://supabase.com/docs/guides/cron/quickstart#run-a-vacuum-every-day", - "title": "Run a vacuum every day" - }, - { - "url": "https://supabase.com/docs/guides/cron/quickstart#delete-data-every-week", - "title": "Delete data every week" - }, - { - "url": "https://supabase.com/docs/guides/cron/quickstart#caution-scheduling-system-maintenance", - "title": "Caution: Scheduling system maintenance" - }, - { - "url": "https://supabase.com/docs/guides/cron/quickstart#examples", - "title": "Examples" - }, - { - "url": "https://supabase.com/docs/guides/cron/quickstart#unschedule-a-job", - "title": "Unschedule a job" - }, - { - "url": "https://supabase.com/docs/guides/cron/quickstart#inspecting-job-runs", - "title": "Inspecting job runs" - }, - { - "url": "https://supabase.com/docs/guides/cron/quickstart#activatedeactivate-a-job", - "title": "Activate/Deactivate a job" - }, - { - "url": "https://supabase.com/docs/guides/cron/quickstart#edit-a-job", - "title": "Edit a job" - }, - { - "url": "https://supabase.com/docs/guides/cron/quickstart#schedule-a-job", - "title": "Schedule a job" - }, - { - "url": "https://supabase.com/docs/guides/cron/quickstart#invoke-supabase-edge-function-every-30-seconds", - "title": "Invoke Supabase Edge Function every 30 seconds" + "title": "pg_net: Async Networking" }, { - "url": "https://supabase.com/docs/guides/cron/quickstart#call-a-database-stored-procedure", - "title": "Call a database stored procedure" + "url": "https://supabase.com/docs/guides/cron", + "title": "Cron" }, { - "url": "https://supabase.com/docs/guides/cron/quickstart#call-a-database-function-every-5-minutes", - "title": "Call a database function every 5 minutes" + "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", + "title": "Automatic embeddings" }, { - "url": "https://supabase.com/docs/guides/cron/install", - "title": "Install" + "url": "https://supabase.com/docs/guides/auth/auth-hooks/send-email-hook", + "title": "Send Email Hook" }, { - "url": "https://supabase.com/docs/guides/cron/install#uninstall", - "title": "Uninstall" + "url": "https://supabase.com/docs/guides/auth/auth-hooks/send-sms-hook", + "title": "Send SMS Hook" } ], - "resultChars": 136987 + "resultChars": 118669 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"Queues API pgmq_public pop read delete send queue_name message_id JavaScript rpc\", limit: 10) { nodes { __typename title href content ... on Guide { subsections { nodes { title href content } } } } } }", + "query": "query { searchDocs(query: \"Supabase queues pgmq pop archive send edge function drain queue\", limit: 5) { nodes { __typename ... on Guide { title href content } ... on CLICommandReference { title href content } ... on ManagementApiReference { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } ... on TroubleshootingGuide { title href content } } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/queues/pgmq", - "title": "PGMQ Extension" - }, - { - "url": "https://supabase.com/docs/guides/queues/pgmq#drop_queue", - "title": "drop_queue" - }, - { - "url": "https://supabase.com/docs/guides/queues/pgmq#sending-messages", - "title": "Sending messages" - }, - { - "url": "https://supabase.com/docs/guides/queues/pgmq#send", - "title": "send" - }, - { - "url": "https://supabase.com/docs/guides/queues/pgmq#send_batch", - "title": "send_batch" - }, - { - "url": "https://supabase.com/docs/guides/queues/pgmq#reading-messages", - "title": "Reading messages" - }, - { - "url": "https://supabase.com/docs/guides/queues/pgmq#read", - "title": "read" - }, - { - "url": "https://supabase.com/docs/guides/queues/pgmq#read_with_poll", - "title": "read_with_poll" - }, - { - "url": "https://supabase.com/docs/guides/queues/pgmq#pop", - "title": "pop" - }, - { - "url": "https://supabase.com/docs/guides/queues/pgmq#deletingarchiving-messages", - "title": "Deleting/Archiving messages" - }, - { - "url": "https://supabase.com/docs/guides/queues/pgmq#delete-single", - "title": "delete (single)" - }, - { - "url": "https://supabase.com/docs/guides/queues/pgmq#delete-batch", - "title": "delete (batch)" - }, - { - "url": "https://supabase.com/docs/guides/queues/pgmq#purge_queue", - "title": "purge_queue" - }, - { - "url": "https://supabase.com/docs/guides/queues/pgmq#archive-single", - "title": "archive (single)" - }, - { - "url": "https://supabase.com/docs/guides/queues/pgmq#archive-batch", - "title": "archive (batch)" - }, - { - "url": "https://supabase.com/docs/guides/queues/pgmq#utilities", - "title": "Utilities" - }, - { - "url": "https://supabase.com/docs/guides/queues/pgmq#set_vt", - "title": "set_vt" - }, - { - "url": "https://supabase.com/docs/guides/queues/pgmq#list_queues", - "title": "list_queues" - }, - { - "url": "https://supabase.com/docs/guides/queues/pgmq#metrics", - "title": "metrics" - }, - { - "url": "https://supabase.com/docs/guides/queues/pgmq#metrics_all", - "title": "metrics_all" - }, - { - "url": "https://supabase.com/docs/guides/queues/pgmq#types", - "title": "Types" - }, - { - "url": "https://supabase.com/docs/guides/queues/pgmq#message_record", - "title": "message_record" - }, - { - "url": "https://supabase.com/docs/guides/queues/pgmq#resources", - "title": "Resources" - }, - { - "url": "https://supabase.com/docs/guides/queues/pgmq#features", - "title": "Features" - }, - { - "url": "https://supabase.com/docs/guides/queues/pgmq#enable-the-extension", - "title": "Enable the extension" - }, - { - "url": "https://supabase.com/docs/guides/queues/pgmq#get-usage", - "title": "Usage " - }, - { - "url": "https://supabase.com/docs/guides/queues/pgmq#queue-management", - "title": "Queue management" - }, - { - "url": "https://supabase.com/docs/guides/queues/pgmq#create", - "title": "create" - }, - { - "url": "https://supabase.com/docs/guides/queues/pgmq#create_unlogged", - "title": "create_unlogged" - }, - { - "url": "https://supabase.com/docs/guides/queues/pgmq#detach_archive", - "title": "detach_archive" + "url": "https://supabase.com/docs/guides/queues/consuming-messages-with-edge-functions", + "title": "Consuming Supabase Queue Messages with Edge Functions" }, { "url": "https://supabase.com/docs/guides/queues", "title": "Supabase Queues" }, - { - "url": "https://supabase.com/docs/guides/queues#features", - "title": "Features" - }, - { - "url": "https://supabase.com/docs/guides/queues#resources", - "title": "Resources" - }, { "url": "https://supabase.com/docs/guides/queues/quickstart", "title": "Quickstart" }, - { - "url": "https://supabase.com/docs/guides/queues/quickstart#queue-types", - "title": "Queue types" - }, - { - "url": "https://supabase.com/docs/guides/queues/quickstart#concepts", - "title": "Concepts" - }, - { - "url": "https://supabase.com/docs/guides/queues/quickstart#pull-based-queue", - "title": "Pull-Based Queue" - }, - { - "url": "https://supabase.com/docs/guides/queues/quickstart#message", - "title": "Message" - }, - { - "url": "https://supabase.com/docs/guides/queues/quickstart#create-queues", - "title": "Create Queues" - }, - { - "url": "https://supabase.com/docs/guides/queues/quickstart#what-happens-when-you-create-a-queue", - "title": "What happens when you create a queue?" - }, - { - "url": "https://supabase.com/docs/guides/queues/quickstart#expose-queues-to-client-side-consumers", - "title": "Expose Queues to client-side consumers" - }, - { - "url": "https://supabase.com/docs/guides/queues/quickstart#enable-rls-on-your-tables-in-pgmq-schema", - "title": "Enable RLS on your tables in pgmq schema" - }, - { - "url": "https://supabase.com/docs/guides/queues/quickstart#grant-permissions-to-pgmq_public-database-functions", - "title": "Grant permissions to pgmq_public database functions" - }, - { - "url": "https://supabase.com/docs/guides/queues/quickstart#enqueueing-and-dequeueing-messages", - "title": "Enqueueing and dequeueing messages" - }, { "url": "https://supabase.com/docs/guides/queues/api", "title": "API" }, { - "url": "https://supabase.com/docs/guides/queues/api#pgmq_publicpopqueue_name", - "title": "pgmq_public.pop(queue_name)" - }, - { - "url": "https://supabase.com/docs/guides/queues/api#pgmq_publicsendqueue_name-message-sleep_seconds", - "title": "pgmq_public.send(queue_name, message, sleep_seconds)" - }, - { - "url": "https://supabase.com/docs/guides/queues/api#pgmq_publicsend_batchqueue_name-messages-sleep_seconds", - "title": "pgmq_public.send_batch(queue_name, messages, sleep_seconds)" - }, - { - "url": "https://supabase.com/docs/guides/queues/api#pgmq_publicreadqueue_name-sleep_seconds-n", - "title": "pgmq_public.read(queue_name, sleep_seconds, n)" - }, - { - "url": "https://supabase.com/docs/guides/queues/api#pgmq_publicarchivequeue_name-message_id", - "title": "pgmq_public.archive(queue_name, message_id)" - }, - { - "url": "https://supabase.com/docs/guides/queues/api#pgmq_publicdeletequeue_name-message_id", - "title": "pgmq_public.delete(queue_name, message_id)" - }, - { - "url": "https://supabase.com/docs/guides/queues/consuming-messages-with-edge-functions", - "title": "Consuming Supabase Queue Messages with Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/queues/consuming-messages-with-edge-functions#consuming-messages-in-an-edge-function", - "title": "Consuming messages in an Edge Function" - }, - { - "url": "https://supabase.com/docs/guides/queues/consuming-messages-with-edge-functions#concepts", - "title": "Concepts" - }, - { - "url": "https://supabase.com/docs/guides/database/functions", - "title": "Database Functions" - }, - { - "url": "https://supabase.com/docs/guides/database/functions#error-handling", - "title": "Error handling" - }, - { - "url": "https://supabase.com/docs/guides/database/functions#general-logging", - "title": "General logging" - }, - { - "url": "https://supabase.com/docs/guides/database/functions#create-database-functions", - "title": "Create Database Functions" - }, - { - "url": "https://supabase.com/docs/guides/database/functions#deep-dive", - "title": "Deep dive" - }, - { - "url": "https://supabase.com/docs/guides/database/functions#resources", - "title": "Resources" - }, - { - "url": "https://supabase.com/docs/guides/database/functions#advanced-logging", - "title": "Advanced logging" - }, - { - "url": "https://supabase.com/docs/guides/database/functions#function-privileges", - "title": "Function privileges" - }, - { - "url": "https://supabase.com/docs/guides/database/functions#security-definer-vs-invoker", - "title": "Security definer vs invoker" - }, - { - "url": "https://supabase.com/docs/guides/database/functions#database-functions-vs-edge-functions", - "title": "Database Functions vs Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/database/functions#suggestions", - "title": "Suggestions" - }, - { - "url": "https://supabase.com/docs/guides/database/functions#passing-parameters", - "title": "Passing parameters" - }, - { - "url": "https://supabase.com/docs/guides/database/functions#returning-data-sets", - "title": "Returning data sets" - }, + "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", + "title": "Automatic embeddings" + } + ], + "resultChars": 51463 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"pgmq.create queue exists if not exists duplicate table create queue pgmq\", limit: 10) { nodes { __typename ... on Guide { title href content } ... on CLICommandReference { title href content } ... on ManagementApiReference { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } ... on TroubleshootingGuide { title href content } } } }", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/guides/database/functions#simple-functions", - "title": "Simple functions" + "url": "https://supabase.com/docs/guides/queues/quickstart", + "title": "Quickstart" }, { - "url": "https://supabase.com/docs/guides/database/functions#getting-started", - "title": "Getting started" + "url": "https://supabase.com/docs/guides/queues/pgmq", + "title": "PGMQ Extension" }, { - "url": "https://supabase.com/docs/guides/database/functions#quick-demo", - "title": "Quick demo" + "url": "https://supabase.com/docs/guides/queues/api", + "title": "API" }, { - "url": "https://supabase.com/docs/guides/database/functions#debugging-functions", - "title": "Debugging functions" + "url": "https://supabase.com/docs/guides/queues", + "title": "Supabase Queues" }, { - "url": "https://supabase.com/docs/guides/database/functions#using-database-functions-to-call-an-external-api", - "title": "Using Database Functions to call an external API" + "url": "https://supabase.com/docs/guides/queues/expose-self-hosted-queues", + "title": "Expose Queues for local and self-hosted Supabase" }, { - "url": "https://supabase.com/docs/guides/database/functions#call-database-functions-using-javascript", - "title": "Call Database Functions using JavaScript" + "url": "https://supabase.com/docs/guides/troubleshooting/42501--permission-denied-for-table-httprequestqueue-KnozmQ", + "title": "42501 : permission denied for table http_request_queue" }, { - "url": "https://supabase.com/docs/reference/javascript/rpc" + "url": "https://supabase.com/docs/guides/functions/examples/resumable-websockets", + "title": "Resumable WebSockets with Edge Functions" } ], - "resultChars": 288356 + "resultChars": 47568 } ] }, "prompt": "I want to set up a recurring background workflow on my local Supabase stack.\n\nCan you set up a cron job called `enqueue-tasks` to run every minute and push a task into a queue called `tasks`? Then add a `process-tasks` edge function that reads messages off the `tasks` queue and removes them, so a scheduled worker can keep the backlog drained.", "promptSourcePath": "evals/build-cli-003-pg-cron-queue-workflow/PROMPT.md", "attempts": 1, - "sourcePath": "codex-gpt-5.6/build-cli-003-pg-cron-queue-workflow.json" + "sourcePath": "codex-gpt-5.4-mini/build-cli-003-pg-cron-queue-workflow.json" }, { - "experiment": "codex-gpt-5.6", + "experiment": "codex-gpt-5.4-mini", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.6-sol", + "modelId": "gpt-5.4-mini", "reasoningEffort": "medium" }, "eval": "build-database-001-migrate-postgres-to-supabase", @@ -11822,49 +5942,20 @@ ] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"restore existing PostgreSQL pg_dump custom binary dump local Supabase pg_restore roles schema migration\", limit: 5) { nodes { ... on Guide { title href content } ... on CLICommandReference { title href content } ... on TroubleshootingGuide { title href content } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform", - "title": "Restore a Platform Project to Self-Hosted" - }, - { - "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/postgres", - "title": "Migrate from Postgres to Supabase" - }, - { - "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/vercel-postgres", - "title": "Migrate from Vercel Postgres to Supabase" - }, - { - "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/neon", - "title": "Migrate from Neon to Supabase" - }, - { - "url": "https://supabase.com/docs/guides/local-development/restoring-downloaded-backup", - "title": "Restoring a downloaded backup locally" - } - ], - "resultChars": 42818 - } - ] + "calls": [] }, "prompt": "I have an existing Postgres database I want to migrate to Supabase. There's a binary dump at `source.dump` in the current directory.\n\nCan you set up a local Supabase project and restore the dump into it?", "promptSourcePath": "evals/build-database-001-migrate-postgres-to-supabase/PROMPT.md", "attempts": 1, - "sourcePath": "codex-gpt-5.6/build-database-001-migrate-postgres-to-supabase.json" + "sourcePath": "codex-gpt-5.4-mini/build-database-001-migrate-postgres-to-supabase.json" }, { - "experiment": "codex-gpt-5.6", + "experiment": "codex-gpt-5.4-mini", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.6-sol", + "modelId": "gpt-5.4-mini", "reasoningEffort": "medium" }, "eval": "build-functions-004-service-role-bypass", @@ -11901,12 +5992,12 @@ { "name": "user A cannot force-read user B note", "passed": true, - "notes": "status=200" + "notes": "status=403" }, { "name": "user B cannot force-read user A note", "passed": true, - "notes": "status=200" + "notes": "status=403" } ], "skills": { @@ -11922,86 +6013,46 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"Edge Functions authorization JWT getUser Authorization header service role row level security\", limit: 5) { nodes { ... on Guide { title href content } ... on TroubleshootingGuide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs", - "title": "Build a User Management App with Next.js" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth-headers", - "title": "Authorization headers" - }, - { - "url": "https://supabase.com/docs/guides/auth/auth-mfa", - "title": "Multi-Factor Authentication" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", - "title": "New API Keys and Asymmetric Authentication" - } - ], - "resultChars": 85366 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase Edge Functions authenticate user Authorization header createClient getUser RLS\", limit: 8) { nodes { ... on Guide { title href content } ... on TroubleshootingGuide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/why-is-my-service-role-key-client-getting-rls-errors-or-not-returning-data-7_1K9z", - "title": "Why is my service role key client getting RLS errors or not returning data?" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs", - "title": "Build a User Management App with Next.js" - }, + "query": "query { searchDocs(query: \"Edge Functions verify_jwt authorization user JWT supabase-js Deno.serve auth.getUser\", limit: 5) { nodes { title href content ... on ClientLibraryFunctionReference { language methodName } } } }", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" + "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", + "title": "Integrating With Supabase Auth" }, { - "url": "https://supabase.com/docs/guides/database/secure-data", - "title": "Securing your data" + "url": "https://supabase.com/docs/guides/functions", + "title": "Edge Functions" }, { - "url": "https://supabase.com/docs/guides/auth/users", - "title": "Users" + "url": "https://supabase.com/docs/guides/functions/websockets", + "title": "Handling WebSockets" }, { - "url": "https://supabase.com/docs/guides/auth", - "title": "Auth" + "url": "https://supabase.com/docs/guides/functions/examples/resumable-websockets", + "title": "Resumable WebSockets with Edge Functions" }, { - "url": "https://supabase.com/docs/guides/api/securing-your-api", - "title": "Securing your API" + "url": "https://supabase.com/docs/guides/functions/auth-headers", + "title": "Authorization headers" } ], - "resultChars": 98711 + "resultChars": 39423 } ] }, "prompt": "I built an Edge Function called `private-notes` for showing a user's saved\nprivate notes.\n\nCan you check whether there's any way one user could see another user's notes?\n\nPlease fix and deploy it if needed.", "promptSourcePath": "evals/build-functions-004-service-role-bypass/PROMPT.md", "attempts": 1, - "sourcePath": "codex-gpt-5.6/build-functions-004-service-role-bypass.json" + "sourcePath": "codex-gpt-5.4-mini/build-functions-004-service-role-bypass.json" }, { - "experiment": "codex-gpt-5.6", + "experiment": "codex-gpt-5.4-mini", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.6-sol", + "modelId": "gpt-5.4-mini", "reasoningEffort": "medium" }, "eval": "build-functions-005-dual-auth-user-secret", @@ -12029,37 +6080,37 @@ { "name": "rejects request with no credentials", "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" + "notes": "status 401: {\"error\":\"Unauthorized\"}" }, { "name": "user with JWT reads only their own rows", "passed": true, - "notes": "status 200: [{\"user_id\":\"34e9425b-8b58-4aa6-9deb-31c762170653\",\"metric\":\"steps_a_ms6z3nra\",\"value\":111}]" + "notes": "status 200: [{\"user_id\":\"9b924d23-e7e5-4c2a-bfcc-e8dcfa55b5da\",\"metric\":\"steps_a_mrw63j19\",\"value\":111}]" }, { "name": "user cannot read another user's rows by passing user_id", "passed": true, - "notes": "status 200: [{\"user_id\":\"34e9425b-8b58-4aa6-9deb-31c762170653\",\"metric\":\"steps_a_ms6z3nra\",\"value\":111}]" + "notes": "status 200: [{\"user_id\":\"9b924d23-e7e5-4c2a-bfcc-e8dcfa55b5da\",\"metric\":\"steps_a_mrw63j19\",\"value\":111}]" }, { "name": "service key bypasses RLS to read the target user's rows", "passed": true, - "notes": "status 200: [{\"user_id\":\"942dd963-0318-4a34-9185-2acf0f1025e8\",\"metric\":\"steps_b_ms6z3nra\",\"value\":222}]" + "notes": "status 200: [{\"user_id\":\"c8f575bd-5d19-4283-a83a-8b681d9f5185\",\"metric\":\"steps_b_mrw63j19\",\"value\":222}]" }, { "name": "non-service key is not granted service access", "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" + "notes": "status 401: {\"error\":\"Unauthorized\"}" }, { "name": "rejects an unverified (forged) user token", "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" + "notes": "status 500: {\"error\":\"Internal Server Error\"}" }, { "name": "a user token in the apikey slot is not treated as the service key", "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" + "notes": "status 401: {\"error\":\"Unauthorized\"}" }, { "name": "implementation uses @supabase/server", @@ -12080,7 +6131,7 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"Edge Functions authenticate JWT Authorization header service_role apikey verify_jwt Deno serve createClient\", limit: 8) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"Edge Functions verify jwt service_role apikey header anonymous auth.uid request headers Deno.serve\", limit: 5) { nodes { __typename title href content ... on Guide { subsections { totalCount } } } } }", "hasContent": true, "pages": [ { @@ -12088,146 +6139,133 @@ "title": "Handling WebSockets" }, { - "url": "https://supabase.com/docs/guides/functions/examples/resumable-websockets", - "title": "Resumable WebSockets with Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth-headers", - "title": "Authorization headers" + "url": "https://supabase.com/docs/guides/functions/logging", + "title": "Logging" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/why-is-my-service-role-key-client-getting-rls-errors-or-not-returning-data-7_1K9z", - "title": "Why is my service role key client getting RLS errors or not returning data?" + "url": "https://supabase.com/docs/guides/functions/examples/resumable-websockets", + "title": "Resumable WebSockets with Edge Functions" }, { "url": "https://supabase.com/docs/guides/functions", "title": "Edge Functions" }, { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" - }, - { - "url": "https://supabase.com/docs/guides/functions/http-methods", - "title": "Routing" + "url": "https://supabase.com/docs/guides/functions/auth-headers", + "title": "Authorization headers" } ], - "resultChars": 60939 + "resultChars": 39082 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"Edge Functions SUPABASE_SECRET_KEY environment variable secret key apikey header manual JWT verification getUser verify_jwt false\", limit: 10) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"Securing Edge Functions service role apikey verify_jwt false createClient service role user token auth.getUser auth.getClaims\", limit: 10) { nodes { __typename title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" + "url": "https://supabase.com/docs/guides/functions/auth-headers", + "title": "Authorization headers" }, { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" + "url": "https://supabase.com/docs/guides/functions/error-codes", + "title": "Error codes" }, { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" + "url": "https://supabase.com/docs/reference/javascript/auth-getclaims" }, { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" + "url": "https://supabase.com/docs/guides/functions/function-configuration", + "title": "Function Configuration" }, { - "url": "https://supabase.com/docs/guides/functions/secrets", - "title": "Environment Variables" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", + "title": "Self-Hosted Functions" }, { - "url": "https://supabase.com/docs/guides/functions/function-configuration", - "title": "Function Configuration" + "url": "https://supabase.com/docs/guides/auth/jwt-fields", + "title": "JWT Claims Reference" }, { - "url": "https://supabase.com/docs/guides/auth/jwts", - "title": "JSON Web Token (JWT)" + "url": "https://supabase.com/docs/reference/swift/auth-getclaims", + "title": "getClaims()" }, { - "url": "https://supabase.com/docs/guides/auth/signing-keys", - "title": "JWT Signing Keys" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-react", + "title": "Build a User Management App with React" }, { - "url": "https://supabase.com/docs/guides/auth/auth-mfa", - "title": "Multi-Factor Authentication" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs", + "title": "Build a User Management App with Next.js" }, { - "url": "https://supabase.com/docs/guides/functions/websockets", - "title": "Handling WebSockets" + "url": "https://supabase.com/docs/guides/getting-started/api-keys", + "title": "Understanding API keys" } ], - "resultChars": 122455 + "resultChars": 123099 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"@supabase/server auth legacy service_role key apikey Edge Functions service_role JWT\", limit: 10) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"secret key Authorization apikey header PostgREST service_role functions internal APIs\", limit: 10) { nodes { __typename title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", + "title": "Envoy API Gateway" }, { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", + "title": "New API Keys and Asymmetric Authentication" }, { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" + "url": "https://supabase.com/docs/guides/database/postgres/roles", + "title": "Postgres Roles" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", - "title": "New API Keys and Asymmetric Authentication" + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" }, { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" + "url": "https://supabase.com/docs/guides/getting-started/api-keys", + "title": "Understanding API keys" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/why-is-my-service-role-key-client-getting-rls-errors-or-not-returning-data-7_1K9z", - "title": "Why is my service role key client getting RLS errors or not returning data?" + "url": "https://supabase.com/docs/guides/api/securing-your-api", + "title": "Securing your API" }, { - "url": "https://supabase.com/docs/guides/auth/signing-keys", - "title": "JWT Signing Keys" + "url": "https://supabase.com/docs/guides/troubleshooting/why-is-my-service-role-key-client-getting-rls-errors-or-not-returning-data-7_1K9z", + "title": "Why is my service role key client getting RLS errors or not returning data?" }, { - "url": "https://supabase.com/docs/guides/functions/examples/resumable-websockets", - "title": "Resumable WebSockets with Edge Functions" + "url": "https://supabase.com/docs/guides/functions/auth", + "title": "Securing Edge Functions" }, { - "url": "https://supabase.com/docs/guides/auth/jwt-fields", - "title": "JWT Claims Reference" + "url": "https://supabase.com/docs/guides/api/rest/postgrest-error-codes", + "title": "Error Codes" }, { - "url": "https://supabase.com/docs/guides/functions/function-configuration", - "title": "Function Configuration" + "url": "https://supabase.com/docs/guides/database/extensions/pgaudit", + "title": "PGAudit: Postgres Auditing" } ], - "resultChars": 114347 + "resultChars": 139952 } ] }, "prompt": "Build and serve a Supabase Edge Function named `user-stats` for this project,\nreachable over HTTP at `/functions/v1/user-stats`.\n\nOur product stores per-user metrics in the existing `user_stats` table.\n\nTwo very different callers need to hit this one endpoint:\n\n1. **Our mobile app**, acting for a signed-in user. It sends that user's\n Supabase access token. The endpoint should return the user's own stats.\n\n2. **Our internal billing service**, a trusted backend with no signed-in user.\n It authenticates with the project's secret (service-role) key in the `apikey`\n header, and names the target user with a `user_id` in the JSON request body.\n It needs that user's stats.\n\nReturn the matching rows as JSON. The endpoint must be secure: only ever serve\nstats to a caller that is genuinely entitled to them, and turn away callers that\nare not.", "promptSourcePath": "evals/build-functions-005-dual-auth-user-secret/PROMPT.md", "attempts": 2, - "sourcePath": "codex-gpt-5.6/build-functions-005-dual-auth-user-secret.json" + "sourcePath": "codex-gpt-5.4-mini/build-functions-005-dual-auth-user-secret.json" }, { - "experiment": "codex-gpt-5.6", + "experiment": "codex-gpt-5.4-mini", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.6-sol", + "modelId": "gpt-5.4-mini", "reasoningEffort": "medium" }, "eval": "build-storage-001-private-bucket-access", @@ -12259,7 +6297,7 @@ { "name": "user A lists only own files", "passed": true, - "notes": "saw: 019fb117-0187-77b3-8842-0d324488c642/receipt-alpha.pdf, 019fb117-0187-77b3-8842-0d324488c642/receipt-beta.pdf" + "notes": "saw: 019f8a22-4d74-754d-9d92-2953cf986df2/receipt-alpha.pdf, 019f8a22-4d74-754d-9d92-2953cf986df2/receipt-beta.pdf" }, { "name": "user B cannot read user A files", @@ -12280,7 +6318,7 @@ { "name": "configured private per-user storage access", "passed": true, - "judgeNotes": "Meets requirements: private user-files bucket, authenticated owner-scoped SELECT and INSERT policies on storage.objects with WITH CHECK for uploads, no RLS disabling or public access, and supabase-js createSignedUrl with short expiry." + "judgeNotes": "The assistant created a private user-files bucket, enabled/kept RLS on storage.objects, added authenticated owner-scoped SELECT and INSERT policies (plus UPDATE) constrained to the user-files bucket and uid-prefixed paths, and provided supabase-js createSignedUrl code with an expiry. No disqualifying public bucket, permissive policies, anon/public role scoping, getPublicUrl, or client service-role usage." } ], "skills": { @@ -12296,39 +6334,35 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"Storage access control policy foldername auth.uid createSignedUrl private bucket upload download\", limit: 6) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"storage policies bucket owner path auth.uid object name user files signed URLs\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/storage/buckets/fundamentals", - "title": "Storage Buckets" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs", + "title": "Build a User Management App with Next.js" }, { "url": "https://supabase.com/docs/guides/troubleshooting/why-cant-i-uploadlistetc-my-public-bucket-Z6CmGt", "title": "Why can't I upload/list/etc my public bucket?" }, { - "url": "https://supabase.com/docs/guides/security/product-security", - "title": "Secure configuration of Supabase products" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3", - "title": "Configure S3 Storage" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-sveltekit", + "title": "Build a User Management App with SvelteKit" }, { - "url": "https://supabase.com/docs/guides/storage/quickstart", - "title": "Storage Quickstart" + "url": "https://supabase.com/docs/guides/storage/buckets/fundamentals", + "title": "Storage Buckets" }, { - "url": "https://supabase.com/docs/guides/storage/security/access-control", - "title": "Storage Access Control" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-refine", + "title": "Build a User Management App with Refine" } ], - "resultChars": 28532 + "resultChars": 119416 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"JavaScript createSignedUrl storage from expiresIn example supabase-js\", limit: 4) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"Storage access control object policies signed URLs supabase-js createSignedUrl\", limit: 10) { nodes { title href content } } }", "hasContent": true, "pages": [ { @@ -12339,30 +6373,81 @@ "url": "https://supabase.com/docs/guides/storage/serving/image-transformations", "title": "Storage Image Transformations" }, + { + "url": "https://supabase.com/docs/guides/storage/uploads/resumable-uploads", + "title": "Resumable Uploads" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-saml-sso", + "title": "Configure SAML SSO" + }, + { + "url": "https://supabase.com/docs/guides/storage/s3/compatibility", + "title": "S3 Compatibility" + }, + { + "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsignedurls" + }, + { + "url": "https://supabase.com/docs/guides/storage/s3/authentication", + "title": "S3 Authentication" + }, { "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsignedurl" }, { "url": "https://supabase.com/docs/guides/platform/manage-your-usage/storage-image-transformations", "title": "Manage Storage Image Transformations usage" + }, + { + "url": "https://supabase.com/docs/guides/auth/redirect-urls", + "title": "Redirect URLs" + } + ], + "resultChars": 124352 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"storage.objects owner policy prefix split_part name user id path\", limit: 10) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/storage/management/copy-move-objects", + "title": "Copy Objects" + }, + { + "url": "https://supabase.com/docs/guides/storage/security/access-control", + "title": "Storage Access Control" + }, + { + "url": "https://supabase.com/docs/guides/storage/security/ownership", + "title": "Ownership" + }, + { + "url": "https://supabase.com/docs/guides/storage/schema/custom-roles", + "title": "Custom Roles" + }, + { + "url": "https://supabase.com/docs/guides/security/product-security", + "title": "Secure configuration of Supabase products" } ], - "resultChars": 31603 + "resultChars": 14510 } ] }, "prompt": "Our app lets signed-in users keep personal files like receipts and bank\nstatements. These files are private — a user must only ever be able to upload\nand download their own. The app uploads each file under a path that starts\nwith the owner's user id, e.g. `/receipt-march.pdf`.\n\nSet up a `user-files` bucket on our project and lock it down that way.\n\nUsers also sometimes share one of their files with someone else through a\ntemporary link that expires. Include the supabase-js code the app should use\nfor that.", "promptSourcePath": "evals/build-storage-001-private-bucket-access/PROMPT.md", "attempts": 1, - "sourcePath": "codex-gpt-5.6/build-storage-001-private-bucket-access.json" + "sourcePath": "codex-gpt-5.4-mini/build-storage-001-private-bucket-access.json" }, { - "experiment": "codex-gpt-5.6", + "experiment": "codex-gpt-5.4-mini", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.6-sol", + "modelId": "gpt-5.4-mini", "reasoningEffort": "medium" }, "eval": "build-tests-001-rls-tenant-isolation", @@ -12381,17 +6466,17 @@ { "name": "pgTAP test file(s) written under supabase/tests/", "passed": true, - "notes": "1 file(s): supabase/tests/database/tenant_isolation.test.sql" + "notes": "1 file(s): supabase/tests/tenant_isolation_test.sql" }, { "name": "pgTAP isolation tests ran and pass", "passed": true, - "notes": "9 passed, 0 failed" + "notes": "6 passed, 0 failed" }, { "name": "agent correctly identifies the posts isolation bug from test results", "passed": true, - "judgeNotes": "The agent correctly identifies `posts` as having a tenant isolation flaw allowing cross-organization reads, and grounds the conclusion in pgTAP failures/results. It does not blame `notes` or dismiss the tests." + "judgeNotes": "The agent identified `posts` as having a tenant isolation flaw (`posts` checked for any membership rather than membership in the row’s org), did not blame `notes`, and used pgTAP/test results as part of validating the issue and fix." } ], "skills": { @@ -12407,46 +6492,46 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"row level security testing pgTAP auth.uid tenant isolation policies\", limit: 5) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"pgTAP Supabase database tests row level security\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ + { + "url": "https://supabase.com/docs/guides/local-development/testing/overview", + "title": "Testing Overview" + }, { "url": "https://supabase.com/docs/guides/local-development/testing/pgtap-extended", "title": "Advanced pgTAP Testing" }, { - "url": "https://supabase.com/docs/guides/local-development/testing/overview", - "title": "Testing Overview" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", + "title": "Row Level Security" }, { "url": "https://supabase.com/docs/guides/database/extensions/pgtap", "title": "pgTAP: Unit Testing" }, { - "url": "https://supabase.com/docs/guides/database/testing", - "title": "Testing Your Database" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", - "title": "Row Level Security" + "url": "https://supabase.com/docs/guides/storage/security/access-control", + "title": "Storage Access Control" } ], - "resultChars": 70562 + "resultChars": 73033 } ] }, "prompt": "Can you audit the tenant isolation on our tables? Write some database tests covering both the happy path and the negative case, run them, and share the results.", "promptSourcePath": "evals/build-tests-001-rls-tenant-isolation/PROMPT.md", "attempts": 1, - "sourcePath": "codex-gpt-5.6/build-tests-001-rls-tenant-isolation.json" + "sourcePath": "codex-gpt-5.4-mini/build-tests-001-rls-tenant-isolation.json" }, { - "experiment": "codex-gpt-5.6", + "experiment": "codex-gpt-5.4-mini", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.6-sol", + "modelId": "gpt-5.4-mini", "reasoningEffort": "medium" }, "eval": "build-vectors-001-rag-with-permissions", @@ -12508,17 +6593,17 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"pgvector semantic search embeddings match_documents RLS auth.uid vector index HNSW RPC security invoker\", limit: 8) { nodes { ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } } } }", + "query": "query { searchDocs(query: \"semantic search pgvector embeddings RLS documents ownership\", limit: 5) { nodes { __typename title href content } } }", "hasContent": true, "pages": [ - { - "url": "https://supabase.com/docs/guides/ai/semantic-search", - "title": "Semantic search" - }, { "url": "https://supabase.com/docs/guides/ai/rag-with-permissions", "title": "RAG with Permissions" }, + { + "url": "https://supabase.com/docs/guides/ai/semantic-search", + "title": "Semantic search" + }, { "url": "https://supabase.com/docs/guides/ai/hybrid-search", "title": "Hybrid search" @@ -12528,39 +6613,75 @@ "title": "Automatic embeddings" }, { - "url": "https://supabase.com/docs/guides/ai/vector-columns", - "title": "Vector columns" + "url": "https://supabase.com/docs/guides/database/extensions/pgvector", + "title": "pgvector: Embeddings and vector similarity" + } + ], + "resultChars": 77020 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Supabase database call edge function from trigger net.http_post\", limit: 5) { nodes { __typename title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_net", + "title": "pg_net: Async Networking" + }, + { + "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", + "title": "Automatic embeddings" + }, + { + "url": "https://supabase.com/docs/guides/functions/connect-to-postgres", + "title": "Integrating with Supabase Database (Postgres)" }, { - "url": "https://supabase.com/docs/guides/ai/going-to-prod", - "title": "Going to Production" + "url": "https://supabase.com/docs/guides/cron/quickstart", + "title": "Quickstart" }, { - "url": "https://supabase.com/docs/guides/ai/vector-indexes", - "title": "Vector indexes" + "url": "https://supabase.com/docs/guides/functions/schedule-functions", + "title": "Scheduling Edge Functions" + } + ], + "resultChars": 63889 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Supabase trigger call edge function pg_net embedding queue document_sections\", limit: 5) { nodes { __typename title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", + "title": "Automatic embeddings" }, { - "url": "https://supabase.com/docs/guides/database/extensions/pgvector", - "title": "pgvector: Embeddings and vector similarity" + "url": "https://supabase.com/docs/guides/functions/examples/semantic-search", + "title": "Semantic Search" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_net", + "title": "pg_net: Async Networking" } ], - "resultChars": 97234 + "resultChars": 51986 } ] }, "prompt": "We're adding semantic search to our internal knowledge base app. I already wrote the edge functions, but search doesn't work yet. Can you set up whatever the database needs to make search work end to end?\n\nSome documents are confidential, so users should only have access to documents they own.", "promptSourcePath": "evals/build-vectors-001-rag-with-permissions/PROMPT.md", "attempts": 1, - "sourcePath": "codex-gpt-5.6/build-vectors-001-rag-with-permissions.json" + "sourcePath": "codex-gpt-5.4-mini/build-vectors-001-rag-with-permissions.json" }, { - "experiment": "codex-gpt-5.6", + "experiment": "codex-gpt-5.4-mini", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.6-sol", - "reasoningEffort": "low" + "modelId": "gpt-5.4-mini", + "reasoningEffort": "medium" }, "eval": "deploy-database-001-prometheus-metrics", "stage": "deploy", @@ -12571,7 +6692,7 @@ "observability" ], "suite": "benchmark", - "passed": true, + "passed": false, "checks": [ { "name": "preserved existing app scrape job", @@ -12579,13 +6700,13 @@ }, { "name": "configured the Supabase Metrics API scrape correctly", - "passed": true, - "judgeNotes": "Prometheus preserves the app scrape and adds a deployable Supabase scrape over HTTPS to the correct metrics path using HTTP Basic Auth with password_file. The target is a project ref on supabase.co, and docker-compose wires the matching password file via a Compose secret mounted at /run/secrets/supabase_secret_key." + "passed": false, + "judgeNotes": "prometheus.yml does not add a Supabase scrape target. docker-compose.yml does not mount a password_file via volume or Compose secret and instead injects SUPABASE_METRICS_SECRET into a rendered config. Required deployable Basic Auth password_file wiring is missing." }, { "name": "documented live deployment and verification steps", - "passed": true, - "judgeNotes": "README includes Secret API key creation, matching secret file placement, Compose stack deploy/restart guidance, and concrete verification via Prometheus targets, Grafana/Prometheus labels, and curl." + "passed": false, + "judgeNotes": "README includes creating a Supabase sb_secret API key and verifying via Prometheus targets, but it does not instruct placing the matching secret file and restarting/reloading the Compose stack. It describes env vars/secret manager injection instead, which mismatches the required secret-file setup." } ], "skills": { @@ -12601,17 +6722,45 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"Prometheus metrics project endpoint customer v1 privileged metrics service_role authentication hosted Supabase\", limit: 5) { nodes { __typename title href content } } }", + "query": "query { searchDocs(query: \"metrics Prometheus self-hosted Supabase\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ + { + "url": "https://supabase.com/docs/guides/telemetry/metrics", + "title": "Metrics API" + }, { "url": "https://supabase.com/docs/guides/telemetry/metrics/grafana-self-hosted", "title": "Metrics API with Prometheus & Grafana (self-hosted)" }, + { + "url": "https://supabase.com/docs/guides/telemetry/metrics/grafana-cloud", + "title": "Metrics API with Grafana Cloud" + }, + { + "url": "https://supabase.com/docs/guides/telemetry/metrics/vendor-agnostic", + "title": "Vendor-agnostic Metrics API setup" + }, + { + "url": "https://supabase.com/docs/guides/database/connection-management", + "title": "Connection management" + } + ], + "resultChars": 29391 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"project metrics Prometheus observability metrics endpoint\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ { "url": "https://supabase.com/docs/guides/telemetry/metrics", "title": "Metrics API" }, + { + "url": "https://supabase.com/docs/guides/telemetry/metrics/grafana-self-hosted", + "title": "Metrics API with Prometheus & Grafana (self-hosted)" + }, { "url": "https://supabase.com/docs/guides/telemetry/metrics/vendor-agnostic", "title": "Vendor-agnostic Metrics API setup" @@ -12621,26 +6770,26 @@ "title": "Metrics API with Grafana Cloud" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", - "title": "New API Keys and Asymmetric Authentication" + "url": "https://supabase.com/docs/guides/platform/read-replicas", + "title": "Read Replicas" } ], - "resultChars": 39702 + "resultChars": 32694 } ] }, - "prompt": "Can you wire my Supabase project metrics into our existing observability stack and document\nin the observability README what we need to do to make the config live?", + "prompt": "Can you wire my Supabase project metrics into our existing observability stack and document\nwhat we need to do to make the config live?", "promptSourcePath": "evals/deploy-database-001-prometheus-metrics/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.6/deploy-database-001-prometheus-metrics.json" + "attempts": 2, + "sourcePath": "codex-gpt-5.4-mini/deploy-database-001-prometheus-metrics.json" }, { - "experiment": "codex-gpt-5.6", + "experiment": "codex-gpt-5.4-mini", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.6-sol", + "modelId": "gpt-5.4-mini", "reasoningEffort": "medium" }, "eval": "deploy-functions-001-edge-function-secrets", @@ -12685,9 +6834,14 @@ }, "docs": { "calls": [ + { + "source": "web_search", + "query": "site:supabase.com/changelog.md Supabase changelog breaking-change edge functions env secrets", + "pages": [] + }, { "source": "search_docs", - "query": "query { searchDocs(query: \"Edge Functions environment variables secrets supabase secrets set deploy functions invoke CORS\", limit: 6) { nodes { ... on Guide { title href content } ... on CLICommandReference { title href content } } } }", + "query": "query { searchDocs(query: \"Edge Function environment variables secrets deploy WEATHER_API_KEY\", limit: 5) { nodes { __typename title href content ... on Guide { subsections { nodes { title href content } } } ... on ClientLibraryFunctionReference { language methodName } } totalCount } }", "hasContent": true, "pages": [ { @@ -12695,347 +6849,296 @@ "title": "Migrating to publishable and secret API keys" }, { - "url": "https://supabase.com/docs/guides/functions/secrets", - "title": "Environment Variables" + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#before-you-start", + "title": "Before you start" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", - "title": "Self-Hosted Functions" + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#step-1-create-the-new-api-keys", + "title": "Step 1: Create the new API keys" }, { - "url": "https://supabase.com/docs/guides/functions/storage-caching", - "title": "Integrating with Supabase Storage" + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#step-2-swap-the-publishable-key-in-client-code", + "title": "Step 2: Swap the publishable key in client code" }, { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" - } - ], - "resultChars": 36973 - } - ] - }, - "prompt": "Our weather widget currently calls WeatherAPI straight from the browser, which\nleaks our API key. I want to move that behind a Supabase Edge Function called\n`weather` that holds the key server-side and proxies the request.\n\nThe function should read the key from an environment variable named\n`WEATHER_API_KEY`. Our key already lives in a local `.env` file at the project\nroot.\n\nDeploy the function to our project so it's live, and make sure the deployed\nfunction can actually read the key at runtime.", - "promptSourcePath": "evals/deploy-functions-001-edge-function-secrets/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.6/deploy-functions-001-edge-function-secrets.json" - }, - { - "experiment": "codex-gpt-5.6", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.6-sol", - "reasoningEffort": "medium" - }, - "eval": "deploy-self-hosting-001-docker-compose", - "stage": "deploy", - "product": [ - "database", - "auth", - "storage" - ], - "topic": [ - "self-hosting" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "cloned the self-host stack (docker-compose.yml + volumes/db)", - "passed": true - }, - { - "name": "didn't conflate with the CLI (no supabase/config.toml in the stack)", - "passed": true - }, - { - "name": "secrets rotated off the shipped defaults", - "passed": true - }, - { - "name": "ANON_KEY and SERVICE_ROLE_KEY are HS256 JWTs signed by JWT_SECRET", - "passed": true - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"self-hosting Docker compose production secrets JWT_SECRET ANON_KEY SERVICE_ROLE_KEY DASHBOARD_PASSWORD POOLER_TENANT_ID\", limit: 8) { nodes { __typename title href content } } }", - "hasContent": true, - "pages": [ + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#step-3-swap-the-secret-key-in-backend-code", + "title": "Step 3: Swap the secret key in backend code" + }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", - "title": "New API Keys and Asymmetric Authentication" + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#database-webhooks-and-pg_net", + "title": "Database Webhooks and pg_net" }, { - "url": "https://supabase.com/docs/guides/auth/signing-keys", - "title": "JWT Signing Keys" + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#step-4-update-edge-functions", + "title": "Step 4: Update Edge Functions" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", - "title": "Envoy API Gateway" + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#option-2-adopt-the-supabaseserver-sdk", + "title": "Option 2: Adopt the @supabase/server SDK" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth", - "title": "Configure Social Login (OAuth) Providers" + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#step-5-verify-nothing-uses-the-legacy-keys", + "title": "Step 5: Verify nothing uses the legacy keys" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker", - "title": "Self-Hosting with Docker" + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#step-6-deactivate-the-legacy-keys", + "title": "Step 6: Deactivate the legacy keys" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", - "title": "Self-Hosted Functions" + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#known-limitations", + "title": "Known limitations" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#next-steps", + "title": "Next steps" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#option-1-read-the-new-keys-from-the-environment", + "title": "Option 1: Read the new keys from the environment" }, { "url": "https://supabase.com/docs/guides/functions/secrets", "title": "Environment Variables" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3", - "title": "Configure S3 Storage" - } - ], - "resultChars": 154701 - } - ] - }, - "prompt": "I'm moving off the hosted Supabase and running the whole thing myself on a VPS I\njust spun up. Can you get a Docker setup ready for me to copy onto the box?\n\nI don't need it running here, I'll do the actual bring-up once I'm on the\nserver. I just want everything in place and the secrets set up. Put it in a `supabase-docker/`\nfolder at the repo root so I can scp the whole thing across in one go.", - "promptSourcePath": "evals/deploy-self-hosting-001-docker-compose/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.6/deploy-self-hosting-001-docker-compose.json" - }, - { - "experiment": "codex-gpt-5.6", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.6-sol", - "reasoningEffort": "medium" - }, - "eval": "investigate-auth-001-deleted-user-access", - "stage": "investigate", - "product": [ - "auth" - ], - "topic": [ - "security", - "sdk" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "victim session active before delete-account", - "passed": true - }, - { - "name": "delete_account flow ran for the victim", - "passed": true - }, - { - "name": "delete-account revokes the user's sessions", - "passed": true, - "notes": "sessions left: 0" - }, - { - "name": "deleted user's refresh token is rejected", - "passed": true - }, - { - "name": "deleted user cannot sign back in", - "passed": true - }, - { - "name": "other users keep their sessions and access", - "passed": true - }, - { - "name": "diagnosed and explained session revocation", - "passed": true, - "judgeNotes": "Diagnoses soft-delete-only RPC, implements auth user deletion with cascading session/refresh token/identity removal, adds RLS active-session mitigation and explains JWT stateless expiry caveat consistently. Correctly distinguishes publishable frontend key/RLS from secret server-only RLS-bypassing key." - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" - ] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"delete user existing access token session revoke refresh token admin deleteUser publishable secret API keys RLS\", limit: 8) { nodes { title href content } } }", - "hasContent": true, - "pages": [ + "url": "https://supabase.com/docs/guides/functions/secrets#local-secrets", + "title": "Local secrets" + }, { - "url": "https://supabase.com/docs/guides/auth/third-party/clerk", - "title": "Clerk" + "url": "https://supabase.com/docs/guides/functions/secrets#default-secrets", + "title": "Default secrets" }, { - "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security", - "title": "Token Security and Row Level Security" + "url": "https://supabase.com/docs/guides/functions/secrets#accessing-environment-variables", + "title": "Accessing environment variables" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/why-is-my-service-role-key-client-getting-rls-errors-or-not-returning-data-7_1K9z", - "title": "Why is my service role key client getting RLS errors or not returning data?" + "url": "https://supabase.com/docs/guides/functions/secrets#production-secrets", + "title": "Production secrets" }, { - "url": "https://supabase.com/docs/guides/auth/signing-keys", - "title": "JWT Signing Keys" + "url": "https://supabase.com/docs/guides/troubleshooting/inspecting-edge-function-environment-variables-wg5qOQ", + "title": "Inspecting edge function environment variables" }, { - "url": "https://supabase.com/docs/guides/storage/buckets/fundamentals", - "title": "Storage Buckets" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin", + "title": "Build a Product Management Android App with Jetpack Compose" }, { - "url": "https://supabase.com/docs/guides/storage/security/access-control", - "title": "Storage Access Control" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin#implement-repositories", + "title": "Implement repositories" }, { - "url": "https://supabase.com/docs/guides/storage/security/ownership", - "title": "Ownership" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin#create-the-success-screen", + "title": "Create the success screen" }, { - "url": "https://supabase.com/docs/guides/auth/oauth-server/getting-started", - "title": "Getting Started with OAuth 2.1 Server" - } - ], - "resultChars": 95765 - }, - { - "source": "web_search", - "query": "https://supabase.com/changelog.md", - "pages": [ + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin#implement-the-mainactivity", + "title": "Implement the MainActivity" + }, { - "url": "https://supabase.com/changelog.md" - } - ] - }, - { - "source": "web_search", - "query": "site:supabase.com/changelog Supabase auth delete user session API keys publishable secret", - "pages": [] - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"auth.sessions session_id JWT claim RLS revoke session access token cannot revoke delete user\", limit: 10) { nodes { title href content } } }", - "hasContent": true, - "pages": [ + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin#implement-screens", + "title": "Implement screens" + }, { - "url": "https://supabase.com/docs/guides/storage/security/ownership", - "title": "Ownership" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin#project-setup", + "title": "Project setup" }, { - "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security", - "title": "Token Security and Row Level Security" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin#create-a-project", + "title": "Create a project" }, { - "url": "https://supabase.com/docs/guides/auth/third-party/clerk", - "title": "Clerk" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin#set-up-the-database-schema", + "title": "Set up the database schema" }, { - "url": "https://supabase.com/docs/guides/auth/sessions", - "title": "User sessions" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin#get-api-details", + "title": "Get API details" }, { - "url": "https://supabase.com/docs/guides/auth/auth-anonymous", - "title": "Anonymous Sign-Ins" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin#set-up-google-authentication", + "title": "Set up Google authentication" }, { - "url": "https://supabase.com/docs/guides/ai/rag-with-permissions", - "title": "RAG with Permissions" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin#building-the-app", + "title": "Building the app" }, { - "url": "https://supabase.com/docs/guides/auth/oauth-server/getting-started", - "title": "Getting Started with OAuth 2.1 Server" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin#create-new-android-project", + "title": "Create new Android project" }, { - "url": "https://supabase.com/docs/guides/resources/glossary", - "title": "Glossary" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin#set-up-api-key-and-secret-securely", + "title": "Set up API key and secret securely" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/why-is-my-service-role-key-client-getting-rls-errors-or-not-returning-data-7_1K9z", - "title": "Why is my service role key client getting RLS errors or not returning data?" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin#create-local-environment-secret", + "title": "Create local environment secret" }, { - "url": "https://supabase.com/docs/guides/auth/signing-keys", - "title": "JWT Signing Keys" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin#read-and-set-value-to-buildconfig", + "title": "Read and set value to BuildConfig" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin#use-value-from-buildconfig", + "title": "Use value from BuildConfig" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin#set-up-supabase-dependencies", + "title": "Set up Supabase dependencies" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin#set-up-hilt-for-dependency-injection", + "title": "Set up Hilt for dependency injection" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin#provide-supabase-instances-with-hilt", + "title": "Provide Supabase instances with Hilt" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin#create-a-data-transfer-object", + "title": "Create a data transfer object" + }, + { + "url": "https://supabase.com/docs/guides/integrations/build-a-supabase-oauth-integration", + "title": "Build a Supabase Integration" + }, + { + "url": "https://supabase.com/docs/guides/integrations/build-a-supabase-oauth-integration#create-an-oauth-app", + "title": "Create an OAuth app" + }, + { + "url": "https://supabase.com/docs/guides/integrations/build-a-supabase-oauth-integration#implementing-the-oauth-20-flow", + "title": "Implementing the OAuth 2.0 flow" + }, + { + "url": "https://supabase.com/docs/guides/integrations/build-a-supabase-oauth-integration#redirecting-to-the-authorize-url", + "title": "Redirecting to the authorize URL" + }, + { + "url": "https://supabase.com/docs/guides/integrations/build-a-supabase-oauth-integration#handling-the-callback", + "title": "Handling the callback" + }, + { + "url": "https://supabase.com/docs/guides/integrations/build-a-supabase-oauth-integration#refreshing-an-access-token", + "title": "Refreshing an access token" + }, + { + "url": "https://supabase.com/docs/guides/integrations/build-a-supabase-oauth-integration#calling-the-management-api", + "title": "Calling the Management API" + }, + { + "url": "https://supabase.com/docs/guides/integrations/build-a-supabase-oauth-integration#use-the-javascript-typescript-sdk", + "title": "Use the JavaScript (TypeScript) SDK" + }, + { + "url": "https://supabase.com/docs/guides/integrations/build-a-supabase-oauth-integration#integration-recommendations", + "title": "Integration recommendations" + }, + { + "url": "https://supabase.com/docs/guides/integrations/build-a-supabase-oauth-integration#store-api-keys-in-env-variables", + "title": "Store API keys in env variables" + }, + { + "url": "https://supabase.com/docs/guides/integrations/build-a-supabase-oauth-integration#pre-fill-database-connection-details", + "title": "Pre-fill database connection details" + }, + { + "url": "https://supabase.com/docs/guides/integrations/build-a-supabase-oauth-integration#create-new-projects", + "title": "Create new projects" + }, + { + "url": "https://supabase.com/docs/guides/integrations/build-a-supabase-oauth-integration#configure-custom-auth-smtp", + "title": "Configure custom Auth SMTP" + }, + { + "url": "https://supabase.com/docs/guides/integrations/build-a-supabase-oauth-integration#handling-dynamic-redirect-urls", + "title": "Handling dynamic redirect URLs" + }, + { + "url": "https://supabase.com/docs/guides/integrations/build-a-supabase-oauth-integration#current-limitations", + "title": "Current limitations" + }, + { + "url": "https://supabase.com/docs/guides/integrations/build-a-supabase-oauth-integration#show-a-connect-supabase-button" + } + ], + "resultChars": 185615 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Managing secrets Edge Functions supabase secrets set env file deploy\", limit: 5) { nodes { __typename title href content } totalCount } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/functions/secrets", + "title": "Environment Variables" + }, + { + "url": "https://supabase.com/docs/guides/local-development/database-migrations", + "title": "Database migrations" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" + }, + { + "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", + "title": "Automatic embeddings" + }, + { + "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream", + "title": "Streaming Speech with ElevenLabs" } ], - "resultChars": 137238 + "resultChars": 73210 } ] }, - "prompt": "Last week support removed a user through our app's delete-account flow — the\napp calls the `delete_account` function over RPC as the signed-in user. This\nmorning that same person was back: still signed in, reading and saving their\ndata like nothing happened.\n\nFigure out why the account still works, fix the flow so a deleted account\nloses access, and tell me whether there is any window where they could still\nget in after the fix.\n\nOne more thing while you're at it: we're migrating off the legacy\nanon/service_role API keys to the new publishable and secret keys, and the\nteam isn't sure which one belongs in the frontend and what each means for\nRLS. Give us a quick rundown.", - "promptSourcePath": "evals/investigate-auth-001-deleted-user-access/PROMPT.md", + "prompt": "Our weather widget currently calls WeatherAPI straight from the browser, which\nleaks our API key. I want to move that behind a Supabase Edge Function called\n`weather` that holds the key server-side and proxies the request.\n\nThe function should read the key from an environment variable named\n`WEATHER_API_KEY`. Our key already lives in a local `.env` file at the project\nroot.\n\nDeploy the function to our project so it's live, and make sure the deployed\nfunction can actually read the key at runtime.", + "promptSourcePath": "evals/deploy-functions-001-edge-function-secrets/PROMPT.md", "attempts": 1, - "sourcePath": "codex-gpt-5.6/investigate-auth-001-deleted-user-access.json" + "sourcePath": "codex-gpt-5.4-mini/deploy-functions-001-edge-function-secrets.json" }, { - "experiment": "codex-gpt-5.6", + "experiment": "codex-gpt-5.4-mini", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.6-sol", + "modelId": "gpt-5.4-mini", "reasoningEffort": "medium" }, - "eval": "investigate-realtime-001-subscribed-no-events", - "stage": "investigate", + "eval": "deploy-self-hosting-001-docker-compose", + "stage": "deploy", "product": [ - "realtime", - "database" + "database", + "auth", + "storage" ], "topic": [ - "sdk" + "self-hosting" ], "suite": "benchmark", - "interface": "mcp", + "interface": "cli", "passed": true, "checks": [ { - "name": "orders table added to supabase_realtime publication", + "name": "cloned the self-host stack (docker-compose.yml + volumes/db)", "passed": true }, { - "name": "courier_locations still in supabase_realtime publication", + "name": "didn't conflate with the CLI (no supabase/config.toml in the stack)", "passed": true }, { - "name": "publication still publishes INSERT events", + "name": "secrets rotated off the shipped defaults", "passed": true }, { - "name": "RLS still enabled on orders", + "name": "ANON_KEY and SERVICE_ROLE_KEY are HS256 JWTs signed by JWT_SECRET", "passed": true - }, - { - "name": "staff can still read orders through RLS", - "passed": true, - "notes": "authenticated sees 2 of 2 orders" - }, - { - "name": "diagnosed missing publication membership", - "passed": true, - "judgeNotes": "The assistant correctly diagnosed orders missing from supabase_realtime, applied ALTER PUBLICATION supabase_realtime ADD TABLE public.orders via migration, verified courier_locations remained, and did not weaken RLS or policies." } ], "skills": { @@ -13051,111 +7154,79 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"Postgres Changes enable table supabase_realtime publication add table postgres_changes SUBSCRIBED no events\", limit: 5) { nodes { ... on Guide { title href content } ... on TroubleshootingGuide { title href content } } } }", + "query": "query {\n searchDocs(query: \"self-hosted docker compose secrets site_url jwt_secret anon service_role postgres password config.toml\", limit: 5) {\n nodes {\n __typename\n ... on Guide { title href content }\n ... on TroubleshootingGuide { title href content }\n ... on CLICommandReference { title href content }\n ... on ManagementApiReference { title href content }\n ... on ClientLibraryFunctionReference { title href content language methodName }\n }\n }\n}", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/realtime/subscribing-to-database-changes", - "title": "Subscribing to Database Changes" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", + "title": "New API Keys and Asymmetric Authentication" }, { - "url": "https://supabase.com/docs/guides/realtime/postgres-changes", - "title": "Postgres Changes" + "url": "https://supabase.com/docs/guides/self-hosting/custom-email-templates", + "title": "Custom Email Templates" }, { - "url": "https://supabase.com/docs/guides/realtime/benchmarks", - "title": "Benchmarks" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth", + "title": "Configure Social Login (OAuth) Providers" }, { - "url": "https://supabase.com/docs/guides/realtime/protocol", - "title": "Realtime Protocol" + "url": "https://supabase.com/docs/guides/self-hosting/docker", + "title": "Self-Hosting with Docker" }, { - "url": "https://supabase.com/docs/guides/realtime/authorization", - "title": "Realtime Authorization" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-proxy-https", + "title": "Configure Reverse Proxy and HTTPS" } ], - "resultChars": 119071 - } - ] - }, - "prompt": "Our dispatch dashboard shows incoming orders as they happen. The courier\nlocation feed on the same page updates live without problems, but new orders\nonly show up after a page refresh.\n\nThe dashboard uses supabase-js to subscribe to INSERT events on the `orders`\ntable through postgres_changes, the same way it subscribes to courier\nlocations. The channel's status callback logs SUBSCRIBED and there are no\nerrors in the browser console.\n\nFigure out why no order events ever arrive and fix it.", - "promptSourcePath": "evals/investigate-realtime-001-subscribed-no-events/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.6/investigate-realtime-001-subscribed-no-events.json" - }, - { - "experiment": "codex-gpt-5.6", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.6-sol", - "reasoningEffort": "medium" - }, - "eval": "investigate-reliability-003-edge-function-5xx-correlation", - "stage": "investigate", - "product": [ - "edge-functions" - ], - "topic": [ - "observability" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "identified image-transform and the recurring 503 pattern", - "passed": true, - "judgeNotes": "Identified image-transform as the affected function and described the recurring pattern of 8 HTTP 503 responses across 07:00–12:00 UTC on 2026-04-28." - }, - { - "name": "attributed recurring 503s to gateway/platform layer, not function code", - "passed": true, - "judgeNotes": "Attributes the recurring image-transform 503s to the Supabase edge gateway/platform layer rather than function code. Grounds this in valid observations: failures only in gateway/API logs with no function runtime invocations, successful nearby invocations on the same deployment, and distinction from avatar-upload's function-level 500." - }, - { - "name": "recommended a concrete next step", - "passed": true, - "judgeNotes": "The assistant recommended concrete next steps, including opening a Supabase support case with project ref, region, time window, and gateway log IDs, plus investigating specific correlated issues." - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" + "resultChars": 88555 + }, + { + "source": "search_docs", + "query": "query {\n searchDocs(query: \"Self-Hosting with Docker quick start Linux generate-keys.sh docker-compose.yml .env example run.sh reset.sh\", limit: 5) {\n nodes {\n __typename\n ... on Guide { title href content }\n ... on TroubleshootingGuide { title href content }\n }\n }\n}", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/self-hosting/docker", + "title": "Self-Hosting with Docker" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", + "title": "New API Keys and Asymmetric Authentication" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth", + "title": "Configure Social Login (OAuth) Providers" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", + "title": "Envoy API Gateway" + } + ], + "resultChars": 96000 + } ] }, - "docs": { - "calls": [] - }, - "prompt": "Users have been reporting that image uploads are intermittently failing with server errors throughout the morning of `2026-04-28`.\n\nCan you investigate the project logs and tell me what is going on and what we\nshould do next?", - "promptSourcePath": "evals/investigate-reliability-003-edge-function-5xx-correlation/PROMPT.md", - "attempts": 2, - "sourcePath": "codex-gpt-5.6/investigate-reliability-003-edge-function-5xx-correlation.json" + "prompt": "I'm moving off the hosted Supabase and running the whole thing myself on a VPS I\njust spun up. Can you get a Docker setup ready for me to copy onto the box?\n\nI don't need it running here, I'll do the actual bring-up once I'm on the\nserver. I just want everything in place and the secrets set up. Put it in a `supabase-docker/`\nfolder at the repo root so I can scp the whole thing across in one go.", + "promptSourcePath": "evals/deploy-self-hosting-001-docker-compose/PROMPT.md", + "attempts": 1, + "sourcePath": "codex-gpt-5.4-mini/deploy-self-hosting-001-docker-compose.json" }, { - "experiment": "codex-gpt-5.6", + "experiment": "codex-gpt-5.4-mini", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.6-sol", + "modelId": "gpt-5.4-mini", "reasoningEffort": "medium" }, - "eval": "resolve-dataapi-001-empty-results", - "stage": "resolve", + "eval": "investigate-auth-001-deleted-user-access", + "stage": "investigate", "product": [ - "data-api", - "database", "auth" ], "topic": [ - "rls", + "security", "sdk" ], "suite": "benchmark", @@ -13163,33 +7234,34 @@ "passed": true, "checks": [ { - "name": "RLS still enabled on bookmarks", + "name": "victim session active before delete-account", "passed": true }, { - "name": "user A reads own bookmarks", + "name": "delete_account flow ran for the victim", "passed": true }, { - "name": "user B cannot read user A bookmarks", - "passed": true + "name": "delete-account revokes the user's sessions", + "passed": true, + "notes": "sessions left: 0" }, { - "name": "anon reads no bookmarks", + "name": "deleted user's refresh token is rejected", "passed": true }, { - "name": "user A can save a new bookmark", + "name": "deleted user cannot sign back in", "passed": true }, { - "name": "user B cannot insert a bookmark as user A", + "name": "other users keep their sessions and access", "passed": true }, { - "name": "diagnosed RLS and added owner-scoped policies", + "name": "diagnosed and explained session revocation", "passed": true, - "judgeNotes": "The assistant correctly diagnosed RLS enabled with no policies as the cause of empty Data API results, kept RLS enabled, created authenticated SELECT and INSERT owner-scoped policies using auth.uid() = user_id with WITH CHECK for insert, and verified isolation. It did not disable RLS or create permissive/public policies." + "judgeNotes": "Meets rubric: identifies soft-delete-only root cause, implements real auth.users deletion with session/refresh-token cascade equivalent, explains JWT expiry caveat and distinguishes Data API path vs purely local JWT validation, and correctly clarifies publishable vs secret/service_role key usage and RLS bypass." } ], "skills": { @@ -13205,98 +7277,131 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"Data API expose table grant authenticated RLS auth.uid select insert policy\", limit: 5) { nodes { title href content } } }", + "query": "query {\n searchDocs(query: \"delete user revoke sessions auth.admin.deleteUser sign out revoke access token\", limit: 5) {\n nodes {\n __typename\n ... on Guide { title href content }\n ... on ClientLibraryFunctionReference { title href content language methodName }\n ... on TroubleshootingGuide { title href content }\n ... on ManagementApiReference { title href content }\n }\n }\n}", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/api/securing-your-api", - "title": "Securing your API" + "url": "https://supabase.com/docs/reference/javascript/auth-signout" }, { - "url": "https://supabase.com/docs/guides/local-development/testing/pgtap-extended", - "title": "Advanced pgTAP Testing" + "url": "https://supabase.com/docs/guides/auth/managing-user-data", + "title": "User Management" }, { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", - "title": "Row Level Security" + "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows", + "title": "OAuth 2.1 Flows" }, { - "url": "https://supabase.com/docs/guides/storage/schema/helper-functions", - "title": "Storage Helper Functions" + "url": "https://supabase.com/docs/reference/javascript/auth-admin-deleteuser" }, { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-flutter", - "title": "Build a User Management App with Flutter" + "url": "https://supabase.com/docs/reference/javascript/auth-admin-signout" } ], - "resultChars": 114795 + "resultChars": 45466 }, { - "source": "web_search", - "query": "https://supabase.com/changelog.md", + "source": "search_docs", + "query": "query {\n searchDocs(query: \"publishable secret key anon service_role frontend RLS\", limit: 10) {\n nodes {\n __typename\n ... on Guide { title href content }\n ... on ClientLibraryFunctionReference { title href content language methodName }\n }\n }\n}", + "hasContent": true, "pages": [ { - "url": "https://supabase.com/changelog.md" + "url": "https://supabase.com/docs/guides/database/secure-data", + "title": "Securing your data" + }, + { + "url": "https://supabase.com/docs/guides/auth/jwt-fields", + "title": "JWT Claims Reference" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-redwoodjs", + "title": "Build a User Management App with RedwoodJS" + }, + { + "url": "https://supabase.com/docs/guides/auth/signing-keys", + "title": "JWT Signing Keys" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/api-keys", + "title": "Understanding API keys" + }, + { + "url": "https://supabase.com/docs/guides/storage/buckets/fundamentals", + "title": "Storage Buckets" + }, + { + "url": "https://supabase.com/docs/guides/storage/security/access-control", + "title": "Storage Access Control" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", + "title": "Row Level Security" + }, + { + "url": "https://supabase.com/docs/guides/auth/auth-anonymous", + "title": "Anonymous Sign-Ins" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" } - ] - }, - { - "source": "web_search", - "query": "site:supabase.com/changelog Supabase changelog Data API grants RLS", - "pages": [] + ], + "resultChars": 152941 } ] }, - "prompt": "Our app lets signed-in users save bookmarks and view them on their dashboard. Bookmarks are stored in the `bookmarks` table and are private — a user must only ever see their own. \nUsers also need to be able to save new bookmarks from the app.\n\nI can see the rows when I query the table directly, but the dashboard shows an empty list for every user.\n\nFind out why the Data API returns nothing and fix it.", - "promptSourcePath": "evals/resolve-dataapi-001-empty-results/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.6/resolve-dataapi-001-empty-results.json" + "prompt": "Last week support removed a user through our app's delete-account flow — the\napp calls the `delete_account` function over RPC as the signed-in user. This\nmorning that same person was back: still signed in, reading and saving their\ndata like nothing happened.\n\nFigure out why the account still works, fix the flow so a deleted account\nloses access, and tell me whether there is any window where they could still\nget in after the fix.\n\nOne more thing while you're at it: we're migrating off the legacy\nanon/service_role API keys to the new publishable and secret keys, and the\nteam isn't sure which one belongs in the frontend and what each means for\nRLS. Give us a quick rundown.", + "promptSourcePath": "evals/investigate-auth-001-deleted-user-access/PROMPT.md", + "attempts": 2, + "sourcePath": "codex-gpt-5.4-mini/investigate-auth-001-deleted-user-access.json" }, { - "experiment": "codex-gpt-5.6", + "experiment": "codex-gpt-5.4-mini", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.6-sol", + "modelId": "gpt-5.4-mini", "reasoningEffort": "medium" }, - "eval": "resolve-database-001-migration-history-mismatch", - "stage": "resolve", + "eval": "investigate-realtime-001-subscribed-no-events", + "stage": "investigate", "product": [ + "realtime", "database" ], "topic": [ - "migrations" + "sdk" ], "suite": "benchmark", - "interface": "cli", + "interface": "mcp", "passed": true, "checks": [ { - "name": "the avatar_url column is applied on the hosted profiles table", + "name": "orders table added to supabase_realtime publication", "passed": true }, { - "name": "migration 20240220000000 is recorded in the remote history", + "name": "courier_locations still in supabase_realtime publication", "passed": true }, { - "name": "remote migration history matches local migration files", + "name": "publication still publishes INSERT events", "passed": true }, { - "name": "local migrations are a valid reconciled sequence", + "name": "RLS still enabled on orders", "passed": true }, { - "name": "production profile data is intact (not reset)", - "passed": true + "name": "staff can still read orders through RLS", + "passed": true, + "notes": "authenticated sees 2 of 2 orders" }, { - "name": "the avatar migration and history reconciliation were done via the Supabase CLI", + "name": "diagnosed missing publication membership", "passed": true, - "judgeNotes": "Applied the pending avatar_url migration through `supabase db push --db-url \"$DB_URL\" --yes` in action #12, whose output indicates it finished. Reconciled the orphan bio migration by adding `supabase/migrations/20240115000000_add_profile_bio.sql` in action #10 so local and remote migration history matched before the push. No disallowed workaround observed." + "judgeNotes": "The assistant correctly identified that orders was missing from the supabase_realtime publication despite courier_locations being present, added only public.orders via ALTER PUBLICATION, verified both tables remained in the publication, and did not blame or weaken RLS/policies or alter client code." } ], "skills": { @@ -13305,88 +7410,82 @@ "supabase-postgres-best-practices" ], "loaded": [ - "supabase" + "supabase", + "supabase-postgres-best-practices" ] }, "docs": { "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase CLI database migrations db push migration list repair remote migration history\", limit: 5) { nodes { ... on Guide { title href content } ... on CLICommandReference { title href content } } } }", + "query": "query { searchDocs(query: \"realtime postgres_changes publication orders insert events table not receiving events\", limit: 5) { nodes { __typename ... on Guide { title href content } ... on TroubleshootingGuide { title href content } ... on ClientLibraryFunctionReference { title href href language methodName content } } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/reference/cli/supabase-db-push", - "title": "Push new migrations to the remote database" + "url": "https://supabase.com/docs/guides/realtime/postgres-changes", + "title": "Postgres Changes" }, { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows", - "title": "Local development workflow" + "url": "https://supabase.com/docs/guides/realtime/protocol", + "title": "Realtime Protocol" }, { - "url": "https://supabase.com/docs/guides/deployment/database-migrations", - "title": "Database Migrations" + "url": "https://supabase.com/docs/guides/realtime/benchmarks", + "title": "Benchmarks" }, { - "url": "https://supabase.com/docs/reference/cli/supabase-migration-repair", - "title": "Repair the migration history table" + "url": "https://supabase.com/docs/guides/realtime/reports", + "title": "Realtime Reports" }, { - "url": "https://supabase.com/docs/reference/cli/supabase-migration-list", - "title": "List local and remote migrations" + "url": "https://supabase.com/docs/guides/realtime/subscribing-to-database-changes", + "title": "Subscribing to Database Changes" } ], - "resultChars": 50099 + "resultChars": 157912 } ] }, - "prompt": "I'm trying to ship a migration to our hosted project and it's not working. Can you figure out what's wrong and get it deployed?", - "promptSourcePath": "evals/resolve-database-001-migration-history-mismatch/PROMPT.md", + "prompt": "Our dispatch dashboard shows incoming orders as they happen. The courier\nlocation feed on the same page updates live without problems, but new orders\nonly show up after a page refresh.\n\nThe dashboard uses supabase-js to subscribe to INSERT events on the `orders`\ntable through postgres_changes, the same way it subscribes to courier\nlocations. The channel's status callback logs SUBSCRIBED and there are no\nerrors in the browser console.\n\nFigure out why no order events ever arrive and fix it.", + "promptSourcePath": "evals/investigate-realtime-001-subscribed-no-events/PROMPT.md", "attempts": 1, - "sourcePath": "codex-gpt-5.6/resolve-database-001-migration-history-mismatch.json" + "sourcePath": "codex-gpt-5.4-mini/investigate-realtime-001-subscribed-no-events.json" }, { - "experiment": "codex-gpt-5.6", + "experiment": "codex-gpt-5.4-mini", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.6-sol", + "modelId": "gpt-5.4-mini", "reasoningEffort": "medium" }, - "eval": "resolve-performance-001-slow-query-cpu-spike", - "stage": "resolve", + "eval": "investigate-reliability-003-edge-function-5xx-correlation", + "stage": "investigate", "product": [ - "database" + "edge-functions" ], "topic": [ - "observability", - "sql" + "observability" ], "suite": "benchmark", "interface": "mcp", "passed": true, "checks": [ { - "name": "inspected pg_stat_statements for query performance", - "passed": true - }, - { - "name": "ran EXPLAIN on the expensive query", - "passed": true - }, - { - "name": "created index covering user_id and created_at", - "passed": true + "name": "identified image-transform and the recurring 503 pattern", + "passed": true, + "judgeNotes": "Identified image-transform as the primary affected function and listed the recurring 503s across the morning of 2026-04-28, covering all 8 gateway failures from 07:00Z through 12:00Z." }, { - "name": "query plan uses an index and avoids sequential scan", + "name": "attributed recurring 503s to gateway/platform layer, not function code", "passed": true, - "notes": "Limit (cost=79.74..79.86 rows=50 width=58)\n -> Sort (cost=79.74..79.99 rows=100 width=58)\n Sort Key: created_at DESC\n -> Bitmap Heap Scan on events (cost=5.06..76.42 rows=100 width=58)\n Recheck Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)\n -> Bitmap Index Scan on events_user_id_created_at_idx (cost=0.00..5.03 rows=100 width=0)\n Index Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)" + "judgeNotes": "Attributes the 503s as more likely platform-side/gateway unavailability than a code bug, grounded in API 503 timing, nearby successful 200s, unchanged deployment/version 42, and distinction from avatar-upload's separate 500." }, { - "name": "inserts still work", - "passed": true + "name": "recommended a concrete next step", + "passed": true, + "judgeNotes": "The assistant recommended concrete next steps, including inspecting deployment health/cold-start/external dependencies, checking the client routing path, adding retry/backoff, and opening a Supabase support case with exact timestamps and function slug." } ], "skills": { @@ -13399,119 +7498,65 @@ ] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"database query optimization EXPLAIN ANALYZE indexes pg_stat_statements\", limit: 5) { nodes { ... on Guide { title href content } ... on TroubleshootingGuide { title href content } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/database/inspect", - "title": "Debugging and monitoring" - }, - { - "url": "https://supabase.com/docs/guides/database/query-optimization", - "title": "Query Optimization" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/running-explain-analyze-on-functions", - "title": "Running EXPLAIN ANALYZE on functions" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/configuration", - "title": "Database configuration" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/understanding-postgresql-explain-output-Un9dqX", - "title": "Understanding Postgres EXPLAIN Output" - } - ], - "resultChars": 39586 - }, - { - "source": "web_search", - "query": "https://supabase.com/changelog.md", - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ] - }, - { - "source": "web_search", - "query": "site:supabase.com/changelog Supabase changelog breaking change database indexes query performance", - "pages": [] - }, - { - "source": "web_search", - "query": "site:supabase.com/docs/guides/database/postgres/row-level-security Supabase RLS public schema", - "pages": [] - } - ] + "calls": [] }, - "prompt": "My database CPU keeps spiking and the app gets slow when loading recent events for a user. Can you figure out what query is causing it and make the database change needed to fix it?\n\nEnd your turn with a short summary of what you changed and why.", - "promptSourcePath": "evals/resolve-performance-001-slow-query-cpu-spike/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.6/resolve-performance-001-slow-query-cpu-spike.json" + "prompt": "Users have been reporting that image uploads are intermittently failing with server errors throughout the morning of `2026-04-28`.\n\nCan you investigate the project logs and tell me what is going on and what we\nshould do next?", + "promptSourcePath": "evals/investigate-reliability-003-edge-function-5xx-correlation/PROMPT.md", + "attempts": 2, + "sourcePath": "codex-gpt-5.4-mini/investigate-reliability-003-edge-function-5xx-correlation.json" }, { - "experiment": "codex-gpt-5.6", + "experiment": "codex-gpt-5.4-mini", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.6-sol", + "modelId": "gpt-5.4-mini", "reasoningEffort": "medium" }, - "eval": "resolve-security-002-rls-cross-tenant-leak", + "eval": "resolve-dataapi-001-empty-results", "stage": "resolve", "product": [ + "data-api", "database", "auth" ], "topic": [ "rls", - "security" + "sdk" ], "suite": "benchmark", "interface": "mcp", "passed": true, "checks": [ { - "name": "RLS enabled on notes", - "passed": true - }, - { - "name": "tenant A sees only org A notes", - "passed": true - }, - { - "name": "tenant B cannot read org A notes", + "name": "RLS still enabled on bookmarks", "passed": true }, { - "name": "tenant A author can update own note", + "name": "user A reads own bookmarks", "passed": true }, { - "name": "tenant B cannot update org A note", + "name": "user B cannot read user A bookmarks", "passed": true }, { - "name": "tenant B author can delete own note", + "name": "anon reads no bookmarks", "passed": true }, { - "name": "tenant B cannot delete org A note", + "name": "user A can save a new bookmark", "passed": true }, { - "name": "tenant A can insert note in own org", + "name": "user B cannot insert a bookmark as user A", "passed": true }, { - "name": "tenant B cannot insert into org A", - "passed": true + "name": "diagnosed RLS and added owner-scoped policies", + "passed": true, + "judgeNotes": "The assistant correctly diagnosed RLS enabled with no policies causing deny-all/zero Data API rows, kept RLS enabled, and created authenticated-only owner-scoped SELECT and INSERT policies using auth.uid() = user_id / WITH CHECK. It also verified the policies." } ], "skills": { @@ -13527,836 +7572,822 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"row level security multi tenant team workspace membership policies auth.uid() security definer\", limit: 5) { nodes { ... on Guide { title href content } ... on TroubleshootingGuide { title href content } } } }", + "query": "query { searchDocs(query: \"Exposing a Table to the Data API RLS policies authenticated select insert\", limit: 5) { nodes { __typename title href content } } }", "hasContent": true, "pages": [ + { + "url": "https://supabase.com/docs/guides/api/securing-your-api", + "title": "Securing your API" + }, { "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", "title": "Row Level Security" }, { - "url": "https://supabase.com/docs/guides/auth/users", - "title": "Users" + "url": "https://supabase.com/docs/guides/database/database-advisors?queryGroups=lint&lint=0027_pg_graphql_authenticated_table_exposed", + "title": "Database Advisor: Lint 0027_pg_graphql_authenticated_table_exposed" + }, + { + "url": "https://supabase.com/docs/guides/database/secure-data", + "title": "Securing your data" + }, + { + "url": "https://supabase.com/docs/guides/local-development/testing/pgtap-extended", + "title": "Advanced pgTAP Testing" + } + ], + "resultChars": 85553 + }, + { + "source": "web_search", + "query": "https://supabase.com/changelog.md", + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ] + }, + { + "source": "web_search", + "query": "site:supabase.com changelog.md supabase", + "pages": [] + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"auth.uid request.jwt.claim.sub current_setting Supabase\", limit: 5) { nodes { __typename title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/ai/rag-with-permissions", + "title": "RAG with Permissions" + }, + { + "url": "https://supabase.com/docs/guides/realtime/authorization", + "title": "Realtime Authorization" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/rls-performance-and-best-practices-Z5Jjwv", + "title": "RLS Performance and Best Practices" + }, + { + "url": "https://supabase.com/docs/guides/auth/third-party/auth0", + "title": "Auth0" + }, + { + "url": "https://supabase.com/docs/guides/realtime/postgres-changes", + "title": "Postgres Changes" } ], - "resultChars": 38609 + "resultChars": 92588 + }, + { + "source": "web_search", + "query": "site:supabase.com/docs/guides/api/securing-your-api Supabase Data API grants RLS auth.uid request.jwt.claim.sub", + "pages": [] + }, + { + "source": "web_search", + "query": "site:supabase.com/docs \"Under the hood, auth.uid() references current_setting('request.jwt.claim.sub')\"", + "pages": [] } ] }, - "prompt": "A customer reported that notes showed up in the wrong workspace.\n\nCan you investigate what is going on and fix it?", - "promptSourcePath": "evals/resolve-security-002-rls-cross-tenant-leak/PROMPT.md", + "prompt": "Our app lets signed-in users save bookmarks and view them on their dashboard. Bookmarks are stored in the `bookmarks` table and are private — a user must only ever see their own. \nUsers also need to be able to save new bookmarks from the app.\n\nI can see the rows when I query the table directly, but the dashboard shows an empty list for every user.\n\nFind out why the Data API returns nothing and fix it.", + "promptSourcePath": "evals/resolve-dataapi-001-empty-results/PROMPT.md", "attempts": 1, - "sourcePath": "codex-gpt-5.6/resolve-security-002-rls-cross-tenant-leak.json" + "sourcePath": "codex-gpt-5.4-mini/resolve-dataapi-001-empty-results.json" }, { - "experiment": "codex-gpt-5.6-no-skills", - "experimentSuite": "no-skills", + "experiment": "codex-gpt-5.4-mini", + "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.6-sol", + "modelId": "gpt-5.4-mini", "reasoningEffort": "medium" }, - "eval": "build-cli-001-bootstrap-app", - "stage": "build", + "eval": "resolve-database-001-migration-history-mismatch", + "stage": "resolve", "product": [ - "database", - "data-api" + "database" ], "topic": [ - "migrations", - "rls" + "migrations" ], "suite": "benchmark", "interface": "cli", - "passed": true, + "passed": false, "checks": [ { - "name": "supabase project initialised (supabase/config.toml exists)", - "passed": true + "name": "the avatar_url column is applied on the hosted profiles table", + "passed": false, + "notes": "avatar_url not found on public.profiles" }, { - "name": "todos table is created by a migration file", - "passed": true + "name": "migration 20240220000000 is recorded in the remote history", + "passed": false, + "notes": "remote history versions: [\"20240101000000\",\"20240115000000\"]" }, { - "name": "todos table exists with at least 2 seeded rows", - "passed": true, - "notes": "found 2 rows" + "name": "remote migration history matches local migration files", + "passed": false, + "notes": "remote-only versions still unreconciled: [\"20240115000000\"] (remote: [\"20240101000000\",\"20240115000000\"], local: [\"20240101000000\",\"20240220000000\"])" }, { - "name": "row level security is enabled on todos", - "passed": true + "name": "local migrations are a valid reconciled sequence", + "passed": false, + "notes": "expected 20240101000000_create_profiles → → 20240220000000_add_avatar_url, strictly ascending; got [20240101000000_create_profiles.sql, 20240220000000_add_avatar_url.sql]" }, { - "name": "a SELECT policy targets the authenticated role", + "name": "production profile data is intact (not reset)", "passed": true }, { - "name": "REST API returns no todos to anonymous requests", - "passed": true, - "notes": "0 rows" - }, - { - "name": "REST API returns the todos to authenticated requests", - "passed": true, - "notes": "2 rows" + "name": "the avatar migration and history reconciliation were done via the Supabase CLI", + "passed": false, + "judgeNotes": "No qualifying hosted-remote fix occurred. `supabase db push --linked` only failed or was run as dry-run (#13, #17, #44, #61). The only real `supabase db push` appears to be #65 against a temporary local test database via `--db-url postgresql://...127.0.0.1:54322/$TEST_DB`, not the hosted project. No `supabase migration repair`, `supabase db pull`, or other CLI reconciliation of the orphan bio migration was performed. Workaround seen: using the local Docker database / temp DB instead of resolving the hosted linked project connection." } ], "skills": { - "available": [], - "loaded": [] + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase row level security authenticated role select policy migrations seed local development CLI db reset\", limit: 5) { nodes { ... on Guide { title href content } ... on CLICommandReference { title href content } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows", - "title": "Local development workflow" - }, - { - "url": "https://supabase.com/docs/reference/cli/supabase-db-reset", - "title": "Resets the local database to current migrations" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/column-level-security", - "title": "Column Level Security" - }, - { - "url": "https://supabase.com/docs/guides/deployment/branching/troubleshooting", - "title": "Troubleshooting" - }, - { - "url": "https://supabase.com/docs/guides/deployment/branching/working-with-branches", - "title": "Working with branches" - } - ], - "resultChars": 49117 - } - ] + "calls": [] }, - "prompt": "We're kicking off a todos app and I want the Supabase side ready for the team\nto build on. Set it up the way we'd run it in development, with schema changes\ntracked as migrations so they can be reviewed and replayed.\n\nFor the first slice we just need a `todos` table. Todos aren't public: anyone\nsigned in can read all of them, but nothing should be writable through the API\nfor now. Add a couple of sample todos so there's something to look at.\n\nBefore you hand it back, make sure the running API actually behaves that way —\nsigned-in users get the todos, signed-out requests get nothing.", - "promptSourcePath": "evals/build-cli-001-bootstrap-app/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.6-no-skills/build-cli-001-bootstrap-app.json" + "prompt": "I'm trying to ship a migration to our hosted project and it's not working. Can you figure out what's wrong and get it deployed?", + "promptSourcePath": "evals/resolve-database-001-migration-history-mismatch/PROMPT.md", + "attempts": 2, + "sourcePath": "codex-gpt-5.4-mini/resolve-database-001-migration-history-mismatch.json" }, { - "experiment": "codex-gpt-5.6-no-skills", - "experimentSuite": "no-skills", + "experiment": "codex-gpt-5.4-mini", + "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.6-sol", + "modelId": "gpt-5.4-mini", "reasoningEffort": "medium" }, - "eval": "build-cli-002-declarative-schema", - "stage": "build", + "eval": "resolve-performance-001-slow-query-cpu-spike", + "stage": "resolve", "product": [ "database" ], "topic": [ - "declarative-schema", - "migrations" + "observability", + "sql" ], "suite": "benchmark", - "interface": "cli", - "passed": false, + "interface": "mcp", + "passed": true, "checks": [ { - "name": "supabase db diff used to generate the migration", - "passed": false + "name": "inspected pg_stat_statements for query performance", + "passed": true }, { - "name": "schema file updated to include description column", + "name": "ran EXPLAIN on the expensive query", "passed": true }, { - "name": "a new migration was generated for the change", + "name": "created index covering user_id and created_at", "passed": true }, { - "name": "description column exists in the live database", + "name": "query plan uses an index and avoids sequential scan", + "passed": true, + "notes": "Limit (cost=55.55..55.61 rows=25 width=88)\n -> Sort (cost=55.55..55.61 rows=25 width=88)\n Sort Key: created_at DESC\n -> Bitmap Heap Scan on events (cost=4.48..54.97 rows=25 width=88)\n Recheck Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)\n -> Bitmap Index Scan on events_user_id_created_at_idx (cost=0.00..4.47 rows=25 width=0)\n Index Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)" + }, + { + "name": "inserts still work", "passed": true } ], "skills": { - "available": [], - "loaded": [] + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] }, "docs": { "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase CLI create migration alter table add column local database migration\", limit: 3) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"Postgres index partial index query performance Supabase recent events\", limit: 5) { nodes { __typename title href ... on Guide { content } ... on ClientLibraryFunctionReference { content language methodName } ... on CLICommandReference { content } ... on TroubleshootingGuide { content } } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/deployment/database-migrations", - "title": "Database Migrations" + "url": "https://supabase.com/docs/guides/troubleshooting/steps-to-improve-query-performance-with-indexes-q8PoC9", + "title": "Steps to improve query performance with indexes" }, { - "url": "https://supabase.com/docs/guides/local-development/database-migrations", - "title": "Database migrations" + "url": "https://supabase.com/docs/guides/ai/semantic-search", + "title": "Semantic search" }, { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows", - "title": "Local development workflow" + "url": "https://supabase.com/docs/guides/database/inspect", + "title": "Debugging and monitoring" + }, + { + "url": "https://supabase.com/docs/guides/telemetry/logs", + "title": "Logging" + }, + { + "url": "https://supabase.com/docs/guides/platform/performance", + "title": "Performance Tuning" } ], - "resultChars": 54403 + "resultChars": 52711 } ] }, - "prompt": "Add a description text column to the `products` table in my local Supabase stack", - "promptSourcePath": "evals/build-cli-002-declarative-schema/PROMPT.md", - "attempts": 2, - "sourcePath": "codex-gpt-5.6-no-skills/build-cli-002-declarative-schema.json" + "prompt": "My database CPU keeps spiking and the app gets slow when loading recent events for a user. Can you figure out what query is causing it and make the database change needed to fix it?\n\nEnd your turn with a short summary of what you changed and why.", + "promptSourcePath": "evals/resolve-performance-001-slow-query-cpu-spike/PROMPT.md", + "attempts": 1, + "sourcePath": "codex-gpt-5.4-mini/resolve-performance-001-slow-query-cpu-spike.json" }, - { - "experiment": "codex-gpt-5.6-no-skills", - "experimentSuite": "no-skills", + { + "experiment": "codex-gpt-5.4-mini", + "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.6-sol", + "modelId": "gpt-5.4-mini", "reasoningEffort": "medium" }, - "eval": "build-cli-003-pg-cron-queue-workflow", - "stage": "build", + "eval": "resolve-security-002-rls-cross-tenant-leak", + "stage": "resolve", "product": [ "database", - "edge-functions", - "cron", - "queues" + "auth" ], "topic": [ - "sql", - "sdk" + "rls", + "security" ], "suite": "benchmark", - "interface": "cli", + "interface": "mcp", "passed": true, "checks": [ { - "name": "pg_cron job 'enqueue-tasks' scheduled to run every minute", - "passed": true, - "notes": "schedule='* * * * *', active=true" + "name": "RLS enabled on notes", + "passed": true }, { - "name": "cron command enqueues to the 'tasks' queue", - "passed": true, - "notes": "queue depth 0 -> 1" + "name": "tenant A sees only org A notes", + "passed": true }, { - "name": "process-tasks function drains the queue", - "passed": true, - "notes": "function removed the seeded message (id 36) from the queue" + "name": "tenant B cannot read org A notes", + "passed": true + }, + { + "name": "tenant A author can update own note", + "passed": true + }, + { + "name": "tenant B cannot update org A note", + "passed": true + }, + { + "name": "tenant B author can delete own note", + "passed": true + }, + { + "name": "tenant B cannot delete org A note", + "passed": true + }, + { + "name": "tenant A can insert note in own org", + "passed": true + }, + { + "name": "tenant B cannot insert into org A", + "passed": true } ], "skills": { - "available": [], - "loaded": [] + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] }, "docs": { "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase Queues create queue pgmq create send read delete Edge Functions JavaScript\", limit: 8) { nodes { ... on Guide { title href content subsections { nodes { title href content } } } ... on ClientLibraryFunctionReference { title href content language methodName } } } }", + "query": "query {\n searchDocs(query: \"row level security policy auth.uid select same org_id membership table enable rls\", limit: 5) {\n nodes {\n title\n href\n content\n ... on Guide { subsections { nodes { title href content } } }\n }\n }\n}", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/queues", - "title": "Supabase Queues" - }, - { - "url": "https://supabase.com/docs/guides/queues#features", - "title": "Features" - }, - { - "url": "https://supabase.com/docs/guides/queues#resources", - "title": "Resources" - }, - { - "url": "https://supabase.com/docs/guides/queues/consuming-messages-with-edge-functions", - "title": "Consuming Supabase Queue Messages with Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/queues/consuming-messages-with-edge-functions#concepts", - "title": "Concepts" - }, - { - "url": "https://supabase.com/docs/guides/queues/consuming-messages-with-edge-functions#consuming-messages-in-an-edge-function", - "title": "Consuming messages in an Edge Function" - }, - { - "url": "https://supabase.com/docs/guides/queues/quickstart", - "title": "Quickstart" - }, - { - "url": "https://supabase.com/docs/guides/queues/quickstart#enable-rls-on-your-tables-in-pgmq-schema", - "title": "Enable RLS on your tables in pgmq schema" - }, - { - "url": "https://supabase.com/docs/guides/queues/quickstart#queue-types", - "title": "Queue types" - }, - { - "url": "https://supabase.com/docs/guides/queues/quickstart#create-queues", - "title": "Create Queues" - }, - { - "url": "https://supabase.com/docs/guides/queues/quickstart#what-happens-when-you-create-a-queue", - "title": "What happens when you create a queue?" - }, - { - "url": "https://supabase.com/docs/guides/queues/quickstart#expose-queues-to-client-side-consumers", - "title": "Expose Queues to client-side consumers" - }, - { - "url": "https://supabase.com/docs/guides/queues/quickstart#grant-permissions-to-pgmq_public-database-functions", - "title": "Grant permissions to pgmq_public database functions" - }, - { - "url": "https://supabase.com/docs/guides/queues/quickstart#enqueueing-and-dequeueing-messages", - "title": "Enqueueing and dequeueing messages" - }, - { - "url": "https://supabase.com/docs/guides/queues/quickstart#message", - "title": "Message" - }, - { - "url": "https://supabase.com/docs/guides/queues/quickstart#pull-based-queue", - "title": "Pull-Based Queue" - }, - { - "url": "https://supabase.com/docs/guides/queues/quickstart#concepts", - "title": "Concepts" - }, - { - "url": "https://supabase.com/docs/guides/queues/api", - "title": "API" - }, - { - "url": "https://supabase.com/docs/guides/queues/api#pgmq_publicpopqueue_name", - "title": "pgmq_public.pop(queue_name)" - }, - { - "url": "https://supabase.com/docs/guides/queues/api#pgmq_publicsendqueue_name-message-sleep_seconds", - "title": "pgmq_public.send(queue_name, message, sleep_seconds)" - }, - { - "url": "https://supabase.com/docs/guides/queues/api#pgmq_publicsend_batchqueue_name-messages-sleep_seconds", - "title": "pgmq_public.send_batch(queue_name, messages, sleep_seconds)" - }, - { - "url": "https://supabase.com/docs/guides/queues/api#pgmq_publicarchivequeue_name-message_id", - "title": "pgmq_public.archive(queue_name, message_id)" - }, - { - "url": "https://supabase.com/docs/guides/queues/api#pgmq_publicdeletequeue_name-message_id", - "title": "pgmq_public.delete(queue_name, message_id)" - }, - { - "url": "https://supabase.com/docs/guides/queues/api#pgmq_publicreadqueue_name-sleep_seconds-n", - "title": "pgmq_public.read(queue_name, sleep_seconds, n)" - }, - { - "url": "https://supabase.com/docs/guides/database/connecting-to-postgres/serverless-drivers", - "title": "Serverless Drivers" - }, - { - "url": "https://supabase.com/docs/guides/database/connecting-to-postgres/serverless-drivers#supabase-edge-functions", - "title": "Supabase Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/database/connecting-to-postgres/serverless-drivers#quickstart", - "title": "Quickstart" - }, - { - "url": "https://supabase.com/docs/guides/database/connecting-to-postgres/serverless-drivers#vercel-edge-functions", - "title": "Vercel Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/database/connecting-to-postgres/serverless-drivers#cloudflare-workers", - "title": "Cloudflare Workers" - }, - { - "url": "https://supabase.com/docs/guides/database/connecting-to-postgres/serverless-drivers#manual-configuration", - "title": "Manual configuration" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pgmq", - "title": "pgmq: Queues" - }, - { - "url": "https://supabase.com/docs/guides/functions/connect-to-postgres", - "title": "Integrating with Supabase Database (Postgres)" - }, - { - "url": "https://supabase.com/docs/guides/functions/connect-to-postgres#ssl-connections", - "title": "SSL connections" - }, - { - "url": "https://supabase.com/docs/guides/functions/connect-to-postgres#production", - "title": "Production" + "url": "https://supabase.com/docs/guides/api/securing-your-api", + "title": "Securing your API" }, { - "url": "https://supabase.com/docs/guides/functions/connect-to-postgres#local-development", - "title": "Local development" + "url": "https://supabase.com/docs/guides/api/securing-your-api#examples", + "title": "Examples" }, { - "url": "https://supabase.com/docs/guides/functions/connect-to-postgres#using-supabase-js", - "title": "Using supabase-js" + "url": "https://supabase.com/docs/guides/api/securing-your-api#accessing-request-information", + "title": "Accessing request information" }, { - "url": "https://supabase.com/docs/guides/functions/connect-to-postgres#using-a-postgres-client", - "title": "Using a Postgres client" + "url": "https://supabase.com/docs/guides/api/securing-your-api#enforce-additional-rules-on-each-request", + "title": "Enforce additional rules on each request" }, { - "url": "https://supabase.com/docs/guides/functions/connect-to-postgres#using-drizzle", - "title": "Using Drizzle" + "url": "https://supabase.com/docs/guides/api/securing-your-api#add-rls-policies", + "title": "Add RLS policies" }, { - "url": "https://supabase.com/docs/guides/functions", - "title": "Edge Functions" + "url": "https://supabase.com/docs/guides/api/securing-your-api#disable-the-data-api", + "title": "Disable the Data API" }, { - "url": "https://supabase.com/docs/guides/functions#quick-technical-notes", - "title": "Quick technical notes" + "url": "https://supabase.com/docs/guides/api/securing-your-api#use-a-dedicated-api-schema", + "title": "Use a dedicated API schema" }, { - "url": "https://supabase.com/docs/guides/functions#when-to-use-edge-functions", - "title": "When to use Edge Functions" + "url": "https://supabase.com/docs/guides/api/securing-your-api#default-privileges-for-new-tables-and-functions", + "title": "Default privileges for new tables and functions" }, { - "url": "https://supabase.com/docs/guides/functions#examples", - "title": "Examples" + "url": "https://supabase.com/docs/guides/api/securing-your-api#grant-access-explicitly", + "title": "Grant access explicitly" }, { - "url": "https://supabase.com/docs/guides/functions#how-it-works", - "title": "How it works" - } - ], - "resultChars": 151897 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Consuming Supabase Queue Messages with Edge Functions queue.read queue.delete\", limit: 5) { nodes { ... on Guide { title href content subsections { nodes { title href content } } } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/queues/consuming-messages-with-edge-functions", - "title": "Consuming Supabase Queue Messages with Edge Functions" + "url": "https://supabase.com/docs/guides/database/postgres/column-level-security", + "title": "Column Level Security" }, { - "url": "https://supabase.com/docs/guides/queues/consuming-messages-with-edge-functions#consuming-messages-in-an-edge-function", - "title": "Consuming messages in an Edge Function" + "url": "https://supabase.com/docs/guides/database/postgres/column-level-security#policies-at-the-row-level", + "title": "Policies at the row level" }, { - "url": "https://supabase.com/docs/guides/queues/consuming-messages-with-edge-functions#concepts", - "title": "Concepts" + "url": "https://supabase.com/docs/guides/database/postgres/column-level-security#considerations-when-using-column-level-privileges", + "title": "Considerations when using column-level privileges" }, { - "url": "https://supabase.com/docs/guides/functions/architecture", - "title": "Edge Functions Architecture" + "url": "https://supabase.com/docs/guides/database/postgres/column-level-security#manage-column-privileges-in-migrations", + "title": "Manage column privileges in migrations" }, { - "url": "https://supabase.com/docs/guides/functions/architecture#1-understanding-edge-functions-through-an-example-image-filtering", - "title": "1. Understanding Edge Functions through an example: Image filtering" + "url": "https://supabase.com/docs/guides/database/postgres/column-level-security#manage-column-privileges-in-the-dashboard", + "title": "Manage column privileges in the Dashboard" }, { - "url": "https://supabase.com/docs/guides/functions/architecture#2-deployment-process", - "title": "2. Deployment process" + "url": "https://supabase.com/docs/guides/database/postgres/column-level-security#privileges-at-the-column-level", + "title": "Privileges at the column level" }, { - "url": "https://supabase.com/docs/guides/functions/architecture#3-global-distribution-and-routing", - "title": "3. Global distribution and routing" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", + "title": "Row Level Security" }, { - "url": "https://supabase.com/docs/guides/functions/architecture#4-execution-mechanics-fast-and-isolated", - "title": "4. Execution mechanics: Fast and isolated" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#specify-roles-in-your-policies", + "title": "Specify roles in your policies" }, { - "url": "https://supabase.com/docs/guides/functions/architecture#benefits-and-use-cases", - "title": "Benefits and use cases" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#row-level-security-in-supabase", + "title": "Row Level Security in Supabase" }, { - "url": "https://supabase.com/docs/guides/queues/quickstart", - "title": "Quickstart" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#policies", + "title": "Policies" }, { - "url": "https://supabase.com/docs/guides/queues/quickstart#enqueueing-and-dequeueing-messages", - "title": "Enqueueing and dequeueing messages" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#enabling-row-level-security", + "title": "Enabling Row Level Security" }, { - "url": "https://supabase.com/docs/guides/queues/quickstart#grant-permissions-to-pgmq_public-database-functions", - "title": "Grant permissions to pgmq_public database functions" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#auto-enable-rls-for-new-tables", + "title": "Auto-enable RLS for new tables" }, { - "url": "https://supabase.com/docs/guides/queues/quickstart#concepts", - "title": "Concepts" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#authenticated-and-unauthenticated-roles", + "title": "Authenticated and unauthenticated roles" }, { - "url": "https://supabase.com/docs/guides/queues/quickstart#pull-based-queue", - "title": "Pull-Based Queue" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#creating-policies", + "title": "Creating policies" }, { - "url": "https://supabase.com/docs/guides/queues/quickstart#message", - "title": "Message" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#select-policies", + "title": "SELECT policies" }, { - "url": "https://supabase.com/docs/guides/queues/quickstart#queue-types", - "title": "Queue types" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#insert-policies", + "title": "INSERT policies" }, { - "url": "https://supabase.com/docs/guides/queues/quickstart#create-queues", - "title": "Create Queues" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#update-policies", + "title": "UPDATE policies" }, { - "url": "https://supabase.com/docs/guides/queues/quickstart#what-happens-when-you-create-a-queue", - "title": "What happens when you create a queue?" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#delete-policies", + "title": "DELETE policies" }, { - "url": "https://supabase.com/docs/guides/queues/quickstart#expose-queues-to-client-side-consumers", - "title": "Expose Queues to client-side consumers" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#views", + "title": "Views" }, { - "url": "https://supabase.com/docs/guides/queues/quickstart#enable-rls-on-your-tables-in-pgmq-schema", - "title": "Enable RLS on your tables in pgmq schema" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#helper-functions", + "title": "Helper functions" }, { - "url": "https://supabase.com/docs/guides/functions/recursive-functions", - "title": "Recursive / Nested Function Calls" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#authuid", + "title": "auth.uid()" }, { - "url": "https://supabase.com/docs/guides/functions/recursive-functions#what-gets-rate-limited", - "title": "What gets rate limited" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#authjwt", + "title": "auth.jwt()" }, { - "url": "https://supabase.com/docs/guides/functions/recursive-functions#rate-limit-budget", - "title": "Rate limit budget" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#mfa", + "title": "MFA" }, { - "url": "https://supabase.com/docs/guides/functions/recursive-functions#handling-rate-limit-errors", - "title": "Handling rate limit errors" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#bypassing-row-level-security", + "title": "Bypassing Row Level Security" }, { - "url": "https://supabase.com/docs/guides/functions/recursive-functions#tips-for-avoiding-rate-limits", - "title": "Tips for avoiding rate limits" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#rls-performance-recommendations", + "title": "RLS performance recommendations" }, { - "url": "https://supabase.com/docs/guides/functions/recursive-functions#1-batch-operations-instead-of-individual-calls", - "title": "1. Batch operations instead of individual calls" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#add-indexes", + "title": "Add indexes" }, { - "url": "https://supabase.com/docs/guides/functions/recursive-functions#2-limit-recursion-depth", - "title": "2. Limit recursion depth" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#benchmarks", + "title": "Benchmarks" }, { - "url": "https://supabase.com/docs/guides/functions/recursive-functions#3-use-queues-for-large-workloads", - "title": "3. Use queues for large workloads" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#call-functions-with-select", + "title": "Call functions with select" }, { - "url": "https://supabase.com/docs/guides/functions/recursive-functions#4-use-shared-libraries-instead-of-separate-functions", - "title": "4. Use shared libraries instead of separate functions" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#benchmarks-1", + "title": "Benchmarks" }, { - "url": "https://supabase.com/docs/guides/functions/recursive-functions#5-add-delays-for-non-urgent-processing", - "title": "5. Add delays for non-urgent processing" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#add-filters-to-every-query", + "title": "Add filters to every query" }, { - "url": "https://supabase.com/docs/guides/functions/recursive-functions#common-patterns-and-their-impact", - "title": "Common patterns and their impact" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#benchmarks-2", + "title": "Benchmarks" }, { - "url": "https://supabase.com/docs/guides/functions/recursive-functions#increasing-rate-limits", - "title": "Increasing rate limits" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#use-security-definer-functions", + "title": "Use security definer functions" }, { - "url": "https://supabase.com/docs/guides/functions/examples/resumable-websockets", - "title": "Resumable WebSockets with Edge Functions" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#minimize-joins", + "title": "Minimize joins" }, { - "url": "https://supabase.com/docs/guides/functions/examples/resumable-websockets#architecture", - "title": "Architecture" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#benchmarks-3", + "title": "Benchmarks" }, { - "url": "https://supabase.com/docs/guides/functions/examples/resumable-websockets#database-schema", - "title": "Database schema" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#benchmarks-4", + "title": "Benchmarks" }, { - "url": "https://supabase.com/docs/guides/functions/examples/resumable-websockets#edge-function-websocket-proxy", - "title": "Edge Function (WebSocket proxy)" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#more-resources", + "title": "More resources" }, { - "url": "https://supabase.com/docs/guides/functions/examples/resumable-websockets#browser-client", - "title": "Browser client" + "url": "https://supabase.com/docs/guides/resources/glossary", + "title": "Glossary" }, { - "url": "https://supabase.com/docs/guides/functions/examples/resumable-websockets#why-this-pattern-works", - "title": "Why this pattern works" + "url": "https://supabase.com/docs/guides/resources/glossary#authenticator-app", + "title": "Authenticator app" }, { - "url": "https://supabase.com/docs/guides/functions/examples/resumable-websockets#next-steps", - "title": "Next steps" - } - ], - "resultChars": 105925 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase Cron schedule SQL cron.schedule job name every minute\", limit: 5) { nodes { ... on Guide { title href content subsections { nodes { title href content } } } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/cron/quickstart", - "title": "Quickstart" + "url": "https://supabase.com/docs/guides/resources/glossary#authorization", + "title": "Authorization" }, { - "url": "https://supabase.com/docs/guides/cron/quickstart#call-a-database-function-every-5-minutes", - "title": "Call a database function every 5 minutes" + "url": "https://supabase.com/docs/guides/resources/glossary#refresh-token-flow", + "title": "Refresh token flow" }, { - "url": "https://supabase.com/docs/guides/cron/quickstart#schedule-a-job", - "title": "Schedule a job" + "url": "https://supabase.com/docs/guides/resources/glossary#row-level-security-policies-rls", + "title": "Row level security policies (RLS)" }, { - "url": "https://supabase.com/docs/guides/cron/quickstart#edit-a-job", - "title": "Edit a job" + "url": "https://supabase.com/docs/guides/resources/glossary#saml", + "title": "SAML" }, { - "url": "https://supabase.com/docs/guides/cron/quickstart#activatedeactivate-a-job", - "title": "Activate/Deactivate a job" + "url": "https://supabase.com/docs/guides/resources/glossary#session", + "title": "Session" }, { - "url": "https://supabase.com/docs/guides/cron/quickstart#unschedule-a-job", - "title": "Unschedule a job" + "url": "https://supabase.com/docs/guides/resources/glossary#single-sign-on-sso", + "title": "Single-sign on (SSO)" }, { - "url": "https://supabase.com/docs/guides/cron/quickstart#inspecting-job-runs", - "title": "Inspecting job runs" + "url": "https://supabase.com/docs/guides/resources/glossary#replay-attack", + "title": "Replay attack" }, { - "url": "https://supabase.com/docs/guides/cron/quickstart#examples", - "title": "Examples" + "url": "https://supabase.com/docs/guides/resources/glossary#jwt-signing-secret", + "title": "JWT signing secret" }, { - "url": "https://supabase.com/docs/guides/cron/quickstart#delete-data-every-week", - "title": "Delete data every week" + "url": "https://supabase.com/docs/guides/resources/glossary#json-web-token-jwt", + "title": "JSON Web Token (JWT)" }, { - "url": "https://supabase.com/docs/guides/cron/quickstart#run-a-vacuum-every-day", - "title": "Run a vacuum every day" + "url": "https://supabase.com/docs/guides/resources/glossary#identity-provider", + "title": "Identity provider" }, { - "url": "https://supabase.com/docs/guides/cron/quickstart#call-a-database-stored-procedure", - "title": "Call a database stored procedure" + "url": "https://supabase.com/docs/guides/resources/glossary#time-based-one-time-password-totp", + "title": "Time-based one-time password (TOTP)" }, { - "url": "https://supabase.com/docs/guides/cron/quickstart#invoke-supabase-edge-function-every-30-seconds", - "title": "Invoke Supabase Edge Function every 30 seconds" + "url": "https://supabase.com/docs/guides/resources/glossary#nonce", + "title": "Nonce" }, { - "url": "https://supabase.com/docs/guides/cron/quickstart#caution-scheduling-system-maintenance", - "title": "Caution: Scheduling system maintenance" + "url": "https://supabase.com/docs/guides/resources/glossary#multi-factor-authentication-mfa-or-2fa", + "title": "Multi-factor authentication (MFA or 2FA)" }, { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net", - "title": "pg_net: Async Networking" + "url": "https://supabase.com/docs/guides/resources/glossary#oauth", + "title": "OAuth" }, { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#delete-usage", - "title": "Usage " + "url": "https://supabase.com/docs/guides/resources/glossary#oidc", + "title": "OIDC" }, { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#enable-the-extension", - "title": "Enable the extension" + "url": "https://supabase.com/docs/guides/resources/glossary#one-time-password-otp", + "title": "One-time password (OTP)" }, { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#debugging-requests", - "title": "Debugging requests" + "url": "https://supabase.com/docs/guides/resources/glossary#password-hashing-function", + "title": "Password hashing function" }, { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#analyzing-responses", - "title": "Analyzing responses" + "url": "https://supabase.com/docs/guides/resources/glossary#password-strength", + "title": "Password strength" }, { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#http_get", - "title": "http_get" + "url": "https://supabase.com/docs/guides/resources/glossary#pkce", + "title": "PKCE" }, { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#get-signature", - "title": "Signature " + "url": "https://supabase.com/docs/guides/resources/glossary#provider-refresh-token", + "title": "Provider refresh token" }, { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#get-usage", - "title": "Usage " + "url": "https://supabase.com/docs/guides/resources/glossary#provider-tokens", + "title": "Provider tokens" }, { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#http_post", - "title": "http_post" + "url": "https://supabase.com/docs/guides/resources/glossary#refresh-token", + "title": "Refresh token" }, { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#post-signature", - "title": "Signature " + "url": "https://supabase.com/docs/guides/resources/glossary#access-token", + "title": "Access token" }, { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#post-usage", - "title": "Usage " + "url": "https://supabase.com/docs/guides/resources/glossary#authentication", + "title": "Authentication" }, { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#http_delete", - "title": "http_delete" + "url": "https://supabase.com/docs/guides/realtime/getting_started", + "title": "Getting Started with Realtime" }, { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#post-signature-1", - "title": "Signature " + "url": "https://supabase.com/docs/guides/realtime/getting_started#2-initialize-the-client", + "title": "2. Initialize the client" }, { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#inspecting-request-data", - "title": "Inspecting request data" + "url": "https://supabase.com/docs/guides/realtime/getting_started#framework-examples", + "title": "Framework examples" }, { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#inspecting-failed-requests", - "title": "Inspecting failed requests" + "url": "https://supabase.com/docs/guides/realtime/getting_started#integration-guides", + "title": "Integration guides" }, { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#configuration", - "title": "Configuration" + "url": "https://supabase.com/docs/guides/realtime/getting_started#advanced-topics", + "title": "Advanced topics" }, { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#get-current-settings", - "title": "Get current settings" + "url": "https://supabase.com/docs/guides/realtime/getting_started#security--configuration", + "title": "Security & configuration" }, { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#alter-settings", - "title": "Alter settings" + "url": "https://supabase.com/docs/guides/realtime/getting_started#core-features", + "title": "Core features" }, { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#examples", - "title": "Examples" + "url": "https://supabase.com/docs/guides/realtime/getting_started#next-steps", + "title": "Next steps" }, { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#invoke-a-supabase-edge-function", - "title": "Invoke a Supabase Edge Function" + "url": "https://supabase.com/docs/guides/realtime/getting_started#when-to-use-postgres-changes", + "title": "When to use Postgres Changes" }, { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#call-an-endpoint-every-minute-with-pg_cron", - "title": "Call an endpoint every minute with pg_cron" + "url": "https://supabase.com/docs/guides/realtime/getting_started#when-to-use-presence", + "title": "When to use Presence" }, { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#execute-pg_net-in-a-trigger", - "title": "Execute pg_net in a trigger" + "url": "https://supabase.com/docs/guides/realtime/getting_started#when-to-use-broadcast", + "title": "When to use Broadcast" }, { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#send-multiple-table-rows-in-one-request", - "title": "Send multiple table rows in one request" + "url": "https://supabase.com/docs/guides/realtime/getting_started#choose-the-right-feature", + "title": "Choose the right feature" }, { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#resources", - "title": "Resources" + "url": "https://supabase.com/docs/guides/realtime/getting_started#clean-up-subscriptions", + "title": "Clean up subscriptions" }, { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#limitations", - "title": "Limitations" + "url": "https://supabase.com/docs/guides/realtime/getting_started#follow-naming-conventions", + "title": "Follow naming conventions" }, { - "url": "https://supabase.com/docs/guides/functions/schedule-functions", - "title": "Scheduling Edge Functions" + "url": "https://supabase.com/docs/guides/realtime/getting_started#use-private-channels", + "title": "Use private channels" }, { - "url": "https://supabase.com/docs/guides/functions/schedule-functions#resources", - "title": "Resources" + "url": "https://supabase.com/docs/guides/realtime/getting_started#essential-best-practices", + "title": "Essential best practices" }, { - "url": "https://supabase.com/docs/guides/functions/schedule-functions#invoke-an-edge-function-every-minute", - "title": "Invoke an Edge Function every minute" + "url": "https://supabase.com/docs/guides/realtime/getting_started#53-using-database-triggers", + "title": "5.3 using database triggers" }, { - "url": "https://supabase.com/docs/guides/functions/schedule-functions#examples", - "title": "Examples" + "url": "https://supabase.com/docs/guides/realtime/getting_started#52-using-httprest-api", + "title": "5.2 using HTTP/REST API" }, { - "url": "https://supabase.com/docs/guides/cron", - "title": "Cron" + "url": "https://supabase.com/docs/guides/realtime/getting_started#51-using-client-libraries", + "title": "5.1 using client libraries" }, { - "url": "https://supabase.com/docs/guides/cron#resources", - "title": "Resources" + "url": "https://supabase.com/docs/guides/realtime/getting_started#5-send-and-receive-messages", + "title": "5. Send and receive messages" }, { - "url": "https://supabase.com/docs/guides/cron#how-does-cron-work", - "title": "How does Cron work?" + "url": "https://supabase.com/docs/guides/realtime/getting_started#4-set-up-authorization", + "title": "4. Set up authorization" }, { - "url": "https://supabase.com/docs/guides/auth/auth-hooks/send-email-hook", - "title": "Send Email Hook" + "url": "https://supabase.com/docs/guides/realtime/getting_started#3-create-your-first-channel", + "title": "3. Create your first Channel" }, { - "url": "https://supabase.com/docs/guides/auth/auth-hooks/send-email-hook#email-change-behavior-and-token-hash-mapping", - "title": "Email change behavior and token hash mapping" + "url": "https://supabase.com/docs/guides/realtime/getting_started#get-api-details", + "title": "Get API details" }, { - "url": "https://supabase.com/docs/guides/auth/auth-hooks/send-email-hook#email-sending-behavior", - "title": "Email sending behavior" + "url": "https://supabase.com/docs/guides/realtime/getting_started#quick-start", + "title": "Quick start" }, { - "url": "https://supabase.com/docs/guides/auth/auth-hooks/send-email-hook#what-to-send", - "title": "What to send" + "url": "https://supabase.com/docs/guides/realtime/getting_started#1-install-the-client-library", + "title": "1. Install the client library" } ], - "resultChars": 176011 + "resultChars": 242659 } ] }, - "prompt": "I want to set up a recurring background workflow on my local Supabase stack.\n\nCan you set up a cron job called `enqueue-tasks` to run every minute and push a task into a queue called `tasks`? Then add a `process-tasks` edge function that reads messages off the `tasks` queue and removes them, so a scheduled worker can keep the backlog drained.", - "promptSourcePath": "evals/build-cli-003-pg-cron-queue-workflow/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.6-no-skills/build-cli-003-pg-cron-queue-workflow.json" + "prompt": "A customer reported that notes showed up in the wrong workspace.\n\nCan you investigate what is going on and fix it?", + "promptSourcePath": "evals/resolve-security-002-rls-cross-tenant-leak/PROMPT.md", + "attempts": 2, + "sourcePath": "codex-gpt-5.4-mini/resolve-security-002-rls-cross-tenant-leak.json" }, { - "experiment": "codex-gpt-5.6-no-skills", + "experiment": "codex-gpt-5.4-mini-no-skills", "experimentSuite": "no-skills", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.6-sol", + "modelId": "gpt-5.4-mini", "reasoningEffort": "medium" }, - "eval": "build-database-001-migrate-postgres-to-supabase", + "eval": "build-cli-001-bootstrap-app", "stage": "build", "product": [ - "database" + "database", + "data-api" ], "topic": [ - "migrations" + "migrations", + "rls" ], "suite": "benchmark", "interface": "cli", "passed": true, "checks": [ { - "name": "all 3 tables exist (teams, members, tasks)", + "name": "supabase project initialised (supabase/config.toml exists)", "passed": true }, { - "name": "row counts match (teams=5, members=10, tasks=13)", + "name": "todos table is created by a migration file", "passed": true }, { - "name": "foreign key constraints survived the restore", + "name": "todos table exists with at least 2 seeded rows", + "passed": true, + "notes": "found 2 rows" + }, + { + "name": "row level security is enabled on todos", "passed": true }, { - "name": "tasks_team_status_idx index survived the restore", + "name": "a SELECT policy targets the authenticated role", "passed": true }, { - "name": "sequences synced (next insert won't conflict with existing IDs)", + "name": "REST API returns no todos to anonymous requests", + "passed": true, + "notes": "0 rows" + }, + { + "name": "REST API returns the todos to authenticated requests", + "passed": true, + "notes": "2 rows" + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] + }, + "prompt": "We're kicking off a todos app and I want the Supabase side ready for the team\nto build on. Set it up the way we'd run it in development, with schema changes\ntracked as migrations so they can be reviewed and replayed.\n\nFor the first slice we just need a `todos` table. Todos aren't public: anyone\nsigned in can read all of them, but nothing should be writable through the API\nfor now. Add a couple of sample todos so there's something to look at.\n\nBefore you hand it back, make sure the running API actually behaves that way —\nsigned-in users get the todos, signed-out requests get nothing.", + "promptSourcePath": "evals/build-cli-001-bootstrap-app/PROMPT.md", + "attempts": 1, + "sourcePath": "codex-gpt-5.4-mini-no-skills/build-cli-001-bootstrap-app.json" + }, + { + "experiment": "codex-gpt-5.4-mini-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.4-mini", + "reasoningEffort": "medium" + }, + "eval": "build-cli-002-declarative-schema", + "stage": "build", + "product": [ + "database" + ], + "topic": [ + "declarative-schema", + "migrations" + ], + "suite": "benchmark", + "interface": "cli", + "passed": false, + "checks": [ + { + "name": "supabase db diff used to generate the migration", + "passed": false + }, + { + "name": "schema file updated to include description column", + "passed": true + }, + { + "name": "a new migration was generated for the change", + "passed": true + }, + { + "name": "description column exists in the live database", "passed": true } ], @@ -14364,50 +8395,205 @@ "available": [], "loaded": [] }, + "docs": { + "calls": [] + }, + "prompt": "Add a description text column to the `products` table in my local Supabase stack", + "promptSourcePath": "evals/build-cli-002-declarative-schema/PROMPT.md", + "attempts": 2, + "sourcePath": "codex-gpt-5.4-mini-no-skills/build-cli-002-declarative-schema.json" + }, + { + "experiment": "codex-gpt-5.4-mini-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.4-mini", + "reasoningEffort": "medium" + }, + "eval": "build-cli-003-pg-cron-queue-workflow", + "stage": "build", + "product": [ + "database", + "edge-functions", + "cron", + "queues" + ], + "topic": [ + "sql", + "sdk" + ], + "suite": "benchmark", + "interface": "cli", + "passed": true, + "checks": [ + { + "name": "pg_cron job 'enqueue-tasks' scheduled to run every minute", + "passed": true, + "notes": "schedule='* * * * *', active=true" + }, + { + "name": "cron command enqueues to the 'tasks' queue", + "passed": true, + "notes": "queue depth 0 -> 1" + }, + { + "name": "process-tasks function drains the queue", + "passed": true, + "notes": "function removed the seeded message (id 2) from the queue" + } + ], + "skills": { + "available": [], + "loaded": [] + }, "docs": { "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase CLI local development restore pg_dump pg_restore database dump\", limit: 5) { nodes { title href content } } }", + "query": "query {\n searchDocs(query: \"pgmq create queue send pop delete cron schedule local supabase\", limit: 10) {\n nodes {\n ... on Guide { title href content }\n ... on CLICommandReference { title href content }\n ... on ManagementApiReference { title href content }\n ... on ClientLibraryFunctionReference { title href content language methodName }\n ... on TroubleshootingGuide { title href content }\n }\n }\n}", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/postgres", - "title": "Migrate from Postgres to Supabase" + "url": "https://supabase.com/docs/guides/queues", + "title": "Supabase Queues" }, { - "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform", - "title": "Restore a Platform Project to Self-Hosted" + "url": "https://supabase.com/docs/guides/queues/expose-self-hosted-queues", + "title": "Expose Queues for local and self-hosted Supabase" }, { - "url": "https://supabase.com/docs/guides/local-development/restoring-downloaded-backup", - "title": "Restoring a downloaded backup locally" + "url": "https://supabase.com/docs/guides/cron", + "title": "Cron" }, { - "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/vercel-postgres", - "title": "Migrate from Vercel Postgres to Supabase" + "url": "https://supabase.com/docs/guides/queues/pgmq", + "title": "PGMQ Extension" }, { - "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/neon", - "title": "Migrate from Neon to Supabase" + "url": "https://supabase.com/docs/guides/database/extensions/pg_cron", + "title": "pg_cron: Schedule Recurring Jobs with Cron Syntax in Postgres" + }, + { + "url": "https://supabase.com/docs/guides/queues/api", + "title": "API" + }, + { + "url": "https://supabase.com/docs/guides/queues/quickstart", + "title": "Quickstart" + }, + { + "url": "https://supabase.com/docs/guides/cron/install", + "title": "Install" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/pgcron-debugging-guide-n1KTaz", + "title": "pg_cron debugging guide" + }, + { + "url": "https://supabase.com/docs/guides/cron/quickstart", + "title": "Quickstart" } ], - "resultChars": 42818 + "resultChars": 58557 + }, + { + "source": "search_docs", + "query": "query {\n searchDocs(query: \"pgmq create if not exists duplicate queue create queue existing\", limit: 5) {\n nodes {\n ... on Guide { title href content }\n ... on ClientLibraryFunctionReference { title href content language methodName }\n ... on TroubleshootingGuide { title href content }\n }\n }\n}", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/queues/pgmq", + "title": "PGMQ Extension" + }, + { + "url": "https://supabase.com/docs/guides/queues/quickstart", + "title": "Quickstart" + }, + { + "url": "https://supabase.com/docs/guides/queues", + "title": "Supabase Queues" + }, + { + "url": "https://supabase.com/docs/guides/queues/expose-self-hosted-queues", + "title": "Expose Queues for local and self-hosted Supabase" + }, + { + "url": "https://supabase.com/docs/guides/queues/api", + "title": "API" + } + ], + "resultChars": 39378 } ] }, + "prompt": "I want to set up a recurring background workflow on my local Supabase stack.\n\nCan you set up a cron job called `enqueue-tasks` to run every minute and push a task into a queue called `tasks`? Then add a `process-tasks` edge function that reads messages off the `tasks` queue and removes them, so a scheduled worker can keep the backlog drained.", + "promptSourcePath": "evals/build-cli-003-pg-cron-queue-workflow/PROMPT.md", + "attempts": 1, + "sourcePath": "codex-gpt-5.4-mini-no-skills/build-cli-003-pg-cron-queue-workflow.json" + }, + { + "experiment": "codex-gpt-5.4-mini-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.4-mini", + "reasoningEffort": "medium" + }, + "eval": "build-database-001-migrate-postgres-to-supabase", + "stage": "build", + "product": [ + "database" + ], + "topic": [ + "migrations" + ], + "suite": "benchmark", + "interface": "cli", + "passed": true, + "checks": [ + { + "name": "all 3 tables exist (teams, members, tasks)", + "passed": true + }, + { + "name": "row counts match (teams=5, members=10, tasks=13)", + "passed": true + }, + { + "name": "foreign key constraints survived the restore", + "passed": true + }, + { + "name": "tasks_team_status_idx index survived the restore", + "passed": true + }, + { + "name": "sequences synced (next insert won't conflict with existing IDs)", + "passed": true + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] + }, "prompt": "I have an existing Postgres database I want to migrate to Supabase. There's a binary dump at `source.dump` in the current directory.\n\nCan you set up a local Supabase project and restore the dump into it?", "promptSourcePath": "evals/build-database-001-migrate-postgres-to-supabase/PROMPT.md", "attempts": 1, - "sourcePath": "codex-gpt-5.6-no-skills/build-database-001-migrate-postgres-to-supabase.json" + "sourcePath": "codex-gpt-5.4-mini-no-skills/build-database-001-migrate-postgres-to-supabase.json" }, { - "experiment": "codex-gpt-5.6-no-skills", + "experiment": "codex-gpt-5.4-mini-no-skills", "experimentSuite": "no-skills", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.6-sol", + "modelId": "gpt-5.4-mini", "reasoningEffort": "medium" }, "eval": "build-functions-004-service-role-bypass", @@ -14460,42 +8646,46 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"Edge Functions authenticate user Authorization header supabase-js getUser service role RLS\", limit: 5) { nodes { ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } } } }", + "query": "query { searchDocs(query: \"Supabase Edge Function service role key verify_jwt auth.getUser private notes RLS\", limit: 5) { nodes { title href ... on Guide { content } ... on ClientLibraryFunctionReference { language methodName content href } } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs", - "title": "Build a User Management App with Next.js" + "url": "https://supabase.com/docs/guides/troubleshooting/why-is-my-service-role-key-client-getting-rls-errors-or-not-returning-data-7_1K9z", + "title": "Why is my service role key client getting RLS errors or not returning data?" }, { "url": "https://supabase.com/docs/guides/functions/auth", "title": "Securing Edge Functions" }, { - "url": "https://supabase.com/docs/guides/functions", - "title": "Edge Functions" + "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", + "title": "Integrating With Supabase Auth" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", + "title": "Row Level Security" + }, + { + "url": "https://supabase.com/docs/guides/database/secure-data", + "title": "Securing your data" } ], - "resultChars": 70238 + "resultChars": 44453 } ] }, "prompt": "I built an Edge Function called `private-notes` for showing a user's saved\nprivate notes.\n\nCan you check whether there's any way one user could see another user's notes?\n\nPlease fix and deploy it if needed.", "promptSourcePath": "evals/build-functions-004-service-role-bypass/PROMPT.md", "attempts": 1, - "sourcePath": "codex-gpt-5.6-no-skills/build-functions-004-service-role-bypass.json" + "sourcePath": "codex-gpt-5.4-mini-no-skills/build-functions-004-service-role-bypass.json" }, { - "experiment": "codex-gpt-5.6-no-skills", + "experiment": "codex-gpt-5.4-mini-no-skills", "experimentSuite": "no-skills", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.6-sol", + "modelId": "gpt-5.4-mini", "reasoningEffort": "medium" }, "eval": "build-functions-005-dual-auth-user-secret", @@ -14513,7 +8703,7 @@ "suite": "benchmark", "interface": "cli", "cliVersion": "2.109.1", - "passed": true, + "passed": false, "checks": [ { "name": "seed rows present", @@ -14523,42 +8713,42 @@ { "name": "rejects request with no credentials", "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" + "notes": "status 401: {\"error\":\"Missing user access token\"}" }, { "name": "user with JWT reads only their own rows", "passed": true, - "notes": "status 200: [{\"user_id\":\"66ade428-6967-435a-a7db-83fb2df16e59\",\"metric\":\"steps_a_ms6zjqfe\",\"value\":111}]" + "notes": "status 200: [{\"user_id\":\"349f40f1-5cd7-4083-b530-07bcc4d87d05\",\"metric\":\"steps_a_mrw66wdy\",\"value\":111}]" }, { "name": "user cannot read another user's rows by passing user_id", "passed": true, - "notes": "status 200: [{\"user_id\":\"66ade428-6967-435a-a7db-83fb2df16e59\",\"metric\":\"steps_a_ms6zjqfe\",\"value\":111}]" + "notes": "status 200: [{\"user_id\":\"349f40f1-5cd7-4083-b530-07bcc4d87d05\",\"metric\":\"steps_a_mrw66wdy\",\"value\":111}]" }, { "name": "service key bypasses RLS to read the target user's rows", "passed": true, - "notes": "status 200: [{\"user_id\":\"21a6e5f1-81f2-4735-b460-3461be1f3c1d\",\"metric\":\"steps_b_ms6zjqfe\",\"value\":222}]" + "notes": "status 200: [{\"user_id\":\"7c6be301-fb1e-4570-98bd-5a394bf64dac\",\"metric\":\"steps_b_mrw66wdy\",\"value\":222}]" }, { "name": "non-service key is not granted service access", "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" + "notes": "status 401: {\"error\":\"Missing user access token\"}" }, { "name": "rejects an unverified (forged) user token", "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" + "notes": "status 401: {\"error\":\"Invalid user access token\"}" }, { "name": "a user token in the apikey slot is not treated as the service key", "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" + "notes": "status 401: {\"error\":\"Missing user access token\"}" }, { "name": "implementation uses @supabase/server", - "passed": true, - "notes": "imports @supabase/server / withSupabase" + "passed": false, + "notes": "hand-rolled (raw supabase-js or other) — this eval requires @supabase/server" } ], "skills": { @@ -14569,227 +8759,316 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"Edge Functions authenticate JWT getClaims service role apikey header SUPABASE_SERVICE_ROLE_KEY\", limit: 8) { nodes { title href content } } }", + "query": "query {\n searchDocs(query: \"Edge Function verify_jwt SUPABASE_SERVICE_ROLE_KEY environment variable\", limit: 5) {\n nodes {\n __typename\n ... on Guide { title href content }\n ... on CLICommandReference { title href content }\n ... on ClientLibraryFunctionReference { title href content language methodName }\n ... on TroubleshootingGuide { title href content }\n ... on ManagementApiReference { title href content }\n }\n }\n}", "hasContent": true, "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", - "title": "New API Keys and Asymmetric Authentication" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" - }, { "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", "title": "Migrating to publishable and secret API keys" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/why-is-my-service-role-key-client-getting-rls-errors-or-not-returning-data-7_1K9z", - "title": "Why is my service role key client getting RLS errors or not returning data?" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", + "title": "Self-Hosted Functions" }, { - "url": "https://supabase.com/docs/guides/auth/third-party/firebase-auth", - "title": "Firebase Auth" + "url": "https://supabase.com/docs/guides/functions/secrets", + "title": "Environment Variables" }, { - "url": "https://supabase.com/docs/guides/auth/jwt-fields", - "title": "JWT Claims Reference" + "url": "https://supabase.com/docs/guides/functions/function-configuration", + "title": "Function Configuration" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", + "title": "Integrating With Supabase Auth" } ], - "resultChars": 87163 + "resultChars": 37304 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"New API Keys Edge Functions apikey secret key service role validate secret key @supabase/server\", limit: 10) { nodes { title href content } } }", + "query": "query {\n searchDocs(query: \"supabase functions entrypoint default path functions directory config.toml\", limit: 5) {\n nodes {\n __typename\n ... on Guide { title href content }\n ... on CLICommandReference { title href content }\n ... on TroubleshootingGuide { title href content }\n }\n }\n}", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", - "title": "New API Keys and Asymmetric Authentication" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/cloudflare-turnstile", - "title": "CAPTCHA support with Cloudflare Turnstile" - }, - { - "url": "https://supabase.com/docs/guides/functions/secrets", - "title": "Environment Variables" + "url": "https://supabase.com/docs/guides/functions/function-configuration", + "title": "Function Configuration" }, { - "url": "https://supabase.com/docs/guides/auth/passkeys", - "title": "Passkey authentication" + "url": "https://supabase.com/docs/guides/local-development/managing-config", + "title": "Managing config and secrets" }, { - "url": "https://supabase.com/docs/guides/functions/quickstart-dashboard", - "title": "Getting Started with Edge Functions (Dashboard)" + "url": "https://supabase.com/docs/guides/functions/development-tips", + "title": "Development tips" }, { - "url": "https://supabase.com/docs/guides/database/secure-data", - "title": "Securing your data" + "url": "https://supabase.com/docs/guides/deployment", + "title": "Deployment & Branching" }, { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-ionic-angular", - "title": "Build a User Management App with Ionic Angular" + "url": "https://supabase.com/docs/guides/functions/unit-test", + "title": "Testing your Edge Functions" } ], - "resultChars": 114420 - }, + "resultChars": 21535 + } + ] + }, + "prompt": "Build and serve a Supabase Edge Function named `user-stats` for this project,\nreachable over HTTP at `/functions/v1/user-stats`.\n\nOur product stores per-user metrics in the existing `user_stats` table.\n\nTwo very different callers need to hit this one endpoint:\n\n1. **Our mobile app**, acting for a signed-in user. It sends that user's\n Supabase access token. The endpoint should return the user's own stats.\n\n2. **Our internal billing service**, a trusted backend with no signed-in user.\n It authenticates with the project's secret (service-role) key in the `apikey`\n header, and names the target user with a `user_id` in the JSON request body.\n It needs that user's stats.\n\nReturn the matching rows as JSON. The endpoint must be secure: only ever serve\nstats to a caller that is genuinely entitled to them, and turn away callers that\nare not.", + "promptSourcePath": "evals/build-functions-005-dual-auth-user-secret/PROMPT.md", + "attempts": 2, + "sourcePath": "codex-gpt-5.4-mini-no-skills/build-functions-005-dual-auth-user-secret.json" + }, + { + "experiment": "codex-gpt-5.4-mini-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.4-mini", + "reasoningEffort": "medium" + }, + "eval": "build-storage-001-private-bucket-access", + "stage": "build", + "product": [ + "storage", + "database" + ], + "topic": [ + "rls", + "sdk" + ], + "suite": "benchmark", + "interface": "mcp", + "passed": true, + "checks": [ + { + "name": "bucket user-files exists", + "passed": true + }, + { + "name": "bucket user-files is private", + "passed": true + }, + { + "name": "RLS still enabled on storage.objects", + "passed": true + }, + { + "name": "user A lists only own files", + "passed": true, + "notes": "saw: 019f8a21-709e-77fb-a44c-fbc04203137a/receipt-alpha.pdf, 019f8a21-709e-77fb-a44c-fbc04203137a/receipt-beta.pdf" + }, + { + "name": "user B cannot read user A files", + "passed": true + }, + { + "name": "anon reads no files", + "passed": true + }, + { + "name": "user A can upload into own folder", + "passed": true + }, + { + "name": "user B cannot upload into user A folder", + "passed": true + }, + { + "name": "configured private per-user storage access", + "passed": true, + "judgeNotes": "The answer creates a private user-files bucket, keeps/enables RLS on storage.objects, defines authenticated SELECT and INSERT policies scoped to the bucket and first path segment matching auth.uid(), and uses createSignedUrl with a 10-minute expiry. No disallowed public bucket/getPublicUrl/service-role/client-side pattern is present." + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"@supabase/server validateApiKey Edge Function SUPABASE_SECRET_KEYS\", limit: 5) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"storage temporary signed url supabase-js createSignedUrl download temporary link\", limit: 5) { nodes { __typename ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" + "url": "https://supabase.com/docs/guides/storage/serving/downloads", + "title": "Serving assets from Storage" }, { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" + "url": "https://supabase.com/docs/guides/storage/uploads/resumable-uploads", + "title": "Resumable Uploads" }, { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" + "url": "https://supabase.com/docs/guides/storage/serving/image-transformations", + "title": "Storage Image Transformations" }, { - "url": "https://supabase.com/docs/guides/functions/secrets", - "title": "Environment Variables" + "url": "https://supabase.com/docs/guides/platform/manage-your-usage/storage-image-transformations", + "title": "Manage Storage Image Transformations usage" }, { - "url": "https://supabase.com/docs/guides/auth/passkeys", - "title": "Passkey authentication" + "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsignedurl" } ], - "resultChars": 51999 + "resultChars": 47481 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"@supabase/server service_role apikey legacy service role Edge Functions withSupabase auth mode\", limit: 8) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"storage.objects policy owner folder prefix auth.uid Supabase bucket private upload insert select\", limit: 10) { nodes { __typename ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" + "url": "https://supabase.com/docs/guides/storage", + "title": "Storage" }, { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" + "url": "https://supabase.com/docs/guides/platform/delete-project", + "title": "Deleting Your Project" }, { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" + "url": "https://supabase.com/docs/guides/storage/management/download-objects", + "title": "Download Objects" }, { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" + "url": "https://supabase.com/docs/guides/storage/quickstart", + "title": "Storage Quickstart" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", - "title": "New API Keys and Asymmetric Authentication" + "url": "https://supabase.com/docs/guides/storage/schema/custom-roles", + "title": "Custom Roles" }, { - "url": "https://supabase.com/docs/guides/auth/choosing-a-server-package", - "title": "Which package to use" + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/firebase-storage", + "title": "Migrated from Firebase Storage to Supabase" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/why-is-my-service-role-key-client-getting-rls-errors-or-not-returning-data-7_1K9z", - "title": "Why is my service role key client getting RLS errors or not returning data?" + "url": "https://supabase.com/docs/guides/storage/s3/compatibility", + "title": "S3 Compatibility" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", - "title": "Self-Hosted Functions" + "url": "https://supabase.com/docs/guides/storage/schema/helper-functions", + "title": "Storage Helper Functions" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3", + "title": "Configure S3 Storage" } ], - "resultChars": 73001 + "resultChars": 78562 } ] - }, - "prompt": "Build and serve a Supabase Edge Function named `user-stats` for this project,\nreachable over HTTP at `/functions/v1/user-stats`.\n\nOur product stores per-user metrics in the existing `user_stats` table.\n\nTwo very different callers need to hit this one endpoint:\n\n1. **Our mobile app**, acting for a signed-in user. It sends that user's\n Supabase access token. The endpoint should return the user's own stats.\n\n2. **Our internal billing service**, a trusted backend with no signed-in user.\n It authenticates with the project's secret (service-role) key in the `apikey`\n header, and names the target user with a `user_id` in the JSON request body.\n It needs that user's stats.\n\nReturn the matching rows as JSON. The endpoint must be secure: only ever serve\nstats to a caller that is genuinely entitled to them, and turn away callers that\nare not.", - "promptSourcePath": "evals/build-functions-005-dual-auth-user-secret/PROMPT.md", + }, + "prompt": "Our app lets signed-in users keep personal files like receipts and bank\nstatements. These files are private — a user must only ever be able to upload\nand download their own. The app uploads each file under a path that starts\nwith the owner's user id, e.g. `/receipt-march.pdf`.\n\nSet up a `user-files` bucket on our project and lock it down that way.\n\nUsers also sometimes share one of their files with someone else through a\ntemporary link that expires. Include the supabase-js code the app should use\nfor that.", + "promptSourcePath": "evals/build-storage-001-private-bucket-access/PROMPT.md", "attempts": 1, - "sourcePath": "codex-gpt-5.6-no-skills/build-functions-005-dual-auth-user-secret.json" + "sourcePath": "codex-gpt-5.4-mini-no-skills/build-storage-001-private-bucket-access.json" }, { - "experiment": "codex-gpt-5.6-no-skills", + "experiment": "codex-gpt-5.4-mini-no-skills", "experimentSuite": "no-skills", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.6-sol", + "modelId": "gpt-5.4-mini", "reasoningEffort": "medium" }, - "eval": "build-storage-001-private-bucket-access", + "eval": "build-tests-001-rls-tenant-isolation", "stage": "build", "product": [ - "storage", "database" ], "topic": [ - "rls", - "sdk" + "tests", + "rls" ], "suite": "benchmark", - "interface": "mcp", + "interface": "cli", "passed": true, "checks": [ { - "name": "bucket user-files exists", - "passed": true + "name": "pgTAP test file(s) written under supabase/tests/", + "passed": true, + "notes": "1 file(s): supabase/tests/tenant_isolation_test.sql" }, { - "name": "bucket user-files is private", - "passed": true + "name": "pgTAP isolation tests ran and pass", + "passed": true, + "notes": "6 passed, 0 failed" }, { - "name": "RLS still enabled on storage.objects", - "passed": true + "name": "agent correctly identifies the posts isolation bug from test results", + "passed": true, + "judgeNotes": "The agent correctly identifies `posts` as having a broken tenant isolation policy where posts from another organization leak to authenticated users, and grounds this in pgTAP test failures. It does not blame `notes` or dismiss the test results. Extra mention of `memberships` does not invalidate the required conclusion." + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] + }, + "prompt": "Can you audit the tenant isolation on our tables? Write some database tests covering both the happy path and the negative case, run them, and share the results.", + "promptSourcePath": "evals/build-tests-001-rls-tenant-isolation/PROMPT.md", + "attempts": 1, + "sourcePath": "codex-gpt-5.4-mini-no-skills/build-tests-001-rls-tenant-isolation.json" + }, + { + "experiment": "codex-gpt-5.4-mini-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.4-mini", + "reasoningEffort": "medium" + }, + "eval": "build-vectors-001-rag-with-permissions", + "stage": "build", + "product": [ + "database", + "vectors" + ], + "topic": [ + "sql", + "rls" + ], + "suite": "benchmark", + "interface": "mcp", + "passed": true, + "checks": [ + { + "name": "document_sections.embedding is vector(384)", + "passed": true, + "notes": "vector(384)" }, { - "name": "user A lists only own files", + "name": "HNSW index on the embedding column", "passed": true, - "notes": "saw: 019fb129-6dc4-779f-a3a1-a1c86804f6c6/receipt-alpha.pdf, 019fb129-6dc4-779f-a3a1-a1c86804f6c6/receipt-beta.pdf" + "notes": "CREATE INDEX document_sections_embedding_idx ON public.document_sections USING hnsw (embedding vector_cosine_ops)" }, { - "name": "user B cannot read user A files", - "passed": true + "name": "index operator class matches the search operator", + "passed": true, + "notes": "function operators: <=>\nindexes: CREATE INDEX document_sections_embedding_idx ON public.document_sections USING hnsw (embedding vector_cosine_ops)" }, { - "name": "anon reads no files", + "name": "user A search returns only own sections, best match first", "passed": true }, { - "name": "user A can upload into own folder", + "name": "user B search returns only own sections, best match first", "passed": true }, { - "name": "user B cannot upload into user A folder", + "name": "user A reads only own sections through the API", "passed": true }, { - "name": "configured private per-user storage access", - "passed": true, - "judgeNotes": "The answer creates a private user-files bucket, keeps RLS enabled, adds authenticated SELECT and INSERT policies scoped to bucket and owner folder via auth.uid(), and provides supabase-js createSignedUrl with a short expiry. No public bucket, permissive policies, public URL, or client service role usage." + "name": "user A reads only own documents through the API", + "passed": true } ], "skills": { @@ -14800,75 +9079,72 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase Storage RLS policy foldername auth.uid private bucket createSignedUrl JavaScript\", limit: 5) { nodes { title href content } } }", + "query": "query {\n searchDocs(query: \"Supabase vector extension ivfflat hnsw create index vector cosine distance policy row level security auth.uid document sections rpc function\", limit: 5) {\n nodes {\n title\n href\n content\n }\n }\n}", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/storage/buckets/fundamentals", - "title": "Storage Buckets" + "url": "https://supabase.com/docs/guides/ai/vector-indexes/hnsw-indexes", + "title": "HNSW indexes" }, { - "url": "https://supabase.com/docs/guides/storage/schema/helper-functions", - "title": "Storage Helper Functions" + "url": "https://supabase.com/docs/guides/ai/rag-with-permissions", + "title": "RAG with Permissions" }, { - "url": "https://supabase.com/docs/guides/security/product-security", - "title": "Secure configuration of Supabase products" + "url": "https://supabase.com/docs/guides/ai/vector-indexes/ivf-indexes", + "title": "IVFFlat indexes" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3", - "title": "Configure S3 Storage" + "url": "https://supabase.com/docs/guides/ai/semantic-search", + "title": "Semantic search" }, { - "url": "https://supabase.com/docs/guides/storage/serving/downloads", - "title": "Serving assets from Storage" + "url": "https://supabase.com/docs/guides/storage/vector/working-with-indexes", + "title": "Working with Vector Indexes" } ], - "resultChars": 23009 + "resultChars": 50607 } ] }, - "prompt": "Our app lets signed-in users keep personal files like receipts and bank\nstatements. These files are private — a user must only ever be able to upload\nand download their own. The app uploads each file under a path that starts\nwith the owner's user id, e.g. `/receipt-march.pdf`.\n\nSet up a `user-files` bucket on our project and lock it down that way.\n\nUsers also sometimes share one of their files with someone else through a\ntemporary link that expires. Include the supabase-js code the app should use\nfor that.", - "promptSourcePath": "evals/build-storage-001-private-bucket-access/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.6-no-skills/build-storage-001-private-bucket-access.json" + "prompt": "We're adding semantic search to our internal knowledge base app. I already wrote the edge functions, but search doesn't work yet. Can you set up whatever the database needs to make search work end to end?\n\nSome documents are confidential, so users should only have access to documents they own.", + "promptSourcePath": "evals/build-vectors-001-rag-with-permissions/PROMPT.md", + "attempts": 2, + "sourcePath": "codex-gpt-5.4-mini-no-skills/build-vectors-001-rag-with-permissions.json" }, { - "experiment": "codex-gpt-5.6-no-skills", + "experiment": "codex-gpt-5.4-mini-no-skills", "experimentSuite": "no-skills", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.6-sol", + "modelId": "gpt-5.4-mini", "reasoningEffort": "medium" }, - "eval": "build-tests-001-rls-tenant-isolation", - "stage": "build", + "eval": "deploy-database-001-prometheus-metrics", + "stage": "deploy", "product": [ "database" ], "topic": [ - "tests", - "rls" + "observability" ], "suite": "benchmark", - "interface": "cli", - "passed": true, + "passed": false, "checks": [ { - "name": "pgTAP test file(s) written under supabase/tests/", - "passed": true, - "notes": "1 file(s): supabase/tests/tenant_isolation.test.sql" + "name": "preserved existing app scrape job", + "passed": true }, { - "name": "pgTAP isolation tests ran and pass", - "passed": true, - "notes": "8 passed, 3 failed" + "name": "configured the Supabase Metrics API scrape correctly", + "passed": false, + "judgeNotes": "Fails: Supabase scrape uses basic_auth.password with an environment variable instead of basic_auth.password_file, and docker-compose.yml does not mount the password_file via a volume or Compose secret. The app job is preserved and endpoint/HTTPS are correct, but required secret-file wiring is missing." }, { - "name": "agent correctly identifies the posts isolation bug from test results", - "passed": true, - "judgeNotes": "Correctly identifies `posts` as having broken tenant isolation, grounded in pgTAP failures showing cross-tenant post visibility, and distinguishes `notes` as correctly restricted." + "name": "documented live deployment and verification steps", + "passed": false, + "judgeNotes": "README includes the correct Supabase metrics endpoint/auth, restart/reload steps, and Prometheus targets verification, but it does not explain how to create a Secret API key or place a matching secret file. The Compose setup uses environment variables instead of the required secret file setup." } ], "skills": { @@ -14879,92 +9155,207 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"database testing pgTAP row level security auth.uid set_config request.jwt.claims\", limit: 5) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"metrics prometheus\", limit: 5) { nodes { __typename ... on Guide { title href content } ... on TroubleshootingGuide { title href content } ... on ClientLibraryFunctionReference { title href language methodName content } ... on CLICommandReference { title href content } ... on ManagementApiReference { title href content } } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/realtime/authorization", - "title": "Realtime Authorization" + "url": "https://supabase.com/docs/guides/telemetry/metrics/grafana-self-hosted", + "title": "Metrics API with Prometheus & Grafana (self-hosted)" }, { - "url": "https://supabase.com/docs/guides/auth/auth-mfa", - "title": "Multi-Factor Authentication" + "url": "https://supabase.com/docs/guides/telemetry/metrics", + "title": "Metrics API" }, { - "url": "https://supabase.com/docs/guides/ai/rag-with-permissions", - "title": "RAG with Permissions" + "url": "https://supabase.com/docs/guides/telemetry/metrics/grafana-cloud", + "title": "Metrics API with Grafana Cloud" }, { - "url": "https://supabase.com/docs/guides/database/extensions/pgaudit", - "title": "PGAudit: Postgres Auditing" + "url": "https://supabase.com/docs/guides/telemetry/metrics/vendor-agnostic", + "title": "Vendor-agnostic Metrics API setup" }, { - "url": "https://supabase.com/docs/guides/getting-started/features", - "title": "Features" + "url": "https://supabase.com/docs/guides/database/replication/manual-replication-monitoring", + "title": "Manual replication monitoring" } ], - "resultChars": 72236 + "resultChars": 27310 + } + ] + }, + "prompt": "Can you wire my Supabase project metrics into our existing observability stack and document\nwhat we need to do to make the config live?", + "promptSourcePath": "evals/deploy-database-001-prometheus-metrics/PROMPT.md", + "attempts": 2, + "sourcePath": "codex-gpt-5.4-mini-no-skills/deploy-database-001-prometheus-metrics.json" + }, + { + "experiment": "codex-gpt-5.4-mini-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.4-mini", + "reasoningEffort": "medium" + }, + "eval": "deploy-functions-001-edge-function-secrets", + "stage": "deploy", + "product": [ + "edge-functions" + ], + "topic": [ + "security" + ], + "suite": "benchmark", + "interface": "cli", + "passed": true, + "checks": [ + { + "name": "WEATHER_API_KEY is set as a Function secret on the project", + "passed": true + }, + { + "name": "the weather function is deployed to the project", + "passed": true, + "notes": "status ACTIVE" + }, + { + "name": "the weather function reads WEATHER_API_KEY from the environment", + "passed": true, + "judgeNotes": "Reads WEATHER_API_KEY from the runtime environment via Deno.env.get(\"WEATHER_API_KEY\")." + }, + { + "name": "WEATHER_API_KEY value is not committed to the repo", + "passed": true + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] + }, + "prompt": "Our weather widget currently calls WeatherAPI straight from the browser, which\nleaks our API key. I want to move that behind a Supabase Edge Function called\n`weather` that holds the key server-side and proxies the request.\n\nThe function should read the key from an environment variable named\n`WEATHER_API_KEY`. Our key already lives in a local `.env` file at the project\nroot.\n\nDeploy the function to our project so it's live, and make sure the deployed\nfunction can actually read the key at runtime.", + "promptSourcePath": "evals/deploy-functions-001-edge-function-secrets/PROMPT.md", + "attempts": 1, + "sourcePath": "codex-gpt-5.4-mini-no-skills/deploy-functions-001-edge-function-secrets.json" + }, + { + "experiment": "codex-gpt-5.4-mini-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.4-mini", + "reasoningEffort": "medium" + }, + "eval": "deploy-self-hosting-001-docker-compose", + "stage": "deploy", + "product": [ + "database", + "auth", + "storage" + ], + "topic": [ + "self-hosting" + ], + "suite": "benchmark", + "interface": "cli", + "passed": true, + "checks": [ + { + "name": "cloned the self-host stack (docker-compose.yml + volumes/db)", + "passed": true + }, + { + "name": "didn't conflate with the CLI (no supabase/config.toml in the stack)", + "passed": true + }, + { + "name": "secrets rotated off the shipped defaults", + "passed": true + }, + { + "name": "ANON_KEY and SERVICE_ROLE_KEY are HS256 JWTs signed by JWT_SECRET", + "passed": true + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [ + { + "source": "web_search", + "query": "site:supabase.com/docs self-hosting docker compose supabase secrets production docker setup", + "pages": [] + }, + { + "source": "web_search", + "query": "site:raw.githubusercontent.com supabase supabase master docker/utils/generate-keys.sh", + "pages": [] } ] }, - "prompt": "Can you audit the tenant isolation on our tables? Write some database tests covering both the happy path and the negative case, run them, and share the results.", - "promptSourcePath": "evals/build-tests-001-rls-tenant-isolation/PROMPT.md", + "prompt": "I'm moving off the hosted Supabase and running the whole thing myself on a VPS I\njust spun up. Can you get a Docker setup ready for me to copy onto the box?\n\nI don't need it running here, I'll do the actual bring-up once I'm on the\nserver. I just want everything in place and the secrets set up. Put it in a `supabase-docker/`\nfolder at the repo root so I can scp the whole thing across in one go.", + "promptSourcePath": "evals/deploy-self-hosting-001-docker-compose/PROMPT.md", "attempts": 1, - "sourcePath": "codex-gpt-5.6-no-skills/build-tests-001-rls-tenant-isolation.json" + "sourcePath": "codex-gpt-5.4-mini-no-skills/deploy-self-hosting-001-docker-compose.json" }, { - "experiment": "codex-gpt-5.6-no-skills", + "experiment": "codex-gpt-5.4-mini-no-skills", "experimentSuite": "no-skills", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.6-sol", + "modelId": "gpt-5.4-mini", "reasoningEffort": "medium" }, - "eval": "build-vectors-001-rag-with-permissions", - "stage": "build", + "eval": "investigate-auth-001-deleted-user-access", + "stage": "investigate", "product": [ - "database", - "vectors" + "auth" ], "topic": [ - "sql", - "rls" + "security", + "sdk" ], "suite": "benchmark", "interface": "mcp", - "passed": true, + "passed": false, "checks": [ { - "name": "document_sections.embedding is vector(384)", - "passed": true, - "notes": "vector(384)" + "name": "victim session active before delete-account", + "passed": false, + "notes": "permission denied for table users" }, { - "name": "HNSW index on the embedding column", - "passed": true, - "notes": "CREATE INDEX document_sections_embedding_hnsw_idx ON public.document_sections USING hnsw (embedding vector_cosine_ops)" + "name": "delete_account flow ran for the victim", + "passed": true }, { - "name": "index operator class matches the search operator", + "name": "delete-account revokes the user's sessions", "passed": true, - "notes": "function operators: <=>\nindexes: CREATE INDEX document_sections_embedding_hnsw_idx ON public.document_sections USING hnsw (embedding vector_cosine_ops)" + "notes": "sessions left: 0" }, { - "name": "user A search returns only own sections, best match first", + "name": "deleted user's refresh token is rejected", "passed": true }, { - "name": "user B search returns only own sections, best match first", + "name": "deleted user cannot sign back in", "passed": true }, { - "name": "user A reads only own sections through the API", - "passed": true + "name": "other users keep their sessions and access", + "passed": false, + "notes": "permission denied for table users" }, { - "name": "user A reads only own documents through the API", - "passed": true + "name": "diagnosed and explained session revocation", + "passed": true, + "judgeNotes": "The answer identifies the soft-delete-only bug, implements real auth user deletion with session/refresh-token cascade and RLS checks, explains the remaining JWT/local-validation caveat consistently with the implemented Data API mitigation, and correctly distinguishes publishable/anon vs secret/service_role keys including RLS bypass." } ], "skills": { @@ -14975,72 +9366,86 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"pgvector semantic search match_documents RPC row level security auth.uid embeddings\", limit: 5) { nodes { ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } } } }", + "query": "query { searchDocs(query: \"publishable secret keys anon service_role frontend RLS\", limit: 5) { nodes { ... on Guide { title href content } ... on CLICommandReference { title href content } ... on ManagementApiReference { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } ... on TroubleshootingGuide { title href content } } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/ai/rag-with-permissions", - "title": "RAG with Permissions" + "url": "https://supabase.com/docs/guides/database/secure-data", + "title": "Securing your data" }, { - "url": "https://supabase.com/docs/guides/ai/semantic-search", - "title": "Semantic search" + "url": "https://supabase.com/docs/guides/auth/jwt-fields", + "title": "JWT Claims Reference" }, { - "url": "https://supabase.com/docs/guides/ai/hybrid-search", - "title": "Hybrid search" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-redwoodjs", + "title": "Build a User Management App with RedwoodJS" }, { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", - "title": "Automatic embeddings" + "url": "https://supabase.com/docs/guides/getting-started/api-keys", + "title": "Understanding API keys" }, { - "url": "https://supabase.com/docs/guides/ai/vector-columns", - "title": "Vector columns" + "url": "https://supabase.com/docs/guides/auth/signing-keys", + "title": "JWT Signing Keys" } ], - "resultChars": 80299 + "resultChars": 91173 } ] }, - "prompt": "We're adding semantic search to our internal knowledge base app. I already wrote the edge functions, but search doesn't work yet. Can you set up whatever the database needs to make search work end to end?\n\nSome documents are confidential, so users should only have access to documents they own.", - "promptSourcePath": "evals/build-vectors-001-rag-with-permissions/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.6-no-skills/build-vectors-001-rag-with-permissions.json" + "prompt": "Last week support removed a user through our app's delete-account flow — the\napp calls the `delete_account` function over RPC as the signed-in user. This\nmorning that same person was back: still signed in, reading and saving their\ndata like nothing happened.\n\nFigure out why the account still works, fix the flow so a deleted account\nloses access, and tell me whether there is any window where they could still\nget in after the fix.\n\nOne more thing while you're at it: we're migrating off the legacy\nanon/service_role API keys to the new publishable and secret keys, and the\nteam isn't sure which one belongs in the frontend and what each means for\nRLS. Give us a quick rundown.", + "promptSourcePath": "evals/investigate-auth-001-deleted-user-access/PROMPT.md", + "attempts": 2, + "sourcePath": "codex-gpt-5.4-mini-no-skills/investigate-auth-001-deleted-user-access.json" }, { - "experiment": "codex-gpt-5.6-no-skills", + "experiment": "codex-gpt-5.4-mini-no-skills", "experimentSuite": "no-skills", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.6-sol", - "reasoningEffort": "low" + "modelId": "gpt-5.4-mini", + "reasoningEffort": "medium" }, - "eval": "deploy-database-001-prometheus-metrics", - "stage": "deploy", + "eval": "investigate-realtime-001-subscribed-no-events", + "stage": "investigate", "product": [ + "realtime", "database" ], "topic": [ - "observability" + "sdk" ], "suite": "benchmark", + "interface": "mcp", "passed": true, "checks": [ { - "name": "preserved existing app scrape job", + "name": "orders table added to supabase_realtime publication", "passed": true }, { - "name": "configured the Supabase Metrics API scrape correctly", + "name": "courier_locations still in supabase_realtime publication", + "passed": true + }, + { + "name": "publication still publishes INSERT events", + "passed": true + }, + { + "name": "RLS still enabled on orders", + "passed": true + }, + { + "name": "staff can still read orders through RLS", "passed": true, - "judgeNotes": "Meets requirements: app scrape preserved, Supabase HTTPS metrics endpoint configured with Basic Auth password_file, project target present, and docker-compose wires the secret to Prometheus." + "notes": "authenticated sees 2 of 2 orders" }, { - "name": "documented live deployment and verification steps", + "name": "diagnosed missing publication membership", "passed": true, - "judgeNotes": "README includes Secret API key creation, environment-backed Compose secret mounted to the expected password_file path, restart/recreate and reload commands, and concrete verification via Prometheus targets/Grafana/curl." + "judgeNotes": "Identified orders missing from supabase_realtime despite subscription/RLS being otherwise fine, added only public.orders to the existing publication, verified courier_locations remained, and did not weaken RLS/policies." } ], "skills": { @@ -15048,92 +9453,48 @@ "loaded": [] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Prometheus metrics hosted project customer v1 privileged metrics service_role Supabase observability\", limit: 8) { nodes { __typename title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/telemetry/metrics/grafana-self-hosted", - "title": "Metrics API with Prometheus & Grafana (self-hosted)" - }, - { - "url": "https://supabase.com/docs/guides/telemetry/metrics", - "title": "Metrics API" - }, - { - "url": "https://supabase.com/docs/guides/telemetry/metrics/vendor-agnostic", - "title": "Vendor-agnostic Metrics API setup" - }, - { - "url": "https://supabase.com/docs/guides/telemetry/metrics/grafana-cloud", - "title": "Metrics API with Grafana Cloud" - }, - { - "url": "https://supabase.com/docs/guides/platform/read-replicas", - "title": "Read Replicas" - }, - { - "url": "https://supabase.com/docs/guides/database/connection-management", - "title": "Connection management" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/roles", - "title": "Postgres Roles" - }, - { - "url": "https://supabase.com/docs/guides/security/security-testing", - "title": "Security testing of your Supabase projects" - } - ], - "resultChars": 48186 - } - ] + "calls": [] }, - "prompt": "Can you wire my Supabase project metrics into our existing observability stack and document\nin the observability README what we need to do to make the config live?", - "promptSourcePath": "evals/deploy-database-001-prometheus-metrics/PROMPT.md", + "prompt": "Our dispatch dashboard shows incoming orders as they happen. The courier\nlocation feed on the same page updates live without problems, but new orders\nonly show up after a page refresh.\n\nThe dashboard uses supabase-js to subscribe to INSERT events on the `orders`\ntable through postgres_changes, the same way it subscribes to courier\nlocations. The channel's status callback logs SUBSCRIBED and there are no\nerrors in the browser console.\n\nFigure out why no order events ever arrive and fix it.", + "promptSourcePath": "evals/investigate-realtime-001-subscribed-no-events/PROMPT.md", "attempts": 1, - "sourcePath": "codex-gpt-5.6-no-skills/deploy-database-001-prometheus-metrics.json" + "sourcePath": "codex-gpt-5.4-mini-no-skills/investigate-realtime-001-subscribed-no-events.json" }, { - "experiment": "codex-gpt-5.6-no-skills", + "experiment": "codex-gpt-5.4-mini-no-skills", "experimentSuite": "no-skills", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.6-sol", + "modelId": "gpt-5.4-mini", "reasoningEffort": "medium" }, - "eval": "deploy-functions-001-edge-function-secrets", - "stage": "deploy", + "eval": "investigate-reliability-003-edge-function-5xx-correlation", + "stage": "investigate", "product": [ "edge-functions" ], "topic": [ - "security" + "observability" ], "suite": "benchmark", - "interface": "cli", - "passed": true, + "interface": "mcp", + "passed": false, "checks": [ { - "name": "WEATHER_API_KEY is set as a Function secret on the project", - "passed": true - }, - { - "name": "the weather function is deployed to the project", + "name": "identified image-transform and the recurring 503 pattern", "passed": true, - "notes": "status ACTIVE" + "judgeNotes": "Identified `image-transform` as the main affected function and described intermittent/recurring 503s throughout the morning of 2026-04-28, with successful requests in between. It did not explicitly enumerate all 8 failures or the 07:00Z-12:00Z span, but it meets the rubric’s core requirement." }, { - "name": "the weather function reads WEATHER_API_KEY from the environment", - "passed": true, - "judgeNotes": "Reads WEATHER_API_KEY from the runtime environment via Deno.env.get(\"WEATHER_API_KEY\")." + "name": "attributed recurring 503s to gateway/platform layer, not function code", + "passed": false, + "judgeNotes": "The assistant does not clearly attribute the recurring 503s to the gateway/platform layer. It instead frames them as a function-specific issue and suggests startup/cold-start/dependency/runtime problems, code inspection, added function logging, and possible rollback. Although it notes unchanged deployment/version and adjacent successes, it fails the required layer attribution." }, { - "name": "WEATHER_API_KEY value is not committed to the repo", - "passed": true + "name": "recommended a concrete next step", + "passed": true, + "judgeNotes": "The assistant recommended concrete actionable next steps, including checking the specific deployment/rollback window, inspecting the function package for startup/runtime failures, improving error logging, rolling back if deploy-related, and adding retry/backoff." } ], "skills": { @@ -15141,80 +9502,65 @@ "loaded": [] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase Edge Functions environment variables secrets set env file deploy function no verify jwt invoke\", limit: 6) { nodes { ... on Guide { title href content } ... on CLICommandReference { title href content } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", - "title": "Self-Hosted Functions" - }, - { - "url": "https://supabase.com/docs/guides/ai/examples/openai", - "title": "Generating OpenAI GPT3 completions" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/functions/secrets", - "title": "Environment Variables" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/upstash-redis", - "title": "Upstash Redis" - } - ], - "resultChars": 35184 - } - ] + "calls": [] }, - "prompt": "Our weather widget currently calls WeatherAPI straight from the browser, which\nleaks our API key. I want to move that behind a Supabase Edge Function called\n`weather` that holds the key server-side and proxies the request.\n\nThe function should read the key from an environment variable named\n`WEATHER_API_KEY`. Our key already lives in a local `.env` file at the project\nroot.\n\nDeploy the function to our project so it's live, and make sure the deployed\nfunction can actually read the key at runtime.", - "promptSourcePath": "evals/deploy-functions-001-edge-function-secrets/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.6-no-skills/deploy-functions-001-edge-function-secrets.json" + "prompt": "Users have been reporting that image uploads are intermittently failing with server errors throughout the morning of `2026-04-28`.\n\nCan you investigate the project logs and tell me what is going on and what we\nshould do next?", + "promptSourcePath": "evals/investigate-reliability-003-edge-function-5xx-correlation/PROMPT.md", + "attempts": 2, + "sourcePath": "codex-gpt-5.4-mini-no-skills/investigate-reliability-003-edge-function-5xx-correlation.json" }, { - "experiment": "codex-gpt-5.6-no-skills", + "experiment": "codex-gpt-5.4-mini-no-skills", "experimentSuite": "no-skills", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.6-sol", + "modelId": "gpt-5.4-mini", "reasoningEffort": "medium" }, - "eval": "deploy-self-hosting-001-docker-compose", - "stage": "deploy", + "eval": "resolve-dataapi-001-empty-results", + "stage": "resolve", "product": [ + "data-api", "database", - "auth", - "storage" + "auth" ], "topic": [ - "self-hosting" + "rls", + "sdk" ], "suite": "benchmark", - "interface": "cli", + "interface": "mcp", "passed": true, "checks": [ { - "name": "cloned the self-host stack (docker-compose.yml + volumes/db)", + "name": "RLS still enabled on bookmarks", + "passed": true + }, + { + "name": "user A reads own bookmarks", "passed": true }, { - "name": "didn't conflate with the CLI (no supabase/config.toml in the stack)", + "name": "user B cannot read user A bookmarks", "passed": true }, { - "name": "secrets rotated off the shipped defaults", + "name": "anon reads no bookmarks", "passed": true }, { - "name": "ANON_KEY and SERVICE_ROLE_KEY are HS256 JWTs signed by JWT_SECRET", + "name": "user A can save a new bookmark", + "passed": true + }, + { + "name": "user B cannot insert a bookmark as user A", "passed": true + }, + { + "name": "diagnosed RLS and added owner-scoped policies", + "passed": true, + "judgeNotes": "Diagnosed RLS enabled with no policies causing deny-all Data API behavior, then added authenticated SELECT and INSERT owner-scoped policies using auth.uid() = user_id while keeping RLS enabled." } ], "skills": { @@ -15222,129 +9568,62 @@ "loaded": [] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"self-hosting Docker docker-compose .env secrets JWT_SECRET ANON_KEY SERVICE_ROLE_KEY POOLER_TENANT_ID\", limit: 8) { nodes { __typename title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", - "title": "New API Keys and Asymmetric Authentication" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", - "title": "Envoy API Gateway" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker", - "title": "Self-Hosting with Docker" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth", - "title": "Configure Social Login (OAuth) Providers" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", - "title": "Self-Hosted Functions" - }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys", - "title": "JWT Signing Keys" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/custom-email-templates", - "title": "Custom Email Templates" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-saml-sso", - "title": "Configure SAML SSO" - } - ], - "resultChars": 171411 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Self-hosting with Docker Quick Start Linux download docker directory git sparse checkout generate-keys.sh\", limit: 5) { nodes { __typename title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/self-hosting/docker", - "title": "Self-Hosting with Docker" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", - "title": "New API Keys and Asymmetric Authentication" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", - "title": "Envoy API Gateway" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting", - "title": "Self-Hosting" - } - ], - "resultChars": 79242 - } - ] + "calls": [] }, - "prompt": "I'm moving off the hosted Supabase and running the whole thing myself on a VPS I\njust spun up. Can you get a Docker setup ready for me to copy onto the box?\n\nI don't need it running here, I'll do the actual bring-up once I'm on the\nserver. I just want everything in place and the secrets set up. Put it in a `supabase-docker/`\nfolder at the repo root so I can scp the whole thing across in one go.", - "promptSourcePath": "evals/deploy-self-hosting-001-docker-compose/PROMPT.md", + "prompt": "Our app lets signed-in users save bookmarks and view them on their dashboard. Bookmarks are stored in the `bookmarks` table and are private — a user must only ever see their own. \nUsers also need to be able to save new bookmarks from the app.\n\nI can see the rows when I query the table directly, but the dashboard shows an empty list for every user.\n\nFind out why the Data API returns nothing and fix it.", + "promptSourcePath": "evals/resolve-dataapi-001-empty-results/PROMPT.md", "attempts": 1, - "sourcePath": "codex-gpt-5.6-no-skills/deploy-self-hosting-001-docker-compose.json" + "sourcePath": "codex-gpt-5.4-mini-no-skills/resolve-dataapi-001-empty-results.json" }, { - "experiment": "codex-gpt-5.6-no-skills", + "experiment": "codex-gpt-5.4-mini-no-skills", "experimentSuite": "no-skills", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.6-sol", + "modelId": "gpt-5.4-mini", "reasoningEffort": "medium" }, - "eval": "investigate-auth-001-deleted-user-access", - "stage": "investigate", + "eval": "resolve-database-001-migration-history-mismatch", + "stage": "resolve", "product": [ - "auth" + "database" ], "topic": [ - "security", - "sdk" + "migrations" ], "suite": "benchmark", - "interface": "mcp", - "passed": true, + "interface": "cli", + "passed": false, "checks": [ { - "name": "victim session active before delete-account", - "passed": true - }, - { - "name": "delete_account flow ran for the victim", - "passed": true + "name": "the avatar_url column is applied on the hosted profiles table", + "passed": false, + "notes": "avatar_url not found on public.profiles" }, { - "name": "delete-account revokes the user's sessions", - "passed": true, - "notes": "sessions left: 0" + "name": "migration 20240220000000 is recorded in the remote history", + "passed": false, + "notes": "remote history versions: [\"20240101000000\",\"20240115000000\"]" }, { - "name": "deleted user's refresh token is rejected", - "passed": true + "name": "remote migration history matches local migration files", + "passed": false, + "notes": "remote-only versions still unreconciled: [\"20240115000000\"] (remote: [\"20240101000000\",\"20240115000000\"], local: [\"20240101000000\",\"20240220000000\"])" }, { - "name": "deleted user cannot sign back in", - "passed": true + "name": "local migrations are a valid reconciled sequence", + "passed": false, + "notes": "expected 20240101000000_create_profiles → → 20240220000000_add_avatar_url, strictly ascending; got [20240101000000_create_profiles.sql, 20240220000000_add_avatar_url.sql]" }, { - "name": "other users keep their sessions and access", + "name": "production profile data is intact (not reset)", "passed": true }, { - "name": "diagnosed and explained session revocation", - "passed": true, - "judgeNotes": "Meets the rubric: identifies the soft-delete-only RPC as the cause, implements deletion of the auth user with session/refresh-token revocation via cascade plus RLS gates that close the data path for stale JWTs, explains JWTs remain cryptographically valid until expiry while data access is blocked by the shipped RLS fix, and correctly distinguishes publishable frontend keys from secret backend-only RLS-bypassing keys." + "name": "the avatar migration and history reconciliation were done via the Supabase CLI", + "passed": false, + "judgeNotes": "No successful hosted `supabase db push` applied the avatar_url migration. The only applying output was `supabase db reset`, which applied migrations to the local database. The final `supabase db push --db-url postgresql://postgres:postgres@127.0.0.1:54322/postgres --include-all --yes` targeted the local DB and reported `Remote database is up to date`, not an applied remote migration. No `supabase migration repair`, `db pull`, or local bio migration reconciliation was performed. Workarounds seen: edited the eval profile to use 127.0.0.1 and pushed against the local DB URL." } ], "skills": { @@ -15352,137 +9631,55 @@ "loaded": [] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"delete user access token remains valid until expiry revoke sessions JWT RLS\", limit: 5) { nodes { ... on Guide { title href content } ... on TroubleshootingGuide { title href content } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/auth/sessions", - "title": "User sessions" - }, - { - "url": "https://supabase.com/docs/guides/realtime/authorization", - "title": "Realtime Authorization" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/should-i-set-a-shorter-max-age-parameter-on-the-cookies-8sbF4V", - "title": "Should I set a shorter Max-Age parameter on the cookies?" - }, - { - "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security", - "title": "Token Security and Row Level Security" - }, - { - "url": "https://supabase.com/docs/guides/storage/security/ownership", - "title": "Ownership" - } - ], - "resultChars": 40980 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"publishable key secret key frontend RLS bypass legacy anon service_role\", limit: 5) { nodes { ... on Guide { title href content } ... on TroubleshootingGuide { title href content } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/storage/security/access-control", - "title": "Storage Access Control" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", - "title": "Row Level Security" - }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys", - "title": "JWT Signing Keys" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" - } - ], - "resultChars": 91501 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"self delete user database function auth.users security definer delete account\", limit: 8) { nodes { ... on Guide { title href content } ... on TroubleshootingGuide { title href content } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/auth/managing-user-data", - "title": "User Management" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pgaudit", - "title": "PGAudit: Postgres Auditing" - }, - { - "url": "https://supabase.com/docs/guides/auth/auth-anonymous", - "title": "Anonymous Sign-Ins" - } - ], - "resultChars": 38021 - } - ] + "calls": [] }, - "prompt": "Last week support removed a user through our app's delete-account flow — the\napp calls the `delete_account` function over RPC as the signed-in user. This\nmorning that same person was back: still signed in, reading and saving their\ndata like nothing happened.\n\nFigure out why the account still works, fix the flow so a deleted account\nloses access, and tell me whether there is any window where they could still\nget in after the fix.\n\nOne more thing while you're at it: we're migrating off the legacy\nanon/service_role API keys to the new publishable and secret keys, and the\nteam isn't sure which one belongs in the frontend and what each means for\nRLS. Give us a quick rundown.", - "promptSourcePath": "evals/investigate-auth-001-deleted-user-access/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.6-no-skills/investigate-auth-001-deleted-user-access.json" + "prompt": "I'm trying to ship a migration to our hosted project and it's not working. Can you figure out what's wrong and get it deployed?", + "promptSourcePath": "evals/resolve-database-001-migration-history-mismatch/PROMPT.md", + "attempts": 2, + "sourcePath": "codex-gpt-5.4-mini-no-skills/resolve-database-001-migration-history-mismatch.json" }, { - "experiment": "codex-gpt-5.6-no-skills", + "experiment": "codex-gpt-5.4-mini-no-skills", "experimentSuite": "no-skills", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.6-sol", + "modelId": "gpt-5.4-mini", "reasoningEffort": "medium" }, - "eval": "investigate-realtime-001-subscribed-no-events", - "stage": "investigate", + "eval": "resolve-performance-001-slow-query-cpu-spike", + "stage": "resolve", "product": [ - "realtime", "database" ], "topic": [ - "sdk" + "observability", + "sql" ], "suite": "benchmark", "interface": "mcp", "passed": true, "checks": [ { - "name": "orders table added to supabase_realtime publication", - "passed": true - }, - { - "name": "courier_locations still in supabase_realtime publication", + "name": "inspected pg_stat_statements for query performance", "passed": true }, { - "name": "publication still publishes INSERT events", + "name": "ran EXPLAIN on the expensive query", "passed": true }, { - "name": "RLS still enabled on orders", + "name": "created index covering user_id and created_at", "passed": true }, { - "name": "staff can still read orders through RLS", + "name": "query plan uses an index and avoids sequential scan", "passed": true, - "notes": "authenticated sees 2 of 2 orders" + "notes": "Limit (cost=55.55..55.61 rows=25 width=88)\n -> Sort (cost=55.55..55.61 rows=25 width=88)\n Sort Key: created_at DESC\n -> Bitmap Heap Scan on events (cost=4.48..54.97 rows=25 width=88)\n Recheck Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)\n -> Bitmap Index Scan on events_user_id_created_at_desc_idx (cost=0.00..4.47 rows=25 width=0)\n Index Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)" }, { - "name": "diagnosed missing publication membership", - "passed": true, - "judgeNotes": "The assistant correctly diagnosed that the channel could subscribe while INSERT events were silent because public.orders was missing from the supabase_realtime publication, added only public.orders to the existing publication via ALTER PUBLICATION in an idempotent migration, preserved courier_locations and existing RLS/policies, and did not blame or weaken RLS/client/networking." + "name": "inserts still work", + "passed": true } ], "skills": { @@ -15492,46 +9689,69 @@ "docs": { "calls": [] }, - "prompt": "Our dispatch dashboard shows incoming orders as they happen. The courier\nlocation feed on the same page updates live without problems, but new orders\nonly show up after a page refresh.\n\nThe dashboard uses supabase-js to subscribe to INSERT events on the `orders`\ntable through postgres_changes, the same way it subscribes to courier\nlocations. The channel's status callback logs SUBSCRIBED and there are no\nerrors in the browser console.\n\nFigure out why no order events ever arrive and fix it.", - "promptSourcePath": "evals/investigate-realtime-001-subscribed-no-events/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.6-no-skills/investigate-realtime-001-subscribed-no-events.json" + "prompt": "My database CPU keeps spiking and the app gets slow when loading recent events for a user. Can you figure out what query is causing it and make the database change needed to fix it?\n\nEnd your turn with a short summary of what you changed and why.", + "promptSourcePath": "evals/resolve-performance-001-slow-query-cpu-spike/PROMPT.md", + "attempts": 2, + "sourcePath": "codex-gpt-5.4-mini-no-skills/resolve-performance-001-slow-query-cpu-spike.json" }, { - "experiment": "codex-gpt-5.6-no-skills", + "experiment": "codex-gpt-5.4-mini-no-skills", "experimentSuite": "no-skills", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.6-sol", + "modelId": "gpt-5.4-mini", "reasoningEffort": "medium" }, - "eval": "investigate-reliability-003-edge-function-5xx-correlation", - "stage": "investigate", + "eval": "resolve-security-002-rls-cross-tenant-leak", + "stage": "resolve", "product": [ - "edge-functions" + "database", + "auth" ], "topic": [ - "observability" + "rls", + "security" ], "suite": "benchmark", "interface": "mcp", "passed": true, "checks": [ { - "name": "identified image-transform and the recurring 503 pattern", - "passed": true, - "judgeNotes": "The assistant correctly identified `image-transform` as the affected function and described eight intermittent HTTP 503 gateway responses spanning 2026-04-28 07:00–12:00 UTC, including the recurring pattern and isolation from runtime logs." + "name": "RLS enabled on notes", + "passed": true }, { - "name": "attributed recurring 503s to gateway/platform layer, not function code", - "passed": true, - "judgeNotes": "The assistant explicitly attributes the recurring image-transform 503s to the gateway/platform layer rather than function code. It grounds this in valid observations: 503s appear in API gateway logs but not Edge Function runtime logs, nearby successful requests occurred on the same deployment/version, and it distinguishes the separate avatar-upload 500 as a function-level issue to treat separately." + "name": "tenant A sees only org A notes", + "passed": true }, { - "name": "recommended a concrete next step", - "passed": true, - "judgeNotes": "The assistant recommended concrete actionable next steps, including opening a Supabase support case with 503 timestamps, function name, and deployment ID, requesting gateway-level investigation, and capturing request/correlation IDs." + "name": "tenant B cannot read org A notes", + "passed": true + }, + { + "name": "tenant A author can update own note", + "passed": true + }, + { + "name": "tenant B cannot update org A note", + "passed": true + }, + { + "name": "tenant B author can delete own note", + "passed": true + }, + { + "name": "tenant B cannot delete org A note", + "passed": true + }, + { + "name": "tenant A can insert note in own org", + "passed": true + }, + { + "name": "tenant B cannot insert into org A", + "passed": true } ], "skills": { @@ -15541,113 +9761,132 @@ "docs": { "calls": [] }, - "prompt": "Users have been reporting that image uploads are intermittently failing with server errors throughout the morning of `2026-04-28`.\n\nCan you investigate the project logs and tell me what is going on and what we\nshould do next?", - "promptSourcePath": "evals/investigate-reliability-003-edge-function-5xx-correlation/PROMPT.md", + "prompt": "A customer reported that notes showed up in the wrong workspace.\n\nCan you investigate what is going on and fix it?", + "promptSourcePath": "evals/resolve-security-002-rls-cross-tenant-leak/PROMPT.md", "attempts": 1, - "sourcePath": "codex-gpt-5.6-no-skills/investigate-reliability-003-edge-function-5xx-correlation.json" + "sourcePath": "codex-gpt-5.4-mini-no-skills/resolve-security-002-rls-cross-tenant-leak.json" }, { - "experiment": "codex-gpt-5.6-no-skills", - "experimentSuite": "no-skills", + "experiment": "codex-gpt-5.6", + "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", "modelId": "gpt-5.6-sol", - "reasoningEffort": "medium" + "reasoningEffort": "low" }, - "eval": "resolve-dataapi-001-empty-results", - "stage": "resolve", + "eval": "build-cli-001-bootstrap-app", + "stage": "build", "product": [ - "data-api", "database", - "auth" + "data-api" ], "topic": [ - "rls", - "sdk" + "migrations", + "rls" ], "suite": "benchmark", - "interface": "mcp", + "interface": "cli", "passed": true, "checks": [ { - "name": "RLS still enabled on bookmarks", + "name": "supabase project initialised (supabase/config.toml exists)", "passed": true }, { - "name": "user A reads own bookmarks", + "name": "todos table is created by a migration file", "passed": true }, { - "name": "user B cannot read user A bookmarks", - "passed": true + "name": "todos table exists with at least 2 seeded rows", + "passed": true, + "notes": "found 2 rows" }, { - "name": "anon reads no bookmarks", + "name": "row level security is enabled on todos", "passed": true }, { - "name": "user A can save a new bookmark", + "name": "a SELECT policy targets the authenticated role", "passed": true }, { - "name": "user B cannot insert a bookmark as user A", - "passed": true + "name": "REST API returns no todos to anonymous requests", + "passed": true, + "notes": "0 rows" }, { - "name": "diagnosed RLS and added owner-scoped policies", + "name": "REST API returns the todos to authenticated requests", "passed": true, - "judgeNotes": "The assistant correctly diagnosed RLS enabled with no policies causing deny-all Data API behavior, kept RLS enabled, and created authenticated SELECT and INSERT policies scoped to user_id = auth.uid(), with INSERT enforced via WITH CHECK. It also avoided permissive/public policies and verified owner isolation." + "notes": "2 rows" } ], "skills": { - "available": [], - "loaded": [] + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] }, "docs": { "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"Row Level Security auth.uid select insert policy authenticated users own rows\", limit: 3) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"local development initialize project migrations row level security authenticated role select policy grants Data API\", limit: 6) { nodes { ... on Guide { title href content } ... on CLICommandReference { title href content } } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", - "title": "Row Level Security" + "url": "https://supabase.com/docs/guides/api/securing-your-api", + "title": "Securing your API" }, { - "url": "https://supabase.com/docs/guides/getting-started/features", - "title": "Features" + "url": "https://supabase.com/docs/guides/local-development/cli-workflows", + "title": "Local development workflow" }, { - "url": "https://supabase.com/docs/guides/database/postgres/column-level-security", - "title": "Column Level Security" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-redwoodjs", + "title": "Build a User Management App with RedwoodJS" + }, + { + "url": "https://supabase.com/docs/guides/database/secure-data", + "title": "Securing your data" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-flutter", + "title": "Build a User Management App with Flutter" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-solidjs", + "title": "Build a User Management App with SolidJS" } ], - "resultChars": 46550 + "resultChars": 141701 } ] }, - "prompt": "Our app lets signed-in users save bookmarks and view them on their dashboard. Bookmarks are stored in the `bookmarks` table and are private — a user must only ever see their own. \nUsers also need to be able to save new bookmarks from the app.\n\nI can see the rows when I query the table directly, but the dashboard shows an empty list for every user.\n\nFind out why the Data API returns nothing and fix it.", - "promptSourcePath": "evals/resolve-dataapi-001-empty-results/PROMPT.md", + "prompt": "We're kicking off a todos app and I want the Supabase side ready for the team\nto build on. Set it up the way we'd run it in development, with schema changes\ntracked as migrations so they can be reviewed and replayed.\n\nFor the first slice we just need a `todos` table. Todos aren't public: anyone\nsigned in can read all of them, but nothing should be writable through the API\nfor now. Add a couple of sample todos so there's something to look at.\n\nBefore you hand it back, make sure the running API actually behaves that way —\nsigned-in users get the todos, signed-out requests get nothing.", + "promptSourcePath": "evals/build-cli-001-bootstrap-app/PROMPT.md", "attempts": 1, - "sourcePath": "codex-gpt-5.6-no-skills/resolve-dataapi-001-empty-results.json" + "sourcePath": "codex-gpt-5.6/build-cli-001-bootstrap-app.json" }, { - "experiment": "codex-gpt-5.6-no-skills", - "experimentSuite": "no-skills", + "experiment": "codex-gpt-5.6", + "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", "modelId": "gpt-5.6-sol", - "reasoningEffort": "medium" + "reasoningEffort": "low" }, - "eval": "resolve-database-001-migration-history-mismatch", - "stage": "resolve", + "eval": "build-cli-002-declarative-schema", + "stage": "build", "product": [ "database" ], "topic": [ + "declarative-schema", "migrations" ], "suite": "benchmark", @@ -15655,273 +9894,231 @@ "passed": true, "checks": [ { - "name": "the avatar_url column is applied on the hosted profiles table", - "passed": true - }, - { - "name": "migration 20240220000000 is recorded in the remote history", + "name": "supabase db diff used to generate the migration", "passed": true }, { - "name": "remote migration history matches local migration files", + "name": "schema file updated to include description column", "passed": true }, { - "name": "local migrations are a valid reconciled sequence", + "name": "a new migration was generated for the change", "passed": true }, { - "name": "production profile data is intact (not reset)", + "name": "description column exists in the live database", "passed": true - }, - { - "name": "the avatar migration and history reconciliation were done via the Supabase CLI", - "passed": true, - "judgeNotes": "Avatar migration was applied by `supabase db push --db-url \"$DB_URL\" --yes`, with output showing `Applying migration 20240220000000_add_avatar_url.sql...` and `Finished supabase db push.` History was reconciled by adding the missing local migration file `supabase/migrations/20240115000000_add_profile_bio.sql` and then running the Supabase CLI push workflow. No disallowed workaround or direct remote SQL mutation was used." } ], "skills": { - "available": [], - "loaded": [] + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] }, "docs": { "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase CLI db push migration history repair duplicate table hosted project migration list\", limit: 5) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"declarative database schemas schema_paths generate migration local development\", limit: 3) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/reference/cli/supabase-migration-repair", - "title": "Repair the migration history table" - }, - { - "url": "https://supabase.com/docs/guides/platform/migrating-within-supabase/backup-restore", - "title": "Backup and Restore using the CLI" - }, - { - "url": "https://supabase.com/docs/guides/deployment/database-migrations", - "title": "Database Migrations" - }, - { - "url": "https://supabase.com/docs/guides/local-development/database-migrations", - "title": "Database migrations" + "url": "https://supabase.com/docs/guides/local-development/declarative-database-schemas", + "title": "Declarative database schemas" }, { - "url": "https://supabase.com/docs/reference/cli/supabase-db-push", - "title": "Push new migrations to the remote database" + "url": "https://supabase.com/docs/guides/local-development/cli-workflows", + "title": "Local development workflow" } ], - "resultChars": 74715 + "resultChars": 37770 } ] }, - "prompt": "I'm trying to ship a migration to our hosted project and it's not working. Can you figure out what's wrong and get it deployed?", - "promptSourcePath": "evals/resolve-database-001-migration-history-mismatch/PROMPT.md", + "prompt": "Add a description text column to the `products` table in my local Supabase stack", + "promptSourcePath": "evals/build-cli-002-declarative-schema/PROMPT.md", "attempts": 1, - "sourcePath": "codex-gpt-5.6-no-skills/resolve-database-001-migration-history-mismatch.json" + "sourcePath": "codex-gpt-5.6/build-cli-002-declarative-schema.json" }, { - "experiment": "codex-gpt-5.6-no-skills", - "experimentSuite": "no-skills", + "experiment": "codex-gpt-5.6", + "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", "modelId": "gpt-5.6-sol", - "reasoningEffort": "medium" + "reasoningEffort": "low" }, - "eval": "resolve-performance-001-slow-query-cpu-spike", - "stage": "resolve", + "eval": "build-cli-003-pg-cron-queue-workflow", + "stage": "build", "product": [ - "database" + "database", + "edge-functions", + "cron", + "queues" ], "topic": [ - "observability", - "sql" + "sql", + "sdk" ], "suite": "benchmark", - "interface": "mcp", + "interface": "cli", "passed": true, "checks": [ { - "name": "inspected pg_stat_statements for query performance", - "passed": true - }, - { - "name": "ran EXPLAIN on the expensive query", - "passed": true - }, - { - "name": "created index covering user_id and created_at", - "passed": true + "name": "pg_cron job 'enqueue-tasks' scheduled to run every minute", + "passed": true, + "notes": "schedule='* * * * *', active=true" }, { - "name": "query plan uses an index and avoids sequential scan", + "name": "cron command enqueues to the 'tasks' queue", "passed": true, - "notes": "Limit (cost=79.74..79.86 rows=50 width=58)\n -> Sort (cost=79.74..79.99 rows=100 width=58)\n Sort Key: created_at DESC\n -> Bitmap Heap Scan on events (cost=5.06..76.42 rows=100 width=58)\n Recheck Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)\n -> Bitmap Index Scan on events_user_id_created_at_idx (cost=0.00..5.03 rows=100 width=0)\n Index Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)" + "notes": "queue depth 1 -> 2" }, { - "name": "inserts still work", - "passed": true + "name": "process-tasks function drains the queue", + "passed": true, + "notes": "function removed the seeded message (id 36) from the queue" } ], "skills": { - "available": [], - "loaded": [] + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] }, "docs": { "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"Postgres multicolumn index WHERE equality ORDER BY DESC LIMIT query performance\", limit: 3) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"Supabase Queues pgmq create queue send read delete messages SQL Cron pg_cron schedule Edge Function local\", limit: 8) { nodes { __typename title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/queues", + "title": "Supabase Queues" + }, + { + "url": "https://supabase.com/docs/guides/functions/schedule-functions", + "title": "Scheduling Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/cron", + "title": "Cron" + }, + { + "url": "https://supabase.com/docs/guides/queues/consuming-messages-with-edge-functions", + "title": "Consuming Supabase Queue Messages with Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_net", + "title": "pg_net: Async Networking" + }, + { + "url": "https://supabase.com/docs/guides/queues/api", + "title": "API" + }, + { + "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", + "title": "Automatic embeddings" + }, + { + "url": "https://supabase.com/docs/guides/cron/install", + "title": "Install" + } + ], + "resultChars": 59370 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Queues API SQL pgmq_public.send pgmq_public.read pgmq_public.delete queue create\", limit: 10) { nodes { __typename title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/database/orioledb", - "title": "OrioleDB Overview" + "url": "https://supabase.com/docs/guides/queues/api", + "title": "API" + }, + { + "url": "https://supabase.com/docs/guides/queues/pgmq", + "title": "PGMQ Extension" }, { - "url": "https://supabase.com/docs/guides/database/query-optimization", - "title": "Query Optimization" + "url": "https://supabase.com/docs/guides/queues/quickstart", + "title": "Quickstart" }, { - "url": "https://supabase.com/docs/guides/database/inspect", - "title": "Debugging and monitoring" + "url": "https://supabase.com/docs/guides/queues", + "title": "Supabase Queues" + }, + { + "url": "https://supabase.com/docs/guides/queues/expose-self-hosted-queues", + "title": "Expose Queues for local and self-hosted Supabase" + }, + { + "url": "https://supabase.com/docs/guides/queues/consuming-messages-with-edge-functions", + "title": "Consuming Supabase Queue Messages with Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/42501--permission-denied-for-table-httprequestqueue-KnozmQ", + "title": "42501 : permission denied for table http_request_queue" } ], - "resultChars": 26398 + "resultChars": 44395 } ] }, - "prompt": "My database CPU keeps spiking and the app gets slow when loading recent events for a user. Can you figure out what query is causing it and make the database change needed to fix it?\n\nEnd your turn with a short summary of what you changed and why.", - "promptSourcePath": "evals/resolve-performance-001-slow-query-cpu-spike/PROMPT.md", + "prompt": "I want to set up a recurring background workflow on my local Supabase stack.\n\nCan you set up a cron job called `enqueue-tasks` to run every minute and push a task into a queue called `tasks`? Then add a `process-tasks` edge function that reads messages off the `tasks` queue and removes them, so a scheduled worker can keep the backlog drained.", + "promptSourcePath": "evals/build-cli-003-pg-cron-queue-workflow/PROMPT.md", "attempts": 1, - "sourcePath": "codex-gpt-5.6-no-skills/resolve-performance-001-slow-query-cpu-spike.json" + "sourcePath": "codex-gpt-5.6/build-cli-003-pg-cron-queue-workflow.json" }, { - "experiment": "codex-gpt-5.6-no-skills", - "experimentSuite": "no-skills", + "experiment": "codex-gpt-5.6", + "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", "modelId": "gpt-5.6-sol", - "reasoningEffort": "medium" - }, - "eval": "resolve-security-002-rls-cross-tenant-leak", - "stage": "resolve", - "product": [ - "database", - "auth" - ], - "topic": [ - "rls", - "security" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "RLS enabled on notes", - "passed": true - }, - { - "name": "tenant A sees only org A notes", - "passed": true - }, - { - "name": "tenant B cannot read org A notes", - "passed": true - }, - { - "name": "tenant A author can update own note", - "passed": true - }, - { - "name": "tenant B cannot update org A note", - "passed": true - }, - { - "name": "tenant B author can delete own note", - "passed": true - }, - { - "name": "tenant B cannot delete org A note", - "passed": true - }, - { - "name": "tenant A can insert note in own org", - "passed": true - }, - { - "name": "tenant B cannot insert into org A", - "passed": true - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "A customer reported that notes showed up in the wrong workspace.\n\nCan you investigate what is going on and fix it?", - "promptSourcePath": "evals/resolve-security-002-rls-cross-tenant-leak/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.6-no-skills/resolve-security-002-rls-cross-tenant-leak.json" - }, - { - "experiment": "opencode-kimi-k3", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "opencode", - "modelProvider": "moonshotai", - "modelId": "moonshotai/kimi-k3" + "reasoningEffort": "low" }, - "eval": "build-cli-001-bootstrap-app", + "eval": "build-database-001-migrate-postgres-to-supabase", "stage": "build", "product": [ - "database", - "data-api" + "database" ], "topic": [ - "migrations", - "rls" + "migrations" ], "suite": "benchmark", "interface": "cli", "passed": true, "checks": [ { - "name": "supabase project initialised (supabase/config.toml exists)", + "name": "all 3 tables exist (teams, members, tasks)", "passed": true }, { - "name": "todos table is created by a migration file", + "name": "row counts match (teams=5, members=10, tasks=13)", "passed": true }, { - "name": "todos table exists with at least 2 seeded rows", - "passed": true, - "notes": "found 2 rows" - }, - { - "name": "row level security is enabled on todos", + "name": "foreign key constraints survived the restore", "passed": true }, { - "name": "a SELECT policy targets the authenticated role", + "name": "tasks_team_status_idx index survived the restore", "passed": true }, { - "name": "REST API returns no todos to anonymous requests", - "passed": true, - "notes": "0 rows" - }, - { - "name": "REST API returns the todos to authenticated requests", - "passed": true, - "notes": "2 rows" + "name": "sequences synced (next insert won't conflict with existing IDs)", + "passed": true } ], "skills": { @@ -15930,66 +10127,95 @@ "supabase-postgres-best-practices" ], "loaded": [ - "supabase", - "supabase-postgres-best-practices" + "supabase" ] }, "docs": { "calls": [ { - "source": "web_fetch", - "query": "https://supabase.com/changelog.md", + "source": "search_docs", + "query": "query { searchDocs(query: \"restore existing PostgreSQL pg_dump custom format local Supabase CLI database roles ownership\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/changelog.md" + "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform", + "title": "Restore a Platform Project to Self-Hosted" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/remove-superuser-access", + "title": "Remove superuser access from Studio" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/transferring-from-cloud-to-self-host-in-supabase-2oWNvW", + "title": "Transferring from platform to self-hosted Supabase" + }, + { + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/postgres", + "title": "Migrate from Postgres to Supabase" + }, + { + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/vercel-postgres", + "title": "Migrate from Vercel Postgres to Supabase" } ], - "resultChars": 93411 + "resultChars": 40436 } ] }, - "prompt": "We're kicking off a todos app and I want the Supabase side ready for the team\nto build on. Set it up the way we'd run it in development, with schema changes\ntracked as migrations so they can be reviewed and replayed.\n\nFor the first slice we just need a `todos` table. Todos aren't public: anyone\nsigned in can read all of them, but nothing should be writable through the API\nfor now. Add a couple of sample todos so there's something to look at.\n\nBefore you hand it back, make sure the running API actually behaves that way —\nsigned-in users get the todos, signed-out requests get nothing.", - "promptSourcePath": "evals/build-cli-001-bootstrap-app/PROMPT.md", + "prompt": "I have an existing Postgres database I want to migrate to Supabase. There's a binary dump at `source.dump` in the current directory.\n\nCan you set up a local Supabase project and restore the dump into it?", + "promptSourcePath": "evals/build-database-001-migrate-postgres-to-supabase/PROMPT.md", "attempts": 1, - "sourcePath": "opencode-kimi-k3/build-cli-001-bootstrap-app.json" + "sourcePath": "codex-gpt-5.6/build-database-001-migrate-postgres-to-supabase.json" }, { - "experiment": "opencode-kimi-k3", + "experiment": "codex-gpt-5.6", "experimentSuite": "benchmark", "experimentDisplay": { - "agent": "opencode", - "modelProvider": "moonshotai", - "modelId": "moonshotai/kimi-k3" + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.6-sol", + "reasoningEffort": "low" }, - "eval": "build-cli-002-declarative-schema", + "eval": "build-functions-004-service-role-bypass", "stage": "build", "product": [ + "edge-functions", + "auth", "database" ], "topic": [ - "declarative-schema", - "migrations" + "rls", + "security", + "sdk" ], "suite": "benchmark", - "interface": "cli", + "interface": "mcp", "passed": true, "checks": [ { - "name": "supabase db diff used to generate the migration", - "passed": true + "name": "rejects missing auth", + "passed": true, + "notes": "status=401" }, { - "name": "schema file updated to include description column", - "passed": true + "name": "user A reads own note", + "passed": true, + "notes": "status=200" }, { - "name": "a new migration was generated for the change", - "passed": true + "name": "reads only with the caller's JWT", + "passed": true, + "notes": "bearer_tokens=2, all_match=true" }, { - "name": "description column exists in the live database", - "passed": true + "name": "user A cannot force-read user B note", + "passed": true, + "notes": "status=403" + }, + { + "name": "user B cannot force-read user A note", + "passed": true, + "notes": "status=403" } ], "skills": { @@ -16004,61 +10230,110 @@ "docs": { "calls": [ { - "source": "web_fetch", - "query": "https://supabase.com/changelog.md", + "source": "search_docs", + "query": "query { searchDocs(query: \"Edge Functions authenticate user Authorization header getUser service role RLS\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/changelog.md" + "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", + "title": "Integrating With Supabase Auth" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs", + "title": "Build a User Management App with Next.js" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/why-is-my-service-role-key-client-getting-rls-errors-or-not-returning-data-7_1K9z", + "title": "Why is my service role key client getting RLS errors or not returning data?" + }, + { + "url": "https://supabase.com/docs/guides/api/securing-your-api", + "title": "Securing your API" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth-headers", + "title": "Authorization headers" } ], - "resultChars": 93411 + "resultChars": 76332 } ] }, - "prompt": "Add a description text column to the `products` table in my local Supabase stack", - "promptSourcePath": "evals/build-cli-002-declarative-schema/PROMPT.md", + "prompt": "I built an Edge Function called `private-notes` for showing a user's saved\nprivate notes.\n\nCan you check whether there's any way one user could see another user's notes?\n\nPlease fix and deploy it if needed.", + "promptSourcePath": "evals/build-functions-004-service-role-bypass/PROMPT.md", "attempts": 1, - "sourcePath": "opencode-kimi-k3/build-cli-002-declarative-schema.json" + "sourcePath": "codex-gpt-5.6/build-functions-004-service-role-bypass.json" }, { - "experiment": "opencode-kimi-k3", + "experiment": "codex-gpt-5.6", "experimentSuite": "benchmark", "experimentDisplay": { - "agent": "opencode", - "modelProvider": "moonshotai", - "modelId": "moonshotai/kimi-k3" + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.6-sol", + "reasoningEffort": "low" }, - "eval": "build-cli-003-pg-cron-queue-workflow", + "eval": "build-functions-005-dual-auth-user-secret", "stage": "build", "product": [ - "database", "edge-functions", - "cron", - "queues" + "auth", + "database" ], "topic": [ - "sql", - "sdk" + "sdk", + "rls", + "security" ], "suite": "benchmark", "interface": "cli", + "cliVersion": "2.109.1", "passed": true, "checks": [ { - "name": "pg_cron job 'enqueue-tasks' scheduled to run every minute", + "name": "seed rows present", "passed": true, - "notes": "schedule='* * * * *', active=true" + "notes": "found 2/2 seeded rows" }, { - "name": "cron command enqueues to the 'tasks' queue", + "name": "rejects request with no credentials", "passed": true, - "notes": "queue depth 0 -> 1" + "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" }, { - "name": "process-tasks function drains the queue", + "name": "user with JWT reads only their own rows", + "passed": true, + "notes": "status 200: [{\"user_id\":\"01556c30-5c1e-43c8-9fc2-2a5d268e23f6\",\"metric\":\"steps_a_mrw5pedi\",\"value\":111}]" + }, + { + "name": "user cannot read another user's rows by passing user_id", + "passed": true, + "notes": "status 200: [{\"user_id\":\"01556c30-5c1e-43c8-9fc2-2a5d268e23f6\",\"metric\":\"steps_a_mrw5pedi\",\"value\":111}]" + }, + { + "name": "service key bypasses RLS to read the target user's rows", + "passed": true, + "notes": "status 200: [{\"user_id\":\"85c7b076-36e8-4317-9587-bb64387a3046\",\"metric\":\"steps_b_mrw5pedi\",\"value\":222}]" + }, + { + "name": "non-service key is not granted service access", + "passed": true, + "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" + }, + { + "name": "rejects an unverified (forged) user token", + "passed": true, + "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" + }, + { + "name": "a user token in the apikey slot is not treated as the service key", + "passed": true, + "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" + }, + { + "name": "implementation uses @supabase/server", "passed": true, - "notes": "function removed the seeded message (id 37) from the queue" + "notes": "imports @supabase/server / withSupabase" } ], "skills": { @@ -16074,169 +10349,207 @@ "calls": [ { "source": "search_docs", - "query": "{ searchDocs(query: \"pg_cron schedule job send message to pgmq queue every minute\", limit: 5) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"Edge Functions authentication Authorization header getUser service role apikey verify_jwt config\", limit: 8) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net", - "title": "pg_net: Async Networking" + "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", + "title": "Integrating With Supabase Auth" }, { - "url": "https://supabase.com/docs/guides/queues/pgmq", - "title": "PGMQ Extension" + "url": "https://supabase.com/docs/guides/functions/auth-headers", + "title": "Authorization headers" }, { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", - "title": "Automatic embeddings" + "url": "https://supabase.com/docs/guides/getting-started/api-keys", + "title": "Understanding API keys" }, { - "url": "https://supabase.com/docs/guides/functions/schedule-functions", - "title": "Scheduling Edge Functions" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", + "title": "Envoy API Gateway" }, { - "url": "https://supabase.com/docs/guides/cron", - "title": "Cron" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", + "title": "New API Keys and Asymmetric Authentication" + }, + { + "url": "https://supabase.com/docs/guides/functions/function-configuration", + "title": "Function Configuration" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" + }, + { + "url": "https://supabase.com/docs/guides/functions/error-codes", + "title": "Error codes" } ], - "resultChars": 68390 + "resultChars": 89420 }, { "source": "search_docs", - "query": "{ searchDocs(query: \"Supabase Queues pgmq consume messages edge function read delete\", limit: 4) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"Securing Edge Functions new auth none apikey secret key Authorization JWT verify_jwt false service role\", limit: 10) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/queues/consuming-messages-with-edge-functions", - "title": "Consuming Supabase Queue Messages with Edge Functions" + "url": "https://supabase.com/docs/guides/functions/auth", + "title": "Securing Edge Functions" }, { - "url": "https://supabase.com/docs/guides/queues", - "title": "Supabase Queues" + "url": "https://supabase.com/docs/guides/functions/function-configuration", + "title": "Function Configuration" }, { - "url": "https://supabase.com/docs/guides/queues/api", - "title": "API" + "url": "https://supabase.com/docs/guides/functions/auth-headers", + "title": "Authorization headers" }, { - "url": "https://supabase.com/docs/guides/queues/quickstart", - "title": "Quickstart" + "url": "https://supabase.com/docs/guides/getting-started/api-keys", + "title": "Understanding API keys" + }, + { + "url": "https://supabase.com/docs/guides/auth/signing-keys", + "title": "JWT Signing Keys" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" + }, + { + "url": "https://supabase.com/docs/guides/functions/error-codes", + "title": "Error codes" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", + "title": "Self-Hosted Functions" + }, + { + "url": "https://supabase.com/docs/guides/functions/status-codes", + "title": "Status codes" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin", + "title": "Build a Product Management Android App with Jetpack Compose" + } + ], + "resultChars": 149679 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"@supabase/server auth secret legacy service_role key apikey Edge Functions\", limit: 10) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", + "title": "Integrating With Supabase Auth" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth", + "title": "Securing Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/api-keys", + "title": "Understanding API keys" + }, + { + "url": "https://supabase.com/docs/guides/functions/secrets", + "title": "Environment Variables" + }, + { + "url": "https://supabase.com/docs/guides/functions/examples/resumable-websockets", + "title": "Resumable WebSockets with Edge Functions" + }, + { + "url": "https://supabase.com/docs/reference/cli/supabase-functions", + "title": "Manage Supabase Edge functions" + }, + { + "url": "https://supabase.com/docs/guides/database/secure-data", + "title": "Securing your data" + }, + { + "url": "https://supabase.com/docs/guides/auth/choosing-a-server-package", + "title": "Which package to use" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", + "title": "New API Keys and Asymmetric Authentication" } ], - "resultChars": 18714 + "resultChars": 79545 } ] }, - "prompt": "I want to set up a recurring background workflow on my local Supabase stack.\n\nCan you set up a cron job called `enqueue-tasks` to run every minute and push a task into a queue called `tasks`? Then add a `process-tasks` edge function that reads messages off the `tasks` queue and removes them, so a scheduled worker can keep the backlog drained.", - "promptSourcePath": "evals/build-cli-003-pg-cron-queue-workflow/PROMPT.md", + "prompt": "Build and serve a Supabase Edge Function named `user-stats` for this project,\nreachable over HTTP at `/functions/v1/user-stats`.\n\nOur product stores per-user metrics in the existing `user_stats` table.\n\nTwo very different callers need to hit this one endpoint:\n\n1. **Our mobile app**, acting for a signed-in user. It sends that user's\n Supabase access token. The endpoint should return the user's own stats.\n\n2. **Our internal billing service**, a trusted backend with no signed-in user.\n It authenticates with the project's secret (service-role) key in the `apikey`\n header, and names the target user with a `user_id` in the JSON request body.\n It needs that user's stats.\n\nReturn the matching rows as JSON. The endpoint must be secure: only ever serve\nstats to a caller that is genuinely entitled to them, and turn away callers that\nare not.", + "promptSourcePath": "evals/build-functions-005-dual-auth-user-secret/PROMPT.md", "attempts": 1, - "sourcePath": "opencode-kimi-k3/build-cli-003-pg-cron-queue-workflow.json" + "sourcePath": "codex-gpt-5.6/build-functions-005-dual-auth-user-secret.json" }, { - "experiment": "opencode-kimi-k3", + "experiment": "codex-gpt-5.6", "experimentSuite": "benchmark", "experimentDisplay": { - "agent": "opencode", - "modelProvider": "moonshotai", - "modelId": "moonshotai/kimi-k3" + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.6-sol", + "reasoningEffort": "low" }, - "eval": "build-database-001-migrate-postgres-to-supabase", + "eval": "build-storage-001-private-bucket-access", "stage": "build", "product": [ + "storage", "database" ], "topic": [ - "migrations" + "rls", + "sdk" ], "suite": "benchmark", - "interface": "cli", + "interface": "mcp", "passed": true, "checks": [ { - "name": "all 3 tables exist (teams, members, tasks)", + "name": "bucket user-files exists", "passed": true }, { - "name": "row counts match (teams=5, members=10, tasks=13)", + "name": "bucket user-files is private", "passed": true }, { - "name": "foreign key constraints survived the restore", + "name": "RLS still enabled on storage.objects", "passed": true }, { - "name": "tasks_team_status_idx index survived the restore", - "passed": true + "name": "user A lists only own files", + "passed": true, + "notes": "saw: 019f8a22-016d-7487-8013-db6490ba0165/receipt-alpha.pdf, 019f8a22-016d-7487-8013-db6490ba0165/receipt-beta.pdf" }, { - "name": "sequences synced (next insert won't conflict with existing IDs)", + "name": "user B cannot read user A files", "passed": true - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" - ] - }, - "docs": { - "calls": [] - }, - "prompt": "I have an existing Postgres database I want to migrate to Supabase. There's a binary dump at `source.dump` in the current directory.\n\nCan you set up a local Supabase project and restore the dump into it?", - "promptSourcePath": "evals/build-database-001-migrate-postgres-to-supabase/PROMPT.md", - "attempts": 1, - "sourcePath": "opencode-kimi-k3/build-database-001-migrate-postgres-to-supabase.json" - }, - { - "experiment": "opencode-kimi-k3", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "opencode", - "modelProvider": "moonshotai", - "modelId": "moonshotai/kimi-k3" - }, - "eval": "build-functions-004-service-role-bypass", - "stage": "build", - "product": [ - "edge-functions", - "auth", - "database" - ], - "topic": [ - "rls", - "security", - "sdk" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "rejects missing auth", - "passed": true, - "notes": "status=401" }, { - "name": "user A reads own note", - "passed": true, - "notes": "status=200" + "name": "anon reads no files", + "passed": true }, { - "name": "reads only with the caller's JWT", - "passed": true, - "notes": "bearer_tokens=2, all_match=true" + "name": "user A can upload into own folder", + "passed": true }, { - "name": "user A cannot force-read user B note", - "passed": true, - "notes": "status=200" + "name": "user B cannot upload into user A folder", + "passed": true }, { - "name": "user B cannot force-read user A note", + "name": "configured private per-user storage access", "passed": true, - "notes": "status=200" + "judgeNotes": "Meets the rubric: private bucket, authenticated owner-scoped SELECT and INSERT policies with WITH CHECK, no permissive/public access, no RLS disabling, and uses createSignedUrl with expiry." } ], "skills": { @@ -16252,108 +10565,75 @@ "calls": [ { "source": "search_docs", - "query": "{ searchDocs(query: \"edge function get user from Authorization header JWT auth.getUser service role bypass RLS\", limit: 5) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"Storage access control private bucket RLS storage.foldername auth.uid createSignedUrl upload policy\", limit: 6) { nodes { ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/troubleshooting/why-is-my-service-role-key-client-getting-rls-errors-or-not-returning-data-7_1K9z", - "title": "Why is my service role key client getting RLS errors or not returning data?" + "url": "https://supabase.com/docs/guides/storage/buckets/fundamentals", + "title": "Storage Buckets" }, { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" + "url": "https://supabase.com/docs/guides/security/product-security", + "title": "Secure configuration of Supabase products" }, { "url": "https://supabase.com/docs/guides/storage/security/access-control", "title": "Storage Access Control" }, { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3", + "title": "Configure S3 Storage" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/why-is-my-select-returning-an-empty-data-array-and-i-have-data-in-the-table-xvOPgx", - "title": "Why is my select returning an empty data array and I have data in the table?" + "url": "https://supabase.com/docs/guides/storage/schema/design", + "title": "The Storage Schema" } ], - "resultChars": 21680 + "resultChars": 21316 } ] }, - "prompt": "I built an Edge Function called `private-notes` for showing a user's saved\nprivate notes.\n\nCan you check whether there's any way one user could see another user's notes?\n\nPlease fix and deploy it if needed.", - "promptSourcePath": "evals/build-functions-004-service-role-bypass/PROMPT.md", + "prompt": "Our app lets signed-in users keep personal files like receipts and bank\nstatements. These files are private — a user must only ever be able to upload\nand download their own. The app uploads each file under a path that starts\nwith the owner's user id, e.g. `/receipt-march.pdf`.\n\nSet up a `user-files` bucket on our project and lock it down that way.\n\nUsers also sometimes share one of their files with someone else through a\ntemporary link that expires. Include the supabase-js code the app should use\nfor that.", + "promptSourcePath": "evals/build-storage-001-private-bucket-access/PROMPT.md", "attempts": 1, - "sourcePath": "opencode-kimi-k3/build-functions-004-service-role-bypass.json" + "sourcePath": "codex-gpt-5.6/build-storage-001-private-bucket-access.json" }, { - "experiment": "opencode-kimi-k3", + "experiment": "codex-gpt-5.6", "experimentSuite": "benchmark", "experimentDisplay": { - "agent": "opencode", - "modelProvider": "moonshotai", - "modelId": "moonshotai/kimi-k3" + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.6-sol", + "reasoningEffort": "low" }, - "eval": "build-functions-005-dual-auth-user-secret", + "eval": "build-tests-001-rls-tenant-isolation", "stage": "build", "product": [ - "edge-functions", - "auth", "database" ], "topic": [ - "sdk", - "rls", - "security" + "tests", + "rls" ], "suite": "benchmark", "interface": "cli", - "cliVersion": "2.109.1", "passed": true, "checks": [ { - "name": "seed rows present", - "passed": true, - "notes": "found 2/2 seeded rows" - }, - { - "name": "rejects request with no credentials", - "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\"}" - }, - { - "name": "user with JWT reads only their own rows", - "passed": true, - "notes": "status 200: [{\"user_id\":\"37806d0f-f274-4021-aea2-77e34259085f\",\"metric\":\"steps_a_ms6x7jy9\",\"value\":111}]" - }, - { - "name": "user cannot read another user's rows by passing user_id", - "passed": true, - "notes": "status 200: [{\"user_id\":\"37806d0f-f274-4021-aea2-77e34259085f\",\"metric\":\"steps_a_ms6x7jy9\",\"value\":111}]" - }, - { - "name": "service key bypasses RLS to read the target user's rows", - "passed": true, - "notes": "status 200: [{\"user_id\":\"ff362a6e-3573-4069-b30d-24e34c6a5e04\",\"metric\":\"steps_b_ms6x7jy9\",\"value\":222}]" - }, - { - "name": "non-service key is not granted service access", - "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\"}" - }, - { - "name": "rejects an unverified (forged) user token", + "name": "pgTAP test file(s) written under supabase/tests/", "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\"}" + "notes": "1 file(s): supabase/tests/database/tenant_isolation.test.sql" }, { - "name": "a user token in the apikey slot is not treated as the service key", + "name": "pgTAP isolation tests ran and pass", "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\"}" + "notes": "5 passed, 3 failed" }, { - "name": "implementation uses @supabase/server", + "name": "agent correctly identifies the posts isolation bug from test results", "passed": true, - "notes": "imports @supabase/server / withSupabase" + "judgeNotes": "The agent correctly identifies `posts` as having the broken tenant isolation policy, specifically that it checks only some membership rather than membership in `posts.org_id`, allowing cross-tenant reads. It grounds the conclusion in pgTAP results and distinguishes `notes` as correctly isolated." } ], "skills": { @@ -16369,102 +10649,92 @@ "calls": [ { "source": "search_docs", - "query": "{ searchDocs(query: \"edge functions environment variables SUPABASE_SERVICE_ROLE_KEY secret key sb_secret verify_jwt\", limit: 6) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"row level security testing pgTAP auth.uid tenant isolation policies\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", - "title": "Self-Hosted Functions" + "url": "https://supabase.com/docs/guides/local-development/testing/pgtap-extended", + "title": "Advanced pgTAP Testing" }, { - "url": "https://supabase.com/docs/guides/functions/secrets", - "title": "Environment Variables" + "url": "https://supabase.com/docs/guides/local-development/testing/overview", + "title": "Testing Overview" }, { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" + "url": "https://supabase.com/docs/guides/database/extensions/pgtap", + "title": "pgTAP: Unit Testing" }, { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" + "url": "https://supabase.com/docs/guides/database/testing", + "title": "Testing Your Database" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/inspecting-edge-function-environment-variables-wg5qOQ", - "title": "Inspecting edge function environment variables" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", + "title": "Row Level Security" } ], - "resultChars": 42813 + "resultChars": 70562 } ] }, - "prompt": "Build and serve a Supabase Edge Function named `user-stats` for this project,\nreachable over HTTP at `/functions/v1/user-stats`.\n\nOur product stores per-user metrics in the existing `user_stats` table.\n\nTwo very different callers need to hit this one endpoint:\n\n1. **Our mobile app**, acting for a signed-in user. It sends that user's\n Supabase access token. The endpoint should return the user's own stats.\n\n2. **Our internal billing service**, a trusted backend with no signed-in user.\n It authenticates with the project's secret (service-role) key in the `apikey`\n header, and names the target user with a `user_id` in the JSON request body.\n It needs that user's stats.\n\nReturn the matching rows as JSON. The endpoint must be secure: only ever serve\nstats to a caller that is genuinely entitled to them, and turn away callers that\nare not.", - "promptSourcePath": "evals/build-functions-005-dual-auth-user-secret/PROMPT.md", + "prompt": "Can you audit the tenant isolation on our tables? Write some database tests covering both the happy path and the negative case, run them, and share the results.", + "promptSourcePath": "evals/build-tests-001-rls-tenant-isolation/PROMPT.md", "attempts": 1, - "sourcePath": "opencode-kimi-k3/build-functions-005-dual-auth-user-secret.json" + "sourcePath": "codex-gpt-5.6/build-tests-001-rls-tenant-isolation.json" }, { - "experiment": "opencode-kimi-k3", + "experiment": "codex-gpt-5.6", "experimentSuite": "benchmark", "experimentDisplay": { - "agent": "opencode", - "modelProvider": "moonshotai", - "modelId": "moonshotai/kimi-k3" + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.6-sol", + "reasoningEffort": "low" }, - "eval": "build-storage-001-private-bucket-access", + "eval": "build-vectors-001-rag-with-permissions", "stage": "build", "product": [ - "storage", - "database" + "database", + "vectors" ], "topic": [ - "rls", - "sdk" + "sql", + "rls" ], "suite": "benchmark", "interface": "mcp", "passed": true, "checks": [ { - "name": "bucket user-files exists", - "passed": true - }, - { - "name": "bucket user-files is private", - "passed": true + "name": "document_sections.embedding is vector(384)", + "passed": true, + "notes": "vector(384)" }, { - "name": "RLS still enabled on storage.objects", - "passed": true + "name": "HNSW index on the embedding column", + "passed": true, + "notes": "CREATE INDEX document_sections_embedding_hnsw_idx ON public.document_sections USING hnsw (embedding vector_ip_ops) WHERE (embedding IS NOT NULL)" }, { - "name": "user A lists only own files", + "name": "index operator class matches the search operator", "passed": true, - "notes": "saw: 019fb0ea-b2de-76fc-8fc8-c72ce1837e46/receipt-alpha.pdf, 019fb0ea-b2de-76fc-8fc8-c72ce1837e46/receipt-beta.pdf" + "notes": "function operators: <#>\nindexes: CREATE INDEX document_sections_embedding_hnsw_idx ON public.document_sections USING hnsw (embedding vector_ip_ops) WHERE (embedding IS NOT NULL)" }, { - "name": "user B cannot read user A files", + "name": "user A search returns only own sections, best match first", "passed": true }, { - "name": "anon reads no files", + "name": "user B search returns only own sections, best match first", "passed": true }, { - "name": "user A can upload into own folder", + "name": "user A reads only own sections through the API", "passed": true }, { - "name": "user B cannot upload into user A folder", + "name": "user A reads only own documents through the API", "passed": true - }, - { - "name": "configured private per-user storage access", - "passed": true, - "judgeNotes": "Meets all criteria: private user-files bucket, RLS left enabled, authenticated owner-scoped SELECT and INSERT policies using foldername/auth.uid with WITH CHECK, and supabase-js createSignedUrl with expiry. Does not use public bucket/getPublicUrl/service role client-side." } ], "skills": { @@ -16478,198 +10748,208 @@ }, "docs": { "calls": [ - { - "source": "web_fetch", - "query": "https://supabase.com/changelog.md", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 93411 - }, { "source": "search_docs", - "query": "{ searchDocs(query: \"storage access control bucket RLS policies owner folder\") { nodes { title href content } } }", + "query": "query { searchDocs(query: \"pgvector semantic search match_documents row level security function auth.uid HNSW\", limit: 8) { nodes { ... on Guide { title href content } ... on TroubleshootingGuide { title href content } } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/storage/security/access-control", - "title": "Storage Access Control" - }, - { - "url": "https://supabase.com/docs/guides/storage/buckets/fundamentals", - "title": "Storage Buckets" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs", - "title": "Build a User Management App with Next.js" - }, - { - "url": "https://supabase.com/docs/guides/security/product-security", - "title": "Secure configuration of Supabase products" - }, - { - "url": "https://supabase.com/docs/guides/platform/migrating-within-supabase/backup-restore", - "title": "Backup and Restore using the CLI" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3", - "title": "Configure S3 Storage" - }, - { - "url": "https://supabase.com/docs/guides/auth/third-party/clerk", - "title": "Clerk" - }, - { - "url": "https://supabase.com/docs/guides/resources/glossary", - "title": "Glossary" - }, - { - "url": "https://supabase.com/docs/guides/storage/security/ownership", - "title": "Ownership" - }, - { - "url": "https://supabase.com/docs/guides/storage/schema/design", - "title": "The Storage Schema" - }, - { - "url": "https://supabase.com/docs/guides/storage/schema/helper-functions", - "title": "Storage Helper Functions" + "url": "https://supabase.com/docs/guides/ai/rag-with-permissions", + "title": "RAG with Permissions" }, { - "url": "https://supabase.com/docs/guides/storage/schema/custom-roles", - "title": "Custom Roles" + "url": "https://supabase.com/docs/guides/ai/hybrid-search", + "title": "Hybrid search" }, { - "url": "https://supabase.com/docs/guides/local-development/database-migrations", - "title": "Database migrations" + "url": "https://supabase.com/docs/guides/ai/semantic-search", + "title": "Semantic search" }, { - "url": "https://supabase.com/docs/guides/storage/debugging/error-codes", - "title": "Error Codes" + "url": "https://supabase.com/docs/guides/ai/vector-indexes", + "title": "Vector indexes" }, { - "url": "https://supabase.com/docs/guides/storage/quickstart", - "title": "Storage Quickstart" + "url": "https://supabase.com/docs/guides/ai/vector-indexes/hnsw-indexes", + "title": "HNSW indexes" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/supabase-storage-inefficient-folder-operations-and-hierarchical-rls-challenges-b05a4d", - "title": "Supabase Storage: Inefficient folder operations and hierarchical RLS challenges" + "url": "https://supabase.com/docs/guides/ai/vector-indexes/ivf-indexes", + "title": "IVFFlat indexes" }, { - "url": "https://supabase.com/docs/guides/database/postgres/column-level-security", - "title": "Column Level Security" + "url": "https://supabase.com/docs/guides/ai/going-to-prod", + "title": "Going to Production" }, { - "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security", - "title": "Token Security and Row Level Security" - }, + "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", + "title": "Automatic embeddings" + } + ], + "resultChars": 97317 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Supabase.ai.Session gte-small embedding dimensions 384 vector\", limit: 5) { nodes { ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } } } }", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/guides/storage/production/scaling", - "title": "Storage Optimizations" + "url": "https://supabase.com/docs/guides/ai/choosing-compute-addon", + "title": "Choosing your Compute Add-on" }, { - "url": "https://supabase.com/docs/guides/api/securing-your-api", - "title": "Securing your API" + "url": "https://supabase.com/docs/guides/ai/concepts", + "title": "Concepts" }, { - "url": "https://supabase.com/docs/guides/realtime/getting_started", - "title": "Getting Started with Realtime" + "url": "https://supabase.com/docs/guides/functions/ai-models", + "title": "Running AI Models" }, { - "url": "https://supabase.com/docs/guides/storage/cdn/fundamentals", - "title": "Storage CDN" + "url": "https://supabase.com/docs/guides/ai/semantic-search", + "title": "Semantic search" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/why-cant-i-uploadlistetc-my-public-bucket-Z6CmGt", - "title": "Why can't I upload/list/etc my public bucket?" - }, + "url": "https://supabase.com/docs/guides/ai", + "title": "AI & Vectors" + } + ], + "resultChars": 63365 + } + ] + }, + "prompt": "We're adding semantic search to our internal knowledge base app. I already wrote the edge functions, but search doesn't work yet. Can you set up whatever the database needs to make search work end to end?\n\nSome documents are confidential, so users should only have access to documents they own.", + "promptSourcePath": "evals/build-vectors-001-rag-with-permissions/PROMPT.md", + "attempts": 1, + "sourcePath": "codex-gpt-5.6/build-vectors-001-rag-with-permissions.json" + }, + { + "experiment": "codex-gpt-5.6", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.6-sol", + "reasoningEffort": "low" + }, + "eval": "deploy-database-001-prometheus-metrics", + "stage": "deploy", + "product": [ + "database" + ], + "topic": [ + "observability" + ], + "suite": "benchmark", + "passed": true, + "checks": [ + { + "name": "preserved existing app scrape job", + "passed": true + }, + { + "name": "configured the Supabase Metrics API scrape correctly", + "passed": true, + "judgeNotes": "Supabase scrape uses HTTPS, correct metrics_path, basic_auth with password_file, preserves the app scrape, targets the Supabase project host via templated project ref, and docker-compose wires the password file as a Compose secret at /run/secrets/supabase_secret_key." + }, + { + "name": "documented live deployment and verification steps", + "passed": true, + "judgeNotes": "README includes live deployment steps with SUPABASE_PROJECT_REF, creating/storing a Supabase Secret API key in the matching Compose secret file, and starting/recreating the Compose stack. It also provides concrete verification via Prometheus targets and an up{job=\"supabase\"} PromQL API query." + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"project metrics Prometheus endpoint authentication service_role metrics API\", limit: 8) { nodes { title href content } } }", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/guides/storage/management/delete-objects", - "title": "Delete Objects" + "url": "https://supabase.com/docs/guides/telemetry/metrics/grafana-self-hosted", + "title": "Metrics API with Prometheus & Grafana (self-hosted)" }, { - "url": "https://supabase.com/docs/guides/local-development/declarative-database-schemas", - "title": "Declarative database schemas" + "url": "https://supabase.com/docs/guides/telemetry/metrics/vendor-agnostic", + "title": "Vendor-agnostic Metrics API setup" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/storage-error-403-forbidden-new-row-violates-row-level-security-policy-on-upload-a94384", - "title": "Storage error: 403 Forbidden: 'new row violates row-level security policy' on upload" + "url": "https://supabase.com/docs/guides/telemetry/metrics/grafana-cloud", + "title": "Metrics API with Grafana Cloud" }, { - "url": "https://supabase.com/docs/guides/storage/s3/authentication", - "title": "S3 Authentication" + "url": "https://supabase.com/docs/guides/telemetry/metrics", + "title": "Metrics API" }, { - "url": "https://supabase.com/docs/guides/ai-tools/mcp", - "title": "Supabase MCP Server" + "url": "https://supabase.com/docs/guides/database/extensions/pgaudit", + "title": "PGAudit: Postgres Auditing" }, { - "url": "https://supabase.com/docs/guides/realtime/authorization", - "title": "Realtime Authorization" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", + "title": "Envoy API Gateway" }, { - "url": "https://supabase.com/docs/guides/getting-started/features", - "title": "Features" - } - ], - "resultChars": 284508 - }, - { - "source": "web_fetch", - "query": "https://supabase.com/docs/guides/storage/security/access-control.md", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/storage/security/access-control.md" + "url": "https://supabase.com/docs/guides/database/postgres/roles", + "title": "Postgres Roles" } ], - "resultChars": 4169 + "resultChars": 71491 } ] }, - "prompt": "Our app lets signed-in users keep personal files like receipts and bank\nstatements. These files are private — a user must only ever be able to upload\nand download their own. The app uploads each file under a path that starts\nwith the owner's user id, e.g. `/receipt-march.pdf`.\n\nSet up a `user-files` bucket on our project and lock it down that way.\n\nUsers also sometimes share one of their files with someone else through a\ntemporary link that expires. Include the supabase-js code the app should use\nfor that.", - "promptSourcePath": "evals/build-storage-001-private-bucket-access/PROMPT.md", - "attempts": 1, - "sourcePath": "opencode-kimi-k3/build-storage-001-private-bucket-access.json" + "prompt": "Can you wire my Supabase project metrics into our existing observability stack and document\nwhat we need to do to make the config live?", + "promptSourcePath": "evals/deploy-database-001-prometheus-metrics/PROMPT.md", + "attempts": 2, + "sourcePath": "codex-gpt-5.6/deploy-database-001-prometheus-metrics.json" }, { - "experiment": "opencode-kimi-k3", + "experiment": "codex-gpt-5.6", "experimentSuite": "benchmark", "experimentDisplay": { - "agent": "opencode", - "modelProvider": "moonshotai", - "modelId": "moonshotai/kimi-k3" + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.6-sol", + "reasoningEffort": "low" }, - "eval": "build-tests-001-rls-tenant-isolation", - "stage": "build", + "eval": "deploy-functions-001-edge-function-secrets", + "stage": "deploy", "product": [ - "database" + "edge-functions" ], "topic": [ - "tests", - "rls" + "security" ], "suite": "benchmark", "interface": "cli", "passed": true, "checks": [ { - "name": "pgTAP test file(s) written under supabase/tests/", - "passed": true, - "notes": "1 file(s): supabase/tests/rls_tenant_isolation_test.sql" + "name": "WEATHER_API_KEY is set as a Function secret on the project", + "passed": true }, { - "name": "pgTAP isolation tests ran and pass", + "name": "the weather function is deployed to the project", "passed": true, - "notes": "9 passed, 6 failed" + "notes": "status ACTIVE" }, { - "name": "agent correctly identifies the posts isolation bug from test results", + "name": "the weather function reads WEATHER_API_KEY from the environment", "passed": true, - "judgeNotes": "The agent correctly identifies `posts` as having the broken tenant isolation policy, specifically that authenticated members can read posts from organizations they are not members of, and grounds this in pgTAP failures (#6, #12). It treats the test results as authoritative and distinguishes `notes` as correctly denying cross-org reads, while also noting other issues." + "judgeNotes": "The function reads WEATHER_API_KEY from the runtime environment using Deno.env.get(\"WEATHER_API_KEY\")." + }, + { + "name": "WEATHER_API_KEY value is not committed to the repo", + "passed": true } ], "skills": { @@ -16682,64 +10962,79 @@ ] }, "docs": { - "calls": [] + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Edge Functions deploy secrets environment variables Deno.env.get WEATHER_API_KEY verify_jwt CORS\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/troubleshooting/inspecting-edge-function-environment-variables-wg5qOQ", + "title": "Inspecting edge function environment variables" + }, + { + "url": "https://supabase.com/docs/guides/functions", + "title": "Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" + }, + { + "url": "https://supabase.com/docs/guides/functions/secrets", + "title": "Environment Variables" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/features", + "title": "Features" + } + ], + "resultChars": 48176 + } + ] }, - "prompt": "Can you audit the tenant isolation on our tables? Write some database tests covering both the happy path and the negative case, run them, and share the results.", - "promptSourcePath": "evals/build-tests-001-rls-tenant-isolation/PROMPT.md", + "prompt": "Our weather widget currently calls WeatherAPI straight from the browser, which\nleaks our API key. I want to move that behind a Supabase Edge Function called\n`weather` that holds the key server-side and proxies the request.\n\nThe function should read the key from an environment variable named\n`WEATHER_API_KEY`. Our key already lives in a local `.env` file at the project\nroot.\n\nDeploy the function to our project so it's live, and make sure the deployed\nfunction can actually read the key at runtime.", + "promptSourcePath": "evals/deploy-functions-001-edge-function-secrets/PROMPT.md", "attempts": 1, - "sourcePath": "opencode-kimi-k3/build-tests-001-rls-tenant-isolation.json" + "sourcePath": "codex-gpt-5.6/deploy-functions-001-edge-function-secrets.json" }, { - "experiment": "opencode-kimi-k3", + "experiment": "codex-gpt-5.6", "experimentSuite": "benchmark", "experimentDisplay": { - "agent": "opencode", - "modelProvider": "moonshotai", - "modelId": "moonshotai/kimi-k3" + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.6-sol", + "reasoningEffort": "low" }, - "eval": "build-vectors-001-rag-with-permissions", - "stage": "build", + "eval": "deploy-self-hosting-001-docker-compose", + "stage": "deploy", "product": [ "database", - "vectors" + "auth", + "storage" ], "topic": [ - "sql", - "rls" + "self-hosting" ], "suite": "benchmark", - "interface": "mcp", + "interface": "cli", "passed": true, "checks": [ { - "name": "document_sections.embedding is vector(384)", - "passed": true, - "notes": "vector(384)" - }, - { - "name": "HNSW index on the embedding column", - "passed": true, - "notes": "CREATE INDEX document_sections_embedding_idx ON public.document_sections USING hnsw (embedding vector_cosine_ops)" - }, - { - "name": "index operator class matches the search operator", - "passed": true, - "notes": "function operators: <=>\nindexes: CREATE INDEX document_sections_embedding_idx ON public.document_sections USING hnsw (embedding vector_cosine_ops)" - }, - { - "name": "user A search returns only own sections, best match first", + "name": "cloned the self-host stack (docker-compose.yml + volumes/db)", "passed": true }, { - "name": "user B search returns only own sections, best match first", + "name": "didn't conflate with the CLI (no supabase/config.toml in the stack)", "passed": true }, { - "name": "user A reads only own sections through the API", + "name": "secrets rotated off the shipped defaults", "passed": true }, { - "name": "user A reads only own documents through the API", + "name": "ANON_KEY and SERVICE_ROLE_KEY are HS256 JWTs signed by JWT_SECRET", "passed": true } ], @@ -16749,89 +11044,133 @@ "supabase-postgres-best-practices" ], "loaded": [ - "supabase" + "supabase-postgres-best-practices" ] }, "docs": { "calls": [ { - "source": "web_fetch", - "query": "https://supabase.com/changelog.md", + "source": "search_docs", + "query": "query { searchDocs(query: \"self-hosting Docker compose production generate API keys JWT secrets docker .env\", limit: 8) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/changelog.md" + "url": "https://supabase.com/docs/guides/self-hosting/docker", + "title": "Self-Hosting with Docker" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth", + "title": "Configure Social Login (OAuth) Providers" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", + "title": "New API Keys and Asymmetric Authentication" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin", + "title": "Build a Product Management Android App with Jetpack Compose" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", + "title": "Self-Hosted Functions" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", + "title": "Envoy API Gateway" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/custom-email-templates", + "title": "Custom Email Templates" } ], - "resultChars": 93411 + "resultChars": 165420 }, { "source": "search_docs", - "query": "{ searchDocs(query: \"semantic search pgvector match documents function embeddings\", limit: 5) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"self-hosted Envoy default API gateway July 2026 API_EXTERNAL_URL auth v1 Docker configuration\", limit: 6) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/ai/semantic-search", - "title": "Semantic search" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", + "title": "Envoy API Gateway" }, { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", - "title": "Automatic embeddings" + "url": "https://supabase.com/docs/guides/self-hosting/docker", + "title": "Self-Hosting with Docker" }, { - "url": "https://supabase.com/docs/guides/ai/hybrid-search", - "title": "Hybrid search" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-proxy-https", + "title": "Configure Reverse Proxy and HTTPS" }, { - "url": "https://supabase.com/docs/guides/ai/vector-columns", - "title": "Vector columns" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth", + "title": "Configure Social Login (OAuth) Providers" }, { - "url": "https://supabase.com/docs/guides/functions/examples/semantic-search", - "title": "Semantic Search" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-saml-sso", + "title": "Configure SAML SSO" } ], - "resultChars": 71630 + "resultChars": 115177 } ] }, - "prompt": "We're adding semantic search to our internal knowledge base app. I already wrote the edge functions, but search doesn't work yet. Can you set up whatever the database needs to make search work end to end?\n\nSome documents are confidential, so users should only have access to documents they own.", - "promptSourcePath": "evals/build-vectors-001-rag-with-permissions/PROMPT.md", - "attempts": 2, - "sourcePath": "opencode-kimi-k3/build-vectors-001-rag-with-permissions.json" + "prompt": "I'm moving off the hosted Supabase and running the whole thing myself on a VPS I\njust spun up. Can you get a Docker setup ready for me to copy onto the box?\n\nI don't need it running here, I'll do the actual bring-up once I'm on the\nserver. I just want everything in place and the secrets set up. Put it in a `supabase-docker/`\nfolder at the repo root so I can scp the whole thing across in one go.", + "promptSourcePath": "evals/deploy-self-hosting-001-docker-compose/PROMPT.md", + "attempts": 1, + "sourcePath": "codex-gpt-5.6/deploy-self-hosting-001-docker-compose.json" }, { - "experiment": "opencode-kimi-k3", + "experiment": "codex-gpt-5.6", "experimentSuite": "benchmark", "experimentDisplay": { - "agent": "opencode", - "modelProvider": "moonshotai", - "modelId": "moonshotai/kimi-k3" + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.6-sol", + "reasoningEffort": "low" }, - "eval": "deploy-database-001-prometheus-metrics", - "stage": "deploy", + "eval": "investigate-auth-001-deleted-user-access", + "stage": "investigate", "product": [ - "database" + "auth" ], "topic": [ - "observability" + "security", + "sdk" ], "suite": "benchmark", + "interface": "mcp", "passed": true, "checks": [ { - "name": "preserved existing app scrape job", + "name": "victim session active before delete-account", "passed": true }, { - "name": "configured the Supabase Metrics API scrape correctly", + "name": "delete_account flow ran for the victim", + "passed": true + }, + { + "name": "delete-account revokes the user's sessions", "passed": true, - "judgeNotes": "Meets all requirements: HTTPS Supabase Metrics API scrape for the project target, correct metrics path, Basic Auth with password_file, app scrape preserved, and docker-compose mounts the secrets directory containing the password file." + "notes": "sessions left: 0" }, { - "name": "documented live deployment and verification steps", + "name": "deleted user's refresh token is rejected", + "passed": true + }, + { + "name": "deleted user cannot sign back in", + "passed": true + }, + { + "name": "other users keep their sessions and access", + "passed": true + }, + { + "name": "diagnosed and explained session revocation", "passed": true, - "judgeNotes": "README includes concrete steps to obtain a Secret API key, place it in the mounted secret file, reload/restart the Compose stack, and verify via Prometheus targets plus a Grafana/PromQL query." + "judgeNotes": "The answer identifies the soft-delete-only root cause, implements real auth user deletion with cascade/revocation of sessions/refresh tokens, adds session-aware RLS to close the Data API stale-JWT window for the affected tables, and explains the remaining stateless JWT validity caveat consistently. It also correctly distinguishes publishable frontend keys from secret server-only keys and notes secret bypasses RLS." } ], "skills": { @@ -16847,96 +11186,163 @@ "calls": [ { "source": "search_docs", - "query": "{ searchDocs(query: \"metrics endpoint prometheus scrape project metrics\") { nodes { title href content } } }", + "query": "query { searchDocs(query: \"delete user existing access token session invalidate JWT sign out global auth sessions\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/telemetry/metrics/grafana-self-hosted", - "title": "Metrics API with Prometheus & Grafana (self-hosted)" + "url": "https://supabase.com/docs/guides/auth/sessions", + "title": "User sessions" }, { - "url": "https://supabase.com/docs/guides/telemetry/metrics", - "title": "Metrics API" + "url": "https://supabase.com/docs/reference/javascript/auth-signout" }, { - "url": "https://supabase.com/docs/guides/telemetry/metrics/vendor-agnostic", - "title": "Vendor-agnostic Metrics API setup" + "url": "https://supabase.com/docs/guides/auth/signout", + "title": "Signing out" }, { - "url": "https://supabase.com/docs/guides/telemetry/metrics/grafana-cloud", - "title": "Metrics API with Grafana Cloud" + "url": "https://supabase.com/docs/guides/auth/managing-user-data", + "title": "User Management" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/how-to-view-database-metrics-uqf2z_", - "title": "How to View Database Metrics" + "url": "https://supabase.com/docs/guides/resources/glossary", + "title": "Glossary" + } + ], + "resultChars": 33042 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"publishable key secret key legacy anon service_role RLS behavior bypass\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/storage/security/access-control", + "title": "Storage Access Control" }, { - "url": "https://supabase.com/docs/guides/platform/read-replicas", - "title": "Read Replicas" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", + "title": "Row Level Security" }, { - "url": "https://supabase.com/docs/guides/database/connection-management", - "title": "Connection management" + "url": "https://supabase.com/docs/guides/auth/signing-keys", + "title": "JWT Signing Keys" }, { - "url": "https://supabase.com/docs/guides/realtime/benchmarks", - "title": "Benchmarks" + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/api-keys", + "title": "Understanding API keys" + } + ], + "resultChars": 91501 + }, + { + "source": "web_search", + "query": "https://supabase.com/changelog.md", + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ] + }, + { + "source": "web_search", + "query": "site:supabase.com/changelog Supabase changelog API keys publishable secret auth sessions delete user JWT", + "pages": [] + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"delete user auth.users SQL deleteUser access token remains valid until expiry\", limit: 6) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/auth/managing-user-data", + "title": "User Management" + }, + { + "url": "https://supabase.com/docs/guides/auth/auth-anonymous", + "title": "Anonymous Sign-Ins" + }, + { + "url": "https://supabase.com/docs/guides/auth/sessions", + "title": "User sessions" + }, + { + "url": "https://supabase.com/docs/guides/platform/temporary-access", + "title": "Temporary access" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/grafana-not-displaying-data-sXJrMj", - "title": "Grafana not displaying data" + "url": "https://supabase.com/docs/guides/platform/sso/multiple-providers", + "title": "Multiple SSO Providers" }, { - "url": "https://supabase.com/docs/guides/telemetry/reports", - "title": "Reports" + "url": "https://supabase.com/docs/reference/javascript/auth-admin-deleteuser" } ], - "resultChars": 81235 + "resultChars": 59952 + }, + { + "source": "web_search", + "query": "site:supabase.com/docs delete user access token remains valid until expiry auth sessions session_id", + "pages": [] } ] }, - "prompt": "Can you wire my Supabase project metrics into our existing observability stack and document\nin the observability README what we need to do to make the config live?", - "promptSourcePath": "evals/deploy-database-001-prometheus-metrics/PROMPT.md", - "attempts": 1, - "sourcePath": "opencode-kimi-k3/deploy-database-001-prometheus-metrics.json" + "prompt": "Last week support removed a user through our app's delete-account flow — the\napp calls the `delete_account` function over RPC as the signed-in user. This\nmorning that same person was back: still signed in, reading and saving their\ndata like nothing happened.\n\nFigure out why the account still works, fix the flow so a deleted account\nloses access, and tell me whether there is any window where they could still\nget in after the fix.\n\nOne more thing while you're at it: we're migrating off the legacy\nanon/service_role API keys to the new publishable and secret keys, and the\nteam isn't sure which one belongs in the frontend and what each means for\nRLS. Give us a quick rundown.", + "promptSourcePath": "evals/investigate-auth-001-deleted-user-access/PROMPT.md", + "attempts": 2, + "sourcePath": "codex-gpt-5.6/investigate-auth-001-deleted-user-access.json" }, { - "experiment": "opencode-kimi-k3", + "experiment": "codex-gpt-5.6", "experimentSuite": "benchmark", "experimentDisplay": { - "agent": "opencode", - "modelProvider": "moonshotai", - "modelId": "moonshotai/kimi-k3" + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.6-sol", + "reasoningEffort": "low" }, - "eval": "deploy-functions-001-edge-function-secrets", - "stage": "deploy", + "eval": "investigate-realtime-001-subscribed-no-events", + "stage": "investigate", "product": [ - "edge-functions" + "realtime", + "database" ], "topic": [ - "security" + "sdk" ], "suite": "benchmark", - "interface": "cli", + "interface": "mcp", "passed": true, "checks": [ { - "name": "WEATHER_API_KEY is set as a Function secret on the project", + "name": "orders table added to supabase_realtime publication", "passed": true }, { - "name": "the weather function is deployed to the project", - "passed": true, - "notes": "status ACTIVE" + "name": "courier_locations still in supabase_realtime publication", + "passed": true }, { - "name": "the weather function reads WEATHER_API_KEY from the environment", - "passed": true, - "judgeNotes": "The function reads WEATHER_API_KEY from the runtime environment using Deno.env.get(\"WEATHER_API_KEY\")." + "name": "publication still publishes INSERT events", + "passed": true }, { - "name": "WEATHER_API_KEY value is not committed to the repo", + "name": "RLS still enabled on orders", "passed": true + }, + { + "name": "staff can still read orders through RLS", + "passed": true, + "notes": "authenticated sees 2 of 2 orders" + }, + { + "name": "diagnosed missing publication membership", + "passed": true, + "judgeNotes": "The assistant correctly identified that the channel could be SUBSCRIBED while orders INSERT events were silent because public.orders was absent from the supabase_realtime publication. It fixed exactly that with ALTER PUBLICATION supabase_realtime ADD TABLE public.orders, verified courier_locations remained in the publication, and did not disable RLS or weaken policies." } ], "skills": { @@ -16952,68 +11358,74 @@ "calls": [ { "source": "search_docs", - "query": "{ searchDocs(query: \"edge function secrets environment variables supabase secrets set env file\", limit: 3) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"Postgres Changes enable table supabase_realtime publication add table realtime replication\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/troubleshooting/inspecting-edge-function-environment-variables-wg5qOQ", - "title": "Inspecting edge function environment variables" + "url": "https://supabase.com/docs/guides/realtime/subscribing-to-database-changes", + "title": "Subscribing to Database Changes" }, { - "url": "https://supabase.com/docs/guides/functions/secrets", - "title": "Environment Variables" + "url": "https://supabase.com/docs/guides/platform/migrating-within-supabase/backup-restore", + "title": "Backup and Restore using the CLI" }, { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech", - "title": "Transcription Telegram Bot" + "url": "https://supabase.com/docs/guides/realtime/benchmarks", + "title": "Benchmarks" + }, + { + "url": "https://supabase.com/docs/guides/database/replication/pipelines", + "title": "Set up Pipelines" + }, + { + "url": "https://supabase.com/docs/guides/realtime/postgres-changes", + "title": "Postgres Changes" } ], - "resultChars": 18708 + "resultChars": 121815 } ] }, - "prompt": "Our weather widget currently calls WeatherAPI straight from the browser, which\nleaks our API key. I want to move that behind a Supabase Edge Function called\n`weather` that holds the key server-side and proxies the request.\n\nThe function should read the key from an environment variable named\n`WEATHER_API_KEY`. Our key already lives in a local `.env` file at the project\nroot.\n\nDeploy the function to our project so it's live, and make sure the deployed\nfunction can actually read the key at runtime.", - "promptSourcePath": "evals/deploy-functions-001-edge-function-secrets/PROMPT.md", + "prompt": "Our dispatch dashboard shows incoming orders as they happen. The courier\nlocation feed on the same page updates live without problems, but new orders\nonly show up after a page refresh.\n\nThe dashboard uses supabase-js to subscribe to INSERT events on the `orders`\ntable through postgres_changes, the same way it subscribes to courier\nlocations. The channel's status callback logs SUBSCRIBED and there are no\nerrors in the browser console.\n\nFigure out why no order events ever arrive and fix it.", + "promptSourcePath": "evals/investigate-realtime-001-subscribed-no-events/PROMPT.md", "attempts": 1, - "sourcePath": "opencode-kimi-k3/deploy-functions-001-edge-function-secrets.json" + "sourcePath": "codex-gpt-5.6/investigate-realtime-001-subscribed-no-events.json" }, { - "experiment": "opencode-kimi-k3", + "experiment": "codex-gpt-5.6", "experimentSuite": "benchmark", "experimentDisplay": { - "agent": "opencode", - "modelProvider": "moonshotai", - "modelId": "moonshotai/kimi-k3" + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.6-sol", + "reasoningEffort": "low" }, - "eval": "deploy-self-hosting-001-docker-compose", - "stage": "deploy", + "eval": "investigate-reliability-003-edge-function-5xx-correlation", + "stage": "investigate", "product": [ - "database", - "auth", - "storage" + "edge-functions" ], "topic": [ - "self-hosting" + "observability" ], "suite": "benchmark", - "interface": "cli", + "interface": "mcp", "passed": true, "checks": [ { - "name": "cloned the self-host stack (docker-compose.yml + volumes/db)", - "passed": true - }, - { - "name": "didn't conflate with the CLI (no supabase/config.toml in the stack)", - "passed": true + "name": "identified image-transform and the recurring 503 pattern", + "passed": true, + "judgeNotes": "The assistant explicitly identified `image-transform` as the affected function and described 8 intermittent/recurring HTTP 503 responses during the morning of 2026-04-28, spanning roughly 06:00/07:00Z to 12:00Z. It did not misattribute the issue to billing-webhook or provide only a vague error description." }, { - "name": "secrets rotated off the shipped defaults", - "passed": true + "name": "attributed recurring 503s to gateway/platform layer, not function code", + "passed": true, + "judgeNotes": "Attributes the recurring 503s to the Edge Function gateway/platform layer, grounded in gateway-only logs with no function execution rows, nearby successful invocations, stable deployment, and distinction from avatar-upload's separate function-level 500." }, { - "name": "ANON_KEY and SERVICE_ROLE_KEY are HS256 JWTs signed by JWT_SECRET", - "passed": true + "name": "recommended a concrete next step", + "passed": true, + "judgeNotes": "The assistant recommended concrete next steps, including opening a Supabase support ticket with project, region, timestamps, and log IDs, plus including request IDs and response bodies. It also suggested actionable remediation like retry backoff and structured logging." } ], "skills": { @@ -17028,49 +11440,63 @@ "docs": { "calls": [ { - "source": "web_fetch", - "query": "https://supabase.com/changelog.md", + "source": "search_docs", + "query": "query { searchDocs(query: \"Edge Functions 503 gateway error function not invoked logs troubleshooting\", limit: 5) { nodes { ... on Guide { title href content } ... on TroubleshootingGuide { title href content } } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/changelog.md" + "url": "https://supabase.com/docs/guides/troubleshooting/edge-function-503-response", + "title": "Edge Function 503 error response" + }, + { + "url": "https://supabase.com/docs/guides/functions/status-codes", + "title": "Status codes" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/edge-function-504-error-response", + "title": "Edge Function 504 error response" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/edge-function-404-error-response", + "title": "Edge Function 404 error response" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/edge-function-500-error-response", + "title": "Edge Function 500 error response" } ], - "resultChars": 93411 + "resultChars": 40011 }, { - "source": "web_fetch", - "query": "https://supabase.com/docs/guides/self-hosting/docker.md", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/self-hosting/docker.md" - } - ], - "resultChars": 30212 + "source": "web_search", + "query": "site:status.supabase.com April 28 2026 Edge Functions incident 503", + "pages": [] } ] }, - "prompt": "I'm moving off the hosted Supabase and running the whole thing myself on a VPS I\njust spun up. Can you get a Docker setup ready for me to copy onto the box?\n\nI don't need it running here, I'll do the actual bring-up once I'm on the\nserver. I just want everything in place and the secrets set up. Put it in a `supabase-docker/`\nfolder at the repo root so I can scp the whole thing across in one go.", - "promptSourcePath": "evals/deploy-self-hosting-001-docker-compose/PROMPT.md", - "attempts": 1, - "sourcePath": "opencode-kimi-k3/deploy-self-hosting-001-docker-compose.json" + "prompt": "Users have been reporting that image uploads are intermittently failing with server errors throughout the morning of `2026-04-28`.\n\nCan you investigate the project logs and tell me what is going on and what we\nshould do next?", + "promptSourcePath": "evals/investigate-reliability-003-edge-function-5xx-correlation/PROMPT.md", + "attempts": 2, + "sourcePath": "codex-gpt-5.6/investigate-reliability-003-edge-function-5xx-correlation.json" }, { - "experiment": "opencode-kimi-k3", + "experiment": "codex-gpt-5.6", "experimentSuite": "benchmark", "experimentDisplay": { - "agent": "opencode", - "modelProvider": "moonshotai", - "modelId": "moonshotai/kimi-k3" + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.6-sol", + "reasoningEffort": "low" }, - "eval": "investigate-auth-001-deleted-user-access", - "stage": "investigate", + "eval": "resolve-dataapi-001-empty-results", + "stage": "resolve", "product": [ + "data-api", + "database", "auth" ], "topic": [ - "security", + "rls", "sdk" ], "suite": "benchmark", @@ -17078,34 +11504,33 @@ "passed": true, "checks": [ { - "name": "victim session active before delete-account", + "name": "RLS still enabled on bookmarks", "passed": true }, { - "name": "delete_account flow ran for the victim", + "name": "user A reads own bookmarks", "passed": true }, { - "name": "delete-account revokes the user's sessions", - "passed": true, - "notes": "sessions left: 0" + "name": "user B cannot read user A bookmarks", + "passed": true }, { - "name": "deleted user's refresh token is rejected", + "name": "anon reads no bookmarks", "passed": true }, { - "name": "deleted user cannot sign back in", + "name": "user A can save a new bookmark", "passed": true }, { - "name": "other users keep their sessions and access", + "name": "user B cannot insert a bookmark as user A", "passed": true }, { - "name": "diagnosed and explained session revocation", + "name": "diagnosed RLS and added owner-scoped policies", "passed": true, - "judgeNotes": "Meets rubric: identifies soft-delete root cause, implements real auth/session/user deletion with refresh token revocation, correctly explains stateless JWT residual window consistent with remaining RLS behavior and mitigation, and accurately distinguishes publishable frontend/RLS-enforced keys from secret server-only/RLS-bypassing keys." + "judgeNotes": "Diagnosed RLS deny-all due to no policies, kept RLS enabled, and created authenticated SELECT/INSERT policies scoped to user_id = auth.uid() with WITH CHECK for inserts." } ], "skills": { @@ -17121,85 +11546,84 @@ "calls": [ { "source": "search_docs", - "query": "{ searchDocs(query: \"publishable secret API keys anon service_role migration RLS\", limit: 5) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"Row Level Security auth.uid SELECT INSERT policy Data API table grants authenticated\", limit: 5) { nodes { ... on Guide { title href content } } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-redwoodjs", - "title": "Build a User Management App with RedwoodJS" + "url": "https://supabase.com/docs/guides/api/securing-your-api", + "title": "Securing your API" }, { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", + "title": "Row Level Security" }, { - "url": "https://supabase.com/docs/guides/auth/quickstarts/with-expo-react-native-social-auth", - "title": "Build a Social Auth App with Expo React Native" + "url": "https://supabase.com/docs/guides/auth/auth-mfa", + "title": "Multi-Factor Authentication" }, { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-sveltekit", - "title": "Build a User Management App with SvelteKit" + "url": "https://supabase.com/docs/guides/local-development/testing/pgtap-extended", + "title": "Advanced pgTAP Testing" }, { - "url": "https://supabase.com/docs/guides/auth/signing-keys", - "title": "JWT Signing Keys" + "url": "https://supabase.com/docs/guides/getting-started/features", + "title": "Features" } ], - "resultChars": 169611 + "resultChars": 104465 } ] }, - "prompt": "Last week support removed a user through our app's delete-account flow — the\napp calls the `delete_account` function over RPC as the signed-in user. This\nmorning that same person was back: still signed in, reading and saving their\ndata like nothing happened.\n\nFigure out why the account still works, fix the flow so a deleted account\nloses access, and tell me whether there is any window where they could still\nget in after the fix.\n\nOne more thing while you're at it: we're migrating off the legacy\nanon/service_role API keys to the new publishable and secret keys, and the\nteam isn't sure which one belongs in the frontend and what each means for\nRLS. Give us a quick rundown.", - "promptSourcePath": "evals/investigate-auth-001-deleted-user-access/PROMPT.md", + "prompt": "Our app lets signed-in users save bookmarks and view them on their dashboard. Bookmarks are stored in the `bookmarks` table and are private — a user must only ever see their own. \nUsers also need to be able to save new bookmarks from the app.\n\nI can see the rows when I query the table directly, but the dashboard shows an empty list for every user.\n\nFind out why the Data API returns nothing and fix it.", + "promptSourcePath": "evals/resolve-dataapi-001-empty-results/PROMPT.md", "attempts": 1, - "sourcePath": "opencode-kimi-k3/investigate-auth-001-deleted-user-access.json" + "sourcePath": "codex-gpt-5.6/resolve-dataapi-001-empty-results.json" }, { - "experiment": "opencode-kimi-k3", + "experiment": "codex-gpt-5.6", "experimentSuite": "benchmark", "experimentDisplay": { - "agent": "opencode", - "modelProvider": "moonshotai", - "modelId": "moonshotai/kimi-k3" + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.6-sol", + "reasoningEffort": "low" }, - "eval": "investigate-realtime-001-subscribed-no-events", - "stage": "investigate", + "eval": "resolve-database-001-migration-history-mismatch", + "stage": "resolve", "product": [ - "realtime", "database" ], "topic": [ - "sdk" + "migrations" ], "suite": "benchmark", - "interface": "mcp", + "interface": "cli", "passed": true, "checks": [ { - "name": "orders table added to supabase_realtime publication", + "name": "the avatar_url column is applied on the hosted profiles table", "passed": true }, { - "name": "courier_locations still in supabase_realtime publication", + "name": "migration 20240220000000 is recorded in the remote history", "passed": true }, { - "name": "publication still publishes INSERT events", + "name": "remote migration history matches local migration files", "passed": true }, { - "name": "RLS still enabled on orders", + "name": "local migrations are a valid reconciled sequence", "passed": true }, { - "name": "staff can still read orders through RLS", - "passed": true, - "notes": "authenticated sees 2 of 2 orders" + "name": "production profile data is intact (not reset)", + "passed": true }, { - "name": "diagnosed missing publication membership", + "name": "the avatar migration and history reconciliation were done via the Supabase CLI", "passed": true, - "judgeNotes": "Diagnosed the silent subscription as orders missing from supabase_realtime despite SUBSCRIBED status, applied exactly ALTER PUBLICATION supabase_realtime ADD TABLE public.orders, preserved courier_locations and did not weaken RLS/policies." + "judgeNotes": "Applied via Supabase CLI command #11: `supabase db push --db-url \"$DB_URL\" --yes`, after which remote history shows `20240220000000 | add_avatar_url`. Reconciled history by adding the missing local migration file in #9: `supabase/migrations/20240115000000_add_profile_bio.sql`, allowing CLI migration state to align. No prohibited direct SQL mutation or prepared-statement workaround was used." } ], "skills": { @@ -17212,47 +11636,84 @@ ] }, "docs": { - "calls": [] + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Supabase CLI deploy migrations db push migration list repair remote hosted project\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/local-development/cli-workflows", + "title": "Local development workflow" + }, + { + "url": "https://supabase.com/docs/guides/local-development/database-migrations", + "title": "Database migrations" + }, + { + "url": "https://supabase.com/docs/reference/cli/supabase-db-push", + "title": "Push new migrations to the remote database" + }, + { + "url": "https://supabase.com/docs/reference/cli/supabase-migration-list", + "title": "List local and remote migrations" + }, + { + "url": "https://supabase.com/docs/guides/deployment/database-migrations", + "title": "Database Migrations" + } + ], + "resultChars": 58436 + } + ] }, - "prompt": "Our dispatch dashboard shows incoming orders as they happen. The courier\nlocation feed on the same page updates live without problems, but new orders\nonly show up after a page refresh.\n\nThe dashboard uses supabase-js to subscribe to INSERT events on the `orders`\ntable through postgres_changes, the same way it subscribes to courier\nlocations. The channel's status callback logs SUBSCRIBED and there are no\nerrors in the browser console.\n\nFigure out why no order events ever arrive and fix it.", - "promptSourcePath": "evals/investigate-realtime-001-subscribed-no-events/PROMPT.md", - "attempts": 1, - "sourcePath": "opencode-kimi-k3/investigate-realtime-001-subscribed-no-events.json" + "prompt": "I'm trying to ship a migration to our hosted project and it's not working. Can you figure out what's wrong and get it deployed?", + "promptSourcePath": "evals/resolve-database-001-migration-history-mismatch/PROMPT.md", + "attempts": 2, + "sourcePath": "codex-gpt-5.6/resolve-database-001-migration-history-mismatch.json" }, { - "experiment": "opencode-kimi-k3", + "experiment": "codex-gpt-5.6", "experimentSuite": "benchmark", "experimentDisplay": { - "agent": "opencode", - "modelProvider": "moonshotai", - "modelId": "moonshotai/kimi-k3" + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.6-sol", + "reasoningEffort": "low" }, - "eval": "investigate-reliability-003-edge-function-5xx-correlation", - "stage": "investigate", + "eval": "resolve-performance-001-slow-query-cpu-spike", + "stage": "resolve", "product": [ - "edge-functions" + "database" ], "topic": [ - "observability" + "observability", + "sql" ], "suite": "benchmark", "interface": "mcp", - "passed": false, + "passed": true, "checks": [ { - "name": "identified image-transform and the recurring 503 pattern", - "passed": true, - "judgeNotes": "Identified image-transform as the affected function and described the recurring HTTP 503 pattern across the morning of 2026-04-28, including most/all 8 gateway failures from 07:00Z to 12:00Z." + "name": "inspected pg_stat_statements for query performance", + "passed": true }, { - "name": "attributed recurring 503s to gateway/platform layer, not function code", - "passed": false, - "judgeNotes": "The response notes that 503s have only gateway logs and no function execution logs, but it ultimately attributes the incident to the function's unpinned npm dependency/module-load behavior and recommends pinning/redeploying the function as the direct fix. That violates the rubric requirement to attribute the recurring 503s to the gateway/platform layer rather than function code/runtime." + "name": "ran EXPLAIN on the expensive query", + "passed": true }, { - "name": "recommended a concrete next step", + "name": "created index covering user_id and created_at", + "passed": true + }, + { + "name": "query plan uses an index and avoids sequential scan", "passed": true, - "judgeNotes": "The assistant recommended concrete actionable next steps, including pinning a known-good dependency version and redeploying, checking package publish history, adding retries, and alerting on gateway-level 503s without executions." + "notes": "Limit (cost=79.74..79.86 rows=50 width=58)\n -> Sort (cost=79.74..79.99 rows=100 width=58)\n Sort Key: created_at DESC\n -> Bitmap Heap Scan on events (cost=5.06..76.42 rows=100 width=58)\n Recheck Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)\n -> Bitmap Index Scan on events_user_id_created_at_idx (cost=0.00..5.03 rows=100 width=0)\n Index Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)" + }, + { + "name": "inserts still work", + "passed": true } ], "skills": { @@ -17265,64 +11726,96 @@ ] }, "docs": { - "calls": [] + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"query optimization pg_stat_statements EXPLAIN indexes\", limit: 5) { nodes { ... on Guide { title href content } ... on TroubleshootingGuide { title href content } } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/database/inspect", + "title": "Debugging and monitoring" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/configuration", + "title": "Database configuration" + }, + { + "url": "https://supabase.com/docs/guides/database/query-optimization", + "title": "Query Optimization" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_stat_statements", + "title": "pg_stat_statements: Query Performance Monitoring" + } + ], + "resultChars": 27461 + } + ] }, - "prompt": "Users have been reporting that image uploads are intermittently failing with server errors throughout the morning of `2026-04-28`.\n\nCan you investigate the project logs and tell me what is going on and what we\nshould do next?", - "promptSourcePath": "evals/investigate-reliability-003-edge-function-5xx-correlation/PROMPT.md", - "attempts": 2, - "sourcePath": "opencode-kimi-k3/investigate-reliability-003-edge-function-5xx-correlation.json" + "prompt": "My database CPU keeps spiking and the app gets slow when loading recent events for a user. Can you figure out what query is causing it and make the database change needed to fix it?\n\nEnd your turn with a short summary of what you changed and why.", + "promptSourcePath": "evals/resolve-performance-001-slow-query-cpu-spike/PROMPT.md", + "attempts": 1, + "sourcePath": "codex-gpt-5.6/resolve-performance-001-slow-query-cpu-spike.json" }, { - "experiment": "opencode-kimi-k3", + "experiment": "codex-gpt-5.6", "experimentSuite": "benchmark", "experimentDisplay": { - "agent": "opencode", - "modelProvider": "moonshotai", - "modelId": "moonshotai/kimi-k3" + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.6-sol", + "reasoningEffort": "low" }, - "eval": "resolve-dataapi-001-empty-results", + "eval": "resolve-security-002-rls-cross-tenant-leak", "stage": "resolve", "product": [ - "data-api", "database", "auth" ], "topic": [ "rls", - "sdk" + "security" ], "suite": "benchmark", "interface": "mcp", "passed": true, "checks": [ { - "name": "RLS still enabled on bookmarks", + "name": "RLS enabled on notes", "passed": true }, { - "name": "user A reads own bookmarks", + "name": "tenant A sees only org A notes", "passed": true }, { - "name": "user B cannot read user A bookmarks", + "name": "tenant B cannot read org A notes", "passed": true }, { - "name": "anon reads no bookmarks", + "name": "tenant A author can update own note", "passed": true }, { - "name": "user A can save a new bookmark", + "name": "tenant B cannot update org A note", "passed": true }, { - "name": "user B cannot insert a bookmark as user A", + "name": "tenant B author can delete own note", "passed": true }, { - "name": "diagnosed RLS and added owner-scoped policies", - "passed": true, - "judgeNotes": "Diagnosed RLS deny-all due to no policies, kept RLS enabled, and added authenticated SELECT and INSERT policies scoped to user_id = auth.uid() with WITH CHECK for inserts." + "name": "tenant B cannot delete org A note", + "passed": true + }, + { + "name": "tenant A can insert note in own org", + "passed": true + }, + { + "name": "tenant B cannot insert into org A", + "passed": true } ], "skills": { @@ -17335,265 +11828,376 @@ ] }, "docs": { - "calls": [] + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"row level security workspace membership auth.uid exists policy multi tenant\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security", + "title": "Token Security and Row Level Security" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", + "title": "Row Level Security" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/features", + "title": "Features" + }, + { + "url": "https://supabase.com/docs/guides/realtime/getting_started", + "title": "Getting Started with Realtime" + }, + { + "url": "https://supabase.com/docs/guides/auth/auth-mfa", + "title": "Multi-Factor Authentication" + } + ], + "resultChars": 91253 + } + ] }, - "prompt": "Our app lets signed-in users save bookmarks and view them on their dashboard. Bookmarks are stored in the `bookmarks` table and are private — a user must only ever see their own. \nUsers also need to be able to save new bookmarks from the app.\n\nI can see the rows when I query the table directly, but the dashboard shows an empty list for every user.\n\nFind out why the Data API returns nothing and fix it.", - "promptSourcePath": "evals/resolve-dataapi-001-empty-results/PROMPT.md", + "prompt": "A customer reported that notes showed up in the wrong workspace.\n\nCan you investigate what is going on and fix it?", + "promptSourcePath": "evals/resolve-security-002-rls-cross-tenant-leak/PROMPT.md", "attempts": 1, - "sourcePath": "opencode-kimi-k3/resolve-dataapi-001-empty-results.json" + "sourcePath": "codex-gpt-5.6/resolve-security-002-rls-cross-tenant-leak.json" }, { - "experiment": "opencode-kimi-k3", - "experimentSuite": "benchmark", + "experiment": "codex-gpt-5.6-no-skills", + "experimentSuite": "no-skills", "experimentDisplay": { - "agent": "opencode", - "modelProvider": "moonshotai", - "modelId": "moonshotai/kimi-k3" + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.6-sol", + "reasoningEffort": "low" }, - "eval": "resolve-database-001-migration-history-mismatch", - "stage": "resolve", + "eval": "build-cli-001-bootstrap-app", + "stage": "build", "product": [ - "database" + "database", + "data-api" ], "topic": [ - "migrations" + "migrations", + "rls" ], "suite": "benchmark", "interface": "cli", "passed": true, "checks": [ { - "name": "the avatar_url column is applied on the hosted profiles table", + "name": "supabase project initialised (supabase/config.toml exists)", "passed": true }, { - "name": "migration 20240220000000 is recorded in the remote history", + "name": "todos table is created by a migration file", "passed": true }, { - "name": "remote migration history matches local migration files", - "passed": true + "name": "todos table exists with at least 2 seeded rows", + "passed": true, + "notes": "found 2 rows" }, { - "name": "local migrations are a valid reconciled sequence", + "name": "row level security is enabled on todos", "passed": true }, { - "name": "production profile data is intact (not reset)", + "name": "a SELECT policy targets the authenticated role", "passed": true }, { - "name": "the avatar migration and history reconciliation were done via the Supabase CLI", + "name": "REST API returns no todos to anonymous requests", + "passed": true, + "notes": "0 rows" + }, + { + "name": "REST API returns the todos to authenticated requests", "passed": true, - "judgeNotes": "Applied avatar_url with `supabase db push` (#19), which showed `Applying migration 20240220000000_add_avatar_url.sql...` and `Finished supabase db push.` Reconciled orphan bio migration by adding local file `20240115000000_add_profile_bio.sql` (#16), after which `supabase migration list` showed local/remote aligned (#17) and the push succeeded. No disallowed workaround or direct SQL mutation seen." + "notes": "2 rows" } ], "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" - ] + "available": [], + "loaded": [] }, "docs": { - "calls": [] + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Row Level Security authenticated users select policy anon denied migrations seed local development\", limit: 5) { nodes { ... on Guide { title href content } } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/getting-started/features", + "title": "Features" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", + "title": "Row Level Security" + }, + { + "url": "https://supabase.com/docs/guides/auth/auth-mfa", + "title": "Multi-Factor Authentication" + }, + { + "url": "https://supabase.com/docs/guides/auth/auth-anonymous", + "title": "Anonymous Sign-Ins" + }, + { + "url": "https://supabase.com/docs/guides/api/securing-your-api", + "title": "Securing your API" + } + ], + "resultChars": 94489 + } + ] }, - "prompt": "I'm trying to ship a migration to our hosted project and it's not working. Can you figure out what's wrong and get it deployed?", - "promptSourcePath": "evals/resolve-database-001-migration-history-mismatch/PROMPT.md", + "prompt": "We're kicking off a todos app and I want the Supabase side ready for the team\nto build on. Set it up the way we'd run it in development, with schema changes\ntracked as migrations so they can be reviewed and replayed.\n\nFor the first slice we just need a `todos` table. Todos aren't public: anyone\nsigned in can read all of them, but nothing should be writable through the API\nfor now. Add a couple of sample todos so there's something to look at.\n\nBefore you hand it back, make sure the running API actually behaves that way —\nsigned-in users get the todos, signed-out requests get nothing.", + "promptSourcePath": "evals/build-cli-001-bootstrap-app/PROMPT.md", "attempts": 1, - "sourcePath": "opencode-kimi-k3/resolve-database-001-migration-history-mismatch.json" + "sourcePath": "codex-gpt-5.6-no-skills/build-cli-001-bootstrap-app.json" }, { - "experiment": "opencode-kimi-k3", - "experimentSuite": "benchmark", + "experiment": "codex-gpt-5.6-no-skills", + "experimentSuite": "no-skills", "experimentDisplay": { - "agent": "opencode", - "modelProvider": "moonshotai", - "modelId": "moonshotai/kimi-k3" + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.6-sol", + "reasoningEffort": "low" }, - "eval": "resolve-performance-001-slow-query-cpu-spike", - "stage": "resolve", + "eval": "build-cli-002-declarative-schema", + "stage": "build", "product": [ "database" ], "topic": [ - "observability", - "sql" + "declarative-schema", + "migrations" ], "suite": "benchmark", - "interface": "mcp", - "passed": true, + "interface": "cli", + "passed": false, "checks": [ { - "name": "inspected pg_stat_statements for query performance", - "passed": true + "name": "supabase db diff used to generate the migration", + "passed": false }, { - "name": "ran EXPLAIN on the expensive query", + "name": "schema file updated to include description column", "passed": true }, { - "name": "created index covering user_id and created_at", + "name": "a new migration was generated for the change", "passed": true }, { - "name": "query plan uses an index and avoids sequential scan", - "passed": true, - "notes": "Limit (cost=55.55..55.61 rows=25 width=88)\n -> Sort (cost=55.55..55.61 rows=25 width=88)\n Sort Key: created_at DESC\n -> Bitmap Heap Scan on events (cost=4.48..54.97 rows=25 width=88)\n Recheck Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)\n -> Bitmap Index Scan on idx_events_user_created_at (cost=0.00..4.47 rows=25 width=0)\n Index Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)" - }, - { - "name": "inserts still work", + "name": "description column exists in the live database", "passed": true } ], "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] + "available": [], + "loaded": [] }, "docs": { - "calls": [] + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Supabase CLI local database migration add column table\", limit: 3) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/deployment/database-migrations", + "title": "Database Migrations" + }, + { + "url": "https://supabase.com/docs/guides/local-development/database-migrations", + "title": "Database migrations" + }, + { + "url": "https://supabase.com/docs/guides/local-development/cli-workflows", + "title": "Local development workflow" + } + ], + "resultChars": 54403 + } + ] }, - "prompt": "My database CPU keeps spiking and the app gets slow when loading recent events for a user. Can you figure out what query is causing it and make the database change needed to fix it?\n\nEnd your turn with a short summary of what you changed and why.", - "promptSourcePath": "evals/resolve-performance-001-slow-query-cpu-spike/PROMPT.md", - "attempts": 1, - "sourcePath": "opencode-kimi-k3/resolve-performance-001-slow-query-cpu-spike.json" + "prompt": "Add a description text column to the `products` table in my local Supabase stack", + "promptSourcePath": "evals/build-cli-002-declarative-schema/PROMPT.md", + "attempts": 2, + "sourcePath": "codex-gpt-5.6-no-skills/build-cli-002-declarative-schema.json" }, { - "experiment": "opencode-kimi-k3", - "experimentSuite": "benchmark", + "experiment": "codex-gpt-5.6-no-skills", + "experimentSuite": "no-skills", "experimentDisplay": { - "agent": "opencode", - "modelProvider": "moonshotai", - "modelId": "moonshotai/kimi-k3" + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.6-sol", + "reasoningEffort": "low" }, - "eval": "resolve-security-002-rls-cross-tenant-leak", - "stage": "resolve", + "eval": "build-cli-003-pg-cron-queue-workflow", + "stage": "build", "product": [ "database", - "auth" + "edge-functions", + "cron", + "queues" ], "topic": [ - "rls", - "security" + "sql", + "sdk" ], "suite": "benchmark", - "interface": "mcp", + "interface": "cli", "passed": true, "checks": [ { - "name": "RLS enabled on notes", - "passed": true - }, - { - "name": "tenant A sees only org A notes", - "passed": true - }, - { - "name": "tenant B cannot read org A notes", - "passed": true - }, - { - "name": "tenant A author can update own note", - "passed": true - }, - { - "name": "tenant B cannot update org A note", - "passed": true - }, - { - "name": "tenant B author can delete own note", - "passed": true - }, - { - "name": "tenant B cannot delete org A note", - "passed": true + "name": "pg_cron job 'enqueue-tasks' scheduled to run every minute", + "passed": true, + "notes": "schedule='* * * * *', active=true" }, { - "name": "tenant A can insert note in own org", - "passed": true + "name": "cron command enqueues to the 'tasks' queue", + "passed": true, + "notes": "queue depth 0 -> 1" }, { - "name": "tenant B cannot insert into org A", - "passed": true + "name": "process-tasks function drains the queue", + "passed": true, + "notes": "function removed the seeded message (id 4) from the queue" } ], "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" + "available": [], + "loaded": [] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Supabase Queues pgmq create queue send read delete Edge Function JavaScript scheduled cron every minute\", limit: 8) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/functions/schedule-functions", + "title": "Scheduling Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/cron", + "title": "Cron" + }, + { + "url": "https://supabase.com/docs/guides/queues", + "title": "Supabase Queues" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_net", + "title": "pg_net: Async Networking" + }, + { + "url": "https://supabase.com/docs/guides/cron/quickstart", + "title": "Quickstart" + }, + { + "url": "https://supabase.com/docs/guides/queues/consuming-messages-with-edge-functions", + "title": "Consuming Supabase Queue Messages with Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/cron/install", + "title": "Install" + }, + { + "url": "https://supabase.com/docs/guides/queues/quickstart", + "title": "Quickstart" + } + ], + "resultChars": 44996 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Queues API pop removes message pgmq pop Edge Functions service role schema pgmq_public rpc\", limit: 6) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/queues/quickstart", + "title": "Quickstart" + }, + { + "url": "https://supabase.com/docs/guides/queues/api", + "title": "API" + }, + { + "url": "https://supabase.com/docs/guides/queues/pgmq", + "title": "PGMQ Extension" + }, + { + "url": "https://supabase.com/docs/guides/queues/expose-self-hosted-queues", + "title": "Expose Queues for local and self-hosted Supabase" + }, + { + "url": "https://supabase.com/docs/guides/queues/consuming-messages-with-edge-functions", + "title": "Consuming Supabase Queue Messages with Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/queues", + "title": "Supabase Queues" + } + ], + "resultChars": 43576 + } ] }, - "docs": { - "calls": [] - }, - "prompt": "A customer reported that notes showed up in the wrong workspace.\n\nCan you investigate what is going on and fix it?", - "promptSourcePath": "evals/resolve-security-002-rls-cross-tenant-leak/PROMPT.md", + "prompt": "I want to set up a recurring background workflow on my local Supabase stack.\n\nCan you set up a cron job called `enqueue-tasks` to run every minute and push a task into a queue called `tasks`? Then add a `process-tasks` edge function that reads messages off the `tasks` queue and removes them, so a scheduled worker can keep the backlog drained.", + "promptSourcePath": "evals/build-cli-003-pg-cron-queue-workflow/PROMPT.md", "attempts": 1, - "sourcePath": "opencode-kimi-k3/resolve-security-002-rls-cross-tenant-leak.json" + "sourcePath": "codex-gpt-5.6-no-skills/build-cli-003-pg-cron-queue-workflow.json" }, { - "experiment": "opencode-kimi-k3-no-skills", + "experiment": "codex-gpt-5.6-no-skills", "experimentSuite": "no-skills", "experimentDisplay": { - "agent": "opencode", - "modelProvider": "moonshotai", - "modelId": "moonshotai/kimi-k3" + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.6-sol", + "reasoningEffort": "low" }, - "eval": "build-cli-001-bootstrap-app", + "eval": "build-database-001-migrate-postgres-to-supabase", "stage": "build", "product": [ - "database", - "data-api" + "database" ], "topic": [ - "migrations", - "rls" + "migrations" ], "suite": "benchmark", "interface": "cli", "passed": true, "checks": [ { - "name": "supabase project initialised (supabase/config.toml exists)", + "name": "all 3 tables exist (teams, members, tasks)", "passed": true }, { - "name": "todos table is created by a migration file", + "name": "row counts match (teams=5, members=10, tasks=13)", "passed": true }, { - "name": "todos table exists with at least 2 seeded rows", - "passed": true, - "notes": "found 2 rows" - }, - { - "name": "row level security is enabled on todos", + "name": "foreign key constraints survived the restore", "passed": true }, { - "name": "a SELECT policy targets the authenticated role", + "name": "tasks_team_status_idx index survived the restore", "passed": true }, { - "name": "REST API returns no todos to anonymous requests", - "passed": true, - "notes": "0 rows" - }, - { - "name": "REST API returns the todos to authenticated requests", - "passed": true, - "notes": "2 rows" + "name": "sequences synced (next insert won't conflict with existing IDs)", + "passed": true } ], "skills": { @@ -17601,49 +12205,91 @@ "loaded": [] }, "docs": { - "calls": [] + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Supabase CLI local development restore pg_restore existing Postgres database dump\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/postgres", + "title": "Migrate from Postgres to Supabase" + }, + { + "url": "https://supabase.com/docs/guides/local-development/restoring-downloaded-backup", + "title": "Restoring a downloaded backup locally" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform", + "title": "Restore a Platform Project to Self-Hosted" + }, + { + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/vercel-postgres", + "title": "Migrate from Vercel Postgres to Supabase" + }, + { + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/neon", + "title": "Migrate from Neon to Supabase" + } + ], + "resultChars": 42818 + } + ] }, - "prompt": "We're kicking off a todos app and I want the Supabase side ready for the team\nto build on. Set it up the way we'd run it in development, with schema changes\ntracked as migrations so they can be reviewed and replayed.\n\nFor the first slice we just need a `todos` table. Todos aren't public: anyone\nsigned in can read all of them, but nothing should be writable through the API\nfor now. Add a couple of sample todos so there's something to look at.\n\nBefore you hand it back, make sure the running API actually behaves that way —\nsigned-in users get the todos, signed-out requests get nothing.", - "promptSourcePath": "evals/build-cli-001-bootstrap-app/PROMPT.md", + "prompt": "I have an existing Postgres database I want to migrate to Supabase. There's a binary dump at `source.dump` in the current directory.\n\nCan you set up a local Supabase project and restore the dump into it?", + "promptSourcePath": "evals/build-database-001-migrate-postgres-to-supabase/PROMPT.md", "attempts": 1, - "sourcePath": "opencode-kimi-k3-no-skills/build-cli-001-bootstrap-app.json" + "sourcePath": "codex-gpt-5.6-no-skills/build-database-001-migrate-postgres-to-supabase.json" }, { - "experiment": "opencode-kimi-k3-no-skills", + "experiment": "codex-gpt-5.6-no-skills", "experimentSuite": "no-skills", "experimentDisplay": { - "agent": "opencode", - "modelProvider": "moonshotai", - "modelId": "moonshotai/kimi-k3" + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.6-sol", + "reasoningEffort": "low" }, - "eval": "build-cli-002-declarative-schema", + "eval": "build-functions-004-service-role-bypass", "stage": "build", "product": [ + "edge-functions", + "auth", "database" ], "topic": [ - "declarative-schema", - "migrations" + "rls", + "security", + "sdk" ], "suite": "benchmark", - "interface": "cli", + "interface": "mcp", "passed": true, "checks": [ { - "name": "supabase db diff used to generate the migration", - "passed": true + "name": "rejects missing auth", + "passed": true, + "notes": "status=401" }, { - "name": "schema file updated to include description column", - "passed": true + "name": "user A reads own note", + "passed": true, + "notes": "status=200" }, { - "name": "a new migration was generated for the change", - "passed": true + "name": "reads only with the caller's JWT", + "passed": true, + "notes": "bearer_tokens=2, all_match=true" }, { - "name": "description column exists in the live database", - "passed": true + "name": "user A cannot force-read user B note", + "passed": true, + "notes": "status=200" + }, + { + "name": "user B cannot force-read user A note", + "passed": true, + "notes": "status=200" } ], "skills": { @@ -17651,51 +12297,112 @@ "loaded": [] }, "docs": { - "calls": [] + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Edge Functions authenticate user Authorization header createClient auth.getUser service role RLS\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/troubleshooting/why-is-my-service-role-key-client-getting-rls-errors-or-not-returning-data-7_1K9z", + "title": "Why is my service role key client getting RLS errors or not returning data?" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", + "title": "Integrating With Supabase Auth" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth-headers", + "title": "Authorization headers" + }, + { + "url": "https://supabase.com/docs/guides/api/securing-your-api", + "title": "Securing your API" + }, + { + "url": "https://supabase.com/docs/guides/api/custom-claims-and-role-based-access-control-rbac", + "title": "Custom Claims & Role-based Access Control (RBAC)" + } + ], + "resultChars": 42377 + } + ] }, - "prompt": "Add a description text column to the `products` table in my local Supabase stack", - "promptSourcePath": "evals/build-cli-002-declarative-schema/PROMPT.md", + "prompt": "I built an Edge Function called `private-notes` for showing a user's saved\nprivate notes.\n\nCan you check whether there's any way one user could see another user's notes?\n\nPlease fix and deploy it if needed.", + "promptSourcePath": "evals/build-functions-004-service-role-bypass/PROMPT.md", "attempts": 1, - "sourcePath": "opencode-kimi-k3-no-skills/build-cli-002-declarative-schema.json" + "sourcePath": "codex-gpt-5.6-no-skills/build-functions-004-service-role-bypass.json" }, { - "experiment": "opencode-kimi-k3-no-skills", + "experiment": "codex-gpt-5.6-no-skills", "experimentSuite": "no-skills", "experimentDisplay": { - "agent": "opencode", - "modelProvider": "moonshotai", - "modelId": "moonshotai/kimi-k3" + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.6-sol", + "reasoningEffort": "low" }, - "eval": "build-cli-003-pg-cron-queue-workflow", + "eval": "build-functions-005-dual-auth-user-secret", "stage": "build", "product": [ - "database", "edge-functions", - "cron", - "queues" + "auth", + "database" ], "topic": [ - "sql", - "sdk" + "sdk", + "rls", + "security" ], "suite": "benchmark", "interface": "cli", + "cliVersion": "2.109.1", "passed": true, "checks": [ { - "name": "pg_cron job 'enqueue-tasks' scheduled to run every minute", + "name": "seed rows present", "passed": true, - "notes": "schedule='* * * * *', active=true" + "notes": "found 2/2 seeded rows" }, { - "name": "cron command enqueues to the 'tasks' queue", + "name": "rejects request with no credentials", "passed": true, - "notes": "queue depth 1 -> 2" + "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" }, { - "name": "process-tasks function drains the queue", + "name": "user with JWT reads only their own rows", + "passed": true, + "notes": "status 200: [{\"user_id\":\"9fddea39-2989-49cf-bf3d-dfc62aae791f\",\"metric\":\"steps_a_mrw5ynw8\",\"value\":111}]" + }, + { + "name": "user cannot read another user's rows by passing user_id", + "passed": true, + "notes": "status 200: [{\"user_id\":\"9fddea39-2989-49cf-bf3d-dfc62aae791f\",\"metric\":\"steps_a_mrw5ynw8\",\"value\":111}]" + }, + { + "name": "service key bypasses RLS to read the target user's rows", + "passed": true, + "notes": "status 200: [{\"user_id\":\"57cf856d-ca80-46e6-84b2-cea5eaa0f57a\",\"metric\":\"steps_b_mrw5ynw8\",\"value\":222}]" + }, + { + "name": "non-service key is not granted service access", + "passed": true, + "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" + }, + { + "name": "rejects an unverified (forged) user token", + "passed": true, + "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" + }, + { + "name": "a user token in the apikey slot is not treated as the service key", "passed": true, - "notes": "function removed the seeded message (id 8) from the queue" + "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" + }, + { + "name": "implementation uses @supabase/server", + "passed": true, + "notes": "imports @supabase/server / withSupabase" } ], "skills": { @@ -17706,448 +12413,1381 @@ "calls": [ { "source": "search_docs", - "query": "{\n searchDocs(query: \"pgmq queues consume messages edge function pop\", limit: 5) {\n nodes {\n title\n href\n content\n }\n }\n}", + "query": "query { searchDocs(query: \"Edge Functions verify_jwt false Authorization bearer getUser service role key apikey header secrets SUPABASE_SERVICE_ROLE_KEY\", limit: 8) { nodes { title href content ... on Guide { subsections { nodes { title href content } } } } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/getting-started/api-keys", + "title": "Understanding API keys" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/api-keys#security-considerations", + "title": "Security considerations" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/api-keys#overview", + "title": "Overview" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/api-keys#publishable-keys", + "title": "Publishable keys" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/api-keys#interaction-with-supabase-auth", + "title": "Interaction with Supabase Auth" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/api-keys#what-secret-keys-allow-access-to", + "title": "What secret keys allow access to" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/api-keys#best-practices-for-handling-secret-keys", + "title": "Best practices for handling secret keys" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/api-keys#what-to-do-if-a-secret-key-or-service_role-has-been-leaked-or-compromised", + "title": "What to do if a secret key or service_role has been leaked or compromised?" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/api-keys#known-limitations-and-compatibility-differences", + "title": "Known limitations and compatibility differences" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", + "title": "Integrating With Supabase Auth" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt#fetching-the-user", + "title": "Fetching the user" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt#row-level-security", + "title": "Row Level Security" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt#example", + "title": "Example" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt#setting-up-auth-context", + "title": "Setting up auth context" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth", + "title": "Securing Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth#external-webhooks", + "title": "External webhooks" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth#public-functions", + "title": "Public functions" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth#service-to-service-calls", + "title": "Service-to-service calls" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth#authenticated-user-calls", + "title": "Authenticated user calls" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth#environment-variables", + "title": "Environment variables" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth#custom-error-responses", + "title": "Custom error responses" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth#combining-modes", + "title": "Combining modes" + }, + { + "url": "https://supabase.com/docs/guides/functions/function-configuration", + "title": "Function Configuration" + }, + { + "url": "https://supabase.com/docs/guides/functions/function-configuration#configuration", + "title": "Configuration" + }, + { + "url": "https://supabase.com/docs/guides/functions/function-configuration#custom-entrypoints", + "title": "Custom entrypoints" + }, + { + "url": "https://supabase.com/docs/guides/functions/function-configuration#skipping-authorization-checks", + "title": "Skipping authorization checks" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#before-you-start", + "title": "Before you start" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#database-webhooks-and-pg_net", + "title": "Database Webhooks and pg_net" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#step-4-update-edge-functions", + "title": "Step 4: Update Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#option-1-read-the-new-keys-from-the-environment", + "title": "Option 1: Read the new keys from the environment" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#option-2-adopt-the-supabaseserver-sdk", + "title": "Option 2: Adopt the @supabase/server SDK" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#step-5-verify-nothing-uses-the-legacy-keys", + "title": "Step 5: Verify nothing uses the legacy keys" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#step-6-deactivate-the-legacy-keys", + "title": "Step 6: Deactivate the legacy keys" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#known-limitations", + "title": "Known limitations" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#next-steps", + "title": "Next steps" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#step-1-create-the-new-api-keys", + "title": "Step 1: Create the new API keys" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#step-2-swap-the-publishable-key-in-client-code", + "title": "Step 2: Swap the publishable key in client code" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#step-3-swap-the-secret-key-in-backend-code", + "title": "Step 3: Swap the secret key in backend code" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", + "title": "New API Keys and Asymmetric Authentication" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#backward-compatibility", + "title": "Backward compatibility" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#differences-from-the-supabase-platform", + "title": "Differences from the Supabase platform" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#environment-variables-configuration", + "title": "Environment variables configuration" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#verifying-the-setup", + "title": "Verifying the setup" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#how-it-works", + "title": "How it works" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#unauthenticated-requests-api-key-only-no-user-session-jwt", + "title": "Unauthenticated requests (API key only, no user session JWT)" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#authenticated-requests-user-session-jwt", + "title": "Authenticated requests (user session JWT)" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#additional-resources", + "title": "Additional resources" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#before-you-begin", + "title": "Before you begin" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#adding-the-new-keys", + "title": "Adding the new keys" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#new-api-keys-format", + "title": "New API keys format" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#regenerating-asymmetric-key-pair", + "title": "Regenerating asymmetric key pair" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#rotating-the-new-api-keys", + "title": "Rotating the new API keys" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#request-flows", + "title": "Request flows" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#kong-api-gateway-routing", + "title": "Kong API gateway routing" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#what-client-sdk-sends", + "title": "What client SDK sends" + }, + { + "url": "https://supabase.com/docs/guides/functions/secrets", + "title": "Environment Variables" + }, + { + "url": "https://supabase.com/docs/guides/functions/secrets#default-secrets", + "title": "Default secrets" + }, + { + "url": "https://supabase.com/docs/guides/functions/secrets#local-secrets", + "title": "Local secrets" + }, + { + "url": "https://supabase.com/docs/guides/functions/secrets#accessing-environment-variables", + "title": "Accessing environment variables" + }, + { + "url": "https://supabase.com/docs/guides/functions/secrets#production-secrets", + "title": "Production secrets" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", + "title": "Envoy API Gateway" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#before-you-begin", + "title": "Before you begin" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#enabling-the-envoy-gateway", + "title": "Enabling the Envoy gateway" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#verify", + "title": "Verify" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#architecture", + "title": "Architecture" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#configuration-file-structure", + "title": "Configuration file structure" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#how-the-configuration-is-rendered-at-startup", + "title": "How the configuration is rendered at startup" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#routes", + "title": "Routes" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#authentication", + "title": "Authentication" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#dashboard-basic-auth", + "title": "Dashboard basic auth" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#api-key-enforcement-on-protected-routes", + "title": "API key enforcement on protected routes" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#opaque-key-translation", + "title": "Opaque key translation" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#forwarded-headers-and-cors", + "title": "Forwarded headers and CORS" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#x-forwarded-headers", + "title": "X-Forwarded headers" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#cors", + "title": "CORS" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#security-hardening", + "title": "Security hardening" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#customizing-the-configuration", + "title": "Customizing the configuration" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#admin-interface", + "title": "Admin interface" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#troubleshooting", + "title": "Troubleshooting" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#logs", + "title": "Logs" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#common-issues", + "title": "Common issues" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#see-also", + "title": "See also" + } + ], + "resultChars": 238678 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Edge Functions JWT verification getUser Authorization header config.toml verify_jwt\", limit: 6) { nodes { title href content ... on Guide { subsections { nodes { title href content } } } } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/functions/function-configuration", + "title": "Function Configuration" + }, + { + "url": "https://supabase.com/docs/guides/functions/function-configuration#skipping-authorization-checks", + "title": "Skipping authorization checks" + }, + { + "url": "https://supabase.com/docs/guides/functions/function-configuration#custom-entrypoints", + "title": "Custom entrypoints" + }, + { + "url": "https://supabase.com/docs/guides/functions/function-configuration#configuration", + "title": "Configuration" + }, + { + "url": "https://supabase.com/docs/guides/functions/deploy", + "title": "Deploy to Production" + }, + { + "url": "https://supabase.com/docs/guides/functions/deploy#example", + "title": "Example" + }, + { + "url": "https://supabase.com/docs/guides/functions/deploy#step-1-authenticate", + "title": "Step 1: Authenticate" + }, + { + "url": "https://supabase.com/docs/guides/functions/deploy#step-2-connect-your-project", + "title": "Step 2: Connect your project" + }, + { + "url": "https://supabase.com/docs/guides/functions/deploy#step-3-deploy-functions", + "title": "Step 3: Deploy Functions" + }, + { + "url": "https://supabase.com/docs/guides/functions/deploy#step-4-verify-successful-deployment", + "title": "Step 4: Verify successful deployment" + }, + { + "url": "https://supabase.com/docs/guides/functions/deploy#step-5-test-your-live-function", + "title": "Step 5: Test your live function" + }, + { + "url": "https://supabase.com/docs/guides/functions/deploy#cicd-deployment", + "title": "CI/CD deployment" + }, + { + "url": "https://supabase.com/docs/guides/functions/deploy#github-actions", + "title": "GitHub Actions" + }, + { + "url": "https://supabase.com/docs/guides/functions/deploy#gitlab-ci", + "title": "GitLab CI" + }, + { + "url": "https://supabase.com/docs/guides/functions/deploy#bitbucket-pipelines", + "title": "Bitbucket Pipelines" + }, + { + "url": "https://supabase.com/docs/guides/functions/deploy#function-configuration", + "title": "Function configuration" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth-headers", + "title": "Authorization headers" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth-headers#the-verify_jwt-platform-check", + "title": "The verify_jwt platform check" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth-headers#understanding-authorization-headers", + "title": "Understanding authorization headers" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", + "title": "Integrating With Supabase Auth" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt#setting-up-auth-context", + "title": "Setting up auth context" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt#row-level-security", + "title": "Row Level Security" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt#example", + "title": "Example" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt#fetching-the-user", + "title": "Fetching the user" + }, + { + "url": "https://supabase.com/docs/guides/functions/development-tips", + "title": "Development tips" + }, + { + "url": "https://supabase.com/docs/guides/functions/development-tips#error-handling", + "title": "Error handling" + }, + { + "url": "https://supabase.com/docs/guides/functions/development-tips#using-http-methods", + "title": "Using HTTP methods" + }, + { + "url": "https://supabase.com/docs/guides/functions/development-tips#naming-edge-functions", + "title": "Naming Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/functions/development-tips#organizing-your-edge-functions", + "title": "Organizing your Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/functions/development-tips#using-configtoml", + "title": "Using config.toml" + }, + { + "url": "https://supabase.com/docs/guides/functions/development-tips#not-using-typescript", + "title": "Not using TypeScript" + }, + { + "url": "https://supabase.com/docs/guides/functions/development-tips#database-functions-vs-edge-functions", + "title": "Database Functions vs Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/functions/error-codes", + "title": "Error codes" + }, + { + "url": "https://supabase.com/docs/guides/functions/error-codes#authentication-errors", + "title": "Authentication Errors" + }, + { + "url": "https://supabase.com/docs/guides/functions/error-codes#unauthorized_no_auth_header", + "title": "UNAUTHORIZED_NO_AUTH_HEADER" + }, + { + "url": "https://supabase.com/docs/guides/functions/error-codes#unauthorized_asymmetric_jwt", + "title": "UNAUTHORIZED_ASYMMETRIC_JWT" + }, + { + "url": "https://supabase.com/docs/guides/functions/error-codes#unauthorized_legacy_jwt", + "title": "UNAUTHORIZED_LEGACY_JWT" + }, + { + "url": "https://supabase.com/docs/guides/functions/error-codes#unauthorized_unsupported_token_algorithm", + "title": "UNAUTHORIZED_UNSUPPORTED_TOKEN_ALGORITHM" + }, + { + "url": "https://supabase.com/docs/guides/functions/error-codes#unauthorized_invalid_jwt_format", + "title": "UNAUTHORIZED_INVALID_JWT_FORMAT" + }, + { + "url": "https://supabase.com/docs/guides/functions/error-codes#load_function_invalid_entrypoint_path_error", + "title": "LOAD_FUNCTION_INVALID_ENTRYPOINT_PATH_ERROR" + }, + { + "url": "https://supabase.com/docs/guides/functions/error-codes#request-errors", + "title": "Request Errors" + }, + { + "url": "https://supabase.com/docs/guides/functions/error-codes#rate_limit_exceeded", + "title": "RATE_LIMIT_EXCEEDED" + }, + { + "url": "https://supabase.com/docs/guides/functions/error-codes#edge_function_error", + "title": "EDGE_FUNCTION_ERROR" + }, + { + "url": "https://supabase.com/docs/guides/functions/error-codes#load_function_metadata_error", + "title": "LOAD_FUNCTION_METADATA_ERROR" + }, + { + "url": "https://supabase.com/docs/guides/functions/error-codes#load_function_error", + "title": "LOAD_FUNCTION_ERROR" + }, + { + "url": "https://supabase.com/docs/guides/functions/error-codes#boot_error", + "title": "BOOT_ERROR" + }, + { + "url": "https://supabase.com/docs/guides/functions/error-codes#not_found", + "title": "NOT_FOUND" + }, + { + "url": "https://supabase.com/docs/guides/functions/error-codes#server-errors", + "title": "Server Errors" + }, + { + "url": "https://supabase.com/docs/guides/functions/error-codes#invalid_url", + "title": "INVALID_URL" + }, + { + "url": "https://supabase.com/docs/guides/functions/error-codes#bad-implementation-errors", + "title": "Bad Implementation Errors" + }, + { + "url": "https://supabase.com/docs/guides/functions/error-codes#idle_timeout", + "title": "IDLE_TIMEOUT" + }, + { + "url": "https://supabase.com/docs/guides/functions/error-codes#worker_resource_limit-worker_limit", + "title": "WORKER_RESOURCE_LIMIT, WORKER_LIMIT" + }, + { + "url": "https://supabase.com/docs/guides/functions/error-codes#worker_error", + "title": "WORKER_ERROR" + }, + { + "url": "https://supabase.com/docs/guides/functions/error-codes#invalid_response_status_code", + "title": "INVALID_RESPONSE_STATUS_CODE" + } + ], + "resultChars": 81389 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Edge Functions default secrets SUPABASE_SERVICE_ROLE_KEY SUPABASE_SECRET_KEY secret API key environment variables\", limit: 8) { nodes { title href content ... on Guide { subsections { nodes { title href content } } } } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#before-you-start", + "title": "Before you start" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#step-1-create-the-new-api-keys", + "title": "Step 1: Create the new API keys" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#step-2-swap-the-publishable-key-in-client-code", + "title": "Step 2: Swap the publishable key in client code" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#step-3-swap-the-secret-key-in-backend-code", + "title": "Step 3: Swap the secret key in backend code" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#database-webhooks-and-pg_net", + "title": "Database Webhooks and pg_net" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#step-4-update-edge-functions", + "title": "Step 4: Update Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#option-1-read-the-new-keys-from-the-environment", + "title": "Option 1: Read the new keys from the environment" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#option-2-adopt-the-supabaseserver-sdk", + "title": "Option 2: Adopt the @supabase/server SDK" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#step-5-verify-nothing-uses-the-legacy-keys", + "title": "Step 5: Verify nothing uses the legacy keys" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#step-6-deactivate-the-legacy-keys", + "title": "Step 6: Deactivate the legacy keys" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#known-limitations", + "title": "Known limitations" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#next-steps", + "title": "Next steps" + }, + { + "url": "https://supabase.com/docs/guides/functions/secrets", + "title": "Environment Variables" + }, + { + "url": "https://supabase.com/docs/guides/functions/secrets#production-secrets", + "title": "Production secrets" + }, + { + "url": "https://supabase.com/docs/guides/functions/secrets#accessing-environment-variables", + "title": "Accessing environment variables" + }, + { + "url": "https://supabase.com/docs/guides/functions/secrets#default-secrets", + "title": "Default secrets" + }, + { + "url": "https://supabase.com/docs/guides/functions/secrets#local-secrets", + "title": "Local secrets" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", + "title": "Self-Hosted Functions" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#internal-vs-external-urls", + "title": "Internal vs external URLs" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#calling-supabase-services-from-functions", + "title": "Calling Supabase services from functions" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#accessing-variables-in-functions", + "title": "Accessing variables in functions" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#using-inline-environment-variables", + "title": "Using inline environment variables" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#using-an-env-file-recommended", + "title": "Using an env file (recommended)" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#custom-environment-variables", + "title": "Custom environment variables" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#step-3-invoke-your-function", + "title": "Step 3: Invoke your function" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#step-2-restart-the-functions-service-to-pick-up-the-new-function", + "title": "Step 2: Restart the functions service to pick up the new function" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#step-1-add-a-new-function-directory-and-the-function-code", + "title": "Step 1: Add a new function directory and the function code" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#create-a-new-function", + "title": "Create a new function" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#invoke-the-default-function", + "title": "Invoke the default function" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#memory-or-timeout-errors", + "title": "Memory or timeout errors" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#custom-env-vars-not-available-in-functions", + "title": "Custom env vars not available in functions" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#changes-to-function-code-not-reflected-after-editing", + "title": "Changes to function code not reflected after editing" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#500-error-on-invocation", + "title": "500 error on invocation" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#troubleshooting", + "title": "Troubleshooting" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#copying-functions-from-supabase-platform", + "title": "Copying functions from Supabase platform" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#deploying-functions-to-a-remote-server", + "title": "Deploying functions to a remote server" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#managing-functions-via-dashboard", + "title": "Managing functions via dashboard" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", + "title": "Integrating With Supabase Auth" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt#row-level-security", + "title": "Row Level Security" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt#example", + "title": "Example" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt#setting-up-auth-context", + "title": "Setting up auth context" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt#fetching-the-user", + "title": "Fetching the user" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth", + "title": "Securing Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth#authenticated-user-calls", + "title": "Authenticated user calls" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth#service-to-service-calls", + "title": "Service-to-service calls" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth#public-functions", + "title": "Public functions" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth#external-webhooks", + "title": "External webhooks" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth#combining-modes", + "title": "Combining modes" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth#custom-error-responses", + "title": "Custom error responses" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth#environment-variables", + "title": "Environment variables" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", + "title": "New API Keys and Asymmetric Authentication" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#authenticated-requests-user-session-jwt", + "title": "Authenticated requests (user session JWT)" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#unauthenticated-requests-api-key-only-no-user-session-jwt", + "title": "Unauthenticated requests (API key only, no user session JWT)" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#request-flows", + "title": "Request flows" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#kong-api-gateway-routing", + "title": "Kong API gateway routing" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#what-client-sdk-sends", + "title": "What client SDK sends" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#how-it-works", + "title": "How it works" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#regenerating-asymmetric-key-pair", + "title": "Regenerating asymmetric key pair" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#rotating-the-new-api-keys", + "title": "Rotating the new API keys" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#backward-compatibility", + "title": "Backward compatibility" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#differences-from-the-supabase-platform", + "title": "Differences from the Supabase platform" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#environment-variables-configuration", + "title": "Environment variables configuration" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#verifying-the-setup", + "title": "Verifying the setup" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#new-api-keys-format", + "title": "New API keys format" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#adding-the-new-keys", + "title": "Adding the new keys" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#before-you-begin", + "title": "Before you begin" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#additional-resources", + "title": "Additional resources" + }, + { + "url": "https://supabase.com/docs/guides/functions/storage-caching", + "title": "Integrating with Supabase Storage" + }, + { + "url": "https://supabase.com/docs/guides/functions/storage-caching#cache-first-pattern", + "title": "Cache-first pattern" + }, + { + "url": "https://supabase.com/docs/guides/functions/storage-caching#basic-file-operations", + "title": "Basic file operations" + }, + { + "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream", + "title": "Streaming Speech with ElevenLabs" + }, + { + "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#create-a-supabase-project-locally", + "title": "Create a Supabase project locally" + }, + { + "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#setup", + "title": "Setup" + }, + { + "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#requirements", + "title": "Requirements" + }, + { + "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#introduction", + "title": "Introduction" + }, + { + "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#set-up-the-environment-variables", + "title": "Set up the environment variables" + }, + { + "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#create-a-supabase-edge-function-for-speech-generation", + "title": "Create a Supabase Edge Function for speech generation" + }, + { + "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#configure-background-tasks-for-supabase-edge-functions", + "title": "Configure background tasks for Supabase Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#configure-the-storage-bucket", + "title": "Configure the storage bucket" + }, + { + "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#test-the-function", + "title": "Test the function" + }, + { + "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#set-the-function-secrets", + "title": "Set the function secrets" + }, + { + "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#deploy-to-supabase", + "title": "Deploy to Supabase" + }, + { + "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#try-it-out", + "title": "Try it out" + }, + { + "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#run-locally", + "title": "Run locally" + }, + { + "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#code-the-supabase-edge-function", + "title": "Code the Supabase Edge Function" + }, + { + "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#dependencies", + "title": "Dependencies" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#401-invalid-jwt" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#400-missing-function-name-in-request" + } + ], + "resultChars": 188415 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"@supabase/server Edge Functions API key validate SUPABASE_SECRET_KEYS withSupabase auth secret\", limit: 10) { nodes { title href content ... on Guide { subsections { nodes { title href content } } } ... on ClientLibraryFunctionReference { language methodName } } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/queues/pgmq", - "title": "PGMQ Extension" + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#next-steps", + "title": "Next steps" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#known-limitations", + "title": "Known limitations" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#before-you-start", + "title": "Before you start" }, { - "url": "https://supabase.com/docs/guides/queues/consuming-messages-with-edge-functions", - "title": "Consuming Supabase Queue Messages with Edge Functions" + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#step-1-create-the-new-api-keys", + "title": "Step 1: Create the new API keys" }, { - "url": "https://supabase.com/docs/guides/queues/quickstart", - "title": "Quickstart" + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#step-2-swap-the-publishable-key-in-client-code", + "title": "Step 2: Swap the publishable key in client code" }, { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", - "title": "Automatic embeddings" + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#step-3-swap-the-secret-key-in-backend-code", + "title": "Step 3: Swap the secret key in backend code" }, { - "url": "https://supabase.com/docs/guides/queues/api", - "title": "API" - } - ], - "resultChars": 68440 - } - ] - }, - "prompt": "I want to set up a recurring background workflow on my local Supabase stack.\n\nCan you set up a cron job called `enqueue-tasks` to run every minute and push a task into a queue called `tasks`? Then add a `process-tasks` edge function that reads messages off the `tasks` queue and removes them, so a scheduled worker can keep the backlog drained.", - "promptSourcePath": "evals/build-cli-003-pg-cron-queue-workflow/PROMPT.md", - "attempts": 1, - "sourcePath": "opencode-kimi-k3-no-skills/build-cli-003-pg-cron-queue-workflow.json" - }, - { - "experiment": "opencode-kimi-k3-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "opencode", - "modelProvider": "moonshotai", - "modelId": "moonshotai/kimi-k3" - }, - "eval": "build-database-001-migrate-postgres-to-supabase", - "stage": "build", - "product": [ - "database" - ], - "topic": [ - "migrations" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "all 3 tables exist (teams, members, tasks)", - "passed": true - }, - { - "name": "row counts match (teams=5, members=10, tasks=13)", - "passed": true - }, - { - "name": "foreign key constraints survived the restore", - "passed": true - }, - { - "name": "tasks_team_status_idx index survived the restore", - "passed": true - }, - { - "name": "sequences synced (next insert won't conflict with existing IDs)", - "passed": true - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "I have an existing Postgres database I want to migrate to Supabase. There's a binary dump at `source.dump` in the current directory.\n\nCan you set up a local Supabase project and restore the dump into it?", - "promptSourcePath": "evals/build-database-001-migrate-postgres-to-supabase/PROMPT.md", - "attempts": 1, - "sourcePath": "opencode-kimi-k3-no-skills/build-database-001-migrate-postgres-to-supabase.json" - }, - { - "experiment": "opencode-kimi-k3-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "opencode", - "modelProvider": "moonshotai", - "modelId": "moonshotai/kimi-k3" - }, - "eval": "build-functions-004-service-role-bypass", - "stage": "build", - "product": [ - "edge-functions", - "auth", - "database" - ], - "topic": [ - "rls", - "security", - "sdk" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "rejects missing auth", - "passed": true, - "notes": "status=401" - }, - { - "name": "user A reads own note", - "passed": true, - "notes": "status=200" - }, - { - "name": "reads only with the caller's JWT", - "passed": true, - "notes": "bearer_tokens=2, all_match=true" - }, - { - "name": "user A cannot force-read user B note", - "passed": true, - "notes": "status=200" - }, - { - "name": "user B cannot force-read user A note", - "passed": true, - "notes": "status=200" - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "I built an Edge Function called `private-notes` for showing a user's saved\nprivate notes.\n\nCan you check whether there's any way one user could see another user's notes?\n\nPlease fix and deploy it if needed.", - "promptSourcePath": "evals/build-functions-004-service-role-bypass/PROMPT.md", - "attempts": 1, - "sourcePath": "opencode-kimi-k3-no-skills/build-functions-004-service-role-bypass.json" - }, - { - "experiment": "opencode-kimi-k3-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "opencode", - "modelProvider": "moonshotai", - "modelId": "moonshotai/kimi-k3" - }, - "eval": "build-functions-005-dual-auth-user-secret", - "stage": "build", - "product": [ - "edge-functions", - "auth", - "database" - ], - "topic": [ - "sdk", - "rls", - "security" - ], - "suite": "benchmark", - "interface": "cli", - "cliVersion": "2.109.1", - "passed": true, - "checks": [ - { - "name": "seed rows present", - "passed": true, - "notes": "found 2/2 seeded rows" - }, - { - "name": "rejects request with no credentials", - "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" - }, - { - "name": "user with JWT reads only their own rows", - "passed": true, - "notes": "status 200: [{\"user_id\":\"97383d24-8574-40e4-8bbf-893c839f4b1e\",\"metric\":\"steps_a_ms6x19hb\",\"value\":111}]" - }, - { - "name": "user cannot read another user's rows by passing user_id", - "passed": true, - "notes": "status 200: [{\"user_id\":\"97383d24-8574-40e4-8bbf-893c839f4b1e\",\"metric\":\"steps_a_ms6x19hb\",\"value\":111}]" - }, - { - "name": "service key bypasses RLS to read the target user's rows", - "passed": true, - "notes": "status 200: [{\"user_id\":\"76300586-725d-4552-ac8c-717466bca850\",\"metric\":\"steps_b_ms6x19hb\",\"value\":222}]" - }, - { - "name": "non-service key is not granted service access", - "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" - }, - { - "name": "rejects an unverified (forged) user token", - "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" - }, - { - "name": "a user token in the apikey slot is not treated as the service key", - "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" - }, - { - "name": "implementation uses @supabase/server", - "passed": true, - "notes": "imports @supabase/server / withSupabase" - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{ searchDocs(query: \"edge function verify service role key apikey header dual authentication\") { nodes { title href content } } }", - "hasContent": true, - "pages": [ + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#database-webhooks-and-pg_net", + "title": "Database Webhooks and pg_net" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#step-4-update-edge-functions", + "title": "Step 4: Update Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#option-1-read-the-new-keys-from-the-environment", + "title": "Option 1: Read the new keys from the environment" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#option-2-adopt-the-supabaseserver-sdk", + "title": "Option 2: Adopt the @supabase/server SDK" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#step-5-verify-nothing-uses-the-legacy-keys", + "title": "Step 5: Verify nothing uses the legacy keys" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#step-6-deactivate-the-legacy-keys", + "title": "Step 6: Deactivate the legacy keys" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth", + "title": "Securing Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth#service-to-service-calls", + "title": "Service-to-service calls" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth#authenticated-user-calls", + "title": "Authenticated user calls" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth#environment-variables", + "title": "Environment variables" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth#custom-error-responses", + "title": "Custom error responses" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth#combining-modes", + "title": "Combining modes" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth#external-webhooks", + "title": "External webhooks" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth#public-functions", + "title": "Public functions" + }, + { + "url": "https://supabase.com/docs/guides/functions/secrets", + "title": "Environment Variables" + }, + { + "url": "https://supabase.com/docs/guides/functions/secrets#default-secrets", + "title": "Default secrets" + }, + { + "url": "https://supabase.com/docs/guides/functions/secrets#production-secrets", + "title": "Production secrets" + }, + { + "url": "https://supabase.com/docs/guides/functions/secrets#local-secrets", + "title": "Local secrets" + }, + { + "url": "https://supabase.com/docs/guides/functions/secrets#accessing-environment-variables", + "title": "Accessing environment variables" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/api-keys", + "title": "Understanding API keys" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/api-keys#interaction-with-supabase-auth", + "title": "Interaction with Supabase Auth" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/api-keys#what-to-do-if-a-secret-key-or-service_role-has-been-leaked-or-compromised", + "title": "What to do if a secret key or service_role has been leaked or compromised?" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/api-keys#best-practices-for-handling-secret-keys", + "title": "Best practices for handling secret keys" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/api-keys#what-secret-keys-allow-access-to", + "title": "What secret keys allow access to" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/api-keys#security-considerations", + "title": "Security considerations" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/api-keys#known-limitations-and-compatibility-differences", + "title": "Known limitations and compatibility differences" + }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", - "title": "Envoy API Gateway" + "url": "https://supabase.com/docs/guides/getting-started/api-keys#publishable-keys", + "title": "Publishable keys" }, { - "url": "https://supabase.com/docs/guides/functions/auth-headers", - "title": "Authorization headers" + "url": "https://supabase.com/docs/guides/getting-started/api-keys#overview", + "title": "Overview" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", - "title": "New API Keys and Asymmetric Authentication" + "url": "https://supabase.com/docs/guides/auth/jwts", + "title": "JSON Web Token (JWT)" }, { - "url": "https://supabase.com/docs/guides/functions/error-codes", + "url": "https://supabase.com/docs/guides/auth/jwts#introduction", + "title": "Introduction" + }, + { + "url": "https://supabase.com/docs/guides/auth/jwts#supabase-and-jwts", + "title": "Supabase and JWTs" + }, + { + "url": "https://supabase.com/docs/guides/auth/jwts#using-custom-or-third-party-jwts", + "title": "Using custom or third-party JWTs" + }, + { + "url": "https://supabase.com/docs/guides/auth/jwts#verifying-a-jwt-from-supabase", + "title": "Verifying a JWT from Supabase" + }, + { + "url": "https://supabase.com/docs/guides/auth/jwts#verifying-with-a-shared-secret-signing-key", + "title": "Verifying with a shared secret signing key" + }, + { + "url": "https://supabase.com/docs/guides/auth/jwts#resources", + "title": "Resources" + }, + { + "url": "https://supabase.com/docs/guides/auth/passkeys", + "title": "Passkey authentication" + }, + { + "url": "https://supabase.com/docs/guides/auth/passkeys#register-a-passkey", + "title": "Register a passkey" + }, + { + "url": "https://supabase.com/docs/guides/auth/passkeys#sign-in-with-a-passkey", + "title": "Sign in with a passkey" + }, + { + "url": "https://supabase.com/docs/guides/auth/passkeys#two-step-api", + "title": "Two-step API" + }, + { + "url": "https://supabase.com/docs/guides/auth/passkeys#admin-api", + "title": "Admin API" + }, + { + "url": "https://supabase.com/docs/guides/auth/passkeys#error-codes", "title": "Error codes" }, { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" + "url": "https://supabase.com/docs/guides/auth/passkeys#limitations", + "title": "Limitations" }, { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" + "url": "https://supabase.com/docs/guides/auth/passkeys#manage-passkeys", + "title": "Manage passkeys" }, { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" + "url": "https://supabase.com/docs/guides/auth/passkeys#how-does-it-work", + "title": "How does it work?" + }, + { + "url": "https://supabase.com/docs/guides/auth/passkeys#enable-passkey-authentication", + "title": "Enable passkey authentication" + }, + { + "url": "https://supabase.com/docs/guides/auth/passkeys#dashboard", + "title": "Dashboard" + }, + { + "url": "https://supabase.com/docs/guides/auth/passkeys#cli", + "title": "CLI" + }, + { + "url": "https://supabase.com/docs/guides/auth/passkeys#management-api", + "title": "Management API" + }, + { + "url": "https://supabase.com/docs/guides/auth/passkeys#enable-in-the-client", + "title": "Enable in the client" + }, + { + "url": "https://supabase.com/docs/guides/functions/examples/cloudflare-turnstile", + "title": "CAPTCHA support with Cloudflare Turnstile" + }, + { + "url": "https://supabase.com/docs/guides/functions/examples/cloudflare-turnstile#setup", + "title": "Setup" + }, + { + "url": "https://supabase.com/docs/guides/functions/examples/cloudflare-turnstile#deploy-the-server-side-validation-edge-functions", + "title": "Deploy the server-side validation Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/functions/examples/cloudflare-turnstile#invoke-the-function-from-your-site", + "title": "Invoke the function from your site" + }, + { + "url": "https://supabase.com/docs/guides/functions/examples/cloudflare-turnstile#code", + "title": "Code" + }, + { + "url": "https://supabase.com/docs/guides/auth/choosing-a-server-package", + "title": "Which package to use" + }, + { + "url": "https://supabase.com/docs/guides/auth/choosing-a-server-package#which-package-to-use", + "title": "Which package to use" + }, + { + "url": "https://supabase.com/docs/guides/auth/choosing-a-server-package#supabasesupabase-js", + "title": "@supabase/supabase-js" + }, + { + "url": "https://supabase.com/docs/guides/auth/choosing-a-server-package#supabasessr", + "title": "@supabase/ssr" + }, + { + "url": "https://supabase.com/docs/guides/auth/choosing-a-server-package#supabaseserver", + "title": "@supabase/server" + }, + { + "url": "https://supabase.com/docs/guides/auth/choosing-a-server-package#advanced-combining-supabaseserver-and-supabasessr", + "title": "Advanced: Combining @supabase/server and @supabase/ssr" + }, + { + "url": "https://supabase.com/docs/guides/auth/choosing-a-server-package#next-steps", + "title": "Next steps" + }, + { + "url": "https://supabase.com/docs/guides/functions/quickstart-dashboard", + "title": "Getting Started with Edge Functions (Dashboard)" + }, + { + "url": "https://supabase.com/docs/guides/functions/quickstart-dashboard#dashboard", + "title": "Dashboard" + }, + { + "url": "https://supabase.com/docs/guides/functions/quickstart-dashboard#usage", + "title": "Usage" + }, + { + "url": "https://supabase.com/docs/guides/functions/quickstart-dashboard#step-6-get-your-function-url-and-keys", + "title": "Step 6: Get your function URL and keys" + }, + { + "url": "https://supabase.com/docs/guides/functions/quickstart-dashboard#step-5-test-your-function", + "title": "Step 5: Test your function" + }, + { + "url": "https://supabase.com/docs/guides/functions/quickstart-dashboard#step-4-deploy-your-function", + "title": "Step 4: Deploy your function" + }, + { + "url": "https://supabase.com/docs/guides/functions/quickstart-dashboard#step-3-customize-your-function-code", + "title": "Step 3: Customize your function code" + }, + { + "url": "https://supabase.com/docs/guides/functions/quickstart-dashboard#step-2-create-your-first-function", + "title": "Step 2: Create your first function" + }, + { + "url": "https://supabase.com/docs/guides/functions/quickstart-dashboard#step-1-navigate-to-the-edge-functions-tab", + "title": "Step 1: Navigate to the Edge Functions tab" + }, + { + "url": "https://supabase.com/docs/guides/functions/quickstart-dashboard#deploy-via-assistant", + "title": "Deploy via Assistant" + }, + { + "url": "https://supabase.com/docs/guides/functions/quickstart-dashboard#download-edge-functions", + "title": "Download Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/functions/quickstart-dashboard#cli", + "title": "CLI" + }, + { + "url": "https://supabase.com/docs/guides/database/secure-data", + "title": "Securing your data" + }, + { + "url": "https://supabase.com/docs/guides/database/secure-data#data-api", + "title": "Data API" + }, + { + "url": "https://supabase.com/docs/guides/database/secure-data#connecting-your-app-securely", + "title": "Connecting your app securely" + }, + { + "url": "https://supabase.com/docs/guides/database/secure-data#more-information", + "title": "More information" + }, + { + "url": "https://supabase.com/docs/guides/database/secure-data#frontend-access", + "title": "Frontend access" + }, + { + "url": "https://supabase.com/docs/guides/database/secure-data#direct-database-connections", + "title": "Direct database connections" + }, + { + "url": "https://supabase.com/docs/guides/database/secure-data#edge-functions", + "title": "Edge Functions" } ], - "resultChars": 79200 - } - ] - }, - "prompt": "Build and serve a Supabase Edge Function named `user-stats` for this project,\nreachable over HTTP at `/functions/v1/user-stats`.\n\nOur product stores per-user metrics in the existing `user_stats` table.\n\nTwo very different callers need to hit this one endpoint:\n\n1. **Our mobile app**, acting for a signed-in user. It sends that user's\n Supabase access token. The endpoint should return the user's own stats.\n\n2. **Our internal billing service**, a trusted backend with no signed-in user.\n It authenticates with the project's secret (service-role) key in the `apikey`\n header, and names the target user with a `user_id` in the JSON request body.\n It needs that user's stats.\n\nReturn the matching rows as JSON. The endpoint must be secure: only ever serve\nstats to a caller that is genuinely entitled to them, and turn away callers that\nare not.", - "promptSourcePath": "evals/build-functions-005-dual-auth-user-secret/PROMPT.md", - "attempts": 1, - "sourcePath": "opencode-kimi-k3-no-skills/build-functions-005-dual-auth-user-secret.json" - }, - { - "experiment": "opencode-kimi-k3-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "opencode", - "modelProvider": "moonshotai", - "modelId": "moonshotai/kimi-k3" - }, - "eval": "build-storage-001-private-bucket-access", - "stage": "build", - "product": [ - "storage", - "database" - ], - "topic": [ - "rls", - "sdk" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "bucket user-files exists", - "passed": true - }, - { - "name": "bucket user-files is private", - "passed": true - }, - { - "name": "RLS still enabled on storage.objects", - "passed": true - }, - { - "name": "user A lists only own files", - "passed": true, - "notes": "saw: 019fb0e7-791a-732c-b308-f2e318f79d38/receipt-alpha.pdf, 019fb0e7-791a-732c-b308-f2e318f79d38/receipt-beta.pdf" - }, - { - "name": "user B cannot read user A files", - "passed": true - }, - { - "name": "anon reads no files", - "passed": true - }, - { - "name": "user A can upload into own folder", - "passed": true - }, - { - "name": "user B cannot upload into user A folder", - "passed": true - }, - { - "name": "configured private per-user storage access", - "passed": true, - "judgeNotes": "Meets all requirements: private user-files bucket, RLS kept enabled, authenticated owner-scoped SELECT and INSERT policies with WITH CHECK, and supabase-js createSignedUrl with expiry for temporary sharing." - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "Our app lets signed-in users keep personal files like receipts and bank\nstatements. These files are private — a user must only ever be able to upload\nand download their own. The app uploads each file under a path that starts\nwith the owner's user id, e.g. `/receipt-march.pdf`.\n\nSet up a `user-files` bucket on our project and lock it down that way.\n\nUsers also sometimes share one of their files with someone else through a\ntemporary link that expires. Include the supabase-js code the app should use\nfor that.", - "promptSourcePath": "evals/build-storage-001-private-bucket-access/PROMPT.md", - "attempts": 1, - "sourcePath": "opencode-kimi-k3-no-skills/build-storage-001-private-bucket-access.json" - }, - { - "experiment": "opencode-kimi-k3-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "opencode", - "modelProvider": "moonshotai", - "modelId": "moonshotai/kimi-k3" - }, - "eval": "build-tests-001-rls-tenant-isolation", - "stage": "build", - "product": [ - "database" - ], - "topic": [ - "tests", - "rls" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "pgTAP test file(s) written under supabase/tests/", - "passed": true, - "notes": "1 file(s): supabase/tests/rls_tenant_isolation_test.sql" - }, - { - "name": "pgTAP isolation tests ran and pass", - "passed": true, - "notes": "8 passed, 3 failed" - }, - { - "name": "agent correctly identifies the posts isolation bug from test results", - "passed": true, - "judgeNotes": "The agent correctly identifies `posts` as having a broken tenant isolation SELECT policy, explaining that authenticated users with any membership can read posts from other orgs because `m.org_id = posts.org_id` is missing. This conclusion is grounded in the pgTAP failures for cross-org post reads. The agent also notes a memberships exposure, but does not blame `notes` or dismiss the tests." - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] + "resultChars": 251299 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"\\\"SUPABASE_SECRET_KEYS\\\" \\\"@supabase/server\\\" withSupabase Edge Functions\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth", + "title": "Securing Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/auth/choosing-a-server-package", + "title": "Which package to use" + }, + { + "url": "https://supabase.com/docs/guides/functions/secrets", + "title": "Environment Variables" + }, + { + "url": "https://supabase.com/docs/guides/functions/quickstart-dashboard", + "title": "Getting Started with Edge Functions (Dashboard)" + } + ], + "resultChars": 41514 + } + ] }, - "prompt": "Can you audit the tenant isolation on our tables? Write some database tests covering both the happy path and the negative case, run them, and share the results.", - "promptSourcePath": "evals/build-tests-001-rls-tenant-isolation/PROMPT.md", - "attempts": 1, - "sourcePath": "opencode-kimi-k3-no-skills/build-tests-001-rls-tenant-isolation.json" + "prompt": "Build and serve a Supabase Edge Function named `user-stats` for this project,\nreachable over HTTP at `/functions/v1/user-stats`.\n\nOur product stores per-user metrics in the existing `user_stats` table.\n\nTwo very different callers need to hit this one endpoint:\n\n1. **Our mobile app**, acting for a signed-in user. It sends that user's\n Supabase access token. The endpoint should return the user's own stats.\n\n2. **Our internal billing service**, a trusted backend with no signed-in user.\n It authenticates with the project's secret (service-role) key in the `apikey`\n header, and names the target user with a `user_id` in the JSON request body.\n It needs that user's stats.\n\nReturn the matching rows as JSON. The endpoint must be secure: only ever serve\nstats to a caller that is genuinely entitled to them, and turn away callers that\nare not.", + "promptSourcePath": "evals/build-functions-005-dual-auth-user-secret/PROMPT.md", + "attempts": 2, + "sourcePath": "codex-gpt-5.6-no-skills/build-functions-005-dual-auth-user-secret.json" }, { - "experiment": "opencode-kimi-k3-no-skills", + "experiment": "codex-gpt-5.6-no-skills", "experimentSuite": "no-skills", "experimentDisplay": { - "agent": "opencode", - "modelProvider": "moonshotai", - "modelId": "moonshotai/kimi-k3" + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.6-sol", + "reasoningEffort": "low" }, - "eval": "build-vectors-001-rag-with-permissions", + "eval": "build-storage-001-private-bucket-access", "stage": "build", "product": [ - "database", - "vectors" + "storage", + "database" ], "topic": [ - "sql", - "rls" + "rls", + "sdk" ], "suite": "benchmark", "interface": "mcp", "passed": true, "checks": [ { - "name": "document_sections.embedding is vector(384)", - "passed": true, - "notes": "vector(384)" + "name": "bucket user-files exists", + "passed": true }, { - "name": "HNSW index on the embedding column", - "passed": true, - "notes": "CREATE INDEX document_sections_embedding_idx ON public.document_sections USING hnsw (embedding vector_cosine_ops)" + "name": "bucket user-files is private", + "passed": true }, { - "name": "index operator class matches the search operator", + "name": "RLS still enabled on storage.objects", + "passed": true + }, + { + "name": "user A lists only own files", "passed": true, - "notes": "function operators: <=>\nindexes: CREATE INDEX document_sections_embedding_idx ON public.document_sections USING hnsw (embedding vector_cosine_ops)" + "notes": "saw: 019f8a3b-e87c-70ea-a82d-0f71677c838b/receipt-alpha.pdf, 019f8a3b-e87c-70ea-a82d-0f71677c838b/receipt-beta.pdf" }, { - "name": "user A search returns only own sections, best match first", + "name": "user B cannot read user A files", "passed": true }, { - "name": "user B search returns only own sections, best match first", + "name": "anon reads no files", "passed": true }, { - "name": "user A reads only own sections through the API", + "name": "user A can upload into own folder", "passed": true }, { - "name": "user A reads only own documents through the API", + "name": "user B cannot upload into user A folder", "passed": true + }, + { + "name": "configured private per-user storage access", + "passed": true, + "judgeNotes": "Meets requirements: private user-files bucket, authenticated owner-scoped SELECT and INSERT policies with WITH CHECK, no RLS disabling or public access, and supabase-js createSignedUrl with short expiry." } ], "skills": { @@ -18158,63 +13798,75 @@ "calls": [ { "source": "search_docs", - "query": "{ searchDocs(query: \"gte-small embedding dimensions vector 384 semantic search edge function\", limit: 3) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"Storage access control RLS policy foldername auth.uid private bucket createSignedUrl supabase-js\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/functions/examples/semantic-search", - "title": "Semantic Search" + "url": "https://supabase.com/docs/guides/storage/buckets/fundamentals", + "title": "Storage Buckets" }, { - "url": "https://supabase.com/docs/guides/ai/choosing-compute-addon", - "title": "Choosing your Compute Add-on" + "url": "https://supabase.com/docs/guides/storage/serving/downloads", + "title": "Serving assets from Storage" }, { - "url": "https://supabase.com/docs/guides/ai/semantic-search", - "title": "Semantic search" + "url": "https://supabase.com/docs/guides/storage/schema/helper-functions", + "title": "Storage Helper Functions" + }, + { + "url": "https://supabase.com/docs/guides/storage/schema/custom-roles", + "title": "Custom Roles" + }, + { + "url": "https://supabase.com/docs/guides/storage/security/access-control", + "title": "Storage Access Control" } ], - "resultChars": 39484 + "resultChars": 17747 } ] }, - "prompt": "We're adding semantic search to our internal knowledge base app. I already wrote the edge functions, but search doesn't work yet. Can you set up whatever the database needs to make search work end to end?\n\nSome documents are confidential, so users should only have access to documents they own.", - "promptSourcePath": "evals/build-vectors-001-rag-with-permissions/PROMPT.md", - "attempts": 2, - "sourcePath": "opencode-kimi-k3-no-skills/build-vectors-001-rag-with-permissions.json" + "prompt": "Our app lets signed-in users keep personal files like receipts and bank\nstatements. These files are private — a user must only ever be able to upload\nand download their own. The app uploads each file under a path that starts\nwith the owner's user id, e.g. `/receipt-march.pdf`.\n\nSet up a `user-files` bucket on our project and lock it down that way.\n\nUsers also sometimes share one of their files with someone else through a\ntemporary link that expires. Include the supabase-js code the app should use\nfor that.", + "promptSourcePath": "evals/build-storage-001-private-bucket-access/PROMPT.md", + "attempts": 1, + "sourcePath": "codex-gpt-5.6-no-skills/build-storage-001-private-bucket-access.json" }, { - "experiment": "opencode-kimi-k3-no-skills", + "experiment": "codex-gpt-5.6-no-skills", "experimentSuite": "no-skills", "experimentDisplay": { - "agent": "opencode", - "modelProvider": "moonshotai", - "modelId": "moonshotai/kimi-k3" + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.6-sol", + "reasoningEffort": "low" }, - "eval": "deploy-database-001-prometheus-metrics", - "stage": "deploy", + "eval": "build-tests-001-rls-tenant-isolation", + "stage": "build", "product": [ "database" ], "topic": [ - "observability" + "tests", + "rls" ], "suite": "benchmark", + "interface": "cli", "passed": true, "checks": [ { - "name": "preserved existing app scrape job", - "passed": true + "name": "pgTAP test file(s) written under supabase/tests/", + "passed": true, + "notes": "1 file(s): supabase/tests/database/tenant_isolation.test.sql" }, { - "name": "configured the Supabase Metrics API scrape correctly", + "name": "pgTAP isolation tests ran and pass", "passed": true, - "judgeNotes": "Supabase scrape is deployable with HTTPS, correct metrics path, Basic Auth using password_file, project target on supabase.co, app job preserved, and docker-compose mounts the password file." + "notes": "5 passed, 3 failed" }, { - "name": "documented live deployment and verification steps", + "name": "agent correctly identifies the posts isolation bug from test results", "passed": true, - "judgeNotes": "README includes steps to create an sb_secret Secret API key, place it in the matching observability/secrets/supabase-metrics-key file, restart/reload the Compose/Prometheus stack, and verify via Prometheus targets/API query and Grafana. Endpoint/auth and secret setup match the config." + "judgeNotes": "The agent correctly identifies `posts` as having a tenant isolation flaw: its policy checks for any membership but does not match `posts.org_id`, allowing cross-organization reads. This is grounded in the pgTAP test results, while noting `notes` isolation passes." } ], "skills": { @@ -18225,240 +13877,232 @@ "calls": [ { "source": "search_docs", - "query": "{ searchDocs(query: \"metrics endpoint prometheus scrape project metrics\") { nodes { title href content } } }", + "query": "query { searchDocs(query: \"Supabase database testing pgTAP row level security auth.uid set role tests\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/telemetry/metrics/grafana-self-hosted", - "title": "Metrics API with Prometheus & Grafana (self-hosted)" - }, - { - "url": "https://supabase.com/docs/guides/telemetry/metrics", - "title": "Metrics API" - }, - { - "url": "https://supabase.com/docs/guides/telemetry/metrics/vendor-agnostic", - "title": "Vendor-agnostic Metrics API setup" - }, - { - "url": "https://supabase.com/docs/guides/telemetry/metrics/grafana-cloud", - "title": "Metrics API with Grafana Cloud" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/how-to-view-database-metrics-uqf2z_", - "title": "How to View Database Metrics" - }, - { - "url": "https://supabase.com/docs/guides/platform/read-replicas", - "title": "Read Replicas" + "url": "https://supabase.com/docs/guides/local-development/testing/overview", + "title": "Testing Overview" }, { - "url": "https://supabase.com/docs/guides/database/connection-management", - "title": "Connection management" + "url": "https://supabase.com/docs/guides/local-development/testing/pgtap-extended", + "title": "Advanced pgTAP Testing" }, { - "url": "https://supabase.com/docs/guides/realtime/benchmarks", - "title": "Benchmarks" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", + "title": "Row Level Security" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/grafana-not-displaying-data-sXJrMj", - "title": "Grafana not displaying data" + "url": "https://supabase.com/docs/guides/database/testing", + "title": "Testing Your Database" }, { - "url": "https://supabase.com/docs/guides/telemetry/reports", - "title": "Reports" + "url": "https://supabase.com/docs/guides/local-development/cli/testing-and-linting", + "title": "Testing and linting" } ], - "resultChars": 81235 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"management api create secret api key sb_secret\") { nodes { title href content methodName language } } }", - "hasContent": true, - "pages": [] - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"management api create secret api key sb_secret\") { nodes { title href content ... on ManagementApiReference { title href content } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", - "title": "New API Keys and Asymmetric Authentication" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin", - "title": "Build a Product Management Android App with Jetpack Compose" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-angular", - "title": "Build a User Management App with Angular" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-ionic-angular", - "title": "Build a User Management App with Ionic Angular" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-ionic-react", - "title": "Build a User Management App with Ionic React" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-svelte", - "title": "Build a User Management App with Svelte" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-redwoodjs", - "title": "Build a User Management App with RedwoodJS" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-vue-3", - "title": "Build a User Management App with Vue 3" - }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys", - "title": "JWT Signing Keys" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-ionic-vue", - "title": "Build a User Management App with Ionic Vue" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs", - "title": "Build a User Management App with Next.js" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-swift", - "title": "Build a User Management App with Swift and SwiftUI" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-sveltekit", - "title": "Build a User Management App with SvelteKit" - }, - { - "url": "https://supabase.com/docs/guides/auth/quickstarts/with-expo-react-native-social-auth", - "title": "Build a Social Auth App with Expo React Native" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-solidjs", - "title": "Build a User Management App with SolidJS" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-react", - "title": "Build a User Management App with React" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-expo-react-native", - "title": "Build a User Management App with Expo React Native" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-refine", - "title": "Build a User Management App with Refine" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nuxt-3", - "title": "Build a User Management App with Nuxt 3" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-flutter", - "title": "Build a User Management App with Flutter" - }, - { - "url": "https://supabase.com/docs/reference/api/v1-bulk-create-secrets", - "title": "Bulk create secrets" - }, - { - "url": "https://supabase.com/docs/reference/api/v1-create-project-api-key", - "title": "Creates a new API key for the project" - }, - { - "url": "https://supabase.com/docs/guides/telemetry/metrics/vendor-agnostic", - "title": "Vendor-agnostic Metrics API setup" - }, + "resultChars": 68836 + } + ] + }, + "prompt": "Can you audit the tenant isolation on our tables? Write some database tests covering both the happy path and the negative case, run them, and share the results.", + "promptSourcePath": "evals/build-tests-001-rls-tenant-isolation/PROMPT.md", + "attempts": 1, + "sourcePath": "codex-gpt-5.6-no-skills/build-tests-001-rls-tenant-isolation.json" + }, + { + "experiment": "codex-gpt-5.6-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.6-sol", + "reasoningEffort": "low" + }, + "eval": "build-vectors-001-rag-with-permissions", + "stage": "build", + "product": [ + "database", + "vectors" + ], + "topic": [ + "sql", + "rls" + ], + "suite": "benchmark", + "interface": "mcp", + "passed": true, + "checks": [ + { + "name": "document_sections.embedding is vector(384)", + "passed": true, + "notes": "vector(384)" + }, + { + "name": "HNSW index on the embedding column", + "passed": true, + "notes": "CREATE INDEX document_sections_embedding_hnsw_idx ON public.document_sections USING hnsw (embedding vector_cosine_ops)" + }, + { + "name": "index operator class matches the search operator", + "passed": true, + "notes": "function operators: <=>\nindexes: CREATE INDEX document_sections_embedding_hnsw_idx ON public.document_sections USING hnsw (embedding vector_cosine_ops)" + }, + { + "name": "user A search returns only own sections, best match first", + "passed": true + }, + { + "name": "user B search returns only own sections, best match first", + "passed": true + }, + { + "name": "user A reads only own sections through the API", + "passed": true + }, + { + "name": "user A reads only own documents through the API", + "passed": true + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"pgvector semantic search match_documents RPC row level security auth.uid\", limit: 5) { nodes { ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } } } }", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/guides/self-hosting/docker", - "title": "Self-Hosting with Docker" + "url": "https://supabase.com/docs/guides/ai/rag-with-permissions", + "title": "RAG with Permissions" }, { - "url": "https://supabase.com/docs/guides/telemetry/metrics/grafana-cloud", - "title": "Metrics API with Grafana Cloud" + "url": "https://supabase.com/docs/guides/ai/semantic-search", + "title": "Semantic search" }, { - "url": "https://supabase.com/docs/guides/telemetry/metrics/grafana-self-hosted", - "title": "Metrics API with Prometheus & Grafana (self-hosted)" + "url": "https://supabase.com/docs/guides/ai/hybrid-search", + "title": "Hybrid search" }, { - "url": "https://supabase.com/docs/guides/integrations/supabase-for-platforms", - "title": "Supabase for Platforms" + "url": "https://supabase.com/docs/guides/getting-started/features", + "title": "Features" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", - "title": "Self-Hosted Functions" + "url": "https://supabase.com/docs/guides/database/full-text-search", + "title": "Full Text Search" } ], - "resultChars": 729739 - }, + "resultChars": 95111 + } + ] + }, + "prompt": "We're adding semantic search to our internal knowledge base app. I already wrote the edge functions, but search doesn't work yet. Can you set up whatever the database needs to make search work end to end?\n\nSome documents are confidential, so users should only have access to documents they own.", + "promptSourcePath": "evals/build-vectors-001-rag-with-permissions/PROMPT.md", + "attempts": 1, + "sourcePath": "codex-gpt-5.6-no-skills/build-vectors-001-rag-with-permissions.json" + }, + { + "experiment": "codex-gpt-5.6-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.6-sol", + "reasoningEffort": "low" + }, + "eval": "deploy-database-001-prometheus-metrics", + "stage": "deploy", + "product": [ + "database" + ], + "topic": [ + "observability" + ], + "suite": "benchmark", + "passed": true, + "checks": [ + { + "name": "preserved existing app scrape job", + "passed": true + }, + { + "name": "configured the Supabase Metrics API scrape correctly", + "passed": true, + "judgeNotes": "Supabase scrape uses HTTPS, correct metrics path, Basic Auth with password_file, preserves the app job, targets the Supabase project host via placeholder substitution, and wires the Compose secret to /run/secrets/supabase_metrics_secret." + }, + { + "name": "documented live deployment and verification steps", + "passed": true, + "judgeNotes": "README includes required live setup: create/copy Supabase Secret API key, write it to the Compose secret file path, configure project ref and UID/GID, and recreate the Compose stack. Verification is concrete via Prometheus targets API, promtool, and Grafana/PromQL up{job=\"supabase\"}. Endpoint/auth and secret path match the provided config." + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [ { "source": "search_docs", - "query": "{ searchDocs(query: \"POST api-keys create new secret key management api endpoint\", limit: 8) { nodes { title href content ... on ManagementApiReference { title href content } } } }", + "query": "query { searchDocs(query: \"Prometheus metrics endpoint hosted project customer v1 privileged metrics service role basic auth\", limit: 8) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin", - "title": "Build a Product Management Android App with Jetpack Compose" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" + "url": "https://supabase.com/docs/guides/telemetry/metrics/grafana-self-hosted", + "title": "Metrics API with Prometheus & Grafana (self-hosted)" }, { - "url": "https://supabase.com/docs/reference/api/v1-create-project-api-key", - "title": "Creates a new API key for the project" + "url": "https://supabase.com/docs/guides/telemetry/metrics/vendor-agnostic", + "title": "Vendor-agnostic Metrics API setup" }, { - "url": "https://supabase.com/docs/guides/integrations/supabase-for-platforms", - "title": "Supabase for Platforms" + "url": "https://supabase.com/docs/guides/telemetry/metrics/grafana-cloud", + "title": "Metrics API with Grafana Cloud" }, { - "url": "https://supabase.com/docs/reference/api/v1-get-project-api-keys", - "title": "Get project api keys" + "url": "https://supabase.com/docs/guides/telemetry/metrics", + "title": "Metrics API" }, { - "url": "https://supabase.com/docs/reference/api/v1-get-project-api-key", - "title": "Get API key" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", + "title": "Envoy API Gateway" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", - "title": "New API Keys and Asymmetric Authentication" + "url": "https://supabase.com/docs/guides/security/security-testing", + "title": "Security testing of your Supabase projects" }, { - "url": "https://supabase.com/docs/reference/api/v1-create-project-signing-key", - "title": "Create a new signing key for the project in standby status" + "url": "https://supabase.com/docs/guides/database/extensions/pgaudit", + "title": "PGAudit: Postgres Auditing" } ], - "resultChars": 99738 + "resultChars": 67558 + }, + { + "source": "web_search", + "query": "site:supabase.com/docs/guides/telemetry/metrics/grafana-self-hosted Supabase Metrics API Prometheus service_role sb_secret", + "pages": [] } ] }, - "prompt": "Can you wire my Supabase project metrics into our existing observability stack and document\nin the observability README what we need to do to make the config live?", + "prompt": "Can you wire my Supabase project metrics into our existing observability stack and document\nwhat we need to do to make the config live?", "promptSourcePath": "evals/deploy-database-001-prometheus-metrics/PROMPT.md", "attempts": 1, - "sourcePath": "opencode-kimi-k3-no-skills/deploy-database-001-prometheus-metrics.json" + "sourcePath": "codex-gpt-5.6-no-skills/deploy-database-001-prometheus-metrics.json" }, { - "experiment": "opencode-kimi-k3-no-skills", + "experiment": "codex-gpt-5.6-no-skills", "experimentSuite": "no-skills", "experimentDisplay": { - "agent": "opencode", - "modelProvider": "moonshotai", - "modelId": "moonshotai/kimi-k3" + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.6-sol", + "reasoningEffort": "low" }, "eval": "deploy-functions-001-edge-function-secrets", "stage": "deploy", @@ -18496,20 +14140,62 @@ "loaded": [] }, "docs": { - "calls": [] + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Supabase Edge Functions environment variables secrets set CLI deploy no verify jwt Deno.env.get\", limit: 8) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", + "title": "Self-Hosted Functions" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/inspecting-edge-function-environment-variables-wg5qOQ", + "title": "Inspecting edge function environment variables" + }, + { + "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech", + "title": "Transcription Telegram Bot" + }, + { + "url": "https://supabase.com/docs/guides/functions/secrets", + "title": "Environment Variables" + }, + { + "url": "https://supabase.com/docs/guides/local-development/cli/testing-and-linting", + "title": "Testing and linting" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth", + "title": "Securing Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/functions/quickstart", + "title": "Getting Started with Edge Functions" + } + ], + "resultChars": 62400 + } + ] }, "prompt": "Our weather widget currently calls WeatherAPI straight from the browser, which\nleaks our API key. I want to move that behind a Supabase Edge Function called\n`weather` that holds the key server-side and proxies the request.\n\nThe function should read the key from an environment variable named\n`WEATHER_API_KEY`. Our key already lives in a local `.env` file at the project\nroot.\n\nDeploy the function to our project so it's live, and make sure the deployed\nfunction can actually read the key at runtime.", "promptSourcePath": "evals/deploy-functions-001-edge-function-secrets/PROMPT.md", "attempts": 1, - "sourcePath": "opencode-kimi-k3-no-skills/deploy-functions-001-edge-function-secrets.json" + "sourcePath": "codex-gpt-5.6-no-skills/deploy-functions-001-edge-function-secrets.json" }, { - "experiment": "opencode-kimi-k3-no-skills", + "experiment": "codex-gpt-5.6-no-skills", "experimentSuite": "no-skills", "experimentDisplay": { - "agent": "opencode", - "modelProvider": "moonshotai", - "modelId": "moonshotai/kimi-k3" + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.6-sol", + "reasoningEffort": "low" }, "eval": "deploy-self-hosting-001-docker-compose", "stage": "deploy", @@ -18550,146 +14236,87 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"self-hosting with docker generate API keys anon service_role JWT secret\") { nodes { title href content } } }", + "query": "query { searchDocs(query: \"self-hosting Docker docker-compose .env secrets JWT_SECRET ANON_KEY SERVICE_ROLE_KEY POSTGRES_PASSWORD DASHBOARD_PASSWORD POOLER_TENANT_ID\", limit: 8) { nodes { title href content } } }", "hasContent": true, "pages": [ { "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", "title": "New API Keys and Asymmetric Authentication" }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys", - "title": "JWT Signing Keys" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", - "title": "Envoy API Gateway" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, { "url": "https://supabase.com/docs/guides/self-hosting/docker", "title": "Self-Hosting with Docker" }, { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin", - "title": "Build a Product Management Android App with Jetpack Compose" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/rotating-anon-service-and-jwt-secrets-1Jq6yd", - "title": "Rotating Anon, Service, and JWT Secrets" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-saml-sso", - "title": "Configure SAML SSO" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-proxy-https", - "title": "Configure Reverse Proxy and HTTPS" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform", - "title": "Restore a Platform Project to Self-Hosted" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-ionic-react", - "title": "Build a User Management App with Ionic React" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs", - "title": "Build a User Management App with Next.js" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-solidjs", - "title": "Build a User Management App with SolidJS" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-react", - "title": "Build a User Management App with React" + "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17", + "title": "Upgrade to Postgres 17" }, { - "url": "https://supabase.com/docs/guides/storage/serving/image-transformations", - "title": "Storage Image Transformations" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", + "title": "Envoy API Gateway" }, { "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth", - "title": "Configure Social Login (OAuth) Providers" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-angular", - "title": "Build a User Management App with Angular" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-ionic-angular", - "title": "Build a User Management App with Ionic Angular" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-phone-mfa", - "title": "Configure Phone Login & MFA" - }, - { - "url": "https://supabase.com/docs/guides/auth/server-side/creating-a-client", - "title": "Creating a Supabase client for SSR" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-refine", - "title": "Build a User Management App with Refine" + "title": "Configure Social Login (OAuth) Providers" }, { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-sveltekit", - "title": "Build a User Management App with SvelteKit" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", + "title": "Self-Hosted Functions" }, { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-svelte", - "title": "Build a User Management App with Svelte" + "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform", + "title": "Restore a Platform Project to Self-Hosted" }, { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-swift", - "title": "Build a User Management App with Swift and SwiftUI" + "url": "https://supabase.com/docs/guides/self-hosting/remove-superuser-access", + "title": "Remove superuser access from Studio" + } + ], + "resultChars": 132450 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Docker self-hosting quick start Linux clone supabase docker cp .env.example generate-keys.sh docker compose pull\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/self-hosting/docker", + "title": "Self-Hosting with Docker" }, { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-flutter", - "title": "Build a User Management App with Flutter" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", + "title": "New API Keys and Asymmetric Authentication" }, { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nuxt-3", - "title": "Build a User Management App with Nuxt 3" + "url": "https://supabase.com/docs/guides/local-development/cli/getting-started", + "title": "Supabase CLI" }, { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-ionic-vue", - "title": "Build a User Management App with Ionic Vue" + "url": "https://supabase.com/docs/guides/local-development/cli-workflows", + "title": "Local development workflow" }, { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-vue-3", - "title": "Build a User Management App with Vue 3" + "url": "https://supabase.com/docs/guides/self-hosting", + "title": "Self-Hosting" } ], - "resultChars": 730694 + "resultChars": 92621 } ] }, "prompt": "I'm moving off the hosted Supabase and running the whole thing myself on a VPS I\njust spun up. Can you get a Docker setup ready for me to copy onto the box?\n\nI don't need it running here, I'll do the actual bring-up once I'm on the\nserver. I just want everything in place and the secrets set up. Put it in a `supabase-docker/`\nfolder at the repo root so I can scp the whole thing across in one go.", "promptSourcePath": "evals/deploy-self-hosting-001-docker-compose/PROMPT.md", "attempts": 1, - "sourcePath": "opencode-kimi-k3-no-skills/deploy-self-hosting-001-docker-compose.json" + "sourcePath": "codex-gpt-5.6-no-skills/deploy-self-hosting-001-docker-compose.json" }, { - "experiment": "opencode-kimi-k3-no-skills", + "experiment": "codex-gpt-5.6-no-skills", "experimentSuite": "no-skills", "experimentDisplay": { - "agent": "opencode", - "modelProvider": "moonshotai", - "modelId": "moonshotai/kimi-k3" + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.6-sol", + "reasoningEffort": "low" }, "eval": "investigate-auth-001-deleted-user-access", "stage": "investigate", @@ -18732,7 +14359,7 @@ { "name": "diagnosed and explained session revocation", "passed": true, - "judgeNotes": "The answer correctly identifies the soft-delete-only root cause, implements real auth user deletion with cascading session/refresh token/identity removal, and explains the remaining stateless JWT access-token window consistently with that implementation. It also accurately distinguishes publishable/frontend/RLS-enforced keys from secret/backend/RLS-bypassing keys and warns not to expose secret keys." + "judgeNotes": "The answer identifies the soft-delete-only root cause, implements real auth-user deletion with cascading session/identity removal, and adds RLS checks that verify the auth user/profile still exists, so its claim of no Data API stale-JWT window is consistent. It still correctly notes JWTs remain cryptographically valid until expiry for local validation/client state. It also correctly distinguishes publishable frontend keys with RLS from secret/server-only keys that bypass RLS." } ], "skills": { @@ -18743,32 +14370,161 @@ "calls": [ { "source": "search_docs", - "query": "{ searchDocs(query: \"publishable key secret key vs anon service_role API keys RLS\", limit: 5) { nodes { title content } } }", + "query": "query { searchDocs(query: \"API keys publishable secret legacy anon service_role RLS JWT session delete user revoke access\", limit: 8) { nodes { ... on Guide { title href content } ... on TroubleshootingGuide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/auth/signing-keys", + "title": "JWT Signing Keys" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/api-keys", + "title": "Understanding API keys" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/why-is-my-service-role-key-client-getting-rls-errors-or-not-returning-data-7_1K9z", + "title": "Why is my service role key client getting RLS errors or not returning data?" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", + "title": "New API Keys and Asymmetric Authentication" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/rotating-anon-service-and-jwt-secrets-1Jq6yd", + "title": "Rotating Anon, Service, and JWT Secrets" + }, + { + "url": "https://supabase.com/docs/guides/auth/jwt-fields", + "title": "JWT Claims Reference" + }, + { + "url": "https://supabase.com/docs/guides/database/secure-data", + "title": "Securing your data" + } + ], + "resultChars": 95277 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"delete user auth.users SQL existing JWT remains valid until expiration sessions revoke refresh token delete account\", limit: 10) { nodes { ... on Guide { title href content } ... on TroubleshootingGuide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/auth/managing-user-data", + "title": "User Management" + }, + { + "url": "https://supabase.com/docs/guides/auth/sessions", + "title": "User sessions" + }, + { + "url": "https://supabase.com/docs/guides/auth/auth-anonymous", + "title": "Anonymous Sign-Ins" + }, + { + "url": "https://supabase.com/docs/guides/auth/signing-keys", + "title": "JWT Signing Keys" + }, + { + "url": "https://supabase.com/docs/guides/auth/jwt-fields", + "title": "JWT Claims Reference" + }, + { + "url": "https://supabase.com/docs/reference/javascript/auth-admin-deleteuser" + }, + { + "url": "https://supabase.com/docs/guides/auth/jwts", + "title": "JSON Web Token (JWT)" + }, + { + "url": "https://supabase.com/docs/reference/swift/auth-admin-deleteuser", + "title": "deleteUser()" + }, + { + "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows", + "title": "OAuth 2.1 Flows" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform", + "title": "Restore a Platform Project to Self-Hosted" + } + ], + "resultChars": 140501 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Deleting users JWT remains valid until expiry deleteUser should sign out\", limit: 5) { nodes { ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } ... on TroubleshootingGuide { title href content } } } }", "hasContent": true, - "pages": [], - "resultChars": 64643 + "pages": [ + { + "url": "https://supabase.com/docs/guides/auth/sessions", + "title": "User sessions" + }, + { + "url": "https://supabase.com/docs/guides/auth/managing-user-data", + "title": "User Management" + }, + { + "url": "https://supabase.com/docs/reference/javascript/auth-signout" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/should-i-set-a-shorter-max-age-parameter-on-the-cookies-8sbF4V", + "title": "Should I set a shorter Max-Age parameter on the cookies?" + }, + { + "url": "https://supabase.com/docs/guides/auth/signing-keys", + "title": "JWT Signing Keys" + } + ], + "resultChars": 54847 }, { "source": "search_docs", - "query": "{ searchDocs(query: \"Understanding API keys publishable anonymous RLS\", limit: 3) { nodes { title content } } }", + "query": "query { searchDocs(query: \"publishable key secret key RLS bypass frontend sb_publishable sb_secret role behavior\", limit: 5) { nodes { ... on Guide { title href content } } } }", "hasContent": true, - "pages": [], - "resultChars": 52294 + "pages": [ + { + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", + "title": "Row Level Security" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/api-keys", + "title": "Understanding API keys" + }, + { + "url": "https://supabase.com/docs/guides/database/secure-data", + "title": "Securing your data" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", + "title": "New API Keys and Asymmetric Authentication" + } + ], + "resultChars": 73182 } ] }, "prompt": "Last week support removed a user through our app's delete-account flow — the\napp calls the `delete_account` function over RPC as the signed-in user. This\nmorning that same person was back: still signed in, reading and saving their\ndata like nothing happened.\n\nFigure out why the account still works, fix the flow so a deleted account\nloses access, and tell me whether there is any window where they could still\nget in after the fix.\n\nOne more thing while you're at it: we're migrating off the legacy\nanon/service_role API keys to the new publishable and secret keys, and the\nteam isn't sure which one belongs in the frontend and what each means for\nRLS. Give us a quick rundown.", "promptSourcePath": "evals/investigate-auth-001-deleted-user-access/PROMPT.md", "attempts": 1, - "sourcePath": "opencode-kimi-k3-no-skills/investigate-auth-001-deleted-user-access.json" + "sourcePath": "codex-gpt-5.6-no-skills/investigate-auth-001-deleted-user-access.json" }, { - "experiment": "opencode-kimi-k3-no-skills", + "experiment": "codex-gpt-5.6-no-skills", "experimentSuite": "no-skills", "experimentDisplay": { - "agent": "opencode", - "modelProvider": "moonshotai", - "modelId": "moonshotai/kimi-k3" + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.6-sol", + "reasoningEffort": "low" }, "eval": "investigate-realtime-001-subscribed-no-events", "stage": "investigate", @@ -18807,7 +14563,7 @@ { "name": "diagnosed missing publication membership", "passed": true, - "judgeNotes": "Identifies missing orders table in supabase_realtime as root cause, applies ALTER PUBLICATION ADD TABLE public.orders, preserves existing publication/RLS/policies and courier_locations, and avoids disallowed fixes." + "judgeNotes": "The assistant correctly identified the root cause as `orders` missing from the `supabase_realtime` publication despite subscriptions reaching SUBSCRIBED, added only `public.orders` to the existing publication, verified `courier_locations` remained, and did not alter RLS or policies." } ], "skills": { @@ -18820,15 +14576,16 @@ "prompt": "Our dispatch dashboard shows incoming orders as they happen. The courier\nlocation feed on the same page updates live without problems, but new orders\nonly show up after a page refresh.\n\nThe dashboard uses supabase-js to subscribe to INSERT events on the `orders`\ntable through postgres_changes, the same way it subscribes to courier\nlocations. The channel's status callback logs SUBSCRIBED and there are no\nerrors in the browser console.\n\nFigure out why no order events ever arrive and fix it.", "promptSourcePath": "evals/investigate-realtime-001-subscribed-no-events/PROMPT.md", "attempts": 1, - "sourcePath": "opencode-kimi-k3-no-skills/investigate-realtime-001-subscribed-no-events.json" + "sourcePath": "codex-gpt-5.6-no-skills/investigate-realtime-001-subscribed-no-events.json" }, { - "experiment": "opencode-kimi-k3-no-skills", + "experiment": "codex-gpt-5.6-no-skills", "experimentSuite": "no-skills", "experimentDisplay": { - "agent": "opencode", - "modelProvider": "moonshotai", - "modelId": "moonshotai/kimi-k3" + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.6-sol", + "reasoningEffort": "low" }, "eval": "investigate-reliability-003-edge-function-5xx-correlation", "stage": "investigate", @@ -18845,17 +14602,17 @@ { "name": "identified image-transform and the recurring 503 pattern", "passed": true, - "judgeNotes": "The assistant identified image-transform as the affected function and described the recurring 8 HTTP 503 gateway failures across the morning of 2026-04-28 from 07:00Z–12:00Z, while distinguishing unrelated billing-webhook noise." + "judgeNotes": "Assistant identified image-transform as the affected function and described recurring 503 responses on 2026-04-28 from 07:00 through 12:00 UTC, matching the required pattern." }, { "name": "attributed recurring 503s to gateway/platform layer, not function code", "passed": true, - "judgeNotes": "The assistant clearly attributes the recurring image-transform 503s to the gateway/platform layer, not function code, and grounds this in valid observations: gateway identifiers with no corresponding runtime execution records, nearby successful executions on unchanged deployment, and distinction from avatar-upload's function-level 500." + "judgeNotes": "The assistant attributes the 503s to the Edge Function gateway/platform layer rather than function code, and grounds this in valid observations: failed requests do not appear in execution logs, successful executions used the same deployment and were fast, and the separate avatar-upload 500 is treated as distinct." }, { "name": "recommended a concrete next step", "passed": true, - "judgeNotes": "Recommended concrete next steps including opening a Supabase support ticket with gateway request IDs and timestamps, plus retry logic and alerting." + "judgeNotes": "The assistant recommended concrete next steps, including escalating to Supabase Support with project/function/deployment details and timestamps, adding retry/backoff, alerting on gateway responses without execution records, and queueing transformations." } ], "skills": { @@ -18867,16 +14624,17 @@ }, "prompt": "Users have been reporting that image uploads are intermittently failing with server errors throughout the morning of `2026-04-28`.\n\nCan you investigate the project logs and tell me what is going on and what we\nshould do next?", "promptSourcePath": "evals/investigate-reliability-003-edge-function-5xx-correlation/PROMPT.md", - "attempts": 2, - "sourcePath": "opencode-kimi-k3-no-skills/investigate-reliability-003-edge-function-5xx-correlation.json" + "attempts": 1, + "sourcePath": "codex-gpt-5.6-no-skills/investigate-reliability-003-edge-function-5xx-correlation.json" }, { - "experiment": "opencode-kimi-k3-no-skills", + "experiment": "codex-gpt-5.6-no-skills", "experimentSuite": "no-skills", "experimentDisplay": { - "agent": "opencode", - "modelProvider": "moonshotai", - "modelId": "moonshotai/kimi-k3" + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.6-sol", + "reasoningEffort": "low" }, "eval": "resolve-dataapi-001-empty-results", "stage": "resolve", @@ -18920,7 +14678,7 @@ { "name": "diagnosed RLS and added owner-scoped policies", "passed": true, - "judgeNotes": "Diagnosed RLS deny-all due to no policies, kept RLS enabled, and created authenticated SELECT and INSERT policies scoped to user_id = auth.uid() with WITH CHECK for inserts." + "judgeNotes": "The assistant correctly diagnosed RLS enabled with no policies as deny-all for Data API, kept RLS enabled, and created authenticated-only SELECT and INSERT policies scoped to user_id = auth.uid(), with INSERT enforced via WITH CHECK." } ], "skills": { @@ -18928,20 +14686,50 @@ "loaded": [] }, "docs": { - "calls": [] + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Row Level Security authenticated users Data API table grants SELECT INSERT auth.uid policies\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/api/securing-your-api", + "title": "Securing your API" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", + "title": "Row Level Security" + }, + { + "url": "https://supabase.com/docs/guides/local-development/testing/pgtap-extended", + "title": "Advanced pgTAP Testing" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/features", + "title": "Features" + }, + { + "url": "https://supabase.com/docs/guides/auth/auth-mfa", + "title": "Multi-Factor Authentication" + } + ], + "resultChars": 104465 + } + ] }, "prompt": "Our app lets signed-in users save bookmarks and view them on their dashboard. Bookmarks are stored in the `bookmarks` table and are private — a user must only ever see their own. \nUsers also need to be able to save new bookmarks from the app.\n\nI can see the rows when I query the table directly, but the dashboard shows an empty list for every user.\n\nFind out why the Data API returns nothing and fix it.", "promptSourcePath": "evals/resolve-dataapi-001-empty-results/PROMPT.md", "attempts": 1, - "sourcePath": "opencode-kimi-k3-no-skills/resolve-dataapi-001-empty-results.json" + "sourcePath": "codex-gpt-5.6-no-skills/resolve-dataapi-001-empty-results.json" }, { - "experiment": "opencode-kimi-k3-no-skills", + "experiment": "codex-gpt-5.6-no-skills", "experimentSuite": "no-skills", "experimentDisplay": { - "agent": "opencode", - "modelProvider": "moonshotai", - "modelId": "moonshotai/kimi-k3" + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.6-sol", + "reasoningEffort": "low" }, "eval": "resolve-database-001-migration-history-mismatch", "stage": "resolve", @@ -18978,7 +14766,7 @@ { "name": "the avatar migration and history reconciliation were done via the Supabase CLI", "passed": true, - "judgeNotes": "Avatar migration was applied through `supabase db push` in action #28, which shows `Applying migration 20240220000000_add_avatar_url.sql...` and `Finished supabase db push.` History was reconciled by adding/restoring the local orphan migration file `20240115000000_add_profile_bio.sql` (actions #20-#23), after which `supabase migration list` and the successful `supabase db push` showed local/remote histories aligned. Only read-only `psql` inspection was used; no banned direct SQL mutation or prepared-statement workaround was seen." + "judgeNotes": "Avatar migration was applied via Supabase CLI in #9, whose output includes the tail of “Finished supabase db push.” The orphan bio migration was reconciled by adding local file supabase/migrations/20240115000000_add_profile_bio.sql and then running the CLI push. No disallowed direct SQL mutation or prepared-statement workaround was shown." } ], "skills": { @@ -18986,20 +14774,50 @@ "loaded": [] }, "docs": { - "calls": [] + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Supabase CLI migration deploy db push repair migration history linked project\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/deployment/database-migrations", + "title": "Database Migrations" + }, + { + "url": "https://supabase.com/docs/guides/local-development/cli-workflows", + "title": "Local development workflow" + }, + { + "url": "https://supabase.com/docs/guides/local-development/database-migrations", + "title": "Database migrations" + }, + { + "url": "https://supabase.com/docs/reference/cli/supabase-migration-repair", + "title": "Repair the migration history table" + }, + { + "url": "https://supabase.com/docs/reference/cli/supabase-db-push", + "title": "Push new migrations to the remote database" + } + ], + "resultChars": 61657 + } + ] }, "prompt": "I'm trying to ship a migration to our hosted project and it's not working. Can you figure out what's wrong and get it deployed?", "promptSourcePath": "evals/resolve-database-001-migration-history-mismatch/PROMPT.md", "attempts": 1, - "sourcePath": "opencode-kimi-k3-no-skills/resolve-database-001-migration-history-mismatch.json" + "sourcePath": "codex-gpt-5.6-no-skills/resolve-database-001-migration-history-mismatch.json" }, { - "experiment": "opencode-kimi-k3-no-skills", + "experiment": "codex-gpt-5.6-no-skills", "experimentSuite": "no-skills", "experimentDisplay": { - "agent": "opencode", - "modelProvider": "moonshotai", - "modelId": "moonshotai/kimi-k3" + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.6-sol", + "reasoningEffort": "low" }, "eval": "resolve-performance-001-slow-query-cpu-spike", "stage": "resolve", @@ -19029,7 +14847,7 @@ { "name": "query plan uses an index and avoids sequential scan", "passed": true, - "notes": "Limit (cost=79.74..79.86 rows=50 width=58)\n -> Sort (cost=79.74..79.99 rows=100 width=58)\n Sort Key: created_at DESC\n -> Bitmap Heap Scan on events (cost=5.06..76.42 rows=100 width=58)\n Recheck Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)\n -> Bitmap Index Scan on idx_events_user_id_created_at (cost=0.00..5.03 rows=100 width=0)\n Index Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)" + "notes": "Limit (cost=79.74..79.86 rows=50 width=58)\n -> Sort (cost=79.74..79.99 rows=100 width=58)\n Sort Key: created_at DESC\n -> Bitmap Heap Scan on events (cost=5.06..76.42 rows=100 width=58)\n Recheck Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)\n -> Bitmap Index Scan on idx_events_user_id_created_at_desc (cost=0.00..5.03 rows=100 width=0)\n Index Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)" }, { "name": "inserts still work", @@ -19046,15 +14864,16 @@ "prompt": "My database CPU keeps spiking and the app gets slow when loading recent events for a user. Can you figure out what query is causing it and make the database change needed to fix it?\n\nEnd your turn with a short summary of what you changed and why.", "promptSourcePath": "evals/resolve-performance-001-slow-query-cpu-spike/PROMPT.md", "attempts": 1, - "sourcePath": "opencode-kimi-k3-no-skills/resolve-performance-001-slow-query-cpu-spike.json" + "sourcePath": "codex-gpt-5.6-no-skills/resolve-performance-001-slow-query-cpu-spike.json" }, { - "experiment": "opencode-kimi-k3-no-skills", + "experiment": "codex-gpt-5.6-no-skills", "experimentSuite": "no-skills", "experimentDisplay": { - "agent": "opencode", - "modelProvider": "moonshotai", - "modelId": "moonshotai/kimi-k3" + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.6-sol", + "reasoningEffort": "low" }, "eval": "resolve-security-002-rls-cross-tenant-leak", "stage": "resolve", @@ -19117,6 +14936,6 @@ "prompt": "A customer reported that notes showed up in the wrong workspace.\n\nCan you investigate what is going on and fix it?", "promptSourcePath": "evals/resolve-security-002-rls-cross-tenant-leak/PROMPT.md", "attempts": 1, - "sourcePath": "opencode-kimi-k3-no-skills/resolve-security-002-rls-cross-tenant-leak.json" + "sourcePath": "codex-gpt-5.6-no-skills/resolve-security-002-rls-cross-tenant-leak.json" } ] diff --git a/apps/web/src/data/regression-eval-results.json b/apps/web/src/data/regression-eval-results.json index 7858eeeb..05472569 100644 --- a/apps/web/src/data/regression-eval-results.json +++ b/apps/web/src/data/regression-eval-results.json @@ -38,17 +38,17 @@ { "name": "user with JWT reads only their own rows", "passed": true, - "notes": "status 200: {\"data\":[{\"user_id\":\"f4d6f7c2-4ad0-4876-be1a-f88b4e08f061\",\"metric\":\"steps_a_msbgsdzz\",\"value\":111}]}" + "notes": "status 200: {\"data\":[{\"user_id\":\"e0467373-82c5-48f3-b182-a7f4996e67e5\",\"metric\":\"steps_a_mrw5pzin\",\"value\":111}]}" }, { "name": "user cannot read another user's rows by passing user_id", "passed": true, - "notes": "status 200: {\"data\":[{\"user_id\":\"f4d6f7c2-4ad0-4876-be1a-f88b4e08f061\",\"metric\":\"steps_a_msbgsdzz\",\"value\":111}]}" + "notes": "status 200: {\"data\":[{\"user_id\":\"e0467373-82c5-48f3-b182-a7f4996e67e5\",\"metric\":\"steps_a_mrw5pzin\",\"value\":111}]}" }, { "name": "service key bypasses RLS to read the target user's rows", "passed": true, - "notes": "status 200: {\"data\":[{\"user_id\":\"a8d12bf4-d51e-4eeb-8cec-e33b0a17fbee\",\"metric\":\"steps_b_msbgsdzz\",\"value\":222}]}" + "notes": "status 200: {\"data\":[{\"user_id\":\"cdabe339-7dff-4ebd-8e3d-40cc6952d6ca\",\"metric\":\"steps_b_mrw5pzin\",\"value\":222}]}" }, { "name": "non-service key is not granted service access", @@ -81,7 +81,84 @@ ] }, "docs": { - "calls": [] + "calls": [ + { + "source": "search_docs", + "query": "{ searchDocs(query: \"@supabase/server withSupabase edge function\", limit: 10) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/auth/choosing-a-server-package", + "title": "Which package to use" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth", + "title": "Securing Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/functions", + "title": "Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/functions/connect-to-postgres", + "title": "Integrating with Supabase Database (Postgres)" + }, + { + "url": "https://supabase.com/docs/guides/functions/quickstart-dashboard", + "title": "Getting Started with Edge Functions (Dashboard)" + }, + { + "url": "https://supabase.com/docs/guides/functions/recursive-functions", + "title": "Recursive / Nested Function Calls" + }, + { + "url": "https://supabase.com/docs/guides/functions/architecture", + "title": "Edge Functions Architecture" + }, + { + "url": "https://supabase.com/docs/guides/database/connecting-to-postgres/serverless-drivers", + "title": "Serverless Drivers" + }, + { + "url": "https://supabase.com/docs/guides/functions/examples/resumable-websockets", + "title": "Resumable WebSockets with Edge Functions" + } + ], + "resultChars": 84300 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"edge functions environment variables SUPABASE_SECRET_KEY SUPABASE_PUBLISHABLE_KEY local development\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/functions/secrets", + "title": "Environment Variables" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth", + "title": "Securing Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", + "title": "Self-Hosted Functions" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/docker", + "title": "Self-Hosting with Docker" + } + ], + "resultChars": 67435 + } + ] }, "prompt": "Build and serve a Supabase Edge Function named `user-stats` for this project,\nreachable over HTTP at `/functions/v1/user-stats`.\n\nImplement it with the **`@supabase/server`** package, which is built for exactly\nthis kind of multi-auth Edge Function. Import it directly in your function:\n\n```ts\nimport { withSupabase } from \"npm:@supabase/server\";\n```\n\nOur product stores per-user metrics in a `user_stats` table that already exists\n(see `supabase/migrations/`), protected by row-level security so a user can read\nonly their own rows.\n\nTwo very different callers need to hit this one endpoint:\n\n1. **Our mobile app**, acting for a signed-in user. It sends that user's\n Supabase access token. The endpoint should return the user's own stats.\n\n2. **Our internal billing service**, a trusted backend with no signed-in user.\n It authenticates with the project's secret (service-role) key in the `apikey`\n header, and names the target user with a `user_id` in the JSON request body.\n It needs that user's stats.\n\nReturn the matching rows as JSON. The endpoint must be secure: only ever serve\nstats to a caller that is genuinely entitled to them, and turn away callers that\nare not.\n\nGet the local stack running so the function is reachable at the path above.", "promptSourcePath": "evals/build-functions-006-dual-auth-with-server/PROMPT.md", @@ -117,7 +194,7 @@ { "name": "did not recommend read replicas for Realtime", "passed": true, - "judgeNotes": "The assistant correctly treated the request as a Supabase Realtime/Postgres Changes setup, enabled the table in the supabase_realtime publication, and provided a postgres_changes client subscription example. It did not recommend or imply read replicas are needed or useful for live updates." + "judgeNotes": "The assistant correctly treated the request as Supabase Realtime/Postgres Changes setup, added the messages table to the supabase_realtime publication, verified it, and provided a postgres_changes client subscription. It did not recommend or imply read replicas are needed or useful for live updates." } ], "skills": { @@ -133,31 +210,31 @@ "calls": [ { "source": "search_docs", - "query": "{ searchDocs(query: \"enable realtime postgres changes replication table\", limit: 5) { nodes { ... on Guide { title href content } } } }", + "query": "{ searchDocs(query: \"enable realtime for a table postgres changes supabase_realtime publication\", limit: 5) { nodes { ... on Guide { title href content } } } }", "hasContent": true, "pages": [ - { - "url": "https://supabase.com/docs/guides/realtime/reports", - "title": "Realtime Reports" - }, { "url": "https://supabase.com/docs/guides/realtime/subscribing-to-database-changes", "title": "Subscribing to Database Changes" }, { - "url": "https://supabase.com/docs/guides/database/replication/pipelines", - "title": "Set up Pipelines" - }, - { - "url": "https://supabase.com/docs/guides/realtime/broadcast", - "title": "Broadcast" + "url": "https://supabase.com/docs/guides/realtime/postgres-changes", + "title": "Postgres Changes" }, { "url": "https://supabase.com/docs/guides/realtime/benchmarks", "title": "Benchmarks" + }, + { + "url": "https://supabase.com/docs/guides/platform/migrating-within-supabase/backup-restore", + "title": "Backup and Restore using the CLI" + }, + { + "url": "https://supabase.com/docs/guides/realtime/architecture", + "title": "Realtime Architecture" } ], - "resultChars": 118402 + "resultChars": 102869 } ] }, @@ -189,17 +266,17 @@ { "name": "identified video-thumbnails and the 546 resource-limit pattern", "passed": true, - "judgeNotes": "Identified video-thumbnails as the affected function and correctly recognized HTTP 546 resource/CPU limit failures rather than 500/503 server errors." + "judgeNotes": "Identified `video-thumbnails` as affected and correctly recognized failures as status_code 546 / WORKER_RESOURCE_LIMIT due to CPUTime resource limit, not 500/503." }, { "name": "attributed the 546s to CPU time exhaustion", "passed": true, - "judgeNotes": "The assistant specifically attributed the 546 failures to CPU time exhaustion, citing log evidence: `shutdown (reason: CPUTime, cpu_time_used: 2000ms, cpu_time_limit: 2000ms)` and the Supabase 2s CPU ceiling." + "judgeNotes": "Assistant specifically attributes 546 failures to CPU time exhaustion, citing log evidence: `shutdown (reason: CPUTime, cpu_time_used: 2000ms, cpu_time_limit: 2000ms)` and explains it is not wall-clock timeout or application logic." }, { "name": "recommended reducing/offloading CPU work as the fix", "passed": true, - "judgeNotes": "The assistant clearly states the fixed CPU limit cannot be raised and recommends reducing CPU work in the hot path and/or offloading thumbnail generation to a background worker/external service." + "judgeNotes": "The assistant clearly identified CPU-intensive thumbnail generation as the root cause and recommended moving decode/encode work off the Edge Function to a background worker/external service, or reducing per-request CPU cost by optimizing/downscaling/lighter decode path. Retry was mentioned only as a short-term mitigation, not the main fix." } ], "skills": { @@ -215,31 +292,31 @@ "calls": [ { "source": "search_docs", - "query": "{ searchDocs(query: \"edge function CPU time limit shutdown\", limit: 5) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"edge function CPU time limit shutdown reason\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ - { - "url": "https://supabase.com/docs/guides/troubleshooting/edge-function-cpu-limits", - "title": "Understanding Edge Function CPU limits" - }, { "url": "https://supabase.com/docs/guides/troubleshooting/edge-function-shutdown-reasons-explained", "title": "Edge Function shutdown reasons explained" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/edge-function-wall-clock-time-limit-reached-Nk38bW", - "title": "Edge Function 'wall clock time limit reached'" + "url": "https://supabase.com/docs/guides/troubleshooting/edge-function-cpu-limits", + "title": "Understanding Edge Function CPU limits" }, { "url": "https://supabase.com/docs/guides/troubleshooting/edge-functions-worker-timeouts-and-websocket-drops", "title": "Edge Functions worker timeouts and WebSocket drops" }, { - "url": "https://supabase.com/docs/guides/functions/limits", - "title": "Limits" + "url": "https://supabase.com/docs/guides/functions/status-codes", + "title": "Status codes" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/edge-function-monitoring-resource-usage", + "title": "Monitoring Edge Function resource usage" } ], - "resultChars": 24082 + "resultChars": 25339 } ] }, @@ -315,7 +392,7 @@ { "name": "diagnosed secure default grants without weakening RLS", "passed": true, - "judgeNotes": "The answer correctly diagnoses missing explicit Data API/table grants as separate from RLS, preserves existing owner-scoped RLS, grants only SELECT and INSERT on public.journal_entries to authenticated, avoids anon/public grants, and keeps RLS enabled. It also verifies behavior." + "judgeNotes": "Diagnosed missing Data API/table privileges under secure-default behavior, distinguished grants from RLS, granted SELECT/INSERT only to authenticated, and kept existing owner-scoped RLS intact." } ], "skills": { @@ -363,7 +440,7 @@ { "name": "user A's update actually checks off their own task", "passed": true, - "notes": "saw: [{\"id\":\"b1ca8789-8db5-4b4f-99b2-794b104932cb\",\"is_done\":true}]" + "notes": "saw: [{\"id\":\"82c4efb5-0b33-46ac-a566-b35a930792a3\",\"is_done\":true}]" }, { "name": "user B cannot update user A's task", @@ -372,7 +449,7 @@ { "name": "diagnosed the missing USING clause and added it", "passed": true, - "judgeNotes": "The assistant correctly diagnosed the missing USING clause on the authenticated UPDATE policy, explained the zero-row behavior, and fixed it with owner-scoped USING while preserving WITH CHECK and RLS." + "judgeNotes": "The answer correctly identifies the missing USING clause on the UPDATE RLS policy as the cause of silent zero-row updates, adds a user_id = auth.uid() USING condition scoped to authenticated owner behavior, preserves WITH CHECK to prevent reassignment, and keeps RLS intact." } ], "skills": { @@ -381,8 +458,7 @@ "supabase-postgres-best-practices" ], "loaded": [ - "supabase", - "supabase-postgres-best-practices" + "supabase" ] }, "docs": { @@ -416,7 +492,7 @@ { "name": "answered unhealthy project recovery question safely", "passed": true, - "judgeNotes": "The answer correctly says pause/restore is not the right first step, recommends restarting the database first, distinguishes pause/restore from restart as a heavier backup-style operation, and suggests checking reports/advisors for overload. It could more explicitly mention logs, scaling/reducing workload, and contacting support if unresolved, but it provides concrete safe recovery guidance and avoids the prohibited recommendations." + "judgeNotes": "The answer clearly says pause/restore is not the right remedy, distinguishes restart from pause/restore, recommends restarting the project, checking status, considering overload/compute/performance tuning, and notes checking logs/status via MCP. It is safe and docs-grounded enough. Minor omission: it does not explicitly mention waiting a few minutes, advisors, or contacting support, but it provides concrete safe first steps and avoids conflating restore with restart." } ], "skills": { @@ -432,7 +508,7 @@ "calls": [ { "source": "search_docs", - "query": "{ searchDocs(query: \"project unhealthy restart pause restore dashboard unresponsive\", limit: 5) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"project unhealthy restart pause resize\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { @@ -440,23 +516,23 @@ "title": "Project Status reports unhealthy services" }, { - "url": "https://supabase.com/docs/guides/platform/backups", - "title": "Database Backups" + "url": "https://supabase.com/docs/guides/platform/read-replicas/getting-started", + "title": "Getting started with Read Replicas" }, { - "url": "https://supabase.com/docs/guides/platform/upgrading", - "title": "Upgrading" + "url": "https://supabase.com/docs/guides/platform/delete-project", + "title": "Deleting Your Project" }, { - "url": "https://supabase.com/docs/guides/platform/free-project-pausing", - "title": "Project Pausing" + "url": "https://supabase.com/docs/guides/self-hosting/custom-email-templates", + "title": "Custom Email Templates" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/failed-to-run-sql-query-connection-terminated-due-to-connection-timeout", - "title": "Error: Failed to run sql query: Connection terminated due to connection timeout" + "url": "https://supabase.com/docs/guides/platform/free-project-pausing", + "title": "Project Pausing" } ], - "resultChars": 28453 + "resultChars": 29375 } ] }, @@ -506,7 +582,7 @@ { "name": "user A can replace their own avatar via upsert", "passed": true, - "notes": "saw: [{\"name\":\"019fc14d-e84c-73a9-90ad-9f2bb2141135/avatar.png\",\"metadata\":{\"version\":\"replacement\"}}]" + "notes": "saw: [{\"name\":\"019f8a21-8307-77b7-a7ca-72de655476e5/avatar.png\",\"metadata\":{\"version\":\"replacement\"}}]" }, { "name": "user B cannot overwrite user A's avatar", @@ -515,7 +591,7 @@ { "name": "added an owner-scoped UPDATE policy without weakening public reads", "passed": true, - "judgeNotes": "Diagnosed missing storage.objects UPDATE policy for Supabase Storage upsert, kept public read/RLS intact, and added an authenticated owner-scoped UPDATE policy with USING and WITH CHECK." + "judgeNotes": "The assistant correctly identified missing UPDATE RLS policy for storage.objects as the cause of avatar replacement/upsert failure, explained that public bucket only affects downloads/public read, and added an authenticated, owner-scoped UPDATE policy with both USING and WITH CHECK based on the user's folder. It kept the bucket public and RLS enabled, without permissive or unsafe changes." } ], "skills": { @@ -574,17 +650,17 @@ { "name": "user with JWT reads only their own rows", "passed": true, - "notes": "status 200: [{\"user_id\":\"421d2f7b-850f-482f-a19b-0ad1810c6ead\",\"metric\":\"steps_a_msbgpkq2\",\"value\":111}]" + "notes": "status 200: {\"user_id\":\"b8a92ee9-ea76-4c43-b0e9-6b46737571bf\",\"stats\":[{\"metric\":\"steps_a_mrw5ngjk\",\"value\":111}]}" }, { "name": "user cannot read another user's rows by passing user_id", "passed": true, - "notes": "status 200: [{\"user_id\":\"421d2f7b-850f-482f-a19b-0ad1810c6ead\",\"metric\":\"steps_a_msbgpkq2\",\"value\":111}]" + "notes": "status 200: {\"user_id\":\"b8a92ee9-ea76-4c43-b0e9-6b46737571bf\",\"stats\":[{\"metric\":\"steps_a_mrw5ngjk\",\"value\":111}]}" }, { "name": "service key bypasses RLS to read the target user's rows", "passed": true, - "notes": "status 200: [{\"user_id\":\"875ca375-3ed1-4c7d-886f-de34025cf021\",\"metric\":\"steps_b_msbgpkq2\",\"value\":222}]" + "notes": "status 200: {\"user_id\":\"cfba7c28-7f8f-41a3-b172-613797821b5d\",\"stats\":[{\"metric\":\"steps_b_mrw5ngjk\",\"value\":222}]}" }, { "name": "non-service key is not granted service access", @@ -648,7 +724,7 @@ { "name": "did not recommend read replicas for Realtime", "passed": true, - "judgeNotes": "The assistant correctly treated the issue as a Supabase Realtime/Postgres Changes setup, added the messages table to the supabase_realtime publication, and did not recommend or conflate read replicas." + "judgeNotes": "The assistant correctly treats the issue as Supabase Realtime/Postgres Changes setup, adds the messages table to the supabase_realtime publication, and provides a postgres_changes client subscription. It does not recommend or imply read replicas are needed or useful." } ], "skills": { @@ -686,17 +762,17 @@ { "name": "identified video-thumbnails and the 546 resource-limit pattern", "passed": true, - "judgeNotes": "Identified `video-thumbnails` as the affected function and correctly recognized failures as HTTP 546 resource/CPU limit responses, not 500/503 server errors." + "judgeNotes": "Identified `video-thumbnails` as the affected function and correctly described failures as HTTP 546 resource/CPU time limit shutdowns, not 500/503 server errors." }, { "name": "attributed the 546s to CPU time exhaustion", "passed": true, - "judgeNotes": "The assistant specifically attributed the 546s to CPU time exhaustion, citing the log evidence: shutdown reason CPUTime and cpu_time_used 2000ms matching cpu_time_limit 2000ms. It explicitly ruled out wall-clock timeout and exceptions." + "judgeNotes": "The assistant specifically attributes the 546 failures to CPU time exhaustion, citing log evidence including `reason: CPUTime` and `cpu_time_used: 2000ms` hitting `cpu_time_limit: 2000ms`. It explicitly distinguishes CPU time from wall-clock/memory/errors." }, { "name": "recommended reducing/offloading CPU work as the fix", "passed": true, - "judgeNotes": "The assistant correctly identifies CPU-bound thumbnail generation as the issue and recommends moving/offloading heavy processing to a worker/external service or reducing CPU work within the function. It does mention retries as a short-term mitigation, but not as the primary fix." + "judgeNotes": "Recommended offloading CPU-heavy thumbnail generation to a background/external worker and reducing per-invocation work if kept on Edge Functions." } ], "skills": { @@ -778,7 +854,7 @@ { "name": "diagnosed secure default grants without weakening RLS", "passed": true, - "judgeNotes": "The answer correctly identifies missing Data API/table grants under secure-by-default behavior, distinguishes grants from RLS, preserves existing owner-scoped RLS, grants only SELECT and INSERT on public.journal_entries to authenticated, and keeps anon/public ungranted with RLS still enforced." + "judgeNotes": "The answer correctly diagnoses secure-by-default/Data API privilege behavior, distinguishes table grants from RLS, preserves owner-scoped RLS, grants only SELECT and INSERT on public.journal_entries to authenticated, and does not grant anon/public or disable/weaken RLS." } ], "skills": { @@ -821,7 +897,7 @@ { "name": "user A's update actually checks off their own task", "passed": true, - "notes": "saw: [{\"id\":\"0fe0bd91-2ed9-481a-a408-e734d1b43462\",\"is_done\":true}]" + "notes": "saw: [{\"id\":\"c1d10598-8b67-463e-bb32-50fb512dcef4\",\"is_done\":true}]" }, { "name": "user B cannot update user A's task", @@ -830,7 +906,7 @@ { "name": "diagnosed the missing USING clause and added it", "passed": true, - "judgeNotes": "Diagnosed the zero-row UPDATE as the tasks UPDATE RLS policy missing a USING clause while only having WITH CHECK. Fixed by recreating the authenticated UPDATE policy with USING (user_id = auth.uid()) and preserving WITH CHECK (user_id = auth.uid()), keeping RLS scoped to task ownership." + "judgeNotes": "The answer correctly identifies the missing UPDATE policy USING clause as the cause of silent zero-row updates, explains WITH CHECK vs USING, and fixes the authenticated tasks policy with USING (user_id = auth.uid()) while preserving WITH CHECK (user_id = auth.uid()) and RLS." } ], "skills": { @@ -841,7 +917,7 @@ "calls": [ { "source": "search_docs", - "query": "{ searchDocs(query: \"RLS policy UPDATE USING WITH CHECK default missing\", limit: 5) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"row level security policy UPDATE USING WITH CHECK default\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { @@ -849,23 +925,23 @@ "title": "Row Level Security" }, { - "url": "https://supabase.com/docs/guides/api/securing-your-api", - "title": "Securing your API" + "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security", + "title": "Token Security and Row Level Security" }, { "url": "https://supabase.com/docs/guides/troubleshooting/rls-simplified-BJTcS8", "title": "RLS Simplified" }, { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-ionic-react", - "title": "Build a User Management App with Ionic React" + "url": "https://supabase.com/docs/guides/local-development/testing/pgtap-extended", + "title": "Advanced pgTAP Testing" }, { - "url": "https://supabase.com/docs/guides/auth/enterprise-sso/auth-sso-saml", - "title": "Single Sign-On with SAML 2.0 for Projects" + "url": "https://supabase.com/docs/guides/api/securing-your-api", + "title": "Securing your API" } ], - "resultChars": 107724 + "resultChars": 87538 } ] }, @@ -897,7 +973,7 @@ { "name": "answered unhealthy project recovery question safely", "passed": true, - "judgeNotes": "The answer clearly recommends restart first, distinguishes restart from pause/restore, warns pause/restore is heavier and not the first step, and offers logs/advisories follow-up. It could be stronger by explicitly mentioning workload/resource checks, troubleshooting docs, and contacting Supabase support if unresolved, but it satisfies the core safety criteria." + "judgeNotes": "The answer recommends restart first and distinguishes restart from pause/restore. It does not fully align with the rubric's wording that pausing/restoring is not the right first step, but it clearly frames it as heavier and only after restart. It gives some concrete safe steps (restart, check status/logs/advisors) though it could mention workload/resource pressure and support." } ], "skills": { @@ -909,7 +985,7 @@ }, "prompt": "My Supabase dashboard says my project is unhealthy, and the dashboard is unusable.\n\nWould restart or pause/restore be better?", "promptSourcePath": "evals/resolve-reliability-001-unhealthy-project-recovery/PROMPT.md", - "attempts": 2, + "attempts": 1, "sourcePath": "claude-code-sonnet-5-no-skills/resolve-reliability-001-unhealthy-project-recovery.json" }, { @@ -953,7 +1029,7 @@ { "name": "user A can replace their own avatar via upsert", "passed": true, - "notes": "saw: [{\"name\":\"019fc14d-d929-709c-8b83-5abab868d2c4/avatar.png\",\"metadata\":{\"version\":\"replacement\"}}]" + "notes": "saw: [{\"name\":\"019f8a21-5cfc-7157-8749-63164119b923/avatar.png\",\"metadata\":{\"version\":\"replacement\"}}]" }, { "name": "user B cannot overwrite user A's avatar", @@ -962,7 +1038,7 @@ { "name": "added an owner-scoped UPDATE policy without weakening public reads", "passed": true, - "judgeNotes": "Diagnosed missing UPDATE RLS policy for upsert replacement, kept public read setup/RLS, and added authenticated owner-scoped UPDATE policy with USING and WITH CHECK." + "judgeNotes": "Diagnoses missing UPDATE RLS policy for upsert replacements, notes public bucket only affects read/download behavior, adds authenticated owner-scoped UPDATE policy with USING and WITH CHECK, and keeps public-read/RLS intact." } ], "skills": { From cc6d9862be4a231a7d805e1cb9ecee5cac1201f8 Mon Sep 17 00:00:00 2001 From: Pedro Rodrigues Date: Wed, 22 Jul 2026 18:53:09 +0100 Subject: [PATCH 12/15] feat: compare PR runs against the latest-main baseline MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Main refreshes (--set-baseline, passed by CI only for full benchmark refreshes on main) stamp metadata.latestMain and demote the previous holder after a successful upload, so exactly one experiment per model is the source-of-truth baseline — stale main history never matches. PR uploads compare against it via baseExperiment; the PR comment table becomes Experiment | passed (n/m · %) | Δ vs main, each delta linking the pre-compared view (?c=), with exactly two footer links: this PR's results and the latest main results. Experiment metadata trimmed to what the filters need (branch, prNumber). Co-Authored-By: Claude Fable 5 --- .github/workflows/eval-refresh.yml | 14 +- apps/framework/scripts/upload-braintrust.ts | 231 +++++++++++++------- 2 files changed, 160 insertions(+), 85 deletions(-) diff --git a/.github/workflows/eval-refresh.yml b/.github/workflows/eval-refresh.yml index f067e0e6..a925c564 100644 --- a/.github/workflows/eval-refresh.yml +++ b/.github/workflows/eval-refresh.yml @@ -461,9 +461,21 @@ jobs: pairs='${{ needs.prepare.outputs.pairs }}' summary_md="$RUNNER_TEMP/braintrust-summary.md" + + # Only a FULL benchmark refresh on main may become the latest-main + # baseline PRs compare against — never PR runs, never dispatches + # that filtered to a subset of evals/experiments. + baseline_args=() + if [ "${{ github.event_name }}" != "pull_request" ] && \ + [ "${{ github.ref_name }}" = "main" ] && \ + [ -z "${{ inputs.experiments }}" ] && [ -z "${{ inputs.eval }}" ]; then + baseline_args+=(--set-baseline) + fi + if jq -e 'any(.[]; .eval_suite == "benchmark")' <<< "$pairs" > /dev/null; then pnpm --filter @supabase-evals/framework upload-braintrust -- \ - --suite benchmark --summary-md "$summary_md" + --suite benchmark --summary-md "$summary_md" \ + ${baseline_args[@]+"${baseline_args[@]}"} fi # Custom-mode pairs aren't tagged benchmark/regression (that field diff --git a/apps/framework/scripts/upload-braintrust.ts b/apps/framework/scripts/upload-braintrust.ts index 3fa6e2d1..45dd5b09 100644 --- a/apps/framework/scripts/upload-braintrust.ts +++ b/apps/framework/scripts/upload-braintrust.ts @@ -19,8 +19,10 @@ * --project Braintrust project name (overrides env) * --name-suffix extra suffix on the Braintrust experiment names * --update append to existing same-named Braintrust experiments - * --summary-md

write a markdown summary (links + passed Δ vs base), + * --summary-md

write a markdown summary (links + passed Δ vs main), * used by CI to post/update a PR comment + * --set-baseline mark these experiments as the latest-main baseline + * (passed by CI only for full refreshes on main) */ import { readFile, writeFile } from "node:fs/promises"; @@ -64,6 +66,7 @@ const UPDATE = rawArgs.includes("--update"); const PROJECT_FLAG = readRepeatedFlag(rawArgs, "project")[0]; const NAME_SUFFIX = readRepeatedFlag(rawArgs, "name-suffix")[0]; const SUMMARY_MD = readRepeatedFlag(rawArgs, "summary-md")[0]; +const SET_BASELINE = rawArgs.includes("--set-baseline"); const shouldIncludeSuite = makeFilterPredicate(SUITE_FILTERS); const shouldIncludeExperimentSuite = makeFilterPredicate( @@ -92,39 +95,36 @@ function ciRunUrl(): string | undefined { } /** - * What produced this upload — the axis that separates scheduled main - * refreshes from PR runs (and local ad-hoc uploads) in Braintrust. Stored as - * experiment metadata/tags so the experiments list can be pre-filtered by - * BTQL, e.g. `metadata.trigger = 'schedule'` or `metadata.prNumber = '101'`. + * Which branch (and PR, if any) produced this upload, stored as experiment + * metadata. `branch` gates whether --set-baseline may stamp `latestMain` + * (the marker PR runs compare against); `prNumber` powers the "results for + * this PR" filter link. */ interface RunContext { - trigger: string; branch?: string; prNumber?: string; - runId?: string; } function runContext(): RunContext { - const event = process.env.GITHUB_EVENT_NAME; - if (!event) { - let branch: string | undefined; + if (!process.env.GITHUB_EVENT_NAME) { try { - branch = execFileSync("git", ["rev-parse", "--abbrev-ref", "HEAD"], { - cwd: ROOT, - encoding: "utf8", - }).trim(); + return { + branch: execFileSync("git", ["rev-parse", "--abbrev-ref", "HEAD"], { + cwd: ROOT, + encoding: "utf8", + }).trim(), + }; } catch { - branch = undefined; + return {}; } - return { trigger: "local", branch }; } // For PR events GITHUB_REF_NAME is "/merge". const prMatch = /^(\d+)\/merge$/.exec(process.env.GITHUB_REF_NAME ?? ""); return { - trigger: event, branch: process.env.GITHUB_HEAD_REF || process.env.GITHUB_REF_NAME, - ...(event === "pull_request" && prMatch ? { prNumber: prMatch[1] } : {}), - runId: process.env.GITHUB_RUN_ID, + ...(process.env.GITHUB_EVENT_NAME === "pull_request" && prMatch + ? { prNumber: prMatch[1] } + : {}), }; } @@ -135,63 +135,109 @@ function runContext(): RunContext { */ function experimentsFilterUrl( anyExperimentUrl: string, - exprs: string[], + expr: string, ): string | undefined { const marker = "/experiments/"; const index = anyExperimentUrl.lastIndexOf(marker); if (index === -1) return undefined; const base = anyExperimentUrl.slice(0, index + marker.length - 1); const search = encodeURIComponent( - JSON.stringify({ filter: exprs.map((expr) => encodeURIComponent(expr)) }), + JSON.stringify({ filter: [encodeURIComponent(expr)] }), ); return `${base}?search=${search}`; } +interface BaselineExperiment { + id: string; + name: string; +} + /** - * Most recent prior Braintrust experiment for the same repo experiment, used - * as the comparison base (score diffs in summarize() and in the UI). - * Experiments are named `@[-suffix]`; prefer plain-sha names - * (real refreshes) over suffixed ad-hoc ones. Best-effort: any failure means - * "no base", never a broken upload. + * The Braintrust experiment marked `latestMain` for this repo experiment — + * the source-of-truth baseline every run compares against. Main refreshes + * mark themselves and demote the previous holder (see setLatestMain), so + * exactly one experiment per model carries the marker; ties (a failed + * demotion) break to the newest. The repo experiment name already encodes + * agent + model + skills, so metadata equality is an exact identity match. + * Best-effort: any failure means "no base", never a broken upload. */ -async function findBaseExperimentName( +async function findMainBaseline( projectId: string | undefined, experiment: string, currentName: string, -): Promise { +): Promise { const apiKey = process.env.BRAINTRUST_API_KEY; if (!projectId || !apiKey) return undefined; try { + const metadata = encodeURIComponent( + JSON.stringify({ experiment, latestMain: true }), + ); const response = await fetch( - `https://api.braintrust.dev/v1/experiment?project_id=${encodeURIComponent(projectId)}&limit=100`, + `https://api.braintrust.dev/v1/experiment?project_id=${encodeURIComponent(projectId)}&metadata=${metadata}&limit=10`, { headers: { Authorization: `Bearer ${apiKey}` } }, ); if (!response.ok) return undefined; const body = (await response.json()) as { - objects?: Array<{ name?: string }>; + objects?: Array<{ id?: string; name?: string; created?: string }>; }; - // Objects arrive newest-first. - const names = (body.objects ?? []) - .map((o) => o.name) + const candidate = (body.objects ?? []) .filter( - (name): name is string => - typeof name === "string" && - name.startsWith(`${experiment}@`) && - name !== currentName, - ); - return names.find((name) => /^.+@[0-9a-f]{7,}$/.test(name)) ?? names[0]; + (o): o is { id: string; name: string; created?: string } => + typeof o.id === "string" && + typeof o.name === "string" && + o.name !== currentName, + ) + .sort((a, b) => (b.created ?? "").localeCompare(a.created ?? ""))[0]; + return candidate ? { id: candidate.id, name: candidate.name } : undefined; } catch { return undefined; } } +/** + * Set or clear an experiment's `latestMain` marker (best-effort). PATCH + * merges metadata, and a null value no longer matches `latestMain = true` + * filters (verified against the API). Promotion happens only after a + * successful upload+summarize, so a crashed main refresh can never win the + * baseline tie-break; a failed demotion leaves two holders, which + * findMainBaseline resolves to the newest. + */ +async function setLatestMain( + experimentId: string, + value: true | null, +): Promise { + const apiKey = process.env.BRAINTRUST_API_KEY; + if (!apiKey) return; + try { + const response = await fetch( + `https://api.braintrust.dev/v1/experiment/${experimentId}`, + { + method: "PATCH", + headers: { + Authorization: `Bearer ${apiKey}`, + "Content-Type": "application/json", + }, + body: JSON.stringify({ metadata: { latestMain: value } }), + }, + ); + if (!response.ok) { + console.warn(`latestMain update failed (${response.status}) for ${experimentId}`); + } + } catch (error) { + console.warn(`latestMain update failed for ${experimentId}: ${String(error)}`); + } +} + interface SummaryRow { - name: string; + /** Repo experiment name (identity: agent + model + skills). */ + experiment: string; url?: string; evalCount: number; - passedScore?: number; + passedCount: number; passedDiff?: number; - comparedTo?: string; + baseName?: string; + /** PR experiment page pre-compared against the main baseline (`?c=`). */ + compareUrl?: string; } function formatSummaryMarkdown( @@ -203,17 +249,25 @@ function formatSummaryMarkdown( "", "### Braintrust benchmark results", "", - "| Experiment | passed | Δ vs base | evals |", - "|---|---|---|---|", + "| Experiment | passed | Δ vs main |", + "|---|---|---|", ]; for (const row of rows) { - const name = row.url ? `[${row.name}](${row.url})` : row.name; - const passed = row.passedScore !== undefined ? pct(row.passedScore) : "–"; - const diff = - row.passedDiff === undefined || !row.comparedTo - ? "–" - : `${row.passedDiff > 0 ? "🟢 +" : row.passedDiff < 0 ? "🔴 " : "⚪ "}${pct(row.passedDiff)} (vs ${row.comparedTo})`; - lines.push(`| ${name} | ${passed} | ${diff} | ${row.evalCount} |`); + const name = row.url ? `[${row.experiment}](${row.url})` : row.experiment; + const passed = `${row.passedCount}/${row.evalCount} · ${pct(row.passedCount / Math.max(row.evalCount, 1))}`; + let diff = "– (no main baseline)"; + if (row.baseName) { + const sha = row.baseName.split("@")[1]; + const base = sha ? `main@${sha}` : row.baseName; + let label = `– vs ${base}`; + if (row.passedDiff !== undefined) { + const points = Math.round(row.passedDiff * 100); + const glyph = points > 0 ? "🟢 +" : points < 0 ? "🔴 " : "⚪ "; + label = `${glyph}${points}% vs ${base}`; + } + diff = row.compareUrl ? `[${label}](${row.compareUrl})` : label; + } + lines.push(`| ${name} | ${passed} | ${diff} |`); } if (filterLinks.length > 0) { lines.push( @@ -359,24 +413,24 @@ async function upload( const context = runContext(); const summaryRows: SummaryRow[] = []; + // Becoming the baseline is opt-in (--set-baseline, passed by CI only for + // full refreshes on main) with belt-and-braces local guards. + const isMainRefresh = + SET_BASELINE && + context.branch === "main" && + !context.prNumber && + !NAME_SUFFIX; + for (const [experiment, pending] of byExperiment) { const name = `${experiment}${sha ? `@${sha}` : ""}${suffix}`; const meta = experimentMetadata.get(experiment); - const baseExperiment = await findBaseExperimentName( - projectId, - experiment, - name, - ); + const baseline = await findMainBaseline(projectId, experiment, name); const btExperiment = init({ ...(projectName ? { project: projectName } : {}), ...(projectId ? { projectId } : {}), experiment: name, update: UPDATE, - ...(baseExperiment ? { baseExperiment } : {}), - tags: [ - context.trigger, - ...(context.prNumber ? [`pr-${context.prNumber}`] : []), - ], + ...(baseline ? { baseExperiment: baseline.name } : {}), metadata: { source: "supabase-evals", experiment, @@ -386,10 +440,8 @@ async function upload( ...(meta?.display ?? {}), ...(sha ? { gitShortSha: sha } : {}), ...(runUrl ? { ciRunUrl: runUrl } : {}), - trigger: context.trigger, ...(context.branch ? { branch: context.branch } : {}), ...(context.prNumber ? { prNumber: context.prNumber } : {}), - ...(context.runId ? { ciRunId: context.runId } : {}), }, }); @@ -457,14 +509,33 @@ async function upload( } const summary = await btExperiment.summarize(); - const passed = summary.scores?.passed; + if (isMainRefresh && summary.experimentId) { + await setLatestMain(summary.experimentId, true); + if (baseline) await setLatestMain(baseline.id, null); + } + // Only trust the diff when it is against the baseline we selected — with + // none set, Braintrust auto-picks a base by git ancestry, which could be + // another PR's experiment. + const baseName = + baseline && summary.comparisonExperimentName === baseline.name + ? baseline.name + : undefined; summaryRows.push({ - name, + experiment, url: summary.experimentUrl, evalCount: pending.length, - passedScore: passed?.score, - passedDiff: passed?.diff, - comparedTo: summary.comparisonExperimentName, + passedCount: pending.filter(({ trace }) => trace.scores.passed === 1) + .length, + ...(baseName ? { passedDiff: summary.scores?.passed?.diff } : {}), + baseName, + // `?c=` opens the experiment pre-compared against the baseline + // (observed from the Braintrust UI's own comparison URLs; not a + // documented param — worst case the link opens uncompared). + ...(summary.experimentUrl && baseName + ? { + compareUrl: `${summary.experimentUrl}?c=${encodeURIComponent(baseName)}`, + } + : {}), }); console.log( `Uploaded ${pending.length} eval trace(s) to Braintrust experiment "${name}"` + @@ -474,30 +545,22 @@ async function upload( await flush(); - // Pre-filtered experiments-list links for this run's slice of the project - // (Braintrust bakes view filters into the URL). + // Pre-filtered experiments-list links (Braintrust bakes view filters into + // the URL): this PR's experiments, and the main-branch baseline history. const anyUrl = summaryRows.find((row) => row.url)?.url; const filterLinks: Array<{ label: string; url: string }> = []; if (anyUrl) { - const add = (label: string, exprs: string[]) => { - const url = experimentsFilterUrl(anyUrl, exprs); + const add = (label: string, expr: string) => { + const url = experimentsFilterUrl(anyUrl, expr); if (url) filterLinks.push({ label, url }); }; - if (context.runId) { - add("All experiments from this CI run", [ - `metadata.ciRunId = '${context.runId}'`, - ]); - } if (context.prNumber) { - add(`All experiments for PR #${context.prNumber}`, [ + add( + `Braintrust results for PR #${context.prNumber}`, `metadata.prNumber = '${context.prNumber}'`, - ]); - } - if (context.trigger === "schedule") { - add("All scheduled main-branch experiments", [ - `metadata.trigger = 'schedule'`, - ]); + ); } + add("Latest main results", `metadata.latestMain = true`); for (const { label, url } of filterLinks) { console.log(`${label}: ${url}`); } From 9eb8d3affcf0bfa5b5f6f42eb8201256ac9fff46 Mon Sep 17 00:00:00 2001 From: Pedro Rodrigues Date: Wed, 22 Jul 2026 19:07:52 +0100 Subject: [PATCH 13/15] =?UTF-8?q?feat:=20batch=20baseline=20rotation=20?= =?UTF-8?q?=E2=80=94=20demote=20old=20holders=20only=20after=20full=20uplo?= =?UTF-8?q?ad?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per the agreed sequence for multi-experiment refreshes: fetch every current latestMain holder per experiment up front, upload and mark the whole new batch, and only then clear the saved old holders. A batch that dies mid-way leaves the previous baselines in place, and clearing every holder (not just the newest) self-heals duplicates left by a past failed demotion. Co-Authored-By: Claude Fable 5 --- apps/framework/scripts/upload-braintrust.ts | 72 +++++++++++++-------- 1 file changed, 45 insertions(+), 27 deletions(-) diff --git a/apps/framework/scripts/upload-braintrust.ts b/apps/framework/scripts/upload-braintrust.ts index 45dd5b09..3cbfb3ce 100644 --- a/apps/framework/scripts/upload-braintrust.ts +++ b/apps/framework/scripts/upload-braintrust.ts @@ -147,50 +147,51 @@ function experimentsFilterUrl( return `${base}?search=${search}`; } -interface BaselineExperiment { - id: string; - name: string; -} - /** - * The Braintrust experiment marked `latestMain` for this repo experiment — - * the source-of-truth baseline every run compares against. Main refreshes - * mark themselves and demote the previous holder (see setLatestMain), so - * exactly one experiment per model carries the marker; ties (a failed - * demotion) break to the newest. The repo experiment name already encodes - * agent + model + skills, so metadata equality is an exact identity match. - * Best-effort: any failure means "no base", never a broken upload. + * The experiments marked `latestMain` for this repo experiment. `baseline` + * (the newest holder's name) is the source-of-truth base every run compares + * against; + * `holderIds` is every current holder, saved BEFORE uploading so a main + * refresh can demote them all only after the new batch is fully uploaded — + * if the upload dies, the old baselines keep the marker. Clearing every + * holder (not just the newest) also self-heals duplicates left by a past + * failed demotion. The repo experiment name already encodes agent + model + + * skills, so metadata equality is an exact identity match. Best-effort: any + * failure means "no base", never a broken upload. */ async function findMainBaseline( projectId: string | undefined, experiment: string, currentName: string, -): Promise { +): Promise<{ baseline?: string; holderIds: string[] }> { const apiKey = process.env.BRAINTRUST_API_KEY; - if (!projectId || !apiKey) return undefined; + if (!projectId || !apiKey) return { holderIds: [] }; try { const metadata = encodeURIComponent( JSON.stringify({ experiment, latestMain: true }), ); const response = await fetch( - `https://api.braintrust.dev/v1/experiment?project_id=${encodeURIComponent(projectId)}&metadata=${metadata}&limit=10`, + `https://api.braintrust.dev/v1/experiment?project_id=${encodeURIComponent(projectId)}&metadata=${metadata}&limit=100`, { headers: { Authorization: `Bearer ${apiKey}` } }, ); - if (!response.ok) return undefined; + if (!response.ok) return { holderIds: [] }; const body = (await response.json()) as { objects?: Array<{ id?: string; name?: string; created?: string }>; }; - const candidate = (body.objects ?? []) + const holders = (body.objects ?? []) .filter( (o): o is { id: string; name: string; created?: string } => typeof o.id === "string" && typeof o.name === "string" && o.name !== currentName, ) - .sort((a, b) => (b.created ?? "").localeCompare(a.created ?? ""))[0]; - return candidate ? { id: candidate.id, name: candidate.name } : undefined; + .sort((a, b) => (b.created ?? "").localeCompare(a.created ?? "")); + return { + ...(holders[0] ? { baseline: holders[0].name } : {}), + holderIds: holders.map((holder) => holder.id), + }; } catch { - return undefined; + return { holderIds: [] }; } } @@ -199,8 +200,8 @@ async function findMainBaseline( * merges metadata, and a null value no longer matches `latestMain = true` * filters (verified against the API). Promotion happens only after a * successful upload+summarize, so a crashed main refresh can never win the - * baseline tie-break; a failed demotion leaves two holders, which - * findMainBaseline resolves to the newest. + * baseline tie-break; a failed demotion leaves extra holders, which + * findMainBaseline resolves to the newest (and clears on the next refresh). */ async function setLatestMain( experimentId: string, @@ -421,16 +422,24 @@ async function upload( !context.prNumber && !NAME_SUFFIX; + // Old baseline holders across the whole batch, demoted only after every + // experiment uploaded (see findMainBaseline). + const supersededHolderIds: string[] = []; + for (const [experiment, pending] of byExperiment) { const name = `${experiment}${sha ? `@${sha}` : ""}${suffix}`; const meta = experimentMetadata.get(experiment); - const baseline = await findMainBaseline(projectId, experiment, name); + const { baseline, holderIds } = await findMainBaseline( + projectId, + experiment, + name, + ); const btExperiment = init({ ...(projectName ? { project: projectName } : {}), ...(projectId ? { projectId } : {}), experiment: name, update: UPDATE, - ...(baseline ? { baseExperiment: baseline.name } : {}), + ...(baseline ? { baseExperiment: baseline } : {}), metadata: { source: "supabase-evals", experiment, @@ -511,14 +520,16 @@ async function upload( const summary = await btExperiment.summarize(); if (isMainRefresh && summary.experimentId) { await setLatestMain(summary.experimentId, true); - if (baseline) await setLatestMain(baseline.id, null); + supersededHolderIds.push( + ...holderIds.filter((id) => id !== summary.experimentId), + ); } // Only trust the diff when it is against the baseline we selected — with // none set, Braintrust auto-picks a base by git ancestry, which could be // another PR's experiment. const baseName = - baseline && summary.comparisonExperimentName === baseline.name - ? baseline.name + baseline && summary.comparisonExperimentName === baseline + ? baseline : undefined; summaryRows.push({ experiment, @@ -545,6 +556,13 @@ async function upload( await flush(); + // The whole batch is uploaded and marked — only now retire the old + // baselines. A crash before this point leaves them in place (and any + // temporary duplicate holders resolve newest-first in findMainBaseline). + for (const id of supersededHolderIds) { + await setLatestMain(id, null); + } + // Pre-filtered experiments-list links (Braintrust bakes view filters into // the URL): this PR's experiments, and the main-branch baseline history. const anyUrl = summaryRows.find((row) => row.url)?.url; From 25e2b4f2253c121d79a7d39161e48735dbecbfb9 Mon Sep 17 00:00:00 2001 From: Pedro Rodrigues Date: Thu, 23 Jul 2026 19:10:54 +0100 Subject: [PATCH 14/15] decouple experiment metadata field in four --- apps/framework/scripts/upload-braintrust.ts | 38 +++++++++++++++------ packages/core/src/eval-metadata.ts | 2 ++ packages/core/src/index.ts | 2 +- 3 files changed, 31 insertions(+), 11 deletions(-) diff --git a/apps/framework/scripts/upload-braintrust.ts b/apps/framework/scripts/upload-braintrust.ts index 3cbfb3ce..43ed44db 100644 --- a/apps/framework/scripts/upload-braintrust.ts +++ b/apps/framework/scripts/upload-braintrust.ts @@ -36,6 +36,7 @@ import { } from "@supabase-evals/core"; import type { EvalSuite, + ExperimentDisplayMetadata, ExperimentSuite, } from "@supabase-evals/core/eval-metadata"; import { @@ -148,27 +149,45 @@ function experimentsFilterUrl( } /** - * The experiments marked `latestMain` for this repo experiment. `baseline` - * (the newest holder's name) is the source-of-truth base every run compares - * against; + * The identity fields that together pin down "the same experiment config" + * across refreshes — decomposed so each is independently filterable/groupable + * in the Braintrust UI, rather than one opaque repo experiment name. `skills` + * is joined into a stable, order-independent string for exact-match + * filtering (the config's own array order is otherwise arbitrary). + */ +function identityKey(display: ExperimentDisplayMetadata | undefined) { + return { + agent: display?.agent, + modelProvider: display?.modelProvider, + modelId: display?.modelId, + ...(display?.reasoningEffort + ? { reasoningEffort: display.reasoningEffort } + : {}), + skills: [...(display?.skills ?? [])].sort().join(","), + }; +} + +/** + * The experiments marked `latestMain` for this identity (agent + model + + * reasoning effort + skills — see `identityKey`). `baseline` (the newest + * holder's name) is the source-of-truth base every run compares against; * `holderIds` is every current holder, saved BEFORE uploading so a main * refresh can demote them all only after the new batch is fully uploaded — * if the upload dies, the old baselines keep the marker. Clearing every * holder (not just the newest) also self-heals duplicates left by a past - * failed demotion. The repo experiment name already encodes agent + model + - * skills, so metadata equality is an exact identity match. Best-effort: any - * failure means "no base", never a broken upload. + * failed demotion. Best-effort: any failure means "no base", never a broken + * upload. */ async function findMainBaseline( projectId: string | undefined, - experiment: string, + display: ExperimentDisplayMetadata | undefined, currentName: string, ): Promise<{ baseline?: string; holderIds: string[] }> { const apiKey = process.env.BRAINTRUST_API_KEY; if (!projectId || !apiKey) return { holderIds: [] }; try { const metadata = encodeURIComponent( - JSON.stringify({ experiment, latestMain: true }), + JSON.stringify({ ...identityKey(display), latestMain: true }), ); const response = await fetch( `https://api.braintrust.dev/v1/experiment?project_id=${encodeURIComponent(projectId)}&metadata=${metadata}&limit=100`, @@ -431,7 +450,7 @@ async function upload( const meta = experimentMetadata.get(experiment); const { baseline, holderIds } = await findMainBaseline( projectId, - experiment, + meta?.display, name, ); const btExperiment = init({ @@ -442,7 +461,6 @@ async function upload( ...(baseline ? { baseExperiment: baseline } : {}), metadata: { source: "supabase-evals", - experiment, ...(meta?.experimentSuite ? { experimentSuite: meta.experimentSuite } : {}), diff --git a/packages/core/src/eval-metadata.ts b/packages/core/src/eval-metadata.ts index 59a645cf..e2505aa0 100644 --- a/packages/core/src/eval-metadata.ts +++ b/packages/core/src/eval-metadata.ts @@ -81,6 +81,8 @@ export const experimentDisplayMetadataSchema = z.object({ modelProvider: modelProviderSchema, modelId: z.string(), reasoningEffort: reasoningEffortSchema.optional(), + /** Skill names installed for this experiment (empty for a no-skills variant). */ + skills: z.array(z.string()).optional(), }); export type ExperimentDisplayMetadata = z.infer< typeof experimentDisplayMetadataSchema diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 4fd3be68..01ff01b5 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -575,7 +575,7 @@ export type ExperimentConfig = { export function getExperimentDisplayMetadata( config: ExperimentConfig ): ExperimentDisplayMetadata { - return config.agent.metadata; + return { ...config.agent.metadata, skills: config.skills }; } export function defineExperiment(config: ExperimentConfig): ExperimentConfig { From 40ad9e246bf379ec5e01b41062488468a7926f66 Mon Sep 17 00:00:00 2001 From: Pedro Rodrigues Date: Fri, 24 Jul 2026 13:24:03 +0100 Subject: [PATCH 15/15] feat: align experiment metadata with the design doc's field table MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Store exactly the doc's experiment-metadata fields: the decomposed config identity (agent/modelProvider/modelId/reasoningEffort/skills), branch, prNumber, experimentSuite, trigger, runId — plus latestMain stamped post-upload. Drops source/gitShortSha/ciRunUrl. Storing skills as the same joined string identityKey queries on also fixes the latestMain lookup: since e978e43 the stored array shape could never match the joined-string filter, so PR runs would silently lose their baseline. Co-Authored-By: Claude Fable 5 --- apps/framework/scripts/upload-braintrust.ts | 40 +++++++++++---------- 1 file changed, 22 insertions(+), 18 deletions(-) diff --git a/apps/framework/scripts/upload-braintrust.ts b/apps/framework/scripts/upload-braintrust.ts index 43ed44db..4a7bd50d 100644 --- a/apps/framework/scripts/upload-braintrust.ts +++ b/apps/framework/scripts/upload-braintrust.ts @@ -87,23 +87,19 @@ function gitShortSha(): string | undefined { } } -function ciRunUrl(): string | undefined { - const { GITHUB_SERVER_URL, GITHUB_REPOSITORY, GITHUB_RUN_ID } = process.env; - if (!GITHUB_SERVER_URL || !GITHUB_REPOSITORY || !GITHUB_RUN_ID) { - return undefined; - } - return `${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}`; -} - /** - * Which branch (and PR, if any) produced this upload, stored as experiment - * metadata. `branch` gates whether --set-baseline may stamp `latestMain` - * (the marker PR runs compare against); `prNumber` powers the "results for - * this PR" filter link. + * How this upload was produced, stored as experiment metadata. `branch` gates + * whether --set-baseline may stamp `latestMain` (the marker PR runs compare + * against); `prNumber` powers the "results for this PR" filter link; + * `trigger` is which of the three workflow entry points ran (`schedule` / + * `pull_request` / `workflow_dispatch`); `runId` is the GitHub Actions run ID, + * kept so a human can jump from a Braintrust row back to the CI run. */ interface RunContext { branch?: string; prNumber?: string; + trigger?: string; + runId?: string; } function runContext(): RunContext { @@ -123,6 +119,10 @@ function runContext(): RunContext { const prMatch = /^(\d+)\/merge$/.exec(process.env.GITHUB_REF_NAME ?? ""); return { branch: process.env.GITHUB_HEAD_REF || process.env.GITHUB_REF_NAME, + trigger: process.env.GITHUB_EVENT_NAME, + ...(process.env.GITHUB_RUN_ID + ? { runId: process.env.GITHUB_RUN_ID } + : {}), ...(process.env.GITHUB_EVENT_NAME === "pull_request" && prMatch ? { prNumber: prMatch[1] } : {}), @@ -153,7 +153,9 @@ function experimentsFilterUrl( * across refreshes — decomposed so each is independently filterable/groupable * in the Braintrust UI, rather than one opaque repo experiment name. `skills` * is joined into a stable, order-independent string for exact-match - * filtering (the config's own array order is otherwise arbitrary). + * filtering (the config's own array order is otherwise arbitrary). Stored + * verbatim as experiment metadata AND used as the `latestMain` lookup filter, + * so the query matches the stored shape by construction. */ function identityKey(display: ExperimentDisplayMetadata | undefined) { return { @@ -429,7 +431,6 @@ async function upload( const projectId = process.env.BRAINTRUST_PROJECT_ID; const sha = gitShortSha(); const suffix = NAME_SUFFIX ? `-${NAME_SUFFIX}` : ""; - const runUrl = ciRunUrl(); const context = runContext(); const summaryRows: SummaryRow[] = []; @@ -459,16 +460,19 @@ async function upload( experiment: name, update: UPDATE, ...(baseline ? { baseExperiment: baseline } : {}), + // Exactly the experiment-metadata fields from the design doc's table + // (plus `latestMain`, stamped separately by setLatestMain after a + // successful main refresh): the decomposed config identity, then the + // run-scoped context. metadata: { - source: "supabase-evals", + ...identityKey(meta?.display), ...(meta?.experimentSuite ? { experimentSuite: meta.experimentSuite } : {}), - ...(meta?.display ?? {}), - ...(sha ? { gitShortSha: sha } : {}), - ...(runUrl ? { ciRunUrl: runUrl } : {}), ...(context.branch ? { branch: context.branch } : {}), ...(context.prNumber ? { prNumber: context.prNumber } : {}), + ...(context.trigger ? { trigger: context.trigger } : {}), + ...(context.runId ? { runId: context.runId } : {}), }, });