From bb95da896d96287fbaffeb674d76aad302b25bcc Mon Sep 17 00:00:00 2001 From: Matt Rossman <22670878+mattrossman@users.noreply.github.com> Date: Fri, 24 Jul 2026 19:32:37 -0400 Subject: [PATCH 01/17] feat: add opencode harness running Kimi K3 via Vercel AI Gateway --- .env.example | 9 +- apps/web/src/App.tsx | 9 +- experiments/opencode-kimi-k3-no-skills.ts | 20 ++ experiments/opencode-kimi-k3.ts | 19 ++ packages/core/package.json | 3 +- packages/core/src/agents/engine.ts | 33 ++- packages/core/src/agents/opencode/index.ts | 43 ++++ .../core/src/agents/opencode/parser.test.ts | 209 ++++++++++++++++ packages/core/src/agents/opencode/parser.ts | 234 ++++++++++++++++++ .../core/src/agents/opencode/runner.test.ts | 144 +++++++++++ packages/core/src/agents/opencode/runner.ts | 220 ++++++++++++++++ packages/core/src/agents/registry.ts | 7 +- packages/core/src/agents/shared.test.ts | 28 +++ packages/core/src/agents/shared.ts | 24 +- packages/core/src/agents/types.ts | 6 + packages/core/src/eval-metadata.ts | 13 +- packages/core/src/index.ts | 3 +- pnpm-lock.yaml | 13 + pnpm-workspace.yaml | 4 + 19 files changed, 1022 insertions(+), 19 deletions(-) create mode 100644 experiments/opencode-kimi-k3-no-skills.ts create mode 100644 experiments/opencode-kimi-k3.ts create mode 100644 packages/core/src/agents/opencode/index.ts create mode 100644 packages/core/src/agents/opencode/parser.test.ts create mode 100644 packages/core/src/agents/opencode/parser.ts create mode 100644 packages/core/src/agents/opencode/runner.test.ts create mode 100644 packages/core/src/agents/opencode/runner.ts create mode 100644 packages/core/src/agents/shared.test.ts diff --git a/.env.example b/.env.example index 19a0696f..53fb4854 100644 --- a/.env.example +++ b/.env.example @@ -1,3 +1,10 @@ -# AI SDK Core direct-provider credentials. +# Claude Code harness and Anthropic ai-sdk experiments. ANTHROPIC_API_KEY= + +# Codex harness and OpenAI ai-sdk experiments. The default LLM judge is an +# OpenAI model, so rubric-scored evals need this key regardless of which +# harness produced the run. OPENAI_API_KEY= + +# Vercel AI Gateway. The opencode harness routes through this. +AI_GATEWAY_API_KEY= diff --git a/apps/web/src/App.tsx b/apps/web/src/App.tsx index a891a817..710ffa07 100644 --- a/apps/web/src/App.tsx +++ b/apps/web/src/App.tsx @@ -198,6 +198,7 @@ const AGENT_LABELS = { "ai-sdk": "AI SDK", "claude-code": "Claude Code", codex: "Codex", + opencode: "OpenCode", } satisfies Record const EXPERIMENT_SUITES = ["benchmark", "no-skills"] as const @@ -313,11 +314,15 @@ function formatOpenAiModel(modelId: string) { } function formatModel(display: ExperimentDisplay) { + // opencode ids are AI Gateway `vendor/model` slugs; format just the model part. + const modelId = display.modelId.replace(/^[a-z-]+\//, "") switch (display.modelProvider) { case "anthropic": - return formatAnthropicModel(display.modelId) + return formatAnthropicModel(modelId) case "openai": - return formatOpenAiModel(display.modelId) + return formatOpenAiModel(modelId) + case "moonshotai": + return modelId } } diff --git a/experiments/opencode-kimi-k3-no-skills.ts b/experiments/opencode-kimi-k3-no-skills.ts new file mode 100644 index 00000000..0a6d2edd --- /dev/null +++ b/experiments/opencode-kimi-k3-no-skills.ts @@ -0,0 +1,20 @@ +import { + defineExperiment, + opencodeAgent, + platformLiteRuntime, + supabaseMcpServer, +} from '@supabase-evals/core'; +import { localStackRuntime } from '@supabase-evals/sandbox'; + +// Same as opencode-kimi-k3 but with no skills, to measure skills' impact. +export default defineExperiment({ + suite: ['no-skills'], + agent: opencodeAgent({ + model: 'moonshotai/kimi-k3', + }), + runtime: platformLiteRuntime({ + mcpServers: [supabaseMcpServer()], + }), + localStack: localStackRuntime(), + skills: [], +}); diff --git a/experiments/opencode-kimi-k3.ts b/experiments/opencode-kimi-k3.ts new file mode 100644 index 00000000..be08d45f --- /dev/null +++ b/experiments/opencode-kimi-k3.ts @@ -0,0 +1,19 @@ +import { + defineExperiment, + opencodeAgent, + platformLiteRuntime, + supabaseMcpServer, +} from '@supabase-evals/core'; +import { localStackRuntime } from '@supabase-evals/sandbox'; + +export default defineExperiment({ + suite: ['benchmark'], + agent: opencodeAgent({ + model: 'moonshotai/kimi-k3', + }), + runtime: platformLiteRuntime({ + mcpServers: [supabaseMcpServer()], + }), + localStack: localStackRuntime(), + skills: ['supabase', 'supabase-postgres-best-practices'], +}); diff --git a/packages/core/package.json b/packages/core/package.json index 4cce8026..51ed3f4d 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -14,11 +14,11 @@ "test": "vitest run" }, "devDependencies": { + "@opencode-ai/sdk": "catalog:", "vitest": "catalog:" }, "dependencies": { "@anthropic-ai/sdk": "catalog:", - "openai": "catalog:", "@ai-sdk/mcp": "catalog:", "@ai-sdk/openai": "catalog:", "@supabase-evals/platform-lite": "workspace:*", @@ -26,6 +26,7 @@ "ai": "catalog:", "executor": "1.4.29", "gray-matter": "^4.0.3", + "openai": "catalog:", "typescript": "catalog:", "zod": "catalog:" } diff --git a/packages/core/src/agents/engine.ts b/packages/core/src/agents/engine.ts index 36dffe39..4e4a874e 100644 --- a/packages/core/src/agents/engine.ts +++ b/packages/core/src/agents/engine.ts @@ -29,6 +29,7 @@ import { SYSTEM_PROMPT_PATH, USER_PROMPT_PATH, processStopReason, + requireEnv, rewriteLoopback, writeSandboxFile, } from './shared.js'; @@ -39,6 +40,10 @@ function modelProviderForAgent(id: AgentRunner['id']): ModelProvider { return 'anthropic'; case 'codex': return 'openai'; + case 'opencode': + throw new Error( + 'opencode is multi-provider; its runner sets `modelProvider` from the model id' + ); case 'ai-sdk': throw new Error('ai-sdk agents are not created through createCliAgent'); } @@ -60,7 +65,9 @@ export function createCliAgent( modelId: options.model, metadata: { agent: runner.id, - modelProvider: modelProviderForAgent(runner.id), + // A multi-provider runner (e.g. opencode) sets its own `modelProvider` + // from the model id; single-provider agents derive it from the agent id. + modelProvider: runner.modelProvider ?? modelProviderForAgent(runner.id), modelId: options.model, ...(options.reasoningEffort ? { reasoningEffort: options.reasoningEffort } @@ -102,6 +109,19 @@ export function createCliAgent( const { events } = raw ? parser.parseTranscript(raw) : { events: [] }; const adapted = adaptTranscript(events); + // Surface run failures that would otherwise be invisible in results + // (visible under --debug): the CLI's own error events, or a run that + // died before streaming any events at all. + const errorEvents = events.filter((e) => e.type === 'error'); + for (const e of errorEvents) { + console.error(`${runner.displayName} error event: ${e.content}`); + } + if (events.length === 0) { + console.error( + `${runner.displayName} produced no transcript events (exit ${command.exitCode}).\nstdout:\n${command.stdout}\nstderr:\n${command.stderr}` + ); + } + return { // The final report is the transcript's closing assistant message — the // CLI's stdout is JSONL, not prose. @@ -117,11 +137,8 @@ export function createCliAgent( } function requireApiKey(runner: AgentRunner): string { - const apiKey = process.env[runner.apiKeyEnvVar]; - if (!apiKey) { - throw new Error( - `Missing ${runner.displayName} credentials. Set ${runner.apiKeyEnvVar} before running ${runner.id} evals.` - ); - } - return apiKey; + return requireEnv( + runner.apiKeyEnvVar, + `Set it to run ${runner.displayName} (${runner.id}) evals.` + ); } diff --git a/packages/core/src/agents/opencode/index.ts b/packages/core/src/agents/opencode/index.ts new file mode 100644 index 00000000..199863e9 --- /dev/null +++ b/packages/core/src/agents/opencode/index.ts @@ -0,0 +1,43 @@ +/** + * OpenCode agent. Owns everything opencode-specific: it wires its own runner + + * parser into the public `opencodeAgent` factory (via the generic + * `createCliAgent` engine) and exports the registry definition the harness uses + * to parse opencode transcripts. + */ + +import type { AgentHarness } from '../../index.js'; +import { createCliAgent } from '../engine.js'; +import type { AgentDefinition } from '../types.js'; +import { + DEFAULT_OPENCODE_MODEL, + createOpencodeRunner, + type OpenCodeModel, +} from './runner.js'; +import { opencodeParser } from './parser.js'; + +/** + * OpenCode as an `AgentHarness`. Every run routes through the Vercel AI + * Gateway (opencode's native `vercel` provider), so the model id is a gateway + * `vendor/model` slug and the only credential is `AI_GATEWAY_API_KEY` — see + * `./runner.ts`. + */ +export function opencodeAgent( + options: { + /** Gateway model slug, `vendor/model` (e.g. `moonshotai/kimi-k3`). */ + model?: OpenCodeModel; + /** Override the pinned CLI version. */ + cliVersion?: string; + } = {} +): AgentHarness { + const model = options.model ?? DEFAULT_OPENCODE_MODEL; + return createCliAgent(createOpencodeRunner(model), opencodeParser, { + model, + cliVersion: options.cliVersion, + }); +} + +/** Runner + parser pairing for the agent registry (id comes from `runner.id`). */ +export const opencodeDefinition: AgentDefinition = { + runner: createOpencodeRunner(DEFAULT_OPENCODE_MODEL), + parser: opencodeParser, +}; diff --git a/packages/core/src/agents/opencode/parser.test.ts b/packages/core/src/agents/opencode/parser.test.ts new file mode 100644 index 00000000..0431b8f0 --- /dev/null +++ b/packages/core/src/agents/opencode/parser.test.ts @@ -0,0 +1,209 @@ +import { describe, expect, it } from 'vitest'; +import { opencodeParser } from './parser.js'; +import { adaptTranscript } from '../../parsers/adapt.js'; + +/** A representative `opencode run --format json` stream (shapes from CLI 1.18.5). */ +const SESSION = [ + JSON.stringify({ type: 'step_start', part: { type: 'step-start' } }), + JSON.stringify({ + type: 'reasoning', + timestamp: 1782295624200, + part: { type: 'reasoning', text: 'I should list the files.' }, + }), + JSON.stringify({ + type: 'text', + timestamp: 1782295624232, + part: { type: 'text', text: 'Listing files.' }, + }), + JSON.stringify({ + type: 'tool_use', + timestamp: 1782295624290, + part: { + type: 'tool', + tool: 'bash', + callID: 'tool_1', + state: { + status: 'completed', + input: { command: 'ls -la', description: 'List files' }, + output: 'file1\nfile2', + metadata: { exit: 0 }, + }, + }, + }), + JSON.stringify({ + type: 'tool_use', + timestamp: 1782295624300, + part: { + type: 'tool', + tool: 'write', + callID: 'tool_2', + state: { + status: 'completed', + input: { filePath: '/work/note.txt', content: 'hi' }, + output: 'written', + }, + }, + }), + JSON.stringify({ + type: 'text', + timestamp: 1782295624400, + part: { type: 'text', text: 'Done.' }, + }), + JSON.stringify({ + type: 'step_finish', + part: { + type: 'step-finish', + reason: 'stop', + tokens: { input: 3, output: 6 }, + }, + }), +].join('\n'); + +describe('opencodeParser', () => { + it("normalizes MCP tools (opencode's `_` names) to tool_use", () => { + const record = JSON.stringify({ + type: 'tool_use', + part: { + type: 'tool', + tool: 'supabase-mcp_list_tables', + callID: 'tool_mcp', + state: { status: 'completed', input: { schemas: ['public'] } }, + }, + }); + const { events } = opencodeParser.parseTranscript(record); + const call = events.find((e) => e.type === 'tool_call'); + expect(call?.tool?.name).toBe('tool_use'); + expect(call?.tool?.originalName).toBe('supabase-mcp_list_tables'); + }); + + it('maps bash + write to canonical tool calls, paired with results by callID', () => { + const { events, errors } = opencodeParser.parseTranscript(SESSION); + expect(errors).toEqual([]); + + const calls = events.filter((e) => e.type === 'tool_call'); + expect(calls.map((e) => e.tool?.name)).toEqual(['shell', 'file_write']); + expect(calls.map((e) => e.tool?.originalName)).toEqual(['bash', 'write']); + expect(calls.map((e) => e.tool?.id)).toEqual(['tool_1', 'tool_2']); + // Normalized views on the event; raw args untouched. + expect(calls[0].tool?.command).toBe('ls -la'); + expect(calls[1].tool?.path).toBe('/work/note.txt'); + + const results = events.filter((e) => e.type === 'tool_result'); + expect(results.map((e) => e.tool?.id)).toEqual(['tool_1', 'tool_2']); + expect(results.every((e) => e.tool?.success === true)).toBe(true); + }); + + it('surfaces reasoning + the assistant report via the adapter', () => { + const events = opencodeParser.parseTranscript(SESSION).events; + expect( + events.some( + (e) => e.type === 'thinking' && e.content === 'I should list the files.' + ) + ).toBe(true); + + const adapted = adaptTranscript(events); + expect(adapted.agentReport).toBe('Done.'); + expect(adapted.steps).toBe(2); // two assistant text turns + expect(adapted.toolCalls).toEqual([ + { + endpoint: 'bash', + body: { command: 'ls -la', description: 'List files' }, + name: 'shell', + command: 'ls -la', + result: 'file1\nfile2', + error: undefined, + ts: 1782295624290, // epoch ms preserved through toISO -> parseTs + }, + { + endpoint: 'write', + body: { filePath: '/work/note.txt', content: 'hi' }, + name: 'file_write', + path: '/work/note.txt', + result: 'written', + error: undefined, + ts: 1782295624300, + }, + ]); + }); + + it('surfaces skill loads from the skill tool and from SKILL.md reads', () => { + const stream = [ + JSON.stringify({ + type: 'tool_use', + part: { + type: 'tool', + tool: 'skill', + callID: 's1', + state: { + status: 'completed', + input: { name: 'supabase' }, + output: '# Supabase', + }, + }, + }), + JSON.stringify({ + type: 'tool_use', + part: { + type: 'tool', + tool: 'read', + callID: 's2', + state: { + status: 'completed', + input: { + filePath: + '.claude/skills/supabase-postgres-best-practices/SKILL.md', + }, + output: '# Postgres', + }, + }, + }), + ].join('\n'); + const adapted = adaptTranscript( + opencodeParser.parseTranscript(stream).events + ); + expect(adapted.toolCalls.map((call) => call.loadedSkill)).toEqual([ + 'supabase', + 'supabase-postgres-best-practices', + ]); + }); + + it('marks a non-zero shell exit as failed (error surfaced via adapter)', () => { + const stream = JSON.stringify({ + type: 'tool_use', + part: { + type: 'tool', + tool: 'bash', + callID: 'c1', + state: { + status: 'completed', + input: { command: 'false' }, + output: 'nope', + metadata: { exit: 1 }, + }, + }, + }); + const events = opencodeParser.parseTranscript(stream).events; + expect(events.find((e) => e.type === 'tool_result')?.tool?.success).toBe( + false + ); + const adapted = adaptTranscript(events); + expect(adapted.toolCalls[0].error).toBe('nope'); + expect(adapted.toolCalls[0].result).toBeUndefined(); + }); + + it('emits an error event and never throws on malformed lines', () => { + const { events, errors } = opencodeParser.parseTranscript( + 'not json\n' + + JSON.stringify({ type: 'error', error: { message: 'boom' } }) + ); + expect(events).toEqual([ + { + timestamp: undefined, + type: 'error', + content: 'boom', + raw: { type: 'error', error: { message: 'boom' } }, + }, + ]); + expect(errors.length).toBe(1); + }); +}); diff --git a/packages/core/src/agents/opencode/parser.ts b/packages/core/src/agents/opencode/parser.ts new file mode 100644 index 00000000..47b1582b --- /dev/null +++ b/packages/core/src/agents/opencode/parser.ts @@ -0,0 +1,234 @@ +/** + * OpenCode transcript parser — for `opencode run --format json` (CLI ≥ 1.15). + * + * The stream is newline-delimited event records, each `{ type, timestamp, + * sessionID, part }`: + * {"type":"step_start","part":{"type":"step-start"}} + * {"type":"text","part":{"type":"text","text":"…"}} + * {"type":"tool_use","part":{"type":"tool","tool":"bash","callID":"…", + * "state":{"status":"completed","input":{…},"output":"…", + * "metadata":{"exit":0}}}} + * {"type":"reasoning","part":{"type":"reasoning","text":"…"}} + * {"type":"error","error":{"message":"…"}} + * {"type":"step_finish","part":{"type":"step-finish","reason":"stop","tokens":{…}}} + * + * A `tool_use` record is self-contained (input + output + status), so it yields + * a paired tool_call + tool_result correlated by `part.callID`. Step records + * carry token/finish info and produce no transcript event (the runner reads the + * terminal `step_finish` reason for the stop reason). + * + * Adapted from `@supabase/agent-evals` (packages/agent-eval/src/parsers). + */ + +import { isRecord, parseJsonlRecords } from '../../json.js'; +import type { + ParsedTranscript, + TranscriptEvent, +} from '../../transcript/types.js'; +import type { AgentTranscriptParser } from '../../parsers/types.js'; +import { + normalizeToolName, + type AgentToolMap, +} from '../../parsers/shared/normalize.js'; +import { + extractArgs, + extractLoadedSkillFromText, + type ArgFieldMap, + type ExtractedArgs, +} from '../../parsers/shared/extract.js'; + +/** + * opencode's tool names → canonical names. opencode uses lowercase built-in tool + * names. Owned here, not in shared. MCP tools arrive under their server name and + * fall through to `tool_use`. + */ +const OPENCODE_TOOLS: AgentToolMap = { + caseInsensitive: true, + tools: { + read: 'file_read', + write: 'file_write', + edit: 'file_edit', + multiedit: 'file_edit', + patch: 'file_edit', + apply_patch: 'file_edit', + bash: 'shell', + shell: 'shell', + webfetch: 'web_fetch', + websearch: 'web_search', + codesearch: 'grep', + glob: 'glob', + grep: 'grep', + list: 'list_dir', + ls: 'list_dir', + task: 'agent_task', + todowrite: 'agent_task', + skill: 'tool_use', + }, +}; + +/** + * opencode tool args → normalized fields. `bash` carries the command in + * `command`; file tools the path in `filePath` (or `path`); `webfetch` the URL + * in `url`. The shared extractor reads whichever keys this map names. + */ +const OPENCODE_ARG_FIELDS: ArgFieldMap = { + path: ['filePath', 'file_path', 'path'], + command: ['command'], + url: ['url'], +}; + +/** Epoch-ms (or pass-through ISO) → ISO string. */ +function toISO(value: unknown): string | undefined { + if (typeof value === 'number') return new Date(value).toISOString(); + if (typeof value === 'string') return value; + return undefined; +} + +function str(value: unknown): string | undefined { + return typeof value === 'string' ? value : undefined; +} + +/** Whether a completed tool call succeeded: shell keys off its exit code. */ +function toolSuccess( + canonical: string, + status: string | undefined, + metadata: Record | undefined +): boolean | undefined { + if (status === undefined) return undefined; + if (status !== 'completed') return false; + if (canonical === 'shell') { + const exit = metadata?.exit; + return typeof exit === 'number' ? exit === 0 : true; + } + return true; +} + +function partToEvents( + type: string, + part: Record, + timestamp: string | undefined, + raw: unknown +): TranscriptEvent[] { + switch (type) { + case 'text': { + const text = str(part.text); + return text + ? [ + { + timestamp, + type: 'message', + role: 'assistant', + content: text, + raw, + }, + ] + : []; + } + case 'reasoning': { + const text = str(part.text); + return text ? [{ timestamp, type: 'thinking', content: text, raw }] : []; + } + case 'tool_use': { + const originalName = str(part.tool) ?? 'unknown'; + const id = str(part.callID); + const state = isRecord(part.state) ? part.state : {}; + const args = isRecord(state.input) ? state.input : {}; + const status = str(state.status); + const metadata = isRecord(state.metadata) ? state.metadata : undefined; + // The builtin tool set is fully enumerated in OPENCODE_TOOLS, so any + // unmapped name is an MCP/custom tool (`_`, which the + // shared `mcp__` fallback doesn't recognize). + const mapped = normalizeToolName(originalName, OPENCODE_TOOLS); + const name = mapped === 'unknown' ? 'tool_use' : mapped; + const normalized: ExtractedArgs = extractArgs(args, OPENCODE_ARG_FIELDS); + + const tool: NonNullable = { + name, + originalName, + id, + args, + }; + if (normalized.path) tool.path = normalized.path; + if (normalized.command) tool.command = normalized.command; + if (normalized.url) tool.url = normalized.url; + tool.loadedSkill = loadedSkillFromOpencodeCall(tool); + + const events: TranscriptEvent[] = [ + { timestamp, type: 'tool_call', tool, raw }, + ]; + // The result is in the same record; emit it only once the call completed. + if (status && status !== 'running' && status !== 'pending') { + events.push({ + timestamp, + type: 'tool_result', + tool: { + name, + originalName, + id, + result: + state.output ?? (isRecord(state.error) ? state.error : undefined), + success: toolSuccess(name, status, metadata), + }, + raw: state, + }); + } + return events; + } + default: + return []; + } +} + +/** + * Identifies opencode skill loads. opencode's native `skill` tool carries the + * skill name in its args; skills read manually surface as `skills// + * SKILL.md` in a file path or shell command. + */ +function loadedSkillFromOpencodeCall( + tool: NonNullable +): string | undefined { + if (tool.originalName.toLowerCase() === 'skill') { + const name = tool.args?.name ?? tool.args?.skill; + if (typeof name === 'string') return name; + } + if (tool.path) return extractLoadedSkillFromText(tool.path); + if (tool.command) return extractLoadedSkillFromText(tool.command); + return undefined; +} + +function recordToEvents(data: Record): TranscriptEvent[] { + const type = str(data.type); + if (!type) return []; + const timestamp = toISO(data.timestamp); + + if (type === 'error') { + const error = isRecord(data.error) ? data.error : undefined; + const message = + str(error?.message) ?? + str(data.message) ?? + JSON.stringify(data.error ?? data); + return [{ timestamp, type: 'error', content: message, raw: data }]; + } + // step_start / step_finish carry no transcript content (tokens + finish reason + // only; the runner reads the terminal step_finish reason for the stop reason). + if (type === 'step_start' || type === 'step_finish') return []; + + const part = isRecord(data.part) ? data.part : undefined; + if (!part) return []; + return partToEvents(type, part, timestamp, data); +} + +export const opencodeParser: AgentTranscriptParser = { + parseTranscript(raw: string): ParsedTranscript { + const { records, errors } = parseJsonlRecords(raw); + const events: TranscriptEvent[] = []; + for (const record of records) { + try { + events.push(...recordToEvents(record)); + } catch (e) { + errors.push(e instanceof Error ? e.message : String(e)); + } + } + return { events, errors }; + }, +}; diff --git a/packages/core/src/agents/opencode/runner.test.ts b/packages/core/src/agents/opencode/runner.test.ts new file mode 100644 index 00000000..fce38221 --- /dev/null +++ b/packages/core/src/agents/opencode/runner.test.ts @@ -0,0 +1,144 @@ +import { describe, expect, it } from 'vitest'; +import type { CommandResult } from '../../index.js'; +import { + buildOpencodeConfig, + createOpencodeRunner, + providerForModel, +} from './runner.js'; + +/** A run's terminal records: a mid-run `step_finish` (tool-calls) then the final one. */ +const SESSION = [ + JSON.stringify({ + type: 'step_finish', + part: { type: 'step-finish', reason: 'tool-calls' }, + }), + JSON.stringify({ type: 'text', part: { type: 'text', text: 'Done.' } }), + JSON.stringify({ + type: 'step_finish', + part: { type: 'step-finish', reason: 'stop' }, + }), +].join('\n'); + +describe('opencode runner', () => { + it("resolves the results-metadata provider from the slug's vendor prefix", () => { + expect(providerForModel('anthropic/claude-sonnet-5')).toBe('anthropic'); + expect(providerForModel('openai/gpt-5.4')).toBe('openai'); + expect(providerForModel('moonshotai/kimi-k3')).toBe('moonshotai'); + }); + + it('throws a clear error for a vendor missing from the provider enum', () => { + expect(() => providerForModel('mistral/some-model')).toThrowError( + /Unsupported model vendor in "mistral\/some-model".*anthropic, openai, moonshotai/ + ); + }); + + it('carries the provider on the runner for experiment display metadata', () => { + expect(createOpencodeRunner('openai/gpt-5.4').modelProvider).toBe('openai'); + expect(createOpencodeRunner('moonshotai/kimi-k3').modelProvider).toBe( + 'moonshotai' + ); + }); + + it('deriveStopReason reads the terminal step_finish reason', () => { + const runner = createOpencodeRunner('anthropic/claude-sonnet-5'); + const ok = { ok: true, exitCode: 0, stdout: '', stderr: '' }; + expect(runner.deriveStopReason!(SESSION, ok)).toBe('stop'); + // A non-stop terminal reason is surfaced verbatim. + const length = JSON.stringify({ + type: 'step_finish', + part: { reason: 'length' }, + }); + expect(runner.deriveStopReason!(length, ok)).toBe('length'); + // An error event wins regardless of exit code. + const errored = JSON.stringify({ + type: 'error', + error: { message: 'model overloaded' }, + }); + expect(runner.deriveStopReason!(errored, ok)).toBe('error'); + }); + + it("builds opencode's MCP config shape from harness server configs", () => { + const config = JSON.parse( + buildOpencodeConfig('moonshotai/kimi-k3', { + supabase: { command: 'npx', args: ['-y', 'srv'], env: { TOKEN: 't' } }, + docs: { command: 'docs-server' }, + }) + ); + expect(config.mcp).toEqual({ + supabase: { + type: 'local', + command: ['npx', '-y', 'srv'], + enabled: true, + environment: { TOKEN: 't' }, + }, + // No env → no `environment` key. + docs: { type: 'local', command: ['docs-server'], enabled: true }, + }); + }); + + it('declares the model under the vercel provider so resolution skips the models.dev catalog', () => { + const config = JSON.parse(buildOpencodeConfig('moonshotai/kimi-k3', {})); + expect(config.provider).toEqual({ + vercel: { models: { 'moonshotai/kimi-k3': {} } }, + }); + }); +}); + +/** Capture the `--model` flag, run env, and written config from one exec. */ +async function captureExec( + model: string, + opts: { mcp?: boolean } = {} +): Promise<{ + runCommand: string; + runEnv: Record | undefined; + config: Record | undefined; +}> { + const ok: CommandResult = { ok: true, exitCode: 0, stdout: '', stderr: '' }; + let runCommand = ''; + let runEnv: Record | undefined; + let config: Record | undefined; + await createOpencodeRunner(model).exec({ + sandbox: { + workspace: '/w', + exec: async (cmd, options) => { + const write = /^printf %s '([^']+)'/.exec(cmd); + if (write) { + config = JSON.parse(Buffer.from(write[1], 'base64').toString('utf8')); + } else if (cmd.includes(' run ')) { + runCommand = cmd; + runEnv = options?.env; + } + return ok; + }, + readFile: async () => '', + }, + model, + apiKey: 'gw-key', + systemPromptPath: '/s', + userPromptPath: '/u', + mcpServers: opts.mcp ? { supabase: { command: 'srv' } } : {}, + timeoutSec: 1, + }); + return { runCommand, runEnv, config }; +} + +describe('opencode runner exec routing', () => { + it("routes the model through opencode's native vercel provider with the gateway key", async () => { + const { runCommand, runEnv, config } = await captureExec( + 'moonshotai/kimi-k3', + { mcp: true } + ); + // Model is addressed under the vercel provider; the gateway slug stays intact. + expect(runCommand).toContain("--model 'vercel/moonshotai/kimi-k3'"); + expect(runEnv).toEqual({ AI_GATEWAY_API_KEY: 'gw-key' }); + expect(config?.mcp).toHaveProperty('supabase'); + }); + + it('writes the model-declaring config even without MCP servers', async () => { + const { runCommand, config } = await captureExec('moonshotai/kimi-k3'); + expect(config?.provider).toEqual({ + vercel: { models: { 'moonshotai/kimi-k3': {} } }, + }); + expect(runCommand).toContain('OPENCODE_CONFIG='); + }); +}); diff --git a/packages/core/src/agents/opencode/runner.ts b/packages/core/src/agents/opencode/runner.ts new file mode 100644 index 00000000..2750a8a1 --- /dev/null +++ b/packages/core/src/agents/opencode/runner.ts @@ -0,0 +1,220 @@ +/** + * OpenCode runner. Headless via `opencode run --format json` (the CLI + * streams newline-delimited event records to stdout; see ./parser.ts). + * + * Three things are opencode-specific: + * - All model requests route through the **Vercel AI Gateway** using + * opencode's native `vercel` provider (from its models.dev catalog): the + * run's `--model` is `vercel//` and the provider reads + * `AI_GATEWAY_API_KEY`. One key covers every vendor, no per-vendor keys. + * https://vercel.com/docs/ai-gateway/coding-agents/opencode + * - Model ids are gateway `vendor/model` slugs (e.g. `moonshotai/kimi-k3`), + * so the runner is built per-model with `modelProvider` (results metadata) + * parsed from the slug's vendor prefix. + * - `opencode run` blocks waiting on stdin even when the message is passed as + * an argument, so we redirect stdin from /dev/null. + * + * In tools mode Supabase access goes through MCP servers, declared in an + * OPENCODE_CONFIG file outside the scored workspace. + */ + +// opencode's own config schema (type-only; pinned to the installed CLI version +// via the catalog). The transcript stream is deliberately NOT typed from this +// SDK: `run --format json` emits a reduced, differently-shaped record than the +// SDK's server-API `Part`/`Event` entities (no id/sessionID/messageID; different +// discriminants), so the parser stays schema-defensive — see ./parser.ts. +import type { Config, McpLocalConfig } from '@opencode-ai/sdk'; +import type { McpServerConfig } from '../../index.js'; +import type { ModelProvider } from '../../eval-metadata.js'; +import { modelProviderSchema } from '../../eval-metadata.js'; +import { isRecord, parseJsonlRecords } from '../../json.js'; +import type { AgentRunner } from '../types.js'; +import { + SCRATCH, + npmGlobalBin, + npmInstallGlobal, + processStopReason, + shellQuote, + writeSandboxFile, +} from '../shared.js'; + +/** + * opencode model id: a Vercel AI Gateway `vendor/model` slug, where the model + * name is the original vendor's id. The catalog is public: + * GET https://ai-gateway.vercel.sh/v1/models + */ +export type OpenCodeModel = string; + +/** Model used when the caller doesn't pick one. */ +export const DEFAULT_OPENCODE_MODEL: OpenCodeModel = + 'anthropic/claude-sonnet-5'; + +/** + * Config provider id of opencode's native Vercel AI Gateway provider. opencode + * addresses a model as `provider/model`, splitting on the first `/`, so a run's + * `--model` is `vercel//` and the gateway `vendor/model` slug + * stays intact as the model id under it. + */ +const GATEWAY_PROVIDER_ID = 'vercel'; + +/** + * The vendor prefix of a gateway `vendor/model` slug, as results metadata. + * Throws for vendors missing from `modelProviderSchema` — extend that enum when + * adding a model from a new vendor. + */ +export function providerForModel(model: string): ModelProvider { + const vendor = modelProviderSchema.safeParse(model.split('/')[0]); + if (!vendor.success) { + throw new Error( + `Unsupported model vendor in "${model}". ` + + `Expected a Vercel AI Gateway vendor/model slug with one of: ${modelProviderSchema.options.join(', ')}.` + ); + } + return vendor.data; +} + +/** + * Shell path to the config, staged in scratch (outside the workspace). Used both + * as the write target and as the `OPENCODE_CONFIG` env value — the shell expands + * `$HOME` in either position. Holds the model declaration and MCP servers. + */ +const OPENCODE_CONFIG_PATH = '"$HOME/.eval/opencode.json"'; + +/** + * Build an opencode runner bound to one gateway model slug. A single run + * targets one model, so the runner resolves `modelProvider` (results metadata) + * from the slug's vendor prefix at build time. + */ +export function createOpencodeRunner( + model: OpenCodeModel +): AgentRunner { + return { + id: 'opencode', + displayName: 'OpenCode', + apiKeyEnvVar: 'AI_GATEWAY_API_KEY', + modelProvider: providerForModel(model), + cliPackage: 'opencode-ai', + // Pinned: opencode's --format json event schema evolves; bump deliberately + // and re-check the parser. See ./parser.ts. Must stay >= 1.17.0: earlier + // CLIs don't await the run event loop (opencode #31389) and intermittently + // exit 0 mid-step, ending runs with no final report. + defaultCliVersion: '1.18.5', + defaultModel: DEFAULT_OPENCODE_MODEL, + + async install(sandbox, version) { + await npmInstallGlobal( + sandbox, + `${this.cliPackage}@${version}`, + this.displayName + ); + }, + + async exec({ + sandbox, + model, + apiKey, + systemPromptPath, + userPromptPath, + mcpServers, + timeoutSec, + }) { + const opencode = npmGlobalBin('opencode'); + + // opencode has no system-prompt flag, so prepend the system prompt to the + // task; both are staged files, joined via command substitution into the + // single message argument. + const message = `"$(cat ${systemPromptPath}; printf '\\n\\n'; cat ${userPromptPath})"`; + + // The config declares the model (see `buildOpencodeConfig`) and any MCP + // servers, so it's written for every run. + await sandbox.exec(`mkdir -p ${SCRATCH}`); + await writeSandboxFile( + sandbox, + OPENCODE_CONFIG_PATH, + buildOpencodeConfig(model, mcpServers) + ); + const configPrefix = `OPENCODE_CONFIG=${OPENCODE_CONFIG_PATH} `; + + const flags = [ + 'run', + message, + `--model ${shellQuote(`${GATEWAY_PROVIDER_ID}/${model}`)}`, + // Newline-delimited JSON event records on stdout. + '--format json', + // The sandbox is the isolation boundary, so let opencode act freely. + '--dangerously-skip-permissions', + ].join(' '); + + // `< /dev/null`: opencode run blocks on stdin otherwise, even with the + // message passed as an argument. + const command = await sandbox.exec( + `${configPrefix}${opencode} ${flags} < /dev/null`, + { + timeoutMs: timeoutSec * 1000, + // The native vercel provider reads the gateway key from this env var. + env: { [this.apiKeyEnvVar]: apiKey }, + } + ); + return { command, raw: command.stdout }; + }, + + deriveStopReason(raw, command) { + if (!raw) return processStopReason(command); + const { records } = parseJsonlRecords(raw); + // An error event means the run failed regardless of exit code. + if (records.some((r) => r.type === 'error')) return 'error'; + // The terminal `step_finish` carries the model's finish reason. + for (let i = records.length - 1; i >= 0; i -= 1) { + if (records[i].type !== 'step_finish') continue; + const part = records[i].part; + const reason = + isRecord(part) && typeof part.reason === 'string' + ? part.reason + : undefined; + if (reason === 'stop') return 'stop'; + if (reason && reason !== 'tool-calls') return reason; // e.g. length — surface verbatim + break; + } + return processStopReason(command); + }, + }; +} + +/** + * opencode's `OPENCODE_CONFIG`. + * + * The model is declared under the native `vercel` provider because opencode + * otherwise resolves models against its models.dev catalog, fetched at run + * time: when that fetch fails it silently falls back to the CLI's bundled + * snapshot, and a model newer than the pinned CLI (e.g. Kimi K3 on 1.15.7) + * intermittently dies with "Model not found". Declaring it makes resolution + * deterministic. + * + * MCP servers map onto `{ mcp: { name: { type: "local", command: [...], + * environment } } }` (the harness's `{command,args,env}` → a single `command` + * array plus `environment`). + */ +export function buildOpencodeConfig( + model: string, + servers: Record +): string { + const mcp: Record = {}; + for (const [name, server] of Object.entries(servers)) { + mcp[name] = { + type: 'local', + command: [server.command, ...(server.args ?? [])], + enabled: true, + ...(server.env ? { environment: server.env } : {}), + }; + } + // Typed against opencode's own `Config` schema, so a config-shape change on a + // CLI bump (mcp/provider layout) fails to compile instead of silently at runtime. + const config: Config = { + $schema: 'https://opencode.ai/config.json', + provider: { + [GATEWAY_PROVIDER_ID]: { models: { [model]: {} } }, + }, + mcp, + }; + return JSON.stringify(config, null, 2); +} diff --git a/packages/core/src/agents/registry.ts b/packages/core/src/agents/registry.ts index 6a91a0d5..58db31b8 100644 --- a/packages/core/src/agents/registry.ts +++ b/packages/core/src/agents/registry.ts @@ -13,8 +13,13 @@ import type { AgentHarnessId } from '../eval-metadata.js'; import type { AgentTranscriptParser } from '../parsers/types.js'; import { claudeCodeDefinition } from './claude-code/index.js'; import { codexDefinition } from './codex/index.js'; +import { opencodeDefinition } from './opencode/index.js'; -const AGENTS: AgentDefinition[] = [claudeCodeDefinition, codexDefinition]; +const AGENTS: AgentDefinition[] = [ + claudeCodeDefinition, + codexDefinition, + opencodeDefinition, +]; const byId = new Map(AGENTS.map((agent) => [agent.runner.id, agent])); diff --git a/packages/core/src/agents/shared.test.ts b/packages/core/src/agents/shared.test.ts new file mode 100644 index 00000000..06ab423b --- /dev/null +++ b/packages/core/src/agents/shared.test.ts @@ -0,0 +1,28 @@ +import { afterEach, describe, expect, it } from 'vitest'; +import { requireEnv } from './shared.js'; + +const VAR = 'OPENCODE_TEST_ENV_VAR'; + +describe('requireEnv', () => { + afterEach(() => { + delete process.env[VAR]; + }); + + it('returns the value when set', () => { + process.env[VAR] = 'secret'; + expect(requireEnv(VAR)).toBe('secret'); + }); + + it('throws a clear, variable-naming error when unset, including the hint', () => { + expect(() => requireEnv(VAR, 'Set it to run X.')).toThrowError( + `Environment variable ${VAR} is not set. Set it to run X.` + ); + }); + + it('distinguishes set-but-empty from unset', () => { + process.env[VAR] = ' '; + expect(() => requireEnv(VAR)).toThrowError( + `Environment variable ${VAR} is set but empty.` + ); + }); +}); diff --git a/packages/core/src/agents/shared.ts b/packages/core/src/agents/shared.ts index cf1058cb..2e2685a0 100644 --- a/packages/core/src/agents/shared.ts +++ b/packages/core/src/agents/shared.ts @@ -1,12 +1,30 @@ /** - * Helpers shared across CLI runners: sandbox scratch paths, file staging, - * global npm install, loopback rewriting, and the default process-exit-based - * stop reason. + * Helpers shared across CLI runners: env-var validation, sandbox scratch paths, + * file staging, global npm install, loopback rewriting, and the default + * process-exit-based stop reason. */ import type { CommandResult, McpServerConfig } from '../index.js'; import type { AgentSandbox } from './types.js'; +/** Reads a required environment variable, throwing an error that names it. */ +export function requireEnv(name: string, hint?: string): string { + // `in` distinguishes "never set" from "set but empty" for a clearer message. + const isSet = name in process.env; + const value = process.env[name]; + if (!isSet || value === undefined) { + throw new Error( + `Environment variable ${name} is not set.${hint ? ` ${hint}` : ''}` + ); + } + if (value.trim() === '') { + throw new Error( + `Environment variable ${name} is set but empty.${hint ? ` ${hint}` : ''}` + ); + } + return value; +} + /** Scratch dir + staged files, outside the workspace so they're never scored. */ export const SCRATCH = '"$HOME/.eval"'; export const SYSTEM_PROMPT_PATH = '"$HOME/.eval/system-prompt.txt"'; diff --git a/packages/core/src/agents/types.ts b/packages/core/src/agents/types.ts index aad716ed..baf91962 100644 --- a/packages/core/src/agents/types.ts +++ b/packages/core/src/agents/types.ts @@ -82,6 +82,12 @@ export interface AgentRunner { displayName: string; /** Env var holding the agent's API key (e.g. `ANTHROPIC_API_KEY`). */ apiKeyEnvVar: string; + /** + * The model's provider, for multi-provider CLIs whose runner is built + * per-model (e.g. opencode). When omitted, the engine derives the provider + * from the agent id. + */ + modelProvider?: ModelProvider; /** npm package providing the CLI. */ cliPackage: string; /** Pinned CLI version — pinned so transcript-format drift can't silently break parsing. */ diff --git a/packages/core/src/eval-metadata.ts b/packages/core/src/eval-metadata.ts index 1a8d541c..59a645cf 100644 --- a/packages/core/src/eval-metadata.ts +++ b/packages/core/src/eval-metadata.ts @@ -52,10 +52,19 @@ export const experimentSuiteSchema = z.enum([ export const EXPERIMENT_SUITES = experimentSuiteSchema.options; export type ExperimentSuite = z.infer; -export const agentHarnessIdSchema = z.enum(['ai-sdk', 'claude-code', 'codex']); +export const agentHarnessIdSchema = z.enum([ + 'ai-sdk', + 'claude-code', + 'codex', + 'opencode', +]); export type AgentHarnessId = z.infer; -export const modelProviderSchema = z.enum(['anthropic', 'openai']); +export const modelProviderSchema = z.enum([ + 'anthropic', + 'openai', + 'moonshotai', +]); export type ModelProvider = z.infer; export const reasoningEffortSchema = z.enum([ diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index c3ccfc0e..caa480d2 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -99,10 +99,11 @@ export { rehydrateTruncatedDocsResults, } from './docs-results.js'; export type { DocsResultSandbox } from './docs-results.js'; -// CLI agent harnesses (Claude Code, Codex, and the framework for adding more). +// CLI agent harnesses (Claude Code, Codex, OpenCode, and the framework for adding more). export { createCliAgent } from './agents/engine.js'; export { claudeCodeAgent } from './agents/claude-code/index.js'; export { codexAgent } from './agents/codex/index.js'; +export { opencodeAgent } from './agents/opencode/index.js'; export type { AgentMetadata, AgentSandbox, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index f7b2f128..019ca4b4 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -24,6 +24,9 @@ catalogs: '@electric-sql/pglite-socket': specifier: 0.1.5 version: 0.1.5 + '@opencode-ai/sdk': + specifier: 1.18.5 + version: 1.18.5 '@supabase/lite': specifier: 0.7.1-next.3 version: 0.7.1-next.3 @@ -279,6 +282,9 @@ importers: specifier: 'catalog:' version: 4.4.3 devDependencies: + '@opencode-ai/sdk': + specifier: 'catalog:' + version: 1.18.5 vitest: specifier: 'catalog:' version: 4.1.8(@opentelemetry/api@1.9.1)(@types/node@25.9.3)(happy-dom@20.10.2)(vite@7.3.5(@types/node@25.9.3)(jiti@2.7.0)(lightningcss@1.32.0)(tsx@4.22.4)) @@ -1086,6 +1092,9 @@ packages: resolution: {integrity: sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==} engines: {node: '>= 8'} + '@opencode-ai/sdk@1.18.5': + resolution: {integrity: sha512-7KgMvP5/1oxbhHj6kYBtPSTEdFKYpUeEYOzBTKdzSaRpapUpFFdn6Hkus3rr0rljO0kukWZIgRd3DrVBwTULGA==} + '@opentelemetry/api-logs@0.214.0': resolution: {integrity: sha512-40lSJeqYO8Uz2Yj7u94/SJWE/wONa7rmMKjI1ZcIjgf3MHNHv1OZUCrCETGuaRF62d5pQD1wKIW+L4lmSMTzZA==} engines: {node: '>=8.0.0'} @@ -5160,6 +5169,10 @@ snapshots: '@nodelib/fs.scandir': 2.1.5 fastq: 1.20.1 + '@opencode-ai/sdk@1.18.5': + dependencies: + cross-spawn: 7.0.6 + '@opentelemetry/api-logs@0.214.0': dependencies: '@opentelemetry/api': 1.9.1 diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 7b181ac5..b0f3f97c 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -8,6 +8,10 @@ catalog: '@ai-sdk/anthropic': ^3.0.71 '@ai-sdk/mcp': ^1.0.39 '@ai-sdk/openai': ^3.0.66 + # Type-only: opencode's config/schema types. Pinned exactly to the CLI + # version the opencode runner installs (defaultCliVersion in + # agents/opencode/runner.ts) so the types track the binary that actually runs. + '@opencode-ai/sdk': 1.18.5 '@electric-sql/pglite': 0.4.5 '@electric-sql/pglite-socket': 0.1.5 '@supabase/lite': 0.7.1-next.3 From 1b19fb4054af8e34e8c9b28c7db9cf375f2cf7ba Mon Sep 17 00:00:00 2001 From: Matt Rossman <22670878+mattrossman@users.noreply.github.com> Date: Fri, 24 Jul 2026 19:38:29 -0400 Subject: [PATCH 02/17] refactor: drop redundant model declaration from opencode config --- .../core/src/agents/opencode/runner.test.ts | 17 ++------ packages/core/src/agents/opencode/runner.ts | 41 +++++++------------ pnpm-workspace.yaml | 4 +- 3 files changed, 20 insertions(+), 42 deletions(-) diff --git a/packages/core/src/agents/opencode/runner.test.ts b/packages/core/src/agents/opencode/runner.test.ts index fce38221..8ecc92b8 100644 --- a/packages/core/src/agents/opencode/runner.test.ts +++ b/packages/core/src/agents/opencode/runner.test.ts @@ -59,7 +59,7 @@ describe('opencode runner', () => { it("builds opencode's MCP config shape from harness server configs", () => { const config = JSON.parse( - buildOpencodeConfig('moonshotai/kimi-k3', { + buildOpencodeConfig({ supabase: { command: 'npx', args: ['-y', 'srv'], env: { TOKEN: 't' } }, docs: { command: 'docs-server' }, }) @@ -75,13 +75,6 @@ describe('opencode runner', () => { docs: { type: 'local', command: ['docs-server'], enabled: true }, }); }); - - it('declares the model under the vercel provider so resolution skips the models.dev catalog', () => { - const config = JSON.parse(buildOpencodeConfig('moonshotai/kimi-k3', {})); - expect(config.provider).toEqual({ - vercel: { models: { 'moonshotai/kimi-k3': {} } }, - }); - }); }); /** Capture the `--model` flag, run env, and written config from one exec. */ @@ -134,11 +127,9 @@ describe('opencode runner exec routing', () => { expect(config?.mcp).toHaveProperty('supabase'); }); - it('writes the model-declaring config even without MCP servers', async () => { + it('skips the config file when there are no MCP servers', async () => { const { runCommand, config } = await captureExec('moonshotai/kimi-k3'); - expect(config?.provider).toEqual({ - vercel: { models: { 'moonshotai/kimi-k3': {} } }, - }); - expect(runCommand).toContain('OPENCODE_CONFIG='); + expect(config).toBeUndefined(); + expect(runCommand).not.toContain('OPENCODE_CONFIG='); }); }); diff --git a/packages/core/src/agents/opencode/runner.ts b/packages/core/src/agents/opencode/runner.ts index 2750a8a1..7fcfa7f0 100644 --- a/packages/core/src/agents/opencode/runner.ts +++ b/packages/core/src/agents/opencode/runner.ts @@ -125,15 +125,17 @@ export function createOpencodeRunner( // single message argument. const message = `"$(cat ${systemPromptPath}; printf '\\n\\n'; cat ${userPromptPath})"`; - // The config declares the model (see `buildOpencodeConfig`) and any MCP - // servers, so it's written for every run. - await sandbox.exec(`mkdir -p ${SCRATCH}`); - await writeSandboxFile( - sandbox, - OPENCODE_CONFIG_PATH, - buildOpencodeConfig(model, mcpServers) - ); - const configPrefix = `OPENCODE_CONFIG=${OPENCODE_CONFIG_PATH} `; + // A config file is only needed for MCP servers. + let configPrefix = ''; + if (Object.keys(mcpServers).length > 0) { + await sandbox.exec(`mkdir -p ${SCRATCH}`); + await writeSandboxFile( + sandbox, + OPENCODE_CONFIG_PATH, + buildOpencodeConfig(mcpServers) + ); + configPrefix = `OPENCODE_CONFIG=${OPENCODE_CONFIG_PATH} `; + } const flags = [ 'run', @@ -181,21 +183,11 @@ export function createOpencodeRunner( } /** - * opencode's `OPENCODE_CONFIG`. - * - * The model is declared under the native `vercel` provider because opencode - * otherwise resolves models against its models.dev catalog, fetched at run - * time: when that fetch fails it silently falls back to the CLI's bundled - * snapshot, and a model newer than the pinned CLI (e.g. Kimi K3 on 1.15.7) - * intermittently dies with "Model not found". Declaring it makes resolution - * deterministic. - * - * MCP servers map onto `{ mcp: { name: { type: "local", command: [...], - * environment } } }` (the harness's `{command,args,env}` → a single `command` - * array plus `environment`). + * opencode's `OPENCODE_CONFIG`. MCP servers map onto `{ mcp: { name: { type: + * "local", command: [...], environment } } }` (the harness's `{command,args,env}` + * → a single `command` array plus `environment`). */ export function buildOpencodeConfig( - model: string, servers: Record ): string { const mcp: Record = {}; @@ -208,12 +200,9 @@ export function buildOpencodeConfig( }; } // Typed against opencode's own `Config` schema, so a config-shape change on a - // CLI bump (mcp/provider layout) fails to compile instead of silently at runtime. + // CLI bump (mcp layout) fails to compile instead of silently at runtime. const config: Config = { $schema: 'https://opencode.ai/config.json', - provider: { - [GATEWAY_PROVIDER_ID]: { models: { [model]: {} } }, - }, mcp, }; return JSON.stringify(config, null, 2); diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index b0f3f97c..981db650 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -8,9 +8,7 @@ catalog: '@ai-sdk/anthropic': ^3.0.71 '@ai-sdk/mcp': ^1.0.39 '@ai-sdk/openai': ^3.0.66 - # Type-only: opencode's config/schema types. Pinned exactly to the CLI - # version the opencode runner installs (defaultCliVersion in - # agents/opencode/runner.ts) so the types track the binary that actually runs. + # Pin to defaultCliVersion in agents/opencode/runner.ts '@opencode-ai/sdk': 1.18.5 '@electric-sql/pglite': 0.4.5 '@electric-sql/pglite-socket': 0.1.5 From ddaf4afd3f334fbd4ddfb3a155b83e56be200110 Mon Sep 17 00:00:00 2001 From: Matt Rossman <22670878+mattrossman@users.noreply.github.com> Date: Fri, 24 Jul 2026 19:53:17 -0400 Subject: [PATCH 03/17] fix: provide AI_GATEWAY_API_KEY to eval refresh runs --- .github/workflows/eval-refresh.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/eval-refresh.yml b/.github/workflows/eval-refresh.yml index 42d5e25a..2052c0dd 100644 --- a/.github/workflows/eval-refresh.yml +++ b/.github/workflows/eval-refresh.yml @@ -259,6 +259,7 @@ jobs: env: ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + AI_GATEWAY_API_KEY: ${{ secrets.AI_GATEWAY_API_KEY }} steps: - name: Checkout uses: actions/checkout@9f698171ed81b15d1823a05fc7211befd50c8ae0 # v6.0.3 @@ -286,6 +287,7 @@ jobs: { echo "ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY}" echo "OPENAI_API_KEY=${OPENAI_API_KEY}" + echo "AI_GATEWAY_API_KEY=${AI_GATEWAY_API_KEY}" } > .env - name: Run evals @@ -300,6 +302,14 @@ jobs: --runs "${{ needs.prepare.outputs.runs }}" \ --timeout-sec "${{ needs.prepare.outputs.timeout_sec }}" + # A missing/skipped experiment (e.g. its API key env var is unset) + # exits 0 without writing results; fail loudly instead of uploading + # nothing and breaking publish-results downstream. + if [ ! -f "results/${{ matrix.experiment }}/${{ matrix.eval_id }}.json" ]; then + echo "::error::no result written for ${{ matrix.experiment }} x ${{ matrix.eval_id }} (experiment skipped?)" + exit 1 + fi + - name: Upload raw results uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: From b33bc2a656c811d909ca11aa183ada232ac979e8 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sat, 25 Jul 2026 00:10:10 +0000 Subject: [PATCH 04/17] chore: refresh eval results --- apps/web/src/data/eval-results.json | 1624 +++++++++++++++++++++++++++ 1 file changed, 1624 insertions(+) diff --git a/apps/web/src/data/eval-results.json b/apps/web/src/data/eval-results.json index 556790c2..4c5e82e6 100644 --- a/apps/web/src/data/eval-results.json +++ b/apps/web/src/data/eval-results.json @@ -17824,5 +17824,1629 @@ "promptSourcePath": "evals/resolve-security-002-rls-cross-tenant-leak/PROMPT.md", "attempts": 1, "sourcePath": "codex-gpt-5.6-no-skills/resolve-security-002-rls-cross-tenant-leak.json" + }, + { + "experiment": "opencode-kimi-k3", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "opencode", + "modelProvider": "moonshotai", + "modelId": "moonshotai/kimi-k3" + }, + "eval": "build-cli-001-bootstrap-app", + "stage": "build", + "product": [ + "database", + "data-api" + ], + "topic": [ + "migrations", + "rls" + ], + "suite": "benchmark", + "interface": "cli", + "passed": true, + "checks": [ + { + "name": "supabase project initialised (supabase/config.toml exists)", + "passed": true + }, + { + "name": "todos table is created by a migration file", + "passed": true + }, + { + "name": "todos table exists with at least 2 seeded rows", + "passed": true, + "notes": "found 2 rows" + }, + { + "name": "row level security is enabled on todos", + "passed": true + }, + { + "name": "a SELECT policy targets the authenticated role", + "passed": true + }, + { + "name": "REST API returns no todos to anonymous requests", + "passed": true, + "notes": "error 42501: permission denied for table todos" + }, + { + "name": "REST API returns the todos to authenticated requests", + "passed": true, + "notes": "2 rows" + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [] + }, + "prompt": "We're kicking off a todos app and I want the Supabase side ready for the team\nto build on. Set it up the way we'd run it in development, with schema changes\ntracked as migrations so they can be reviewed and replayed.\n\nFor the first slice we just need a `todos` table. Todos aren't public: anyone\nsigned in can read all of them, but nothing should be writable through the API\nfor now. Add a couple of sample todos so there's something to look at.\n\nBefore you hand it back, make sure the running API actually behaves that way —\nsigned-in users get the todos, signed-out requests get nothing.", + "promptSourcePath": "evals/build-cli-001-bootstrap-app/PROMPT.md", + "attempts": 1, + "sourcePath": "opencode-kimi-k3/build-cli-001-bootstrap-app.json" + }, + { + "experiment": "opencode-kimi-k3", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "opencode", + "modelProvider": "moonshotai", + "modelId": "moonshotai/kimi-k3" + }, + "eval": "build-cli-002-declarative-schema", + "stage": "build", + "product": [ + "database" + ], + "topic": [ + "declarative-schema", + "migrations" + ], + "suite": "benchmark", + "interface": "cli", + "passed": true, + "checks": [ + { + "name": "supabase db diff used to generate the migration", + "passed": true + }, + { + "name": "schema file updated to include description column", + "passed": true + }, + { + "name": "a new migration was generated for the change", + "passed": true + }, + { + "name": "description column exists in the live database", + "passed": true + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [] + }, + "prompt": "Add a description text column to the `products` table in my local Supabase stack", + "promptSourcePath": "evals/build-cli-002-declarative-schema/PROMPT.md", + "attempts": 1, + "sourcePath": "opencode-kimi-k3/build-cli-002-declarative-schema.json" + }, + { + "experiment": "opencode-kimi-k3", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "opencode", + "modelProvider": "moonshotai", + "modelId": "moonshotai/kimi-k3" + }, + "eval": "build-cli-003-pg-cron-queue-workflow", + "stage": "build", + "product": [ + "database", + "edge-functions", + "cron", + "queues" + ], + "topic": [ + "sql", + "sdk" + ], + "suite": "benchmark", + "interface": "cli", + "passed": true, + "checks": [ + { + "name": "pg_cron job 'enqueue-tasks' scheduled to run every minute", + "passed": true, + "notes": "schedule='* * * * *', active=true" + }, + { + "name": "cron command enqueues to the 'tasks' queue", + "passed": true, + "notes": "queue depth 1 -> 2" + }, + { + "name": "process-tasks function drains the queue", + "passed": true, + "notes": "function removed the seeded message (id 3) from the queue" + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [ + { + "source": "web_fetch", + "query": "https://supabase.com/changelog.md", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 92585 + }, + { + "source": "web_fetch", + "query": "https://supabase.com/docs/guides/queues/quickstart.md", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/queues/quickstart.md" + } + ], + "resultChars": 8775 + }, + { + "source": "web_fetch", + "query": "https://supabase.com/docs/guides/cron.md", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/cron.md" + } + ], + "resultChars": 1405 + }, + { + "source": "web_fetch", + "query": "https://supabase.com/docs/guides/cron/quickstart.md", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/cron/quickstart.md" + } + ], + "resultChars": 6155 + }, + { + "source": "web_fetch", + "query": "https://supabase.com/docs/guides/functions/examples/pgmq-serverless-workers.md", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/functions/examples/pgmq-serverless-workers.md" + } + ] + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"consume queue messages with edge function pgmq pop worker\", limit: 6) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/queues/consuming-messages-with-edge-functions", + "title": "Consuming Supabase Queue Messages with Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", + "title": "Automatic embeddings" + }, + { + "url": "https://supabase.com/docs/guides/queues/pgmq", + "title": "PGMQ Extension" + } + ], + "resultChars": 55604 + }, + { + "source": "web_fetch", + "query": "https://supabase.com/docs/guides/queues/consuming-messages-with-edge-functions.md", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/queues/consuming-messages-with-edge-functions.md" + } + ], + "resultChars": 3873 + } + ] + }, + "prompt": "I want to set up a recurring background workflow on my local Supabase stack.\n\nCan you set up a cron job called `enqueue-tasks` to run every minute and push a task into a queue called `tasks`? Then add a `process-tasks` edge function that reads messages off the `tasks` queue and removes them, so a scheduled worker can keep the backlog drained.", + "promptSourcePath": "evals/build-cli-003-pg-cron-queue-workflow/PROMPT.md", + "attempts": 1, + "sourcePath": "opencode-kimi-k3/build-cli-003-pg-cron-queue-workflow.json" + }, + { + "experiment": "opencode-kimi-k3", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "opencode", + "modelProvider": "moonshotai", + "modelId": "moonshotai/kimi-k3" + }, + "eval": "build-database-001-migrate-postgres-to-supabase", + "stage": "build", + "product": [ + "database" + ], + "topic": [ + "migrations" + ], + "suite": "benchmark", + "interface": "cli", + "passed": true, + "checks": [ + { + "name": "all 3 tables exist (teams, members, tasks)", + "passed": true + }, + { + "name": "row counts match (teams=5, members=10, tasks=13)", + "passed": true + }, + { + "name": "foreign key constraints survived the restore", + "passed": true + }, + { + "name": "tasks_team_status_idx index survived the restore", + "passed": true + }, + { + "name": "sequences synced (next insert won't conflict with existing IDs)", + "passed": true + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [] + }, + "prompt": "I have an existing Postgres database I want to migrate to Supabase. There's a binary dump at `source.dump` in the current directory.\n\nCan you set up a local Supabase project and restore the dump into it?", + "promptSourcePath": "evals/build-database-001-migrate-postgres-to-supabase/PROMPT.md", + "attempts": 1, + "sourcePath": "opencode-kimi-k3/build-database-001-migrate-postgres-to-supabase.json" + }, + { + "experiment": "opencode-kimi-k3", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "opencode", + "modelProvider": "moonshotai", + "modelId": "moonshotai/kimi-k3" + }, + "eval": "build-functions-004-service-role-bypass", + "stage": "build", + "product": [ + "edge-functions", + "auth", + "database" + ], + "topic": [ + "rls", + "security", + "sdk" + ], + "suite": "benchmark", + "interface": "mcp", + "passed": true, + "checks": [ + { + "name": "rejects missing auth", + "passed": true, + "notes": "status=401" + }, + { + "name": "user A reads own note", + "passed": true, + "notes": "status=200" + }, + { + "name": "reads only with the caller's JWT", + "passed": true, + "notes": "bearer_tokens=2, all_match=true" + }, + { + "name": "user A cannot force-read user B note", + "passed": true, + "notes": "status=200" + }, + { + "name": "user B cannot force-read user A note", + "passed": true, + "notes": "status=200" + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [ + { + "source": "web_fetch", + "query": "https://supabase.com/changelog.md", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 92585 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"edge function get user from JWT authorization header RLS\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/functions/auth", + "title": "Securing Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", + "title": "Integrating With Supabase Auth" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/why-is-my-service-role-key-client-getting-rls-errors-or-not-returning-data-7_1K9z", + "title": "Why is my service role key client getting RLS errors or not returning data?" + }, + { + "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security", + "title": "Token Security and Row Level Security" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/why-is-my-select-returning-an-empty-data-array-and-i-have-data-in-the-table-xvOPgx", + "title": "Why is my select returning an empty data array and I have data in the table?" + } + ], + "resultChars": 29869 + } + ] + }, + "prompt": "I built an Edge Function called `private-notes` for showing a user's saved\nprivate notes.\n\nCan you check whether there's any way one user could see another user's notes?\n\nPlease fix and deploy it if needed.", + "promptSourcePath": "evals/build-functions-004-service-role-bypass/PROMPT.md", + "attempts": 1, + "sourcePath": "opencode-kimi-k3/build-functions-004-service-role-bypass.json" + }, + { + "experiment": "opencode-kimi-k3", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "opencode", + "modelProvider": "moonshotai", + "modelId": "moonshotai/kimi-k3" + }, + "eval": "build-functions-005-dual-auth-user-secret", + "stage": "build", + "product": [ + "edge-functions", + "auth", + "database" + ], + "topic": [ + "sdk", + "rls", + "security" + ], + "suite": "benchmark", + "interface": "cli", + "cliVersion": "2.109.1", + "passed": true, + "checks": [ + { + "name": "seed rows present", + "passed": true, + "notes": "found 2/2 seeded rows" + }, + { + "name": "rejects request with no credentials", + "passed": true, + "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" + }, + { + "name": "user with JWT reads only their own rows", + "passed": true, + "notes": "status 200: [{\"user_id\":\"17717159-30ef-4e5f-bf1b-40f3b4dae017\",\"metric\":\"steps_a_mrzm2gwa\",\"value\":111}]" + }, + { + "name": "user cannot read another user's rows by passing user_id", + "passed": true, + "notes": "status 200: [{\"user_id\":\"17717159-30ef-4e5f-bf1b-40f3b4dae017\",\"metric\":\"steps_a_mrzm2gwa\",\"value\":111}]" + }, + { + "name": "service key bypasses RLS to read the target user's rows", + "passed": true, + "notes": "status 200: [{\"user_id\":\"9513ef26-e76f-4e48-9439-ef8ed31563ae\",\"metric\":\"steps_b_mrzm2gwa\",\"value\":222}]" + }, + { + "name": "non-service key is not granted service access", + "passed": true, + "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" + }, + { + "name": "rejects an unverified (forged) user token", + "passed": true, + "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" + }, + { + "name": "a user token in the apikey slot is not treated as the service key", + "passed": true, + "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" + }, + { + "name": "implementation uses @supabase/server", + "passed": true, + "notes": "imports @supabase/server / withSupabase" + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [ + { + "source": "web_fetch", + "query": "https://supabase.com/changelog.md", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 92585 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"edge function verify_jwt config.toml skip JWT verification\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/functions/function-configuration", + "title": "Function Configuration" + }, + { + "url": "https://supabase.com/docs/guides/functions/deploy", + "title": "Deploy to Production" + }, + { + "url": "https://supabase.com/docs/guides/functions/development-tips", + "title": "Development tips" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth-headers", + "title": "Authorization headers" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth", + "title": "Securing Edge Functions" + } + ], + "resultChars": 25625 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"edge functions environment variables SUPABASE_SECRET_KEYS publishable secret api keys\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" + }, + { + "url": "https://supabase.com/docs/guides/functions/secrets", + "title": "Environment Variables" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth", + "title": "Securing Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", + "title": "Self-Hosted Functions" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/docker", + "title": "Self-Hosting with Docker" + } + ], + "resultChars": 67435 + } + ] + }, + "prompt": "Build and serve a Supabase Edge Function named `user-stats` for this project,\nreachable over HTTP at `/functions/v1/user-stats`.\n\nOur product stores per-user metrics in the existing `user_stats` table.\n\nTwo very different callers need to hit this one endpoint:\n\n1. **Our mobile app**, acting for a signed-in user. It sends that user's\n Supabase access token. The endpoint should return the user's own stats.\n\n2. **Our internal billing service**, a trusted backend with no signed-in user.\n It authenticates with the project's secret (service-role) key in the `apikey`\n header, and names the target user with a `user_id` in the JSON request body.\n It needs that user's stats.\n\nReturn the matching rows as JSON. The endpoint must be secure: only ever serve\nstats to a caller that is genuinely entitled to them, and turn away callers that\nare not.", + "promptSourcePath": "evals/build-functions-005-dual-auth-user-secret/PROMPT.md", + "attempts": 1, + "sourcePath": "opencode-kimi-k3/build-functions-005-dual-auth-user-secret.json" + }, + { + "experiment": "opencode-kimi-k3", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "opencode", + "modelProvider": "moonshotai", + "modelId": "moonshotai/kimi-k3" + }, + "eval": "build-storage-001-private-bucket-access", + "stage": "build", + "product": [ + "storage", + "database" + ], + "topic": [ + "rls", + "sdk" + ], + "suite": "benchmark", + "interface": "mcp", + "passed": true, + "checks": [ + { + "name": "bucket user-files exists", + "passed": true + }, + { + "name": "bucket user-files is private", + "passed": true + }, + { + "name": "RLS still enabled on storage.objects", + "passed": true + }, + { + "name": "user A lists only own files", + "passed": true, + "notes": "saw: 019f968f-d2df-75d7-8566-9d025f9a1cb6/receipt-alpha.pdf, 019f968f-d2df-75d7-8566-9d025f9a1cb6/receipt-beta.pdf" + }, + { + "name": "user B cannot read user A files", + "passed": true + }, + { + "name": "anon reads no files", + "passed": true + }, + { + "name": "user A can upload into own folder", + "passed": true + }, + { + "name": "user B cannot upload into user A folder", + "passed": true + }, + { + "name": "configured private per-user storage access", + "passed": true, + "judgeNotes": "Creates private user-files bucket, owner-scoped SELECT and INSERT policies for authenticated users with RLS left enabled, avoids public/permissive access, and uses createSignedUrl with expiry for temporary sharing." + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] + }, + "docs": { + "calls": [ + { + "source": "web_fetch", + "query": "https://supabase.com/changelog.md", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 92585 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"storage access control policies bucket private folder user id\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/storage/security/access-control", + "title": "Storage Access Control" + }, + { + "url": "https://supabase.com/docs/guides/storage/buckets/fundamentals", + "title": "Storage Buckets" + }, + { + "url": "https://supabase.com/docs/guides/storage/cdn/fundamentals", + "title": "Storage CDN" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/why-cant-i-uploadlistetc-my-public-bucket-Z6CmGt", + "title": "Why can't I upload/list/etc my public bucket?" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3", + "title": "Configure S3 Storage" + } + ], + "resultChars": 20134 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"createSignedUrl temporary share link storage supabase-js\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/storage/serving/downloads", + "title": "Serving assets from Storage" + }, + { + "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsignedurl" + }, + { + "url": "https://supabase.com/docs/guides/storage/uploads/resumable-uploads", + "title": "Resumable Uploads" + }, + { + "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsignedurls" + }, + { + "url": "https://supabase.com/docs/guides/storage/serving/image-transformations", + "title": "Storage Image Transformations" + } + ], + "resultChars": 38982 + } + ] + }, + "prompt": "Our app lets signed-in users keep personal files like receipts and bank\nstatements. These files are private — a user must only ever be able to upload\nand download their own. The app uploads each file under a path that starts\nwith the owner's user id, e.g. `/receipt-march.pdf`.\n\nSet up a `user-files` bucket on our project and lock it down that way.\n\nUsers also sometimes share one of their files with someone else through a\ntemporary link that expires. Include the supabase-js code the app should use\nfor that.", + "promptSourcePath": "evals/build-storage-001-private-bucket-access/PROMPT.md", + "attempts": 1, + "sourcePath": "opencode-kimi-k3/build-storage-001-private-bucket-access.json" + }, + { + "experiment": "opencode-kimi-k3", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "opencode", + "modelProvider": "moonshotai", + "modelId": "moonshotai/kimi-k3" + }, + "eval": "build-tests-001-rls-tenant-isolation", + "stage": "build", + "product": [ + "database" + ], + "topic": [ + "tests", + "rls" + ], + "suite": "benchmark", + "interface": "cli", + "passed": true, + "checks": [ + { + "name": "pgTAP test file(s) written under supabase/tests/", + "passed": true, + "notes": "1 file(s): supabase/tests/rls_tenant_isolation.sql" + }, + { + "name": "pgTAP isolation tests ran and pass", + "passed": true, + "notes": "8 passed, 3 failed" + }, + { + "name": "agent correctly identifies the posts isolation bug from test results", + "passed": true, + "judgeNotes": "The agent correctly identifies `posts` as having the broken tenant isolation policy, specifically that authenticated members can read posts from organizations they are not members of, and grounds this in the pgTAP failure (test 5). It also treats the test results as authoritative and notes that `notes` isolation is correct." + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [] + }, + "prompt": "Can you audit the tenant isolation on our tables? Write some database tests covering both the happy path and the negative case, run them, and share the results.", + "promptSourcePath": "evals/build-tests-001-rls-tenant-isolation/PROMPT.md", + "attempts": 1, + "sourcePath": "opencode-kimi-k3/build-tests-001-rls-tenant-isolation.json" + }, + { + "experiment": "opencode-kimi-k3", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "opencode", + "modelProvider": "moonshotai", + "modelId": "moonshotai/kimi-k3" + }, + "eval": "build-vectors-001-rag-with-permissions", + "stage": "build", + "product": [ + "database", + "vectors" + ], + "topic": [ + "sql", + "rls" + ], + "suite": "benchmark", + "interface": "mcp", + "passed": true, + "checks": [ + { + "name": "document_sections.embedding is vector(384)", + "passed": true, + "notes": "vector(384)" + }, + { + "name": "HNSW index on the embedding column", + "passed": true, + "notes": "CREATE INDEX document_sections_embedding_idx ON public.document_sections USING hnsw (embedding vector_cosine_ops)" + }, + { + "name": "index operator class matches the search operator", + "passed": true, + "notes": "function operators: <=>\nindexes: CREATE INDEX document_sections_embedding_idx ON public.document_sections USING hnsw (embedding vector_cosine_ops)" + }, + { + "name": "user A search returns only own sections, best match first", + "passed": true + }, + { + "name": "user B search returns only own sections, best match first", + "passed": true + }, + { + "name": "user A reads only own sections through the API", + "passed": true + }, + { + "name": "user A reads only own documents through the API", + "passed": true + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] + }, + "docs": { + "calls": [ + { + "source": "web_fetch", + "query": "https://supabase.com/changelog.md", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 92585 + } + ] + }, + "prompt": "We're adding semantic search to our internal knowledge base app. I already wrote the edge functions, but search doesn't work yet. Can you set up whatever the database needs to make search work end to end?\n\nSome documents are confidential, so users should only have access to documents they own.", + "promptSourcePath": "evals/build-vectors-001-rag-with-permissions/PROMPT.md", + "attempts": 1, + "sourcePath": "opencode-kimi-k3/build-vectors-001-rag-with-permissions.json" + }, + { + "experiment": "opencode-kimi-k3", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "opencode", + "modelProvider": "moonshotai", + "modelId": "moonshotai/kimi-k3" + }, + "eval": "deploy-database-001-prometheus-metrics", + "stage": "deploy", + "product": [ + "database" + ], + "topic": [ + "observability" + ], + "suite": "benchmark", + "passed": true, + "checks": [ + { + "name": "preserved existing app scrape job", + "passed": true + }, + { + "name": "configured the Supabase Metrics API scrape correctly", + "passed": true, + "judgeNotes": "Prometheus config preserves the app job and adds a Supabase scrape over HTTPS to .supabase.co:443 with /customer/v1/privileged/metrics and HTTP Basic Auth using password_file. docker-compose mounts the secrets directory containing that password file into the Prometheus container." + }, + { + "name": "documented live deployment and verification steps", + "passed": true, + "judgeNotes": "README includes live setup steps for project ref, creating a Secret API key, placing it in the matching secrets file, and reloading/restarting Prometheus/Compose. Verification is concrete via Prometheus targets, direct metrics curl, and Grafana dashboard." + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "{ searchDocs(query: \"metrics endpoint prometheus observability export metrics\") { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/telemetry/metrics", + "title": "Metrics API" + }, + { + "url": "https://supabase.com/docs/guides/telemetry/metrics/grafana-self-hosted", + "title": "Metrics API with Prometheus & Grafana (self-hosted)" + }, + { + "url": "https://supabase.com/docs/guides/telemetry/metrics/vendor-agnostic", + "title": "Vendor-agnostic Metrics API setup" + }, + { + "url": "https://supabase.com/docs/guides/telemetry/metrics/grafana-cloud", + "title": "Metrics API with Grafana Cloud" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/how-to-view-database-metrics-uqf2z_", + "title": "How to View Database Metrics" + }, + { + "url": "https://supabase.com/docs/guides/platform/read-replicas", + "title": "Read Replicas" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/data-deletion", + "title": "Deleting data and dropping objects safely" + }, + { + "url": "https://supabase.com/docs/guides/telemetry/reports", + "title": "Reports" + }, + { + "url": "https://supabase.com/docs/guides/database/replication/pipelines-monitoring", + "title": "Monitor pipeline status" + } + ], + "resultChars": 90527 + }, + { + "source": "web_fetch", + "query": "https://supabase.com/docs/guides/telemetry/metrics/grafana-self-hosted.md", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/telemetry/metrics/grafana-self-hosted.md" + } + ], + "resultChars": 3752 + }, + { + "source": "web_fetch", + "query": "https://supabase.com/docs/guides/telemetry/metrics.md", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/telemetry/metrics.md" + } + ], + "resultChars": 4778 + } + ] + }, + "prompt": "Can you wire my Supabase project metrics into our existing observability stack and document\nwhat we need to do to make the config live?", + "promptSourcePath": "evals/deploy-database-001-prometheus-metrics/PROMPT.md", + "attempts": 1, + "sourcePath": "opencode-kimi-k3/deploy-database-001-prometheus-metrics.json" + }, + { + "experiment": "opencode-kimi-k3", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "opencode", + "modelProvider": "moonshotai", + "modelId": "moonshotai/kimi-k3" + }, + "eval": "deploy-functions-001-edge-function-secrets", + "stage": "deploy", + "product": [ + "edge-functions" + ], + "topic": [ + "security" + ], + "suite": "benchmark", + "interface": "cli", + "passed": true, + "checks": [ + { + "name": "WEATHER_API_KEY is set as a Function secret on the project", + "passed": true + }, + { + "name": "the weather function is deployed to the project", + "passed": true, + "notes": "status ACTIVE" + }, + { + "name": "the weather function reads WEATHER_API_KEY from the environment", + "passed": true, + "judgeNotes": "The function reads WEATHER_API_KEY from the runtime environment using Deno.env.get(\"WEATHER_API_KEY\")." + }, + { + "name": "WEATHER_API_KEY value is not committed to the repo", + "passed": true + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [ + { + "source": "web_fetch", + "query": "https://supabase.com/changelog.md", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 92585 + } + ] + }, + "prompt": "Our weather widget currently calls WeatherAPI straight from the browser, which\nleaks our API key. I want to move that behind a Supabase Edge Function called\n`weather` that holds the key server-side and proxies the request.\n\nThe function should read the key from an environment variable named\n`WEATHER_API_KEY`. Our key already lives in a local `.env` file at the project\nroot.\n\nDeploy the function to our project so it's live, and make sure the deployed\nfunction can actually read the key at runtime.", + "promptSourcePath": "evals/deploy-functions-001-edge-function-secrets/PROMPT.md", + "attempts": 1, + "sourcePath": "opencode-kimi-k3/deploy-functions-001-edge-function-secrets.json" + }, + { + "experiment": "opencode-kimi-k3", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "opencode", + "modelProvider": "moonshotai", + "modelId": "moonshotai/kimi-k3" + }, + "eval": "deploy-self-hosting-001-docker-compose", + "stage": "deploy", + "product": [ + "database", + "auth", + "storage" + ], + "topic": [ + "self-hosting" + ], + "suite": "benchmark", + "interface": "cli", + "passed": true, + "checks": [ + { + "name": "cloned the self-host stack (docker-compose.yml + volumes/db)", + "passed": true + }, + { + "name": "didn't conflate with the CLI (no supabase/config.toml in the stack)", + "passed": true + }, + { + "name": "secrets rotated off the shipped defaults", + "passed": true + }, + { + "name": "ANON_KEY and SERVICE_ROLE_KEY are HS256 JWTs signed by JWT_SECRET", + "passed": true + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [] + }, + "prompt": "I'm moving off the hosted Supabase and running the whole thing myself on a VPS I\njust spun up. Can you get a Docker setup ready for me to copy onto the box?\n\nI don't need it running here, I'll do the actual bring-up once I'm on the\nserver. I just want everything in place and the secrets set up. Put it in a `supabase-docker/`\nfolder at the repo root so I can scp the whole thing across in one go.", + "promptSourcePath": "evals/deploy-self-hosting-001-docker-compose/PROMPT.md", + "attempts": 1, + "sourcePath": "opencode-kimi-k3/deploy-self-hosting-001-docker-compose.json" + }, + { + "experiment": "opencode-kimi-k3", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "opencode", + "modelProvider": "moonshotai", + "modelId": "moonshotai/kimi-k3" + }, + "eval": "investigate-auth-001-deleted-user-access", + "stage": "investigate", + "product": [ + "auth" + ], + "topic": [ + "security", + "sdk" + ], + "suite": "benchmark", + "interface": "mcp", + "passed": true, + "checks": [ + { + "name": "victim session active before delete-account", + "passed": true + }, + { + "name": "delete_account flow ran for the victim", + "passed": true + }, + { + "name": "delete-account revokes the user's sessions", + "passed": true, + "notes": "sessions left: 0" + }, + { + "name": "deleted user's refresh token is rejected", + "passed": true + }, + { + "name": "deleted user cannot sign back in", + "passed": true + }, + { + "name": "other users keep their sessions and access", + "passed": true + }, + { + "name": "diagnosed and explained session revocation", + "passed": true, + "judgeNotes": "Meets rubric: diagnoses soft-delete-only root cause, implements deletion of auth user with session/refresh-token revocation via cascades, accurately notes remaining stateless access-token window until expiry with mitigations, and correctly distinguishes publishable frontend/RLS behavior from secret server-only/RLS-bypass behavior." + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "{ searchDocs(query: \"publishable key secret key migrate from anon service_role API keys RLS\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-redwoodjs", + "title": "Build a User Management App with RedwoodJS" + }, + { + "url": "https://supabase.com/docs/guides/auth/quickstarts/with-expo-react-native-social-auth", + "title": "Build a Social Auth App with Expo React Native" + }, + { + "url": "https://supabase.com/docs/guides/auth/signing-keys", + "title": "JWT Signing Keys" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-sveltekit", + "title": "Build a User Management App with SvelteKit" + } + ], + "resultChars": 169611 + } + ] + }, + "prompt": "Last week support removed a user through our app's delete-account flow — the\napp calls the `delete_account` function over RPC as the signed-in user. This\nmorning that same person was back: still signed in, reading and saving their\ndata like nothing happened.\n\nFigure out why the account still works, fix the flow so a deleted account\nloses access, and tell me whether there is any window where they could still\nget in after the fix.\n\nOne more thing while you're at it: we're migrating off the legacy\nanon/service_role API keys to the new publishable and secret keys, and the\nteam isn't sure which one belongs in the frontend and what each means for\nRLS. Give us a quick rundown.", + "promptSourcePath": "evals/investigate-auth-001-deleted-user-access/PROMPT.md", + "attempts": 1, + "sourcePath": "opencode-kimi-k3/investigate-auth-001-deleted-user-access.json" + }, + { + "experiment": "opencode-kimi-k3", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "opencode", + "modelProvider": "moonshotai", + "modelId": "moonshotai/kimi-k3" + }, + "eval": "investigate-realtime-001-subscribed-no-events", + "stage": "investigate", + "product": [ + "realtime", + "database" + ], + "topic": [ + "sdk" + ], + "suite": "benchmark", + "interface": "mcp", + "passed": true, + "checks": [ + { + "name": "orders table added to supabase_realtime publication", + "passed": true + }, + { + "name": "courier_locations still in supabase_realtime publication", + "passed": true + }, + { + "name": "publication still publishes INSERT events", + "passed": true + }, + { + "name": "RLS still enabled on orders", + "passed": true + }, + { + "name": "staff can still read orders through RLS", + "passed": true, + "notes": "authenticated sees 2 of 2 orders" + }, + { + "name": "diagnosed missing publication membership", + "passed": true, + "judgeNotes": "The assistant correctly identified the root cause as orders missing from the supabase_realtime publication, added public.orders to the existing publication, and did not weaken RLS/policies or disrupt courier_locations. It did include verification inserts/log checks, but the core fix is exactly the required publication change." + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [] + }, + "prompt": "Our dispatch dashboard shows incoming orders as they happen. The courier\nlocation feed on the same page updates live without problems, but new orders\nonly show up after a page refresh.\n\nThe dashboard uses supabase-js to subscribe to INSERT events on the `orders`\ntable through postgres_changes, the same way it subscribes to courier\nlocations. The channel's status callback logs SUBSCRIBED and there are no\nerrors in the browser console.\n\nFigure out why no order events ever arrive and fix it.", + "promptSourcePath": "evals/investigate-realtime-001-subscribed-no-events/PROMPT.md", + "attempts": 1, + "sourcePath": "opencode-kimi-k3/investigate-realtime-001-subscribed-no-events.json" + }, + { + "experiment": "opencode-kimi-k3", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "opencode", + "modelProvider": "moonshotai", + "modelId": "moonshotai/kimi-k3" + }, + "eval": "investigate-reliability-003-edge-function-5xx-correlation", + "stage": "investigate", + "product": [ + "edge-functions" + ], + "topic": [ + "observability" + ], + "suite": "benchmark", + "interface": "mcp", + "passed": false, + "checks": [ + { + "name": "identified image-transform and the recurring 503 pattern", + "passed": true, + "judgeNotes": "Assistant identified image-transform as the affected function and described the recurring HTTP 503 pattern across the morning of 2026-04-28, covering the 8 gateway failures from 07:00Z to 12:00Z. It also correctly distinguished the older billing-webhook 503s as unrelated." + }, + { + "name": "attributed recurring 503s to gateway/platform layer, not function code", + "passed": false, + "judgeNotes": "The assistant notes gateway 503s with no execution records, but ultimately attributes the cause to the function's unpinned dependency/boot behavior and recommends pinning/redeploying the function, explicitly saying it is not a platform-side incident. This fails the required gateway/platform-layer attribution." + }, + { + "name": "recommended a concrete next step", + "passed": true, + "judgeNotes": "Recommended concrete next steps: pin/roll back dependencies and redeploy, review upstream package publish history, rotate secrets if compromised, and add alerting." + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [ + { + "source": "web_fetch", + "query": "https://supabase.com/changelog.md", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 92585 + }, + { + "source": "web_fetch", + "query": "https://supabase.com/docs/guides/security/npm-security.md", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/security/npm-security.md" + } + ], + "resultChars": 17839 + } + ] + }, + "prompt": "Users have been reporting that image uploads are intermittently failing with server errors throughout the morning of `2026-04-28`.\n\nCan you investigate the project logs and tell me what is going on and what we\nshould do next?", + "promptSourcePath": "evals/investigate-reliability-003-edge-function-5xx-correlation/PROMPT.md", + "attempts": 2, + "sourcePath": "opencode-kimi-k3/investigate-reliability-003-edge-function-5xx-correlation.json" + }, + { + "experiment": "opencode-kimi-k3", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "opencode", + "modelProvider": "moonshotai", + "modelId": "moonshotai/kimi-k3" + }, + "eval": "resolve-dataapi-001-empty-results", + "stage": "resolve", + "product": [ + "data-api", + "database", + "auth" + ], + "topic": [ + "rls", + "sdk" + ], + "suite": "benchmark", + "interface": "mcp", + "passed": true, + "checks": [ + { + "name": "RLS still enabled on bookmarks", + "passed": true + }, + { + "name": "user A reads own bookmarks", + "passed": true + }, + { + "name": "user B cannot read user A bookmarks", + "passed": true + }, + { + "name": "anon reads no bookmarks", + "passed": true + }, + { + "name": "user A can save a new bookmark", + "passed": true + }, + { + "name": "user B cannot insert a bookmark as user A", + "passed": true + }, + { + "name": "diagnosed RLS and added owner-scoped policies", + "passed": true, + "judgeNotes": "Diagnosed RLS enabled with no policies causing deny-all Data API results, kept RLS enabled, and created authenticated-only owner-scoped SELECT and INSERT policies using auth.uid() with WITH CHECK for inserts." + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [] + }, + "prompt": "Our app lets signed-in users save bookmarks and view them on their dashboard. Bookmarks are stored in the `bookmarks` table and are private — a user must only ever see their own. \nUsers also need to be able to save new bookmarks from the app.\n\nI can see the rows when I query the table directly, but the dashboard shows an empty list for every user.\n\nFind out why the Data API returns nothing and fix it.", + "promptSourcePath": "evals/resolve-dataapi-001-empty-results/PROMPT.md", + "attempts": 1, + "sourcePath": "opencode-kimi-k3/resolve-dataapi-001-empty-results.json" + }, + { + "experiment": "opencode-kimi-k3", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "opencode", + "modelProvider": "moonshotai", + "modelId": "moonshotai/kimi-k3" + }, + "eval": "resolve-database-001-migration-history-mismatch", + "stage": "resolve", + "product": [ + "database" + ], + "topic": [ + "migrations" + ], + "suite": "benchmark", + "interface": "cli", + "passed": true, + "checks": [ + { + "name": "the avatar_url column is applied on the hosted profiles table", + "passed": true + }, + { + "name": "migration 20240220000000 is recorded in the remote history", + "passed": true + }, + { + "name": "remote migration history matches local migration files", + "passed": true + }, + { + "name": "local migrations are a valid reconciled sequence", + "passed": true + }, + { + "name": "production profile data is intact (not reset)", + "passed": true + }, + { + "name": "the avatar migration and history reconciliation were done via the Supabase CLI", + "passed": true, + "judgeNotes": "Applied avatar_url via `supabase db push` in #18, which showed `Applying migration 20240220000000_add_avatar_url.sql...` and `Finished supabase db push.` Reconciled the orphan bio migration by adding the missing local file `supabase/migrations/20240115000000_add_profile_bio.sql` in #16, after which migration list aligned and `db push` succeeded. No disallowed workaround or direct remote mutation was used." + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [] + }, + "prompt": "I'm trying to ship a migration to our hosted project and it's not working. Can you figure out what's wrong and get it deployed?", + "promptSourcePath": "evals/resolve-database-001-migration-history-mismatch/PROMPT.md", + "attempts": 1, + "sourcePath": "opencode-kimi-k3/resolve-database-001-migration-history-mismatch.json" + }, + { + "experiment": "opencode-kimi-k3", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "opencode", + "modelProvider": "moonshotai", + "modelId": "moonshotai/kimi-k3" + }, + "eval": "resolve-performance-001-slow-query-cpu-spike", + "stage": "resolve", + "product": [ + "database" + ], + "topic": [ + "observability", + "sql" + ], + "suite": "benchmark", + "interface": "mcp", + "passed": true, + "checks": [ + { + "name": "inspected pg_stat_statements for query performance", + "passed": true + }, + { + "name": "ran EXPLAIN on the expensive query", + "passed": true + }, + { + "name": "created index covering user_id and created_at", + "passed": true + }, + { + "name": "query plan uses an index and avoids sequential scan", + "passed": true, + "notes": "Limit (cost=79.74..79.86 rows=50 width=58)\n -> Sort (cost=79.74..79.99 rows=100 width=58)\n Sort Key: created_at DESC\n -> Bitmap Heap Scan on events (cost=5.06..76.42 rows=100 width=58)\n Recheck Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)\n -> Bitmap Index Scan on events_user_id_created_at_idx (cost=0.00..5.03 rows=100 width=0)\n Index Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)" + }, + { + "name": "inserts still work", + "passed": true + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] + }, + "docs": { + "calls": [] + }, + "prompt": "My database CPU keeps spiking and the app gets slow when loading recent events for a user. Can you figure out what query is causing it and make the database change needed to fix it?\n\nEnd your turn with a short summary of what you changed and why.", + "promptSourcePath": "evals/resolve-performance-001-slow-query-cpu-spike/PROMPT.md", + "attempts": 1, + "sourcePath": "opencode-kimi-k3/resolve-performance-001-slow-query-cpu-spike.json" + }, + { + "experiment": "opencode-kimi-k3", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "opencode", + "modelProvider": "moonshotai", + "modelId": "moonshotai/kimi-k3" + }, + "eval": "resolve-security-002-rls-cross-tenant-leak", + "stage": "resolve", + "product": [ + "database", + "auth" + ], + "topic": [ + "rls", + "security" + ], + "suite": "benchmark", + "interface": "mcp", + "passed": true, + "checks": [ + { + "name": "RLS enabled on notes", + "passed": true + }, + { + "name": "tenant A sees only org A notes", + "passed": true + }, + { + "name": "tenant B cannot read org A notes", + "passed": true + }, + { + "name": "tenant A author can update own note", + "passed": true + }, + { + "name": "tenant B cannot update org A note", + "passed": true + }, + { + "name": "tenant B author can delete own note", + "passed": true + }, + { + "name": "tenant B cannot delete org A note", + "passed": true + }, + { + "name": "tenant A can insert note in own org", + "passed": true + }, + { + "name": "tenant B cannot insert into org A", + "passed": true + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [] + }, + "prompt": "A customer reported that notes showed up in the wrong workspace.\n\nCan you investigate what is going on and fix it?", + "promptSourcePath": "evals/resolve-security-002-rls-cross-tenant-leak/PROMPT.md", + "attempts": 1, + "sourcePath": "opencode-kimi-k3/resolve-security-002-rls-cross-tenant-leak.json" } ] From d79b20e7c682611f42a9f1c11c0650ef34197b70 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sat, 25 Jul 2026 00:33:27 +0000 Subject: [PATCH 05/17] chore: refresh eval results --- apps/web/src/data/eval-results.json | 1246 +++++++++++++++++++++++++++ 1 file changed, 1246 insertions(+) diff --git a/apps/web/src/data/eval-results.json b/apps/web/src/data/eval-results.json index 4c5e82e6..1e72268c 100644 --- a/apps/web/src/data/eval-results.json +++ b/apps/web/src/data/eval-results.json @@ -19448,5 +19448,1251 @@ "promptSourcePath": "evals/resolve-security-002-rls-cross-tenant-leak/PROMPT.md", "attempts": 1, "sourcePath": "opencode-kimi-k3/resolve-security-002-rls-cross-tenant-leak.json" + }, + { + "experiment": "opencode-kimi-k3-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "opencode", + "modelProvider": "moonshotai", + "modelId": "moonshotai/kimi-k3" + }, + "eval": "build-cli-001-bootstrap-app", + "stage": "build", + "product": [ + "database", + "data-api" + ], + "topic": [ + "migrations", + "rls" + ], + "suite": "benchmark", + "interface": "cli", + "passed": true, + "checks": [ + { + "name": "supabase project initialised (supabase/config.toml exists)", + "passed": true + }, + { + "name": "todos table is created by a migration file", + "passed": true + }, + { + "name": "todos table exists with at least 2 seeded rows", + "passed": true, + "notes": "found 2 rows" + }, + { + "name": "row level security is enabled on todos", + "passed": true + }, + { + "name": "a SELECT policy targets the authenticated role", + "passed": true + }, + { + "name": "REST API returns no todos to anonymous requests", + "passed": true, + "notes": "0 rows" + }, + { + "name": "REST API returns the todos to authenticated requests", + "passed": true, + "notes": "2 rows" + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] + }, + "prompt": "We're kicking off a todos app and I want the Supabase side ready for the team\nto build on. Set it up the way we'd run it in development, with schema changes\ntracked as migrations so they can be reviewed and replayed.\n\nFor the first slice we just need a `todos` table. Todos aren't public: anyone\nsigned in can read all of them, but nothing should be writable through the API\nfor now. Add a couple of sample todos so there's something to look at.\n\nBefore you hand it back, make sure the running API actually behaves that way —\nsigned-in users get the todos, signed-out requests get nothing.", + "promptSourcePath": "evals/build-cli-001-bootstrap-app/PROMPT.md", + "attempts": 1, + "sourcePath": "opencode-kimi-k3-no-skills/build-cli-001-bootstrap-app.json" + }, + { + "experiment": "opencode-kimi-k3-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "opencode", + "modelProvider": "moonshotai", + "modelId": "moonshotai/kimi-k3" + }, + "eval": "build-cli-002-declarative-schema", + "stage": "build", + "product": [ + "database" + ], + "topic": [ + "declarative-schema", + "migrations" + ], + "suite": "benchmark", + "interface": "cli", + "passed": true, + "checks": [ + { + "name": "supabase db diff used to generate the migration", + "passed": true + }, + { + "name": "schema file updated to include description column", + "passed": true + }, + { + "name": "a new migration was generated for the change", + "passed": true + }, + { + "name": "description column exists in the live database", + "passed": true + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] + }, + "prompt": "Add a description text column to the `products` table in my local Supabase stack", + "promptSourcePath": "evals/build-cli-002-declarative-schema/PROMPT.md", + "attempts": 1, + "sourcePath": "opencode-kimi-k3-no-skills/build-cli-002-declarative-schema.json" + }, + { + "experiment": "opencode-kimi-k3-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "opencode", + "modelProvider": "moonshotai", + "modelId": "moonshotai/kimi-k3" + }, + "eval": "build-cli-003-pg-cron-queue-workflow", + "stage": "build", + "product": [ + "database", + "edge-functions", + "cron", + "queues" + ], + "topic": [ + "sql", + "sdk" + ], + "suite": "benchmark", + "interface": "cli", + "passed": true, + "checks": [ + { + "name": "pg_cron job 'enqueue-tasks' scheduled to run every minute", + "passed": true, + "notes": "schedule='* * * * *', active=true" + }, + { + "name": "cron command enqueues to the 'tasks' queue", + "passed": true, + "notes": "queue depth 0 -> 1" + }, + { + "name": "process-tasks function drains the queue", + "passed": true, + "notes": "function removed the seeded message (id 5) from the queue" + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "{ searchDocs(query: \"pg_cron schedule job every minute pgmq send queue\") { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_net", + "title": "pg_net: Async Networking" + }, + { + "url": "https://supabase.com/docs/guides/queues/pgmq", + "title": "PGMQ Extension" + }, + { + "url": "https://supabase.com/docs/guides/cron", + "title": "Cron" + }, + { + "url": "https://supabase.com/docs/guides/functions/schedule-functions", + "title": "Scheduling Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/queues/quickstart", + "title": "Quickstart" + }, + { + "url": "https://supabase.com/docs/guides/cron/install", + "title": "Install" + }, + { + "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", + "title": "Automatic embeddings" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/pgcron-debugging-guide-n1KTaz", + "title": "pg_cron debugging guide" + }, + { + "url": "https://supabase.com/docs/guides/platform/upgrading", + "title": "Upgrading" + }, + { + "url": "https://supabase.com/docs/guides/cron/quickstart", + "title": "Quickstart" + }, + { + "url": "https://supabase.com/docs/guides/database/migrating-to-pg-partman", + "title": "Migrate from TimescaleDB to pg_partman" + }, + { + "url": "https://supabase.com/docs/guides/queues", + "title": "Supabase Queues" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/pg_cron-launcher-crashes-with-duplicate-key-value-violates-unique-constraint-cc6472", + "title": "`pg_cron launcher crashes with 'duplicate key value violates unique constraint'`" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/wrappers/overview", + "title": "Foreign Data Wrappers" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_partman", + "title": "pg_partman: partition management" + } + ], + "resultChars": 125276 + } + ] + }, + "prompt": "I want to set up a recurring background workflow on my local Supabase stack.\n\nCan you set up a cron job called `enqueue-tasks` to run every minute and push a task into a queue called `tasks`? Then add a `process-tasks` edge function that reads messages off the `tasks` queue and removes them, so a scheduled worker can keep the backlog drained.", + "promptSourcePath": "evals/build-cli-003-pg-cron-queue-workflow/PROMPT.md", + "attempts": 1, + "sourcePath": "opencode-kimi-k3-no-skills/build-cli-003-pg-cron-queue-workflow.json" + }, + { + "experiment": "opencode-kimi-k3-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "opencode", + "modelProvider": "moonshotai", + "modelId": "moonshotai/kimi-k3" + }, + "eval": "build-database-001-migrate-postgres-to-supabase", + "stage": "build", + "product": [ + "database" + ], + "topic": [ + "migrations" + ], + "suite": "benchmark", + "interface": "cli", + "passed": true, + "checks": [ + { + "name": "all 3 tables exist (teams, members, tasks)", + "passed": true + }, + { + "name": "row counts match (teams=5, members=10, tasks=13)", + "passed": true + }, + { + "name": "foreign key constraints survived the restore", + "passed": true + }, + { + "name": "tasks_team_status_idx index survived the restore", + "passed": true + }, + { + "name": "sequences synced (next insert won't conflict with existing IDs)", + "passed": true + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] + }, + "prompt": "I have an existing Postgres database I want to migrate to Supabase. There's a binary dump at `source.dump` in the current directory.\n\nCan you set up a local Supabase project and restore the dump into it?", + "promptSourcePath": "evals/build-database-001-migrate-postgres-to-supabase/PROMPT.md", + "attempts": 1, + "sourcePath": "opencode-kimi-k3-no-skills/build-database-001-migrate-postgres-to-supabase.json" + }, + { + "experiment": "opencode-kimi-k3-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "opencode", + "modelProvider": "moonshotai", + "modelId": "moonshotai/kimi-k3" + }, + "eval": "build-functions-004-service-role-bypass", + "stage": "build", + "product": [ + "edge-functions", + "auth", + "database" + ], + "topic": [ + "rls", + "security", + "sdk" + ], + "suite": "benchmark", + "interface": "mcp", + "passed": true, + "checks": [ + { + "name": "rejects missing auth", + "passed": true, + "notes": "status=401" + }, + { + "name": "user A reads own note", + "passed": true, + "notes": "status=200" + }, + { + "name": "reads only with the caller's JWT", + "passed": true, + "notes": "bearer_tokens=2, all_match=true" + }, + { + "name": "user A cannot force-read user B note", + "passed": true, + "notes": "status=200" + }, + { + "name": "user B cannot force-read user A note", + "passed": true, + "notes": "status=200" + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] + }, + "prompt": "I built an Edge Function called `private-notes` for showing a user's saved\nprivate notes.\n\nCan you check whether there's any way one user could see another user's notes?\n\nPlease fix and deploy it if needed.", + "promptSourcePath": "evals/build-functions-004-service-role-bypass/PROMPT.md", + "attempts": 1, + "sourcePath": "opencode-kimi-k3-no-skills/build-functions-004-service-role-bypass.json" + }, + { + "experiment": "opencode-kimi-k3-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "opencode", + "modelProvider": "moonshotai", + "modelId": "moonshotai/kimi-k3" + }, + "eval": "build-functions-005-dual-auth-user-secret", + "stage": "build", + "product": [ + "edge-functions", + "auth", + "database" + ], + "topic": [ + "sdk", + "rls", + "security" + ], + "suite": "benchmark", + "interface": "cli", + "cliVersion": "2.109.1", + "passed": false, + "checks": [ + { + "name": "seed rows present", + "passed": true, + "notes": "found 2/2 seeded rows" + }, + { + "name": "rejects request with no credentials", + "passed": true, + "notes": "status 401: {\"error\":\"Missing credentials\"}" + }, + { + "name": "user with JWT reads only their own rows", + "passed": true, + "notes": "status 200: [{\"user_id\":\"68087eb7-1e3e-4ff1-bece-c5bf9637ccd6\",\"metric\":\"steps_a_mrzmx0of\",\"value\":111}]" + }, + { + "name": "user cannot read another user's rows by passing user_id", + "passed": true, + "notes": "status 200: [{\"user_id\":\"68087eb7-1e3e-4ff1-bece-c5bf9637ccd6\",\"metric\":\"steps_a_mrzmx0of\",\"value\":111}]" + }, + { + "name": "service key bypasses RLS to read the target user's rows", + "passed": true, + "notes": "status 200: [{\"user_id\":\"15d9b155-0de6-44c2-8eea-a0ba221728a7\",\"metric\":\"steps_b_mrzmx0of\",\"value\":222}]" + }, + { + "name": "non-service key is not granted service access", + "passed": true, + "notes": "status 401: {\"error\":\"Missing credentials\"}" + }, + { + "name": "rejects an unverified (forged) user token", + "passed": true, + "notes": "status 401: {\"error\":\"Invalid or expired access token\"}" + }, + { + "name": "a user token in the apikey slot is not treated as the service key", + "passed": true, + "notes": "status 401: {\"error\":\"Missing credentials\"}" + }, + { + "name": "implementation uses @supabase/server", + "passed": false, + "notes": "hand-rolled (raw supabase-js or other) — this eval requires @supabase/server" + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] + }, + "prompt": "Build and serve a Supabase Edge Function named `user-stats` for this project,\nreachable over HTTP at `/functions/v1/user-stats`.\n\nOur product stores per-user metrics in the existing `user_stats` table.\n\nTwo very different callers need to hit this one endpoint:\n\n1. **Our mobile app**, acting for a signed-in user. It sends that user's\n Supabase access token. The endpoint should return the user's own stats.\n\n2. **Our internal billing service**, a trusted backend with no signed-in user.\n It authenticates with the project's secret (service-role) key in the `apikey`\n header, and names the target user with a `user_id` in the JSON request body.\n It needs that user's stats.\n\nReturn the matching rows as JSON. The endpoint must be secure: only ever serve\nstats to a caller that is genuinely entitled to them, and turn away callers that\nare not.", + "promptSourcePath": "evals/build-functions-005-dual-auth-user-secret/PROMPT.md", + "attempts": 2, + "sourcePath": "opencode-kimi-k3-no-skills/build-functions-005-dual-auth-user-secret.json" + }, + { + "experiment": "opencode-kimi-k3-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "opencode", + "modelProvider": "moonshotai", + "modelId": "moonshotai/kimi-k3" + }, + "eval": "build-storage-001-private-bucket-access", + "stage": "build", + "product": [ + "storage", + "database" + ], + "topic": [ + "rls", + "sdk" + ], + "suite": "benchmark", + "interface": "mcp", + "passed": true, + "checks": [ + { + "name": "bucket user-files exists", + "passed": true + }, + { + "name": "bucket user-files is private", + "passed": true + }, + { + "name": "RLS still enabled on storage.objects", + "passed": true + }, + { + "name": "user A lists only own files", + "passed": true, + "notes": "saw: 019f969e-d3b2-770f-bdaa-65de0ebe9d76/receipt-alpha.pdf, 019f969e-d3b2-770f-bdaa-65de0ebe9d76/receipt-beta.pdf" + }, + { + "name": "user B cannot read user A files", + "passed": true + }, + { + "name": "anon reads no files", + "passed": true + }, + { + "name": "user A can upload into own folder", + "passed": true + }, + { + "name": "user B cannot upload into user A folder", + "passed": true + }, + { + "name": "configured private per-user storage access", + "passed": true, + "judgeNotes": "Meets all requirements: private user-files bucket, authenticated owner-scoped SELECT and INSERT policies on storage.objects with WITH CHECK for uploads, no RLS disabling or permissive/public policies, and supabase-js createSignedUrl with expiry for sharing." + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "{ searchDocs(query: \"storage access control RLS policies private bucket signed URL\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/storage/buckets/fundamentals", + "title": "Storage Buckets" + }, + { + "url": "https://supabase.com/docs/guides/storage/security/access-control", + "title": "Storage Access Control" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-angular", + "title": "Build a User Management App with Angular" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-ionic-angular", + "title": "Build a User Management App with Ionic Angular" + }, + { + "url": "https://supabase.com/docs/guides/security/product-security", + "title": "Secure configuration of Supabase products" + } + ], + "resultChars": 71040 + } + ] + }, + "prompt": "Our app lets signed-in users keep personal files like receipts and bank\nstatements. These files are private — a user must only ever be able to upload\nand download their own. The app uploads each file under a path that starts\nwith the owner's user id, e.g. `/receipt-march.pdf`.\n\nSet up a `user-files` bucket on our project and lock it down that way.\n\nUsers also sometimes share one of their files with someone else through a\ntemporary link that expires. Include the supabase-js code the app should use\nfor that.", + "promptSourcePath": "evals/build-storage-001-private-bucket-access/PROMPT.md", + "attempts": 1, + "sourcePath": "opencode-kimi-k3-no-skills/build-storage-001-private-bucket-access.json" + }, + { + "experiment": "opencode-kimi-k3-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "opencode", + "modelProvider": "moonshotai", + "modelId": "moonshotai/kimi-k3" + }, + "eval": "build-tests-001-rls-tenant-isolation", + "stage": "build", + "product": [ + "database" + ], + "topic": [ + "tests", + "rls" + ], + "suite": "benchmark", + "interface": "cli", + "passed": true, + "checks": [ + { + "name": "pgTAP test file(s) written under supabase/tests/", + "passed": true, + "notes": "1 file(s): supabase/tests/database/tenant_isolation.test.sql" + }, + { + "name": "pgTAP isolation tests ran and pass", + "passed": true, + "notes": "12 passed, 0 failed" + }, + { + "name": "agent correctly identifies the posts isolation bug from test results", + "passed": true, + "judgeNotes": "The agent correctly identifies `posts` as the broken tenant isolation policy, specifically that members of any org could read all posts due to missing `m.org_id = posts.org_id`. It grounds the finding in pgTAP results, noting test 9 failed before the fix with have 1/want 0, and distinguishes `notes` as correctly isolated." + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] + }, + "prompt": "Can you audit the tenant isolation on our tables? Write some database tests covering both the happy path and the negative case, run them, and share the results.", + "promptSourcePath": "evals/build-tests-001-rls-tenant-isolation/PROMPT.md", + "attempts": 1, + "sourcePath": "opencode-kimi-k3-no-skills/build-tests-001-rls-tenant-isolation.json" + }, + { + "experiment": "opencode-kimi-k3-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "opencode", + "modelProvider": "moonshotai", + "modelId": "moonshotai/kimi-k3" + }, + "eval": "build-vectors-001-rag-with-permissions", + "stage": "build", + "product": [ + "database", + "vectors" + ], + "topic": [ + "sql", + "rls" + ], + "suite": "benchmark", + "interface": "mcp", + "passed": true, + "checks": [ + { + "name": "document_sections.embedding is vector(384)", + "passed": true, + "notes": "vector(384)" + }, + { + "name": "HNSW index on the embedding column", + "passed": true, + "notes": "CREATE INDEX document_sections_embedding_idx ON public.document_sections USING hnsw (embedding vector_cosine_ops)" + }, + { + "name": "index operator class matches the search operator", + "passed": true, + "notes": "function operators: <=>\nindexes: CREATE INDEX document_sections_embedding_idx ON public.document_sections USING hnsw (embedding vector_cosine_ops)" + }, + { + "name": "user A search returns only own sections, best match first", + "passed": true + }, + { + "name": "user B search returns only own sections, best match first", + "passed": true + }, + { + "name": "user A reads only own sections through the API", + "passed": true + }, + { + "name": "user A reads only own documents through the API", + "passed": true + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] + }, + "prompt": "We're adding semantic search to our internal knowledge base app. I already wrote the edge functions, but search doesn't work yet. Can you set up whatever the database needs to make search work end to end?\n\nSome documents are confidential, so users should only have access to documents they own.", + "promptSourcePath": "evals/build-vectors-001-rag-with-permissions/PROMPT.md", + "attempts": 1, + "sourcePath": "opencode-kimi-k3-no-skills/build-vectors-001-rag-with-permissions.json" + }, + { + "experiment": "opencode-kimi-k3-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "opencode", + "modelProvider": "moonshotai", + "modelId": "moonshotai/kimi-k3" + }, + "eval": "deploy-database-001-prometheus-metrics", + "stage": "deploy", + "product": [ + "database" + ], + "topic": [ + "observability" + ], + "suite": "benchmark", + "passed": true, + "checks": [ + { + "name": "preserved existing app scrape job", + "passed": true + }, + { + "name": "configured the Supabase Metrics API scrape correctly", + "passed": true, + "judgeNotes": "Adds Supabase Metrics API scrape over HTTPS with the required metrics path, Basic Auth using password_file, preserves the app scrape, and mounts the secrets directory into Prometheus so the password_file is available." + }, + { + "name": "documented live deployment and verification steps", + "passed": true, + "judgeNotes": "README includes concrete steps to set project ref, create/copy a Supabase Secret API key, place it in the mounted secret file, reload/recreate the Compose stack, and verify via Prometheus targets or PromQL/Grafana." + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "{ searchDocs(query: \"metrics endpoint prometheus scrape project metrics API\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/telemetry/metrics", + "title": "Metrics API" + }, + { + "url": "https://supabase.com/docs/guides/telemetry/metrics/grafana-self-hosted", + "title": "Metrics API with Prometheus & Grafana (self-hosted)" + }, + { + "url": "https://supabase.com/docs/guides/telemetry/metrics/vendor-agnostic", + "title": "Vendor-agnostic Metrics API setup" + }, + { + "url": "https://supabase.com/docs/guides/telemetry/metrics/grafana-cloud", + "title": "Metrics API with Grafana Cloud" + } + ], + "resultChars": 22788 + } + ] + }, + "prompt": "Can you wire my Supabase project metrics into our existing observability stack and document\nwhat we need to do to make the config live?", + "promptSourcePath": "evals/deploy-database-001-prometheus-metrics/PROMPT.md", + "attempts": 1, + "sourcePath": "opencode-kimi-k3-no-skills/deploy-database-001-prometheus-metrics.json" + }, + { + "experiment": "opencode-kimi-k3-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "opencode", + "modelProvider": "moonshotai", + "modelId": "moonshotai/kimi-k3" + }, + "eval": "deploy-functions-001-edge-function-secrets", + "stage": "deploy", + "product": [ + "edge-functions" + ], + "topic": [ + "security" + ], + "suite": "benchmark", + "interface": "cli", + "passed": true, + "checks": [ + { + "name": "WEATHER_API_KEY is set as a Function secret on the project", + "passed": true + }, + { + "name": "the weather function is deployed to the project", + "passed": true, + "notes": "status ACTIVE" + }, + { + "name": "the weather function reads WEATHER_API_KEY from the environment", + "passed": true, + "judgeNotes": "Reads WEATHER_API_KEY from the runtime environment using Deno.env.get(\"WEATHER_API_KEY\")." + }, + { + "name": "WEATHER_API_KEY value is not committed to the repo", + "passed": true + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] + }, + "prompt": "Our weather widget currently calls WeatherAPI straight from the browser, which\nleaks our API key. I want to move that behind a Supabase Edge Function called\n`weather` that holds the key server-side and proxies the request.\n\nThe function should read the key from an environment variable named\n`WEATHER_API_KEY`. Our key already lives in a local `.env` file at the project\nroot.\n\nDeploy the function to our project so it's live, and make sure the deployed\nfunction can actually read the key at runtime.", + "promptSourcePath": "evals/deploy-functions-001-edge-function-secrets/PROMPT.md", + "attempts": 1, + "sourcePath": "opencode-kimi-k3-no-skills/deploy-functions-001-edge-function-secrets.json" + }, + { + "experiment": "opencode-kimi-k3-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "opencode", + "modelProvider": "moonshotai", + "modelId": "moonshotai/kimi-k3" + }, + "eval": "deploy-self-hosting-001-docker-compose", + "stage": "deploy", + "product": [ + "database", + "auth", + "storage" + ], + "topic": [ + "self-hosting" + ], + "suite": "benchmark", + "interface": "cli", + "passed": true, + "checks": [ + { + "name": "cloned the self-host stack (docker-compose.yml + volumes/db)", + "passed": true + }, + { + "name": "didn't conflate with the CLI (no supabase/config.toml in the stack)", + "passed": true + }, + { + "name": "secrets rotated off the shipped defaults", + "passed": true + }, + { + "name": "ANON_KEY and SERVICE_ROLE_KEY are HS256 JWTs signed by JWT_SECRET", + "passed": true + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] + }, + "prompt": "I'm moving off the hosted Supabase and running the whole thing myself on a VPS I\njust spun up. Can you get a Docker setup ready for me to copy onto the box?\n\nI don't need it running here, I'll do the actual bring-up once I'm on the\nserver. I just want everything in place and the secrets set up. Put it in a `supabase-docker/`\nfolder at the repo root so I can scp the whole thing across in one go.", + "promptSourcePath": "evals/deploy-self-hosting-001-docker-compose/PROMPT.md", + "attempts": 1, + "sourcePath": "opencode-kimi-k3-no-skills/deploy-self-hosting-001-docker-compose.json" + }, + { + "experiment": "opencode-kimi-k3-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "opencode", + "modelProvider": "moonshotai", + "modelId": "moonshotai/kimi-k3" + }, + "eval": "investigate-auth-001-deleted-user-access", + "stage": "investigate", + "product": [ + "auth" + ], + "topic": [ + "security", + "sdk" + ], + "suite": "benchmark", + "interface": "mcp", + "passed": true, + "checks": [ + { + "name": "victim session active before delete-account", + "passed": true + }, + { + "name": "delete_account flow ran for the victim", + "passed": true + }, + { + "name": "delete-account revokes the user's sessions", + "passed": true, + "notes": "sessions left: 0" + }, + { + "name": "deleted user's refresh token is rejected", + "passed": true + }, + { + "name": "deleted user cannot sign back in", + "passed": true + }, + { + "name": "other users keep their sessions and access", + "passed": true + }, + { + "name": "diagnosed and explained session revocation", + "passed": true, + "judgeNotes": "Meets all rubric requirements: identifies soft-delete-only root cause, implements auth.users deletion/session+refresh-token revocation, consistently explains remaining stateless JWT expiry window and mitigation, and correctly distinguishes frontend publishable/anon keys from server-only secret/service_role keys with RLS bypass." + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] + }, + "prompt": "Last week support removed a user through our app's delete-account flow — the\napp calls the `delete_account` function over RPC as the signed-in user. This\nmorning that same person was back: still signed in, reading and saving their\ndata like nothing happened.\n\nFigure out why the account still works, fix the flow so a deleted account\nloses access, and tell me whether there is any window where they could still\nget in after the fix.\n\nOne more thing while you're at it: we're migrating off the legacy\nanon/service_role API keys to the new publishable and secret keys, and the\nteam isn't sure which one belongs in the frontend and what each means for\nRLS. Give us a quick rundown.", + "promptSourcePath": "evals/investigate-auth-001-deleted-user-access/PROMPT.md", + "attempts": 1, + "sourcePath": "opencode-kimi-k3-no-skills/investigate-auth-001-deleted-user-access.json" + }, + { + "experiment": "opencode-kimi-k3-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "opencode", + "modelProvider": "moonshotai", + "modelId": "moonshotai/kimi-k3" + }, + "eval": "investigate-realtime-001-subscribed-no-events", + "stage": "investigate", + "product": [ + "realtime", + "database" + ], + "topic": [ + "sdk" + ], + "suite": "benchmark", + "interface": "mcp", + "passed": true, + "checks": [ + { + "name": "orders table added to supabase_realtime publication", + "passed": true + }, + { + "name": "courier_locations still in supabase_realtime publication", + "passed": true + }, + { + "name": "publication still publishes INSERT events", + "passed": true + }, + { + "name": "RLS still enabled on orders", + "passed": true + }, + { + "name": "staff can still read orders through RLS", + "passed": true, + "notes": "authenticated sees 2 of 2 orders" + }, + { + "name": "diagnosed missing publication membership", + "passed": true, + "judgeNotes": "Identified the missing orders table in the supabase_realtime publication as the root cause, added public.orders with ALTER PUBLICATION, and did not weaken RLS/policies or disrupt courier_locations." + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] + }, + "prompt": "Our dispatch dashboard shows incoming orders as they happen. The courier\nlocation feed on the same page updates live without problems, but new orders\nonly show up after a page refresh.\n\nThe dashboard uses supabase-js to subscribe to INSERT events on the `orders`\ntable through postgres_changes, the same way it subscribes to courier\nlocations. The channel's status callback logs SUBSCRIBED and there are no\nerrors in the browser console.\n\nFigure out why no order events ever arrive and fix it.", + "promptSourcePath": "evals/investigate-realtime-001-subscribed-no-events/PROMPT.md", + "attempts": 1, + "sourcePath": "opencode-kimi-k3-no-skills/investigate-realtime-001-subscribed-no-events.json" + }, + { + "experiment": "opencode-kimi-k3-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "opencode", + "modelProvider": "moonshotai", + "modelId": "moonshotai/kimi-k3" + }, + "eval": "investigate-reliability-003-edge-function-5xx-correlation", + "stage": "investigate", + "product": [ + "edge-functions" + ], + "topic": [ + "observability" + ], + "suite": "benchmark", + "interface": "mcp", + "passed": false, + "checks": [ + { + "name": "identified image-transform and the recurring 503 pattern", + "passed": true, + "judgeNotes": "Identified image-transform as the main affected function and described the recurring pattern of 8 HTTP 503 gateway failures across 07:00Z–12:00Z on 2026-04-28, while distinguishing old billing-webhook 503s as unrelated." + }, + { + "name": "attributed recurring 503s to gateway/platform layer, not function code", + "passed": false, + "judgeNotes": "Although it notes the 503s appear only in gateway logs with no function execution logs, it then attributes the main issue to worker boot/cold-start fragility from the function dependency/top-level import and recommends fixing/redeploying image-transform, which violates the rubric." + }, + { + "name": "recommended a concrete next step", + "passed": true, + "judgeNotes": "The assistant recommended concrete actionable next steps, including redeploy/config changes, retry logic, observability improvements, and escalating to Supabase support with specific gateway request IDs if 503s persist." + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] + }, + "prompt": "Users have been reporting that image uploads are intermittently failing with server errors throughout the morning of `2026-04-28`.\n\nCan you investigate the project logs and tell me what is going on and what we\nshould do next?", + "promptSourcePath": "evals/investigate-reliability-003-edge-function-5xx-correlation/PROMPT.md", + "attempts": 2, + "sourcePath": "opencode-kimi-k3-no-skills/investigate-reliability-003-edge-function-5xx-correlation.json" + }, + { + "experiment": "opencode-kimi-k3-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "opencode", + "modelProvider": "moonshotai", + "modelId": "moonshotai/kimi-k3" + }, + "eval": "resolve-dataapi-001-empty-results", + "stage": "resolve", + "product": [ + "data-api", + "database", + "auth" + ], + "topic": [ + "rls", + "sdk" + ], + "suite": "benchmark", + "interface": "mcp", + "passed": true, + "checks": [ + { + "name": "RLS still enabled on bookmarks", + "passed": true + }, + { + "name": "user A reads own bookmarks", + "passed": true + }, + { + "name": "user B cannot read user A bookmarks", + "passed": true + }, + { + "name": "anon reads no bookmarks", + "passed": true + }, + { + "name": "user A can save a new bookmark", + "passed": true + }, + { + "name": "user B cannot insert a bookmark as user A", + "passed": true + }, + { + "name": "diagnosed RLS and added owner-scoped policies", + "passed": true, + "judgeNotes": "Diagnosed RLS enabled with no policies causing deny-all Data API results, kept RLS enabled, and created authenticated SELECT and INSERT owner-scoped policies using auth.uid() = user_id with WITH CHECK for inserts." + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] + }, + "prompt": "Our app lets signed-in users save bookmarks and view them on their dashboard. Bookmarks are stored in the `bookmarks` table and are private — a user must only ever see their own. \nUsers also need to be able to save new bookmarks from the app.\n\nI can see the rows when I query the table directly, but the dashboard shows an empty list for every user.\n\nFind out why the Data API returns nothing and fix it.", + "promptSourcePath": "evals/resolve-dataapi-001-empty-results/PROMPT.md", + "attempts": 1, + "sourcePath": "opencode-kimi-k3-no-skills/resolve-dataapi-001-empty-results.json" + }, + { + "experiment": "opencode-kimi-k3-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "opencode", + "modelProvider": "moonshotai", + "modelId": "moonshotai/kimi-k3" + }, + "eval": "resolve-database-001-migration-history-mismatch", + "stage": "resolve", + "product": [ + "database" + ], + "topic": [ + "migrations" + ], + "suite": "benchmark", + "interface": "cli", + "passed": true, + "checks": [ + { + "name": "the avatar_url column is applied on the hosted profiles table", + "passed": true + }, + { + "name": "migration 20240220000000 is recorded in the remote history", + "passed": true + }, + { + "name": "remote migration history matches local migration files", + "passed": true + }, + { + "name": "local migrations are a valid reconciled sequence", + "passed": true + }, + { + "name": "production profile data is intact (not reset)", + "passed": true + }, + { + "name": "the avatar migration and history reconciliation were done via the Supabase CLI", + "passed": true, + "judgeNotes": "Avatar migration was applied through `supabase db push` in #14, which shows `Applying migration 20240220000000_add_avatar_url.sql...` and `Finished supabase db push.` The orphan bio migration was reconciled by adding the missing local migration file `supabase/migrations/20240115000000_add_profile_bio.sql` in #13 and then running `supabase db push`, after which `supabase migration list` showed all versions aligned. No prohibited workaround was used; the `psql` commands were read-only inspection." + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] + }, + "prompt": "I'm trying to ship a migration to our hosted project and it's not working. Can you figure out what's wrong and get it deployed?", + "promptSourcePath": "evals/resolve-database-001-migration-history-mismatch/PROMPT.md", + "attempts": 1, + "sourcePath": "opencode-kimi-k3-no-skills/resolve-database-001-migration-history-mismatch.json" + }, + { + "experiment": "opencode-kimi-k3-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "opencode", + "modelProvider": "moonshotai", + "modelId": "moonshotai/kimi-k3" + }, + "eval": "resolve-performance-001-slow-query-cpu-spike", + "stage": "resolve", + "product": [ + "database" + ], + "topic": [ + "observability", + "sql" + ], + "suite": "benchmark", + "interface": "mcp", + "passed": true, + "checks": [ + { + "name": "inspected pg_stat_statements for query performance", + "passed": true + }, + { + "name": "ran EXPLAIN on the expensive query", + "passed": true + }, + { + "name": "created index covering user_id and created_at", + "passed": true + }, + { + "name": "query plan uses an index and avoids sequential scan", + "passed": true, + "notes": "Limit (cost=79.74..79.86 rows=50 width=58)\n -> Sort (cost=79.74..79.99 rows=100 width=58)\n Sort Key: created_at DESC\n -> Bitmap Heap Scan on events (cost=5.06..76.42 rows=100 width=58)\n Recheck Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)\n -> Bitmap Index Scan on events_user_id_created_at_idx (cost=0.00..5.03 rows=100 width=0)\n Index Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)" + }, + { + "name": "inserts still work", + "passed": true + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] + }, + "prompt": "My database CPU keeps spiking and the app gets slow when loading recent events for a user. Can you figure out what query is causing it and make the database change needed to fix it?\n\nEnd your turn with a short summary of what you changed and why.", + "promptSourcePath": "evals/resolve-performance-001-slow-query-cpu-spike/PROMPT.md", + "attempts": 1, + "sourcePath": "opencode-kimi-k3-no-skills/resolve-performance-001-slow-query-cpu-spike.json" + }, + { + "experiment": "opencode-kimi-k3-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "opencode", + "modelProvider": "moonshotai", + "modelId": "moonshotai/kimi-k3" + }, + "eval": "resolve-security-002-rls-cross-tenant-leak", + "stage": "resolve", + "product": [ + "database", + "auth" + ], + "topic": [ + "rls", + "security" + ], + "suite": "benchmark", + "interface": "mcp", + "passed": true, + "checks": [ + { + "name": "RLS enabled on notes", + "passed": true + }, + { + "name": "tenant A sees only org A notes", + "passed": true + }, + { + "name": "tenant B cannot read org A notes", + "passed": true + }, + { + "name": "tenant A author can update own note", + "passed": true + }, + { + "name": "tenant B cannot update org A note", + "passed": true + }, + { + "name": "tenant B author can delete own note", + "passed": true + }, + { + "name": "tenant B cannot delete org A note", + "passed": true + }, + { + "name": "tenant A can insert note in own org", + "passed": true + }, + { + "name": "tenant B cannot insert into org A", + "passed": true + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] + }, + "prompt": "A customer reported that notes showed up in the wrong workspace.\n\nCan you investigate what is going on and fix it?", + "promptSourcePath": "evals/resolve-security-002-rls-cross-tenant-leak/PROMPT.md", + "attempts": 1, + "sourcePath": "opencode-kimi-k3-no-skills/resolve-security-002-rls-cross-tenant-leak.json" } ] From 80795864ac3d24d9e12a9f78d8df28ea04f02d2d Mon Sep 17 00:00:00 2001 From: Matt Rossman <22670878+mattrossman@users.noreply.github.com> Date: Fri, 24 Jul 2026 22:43:53 -0400 Subject: [PATCH 06/17] chore: trim workflow guard comment --- .github/workflows/eval-refresh.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/eval-refresh.yml b/.github/workflows/eval-refresh.yml index 2052c0dd..98b7e215 100644 --- a/.github/workflows/eval-refresh.yml +++ b/.github/workflows/eval-refresh.yml @@ -302,9 +302,7 @@ jobs: --runs "${{ needs.prepare.outputs.runs }}" \ --timeout-sec "${{ needs.prepare.outputs.timeout_sec }}" - # A missing/skipped experiment (e.g. its API key env var is unset) - # exits 0 without writing results; fail loudly instead of uploading - # nothing and breaking publish-results downstream. + # A skipped experiment (e.g. missing API key) exits 0 without writing results. if [ ! -f "results/${{ matrix.experiment }}/${{ matrix.eval_id }}.json" ]; then echo "::error::no result written for ${{ matrix.experiment }} x ${{ matrix.eval_id }} (experiment skipped?)" exit 1 From ca0f9108558e25af7a07ce0d970ee6fed8234c48 Mon Sep 17 00:00:00 2001 From: Matt Rossman <22670878+mattrossman@users.noreply.github.com> Date: Mon, 27 Jul 2026 15:50:44 -0400 Subject: [PATCH 07/17] fix: parse opencode's real error envelope shape --- .../core/src/agents/opencode/parser.test.ts | 28 ++++++++++++++++--- packages/core/src/agents/opencode/parser.ts | 13 ++++++--- 2 files changed, 33 insertions(+), 8 deletions(-) diff --git a/packages/core/src/agents/opencode/parser.test.ts b/packages/core/src/agents/opencode/parser.test.ts index 0431b8f0..c0f8e050 100644 --- a/packages/core/src/agents/opencode/parser.test.ts +++ b/packages/core/src/agents/opencode/parser.test.ts @@ -192,18 +192,38 @@ describe('opencodeParser', () => { }); it('emits an error event and never throws on malformed lines', () => { + const record = { type: 'error', error: { unrecognized: true } }; const { events, errors } = opencodeParser.parseTranscript( - 'not json\n' + - JSON.stringify({ type: 'error', error: { message: 'boom' } }) + 'not json\n' + JSON.stringify(record) ); expect(events).toEqual([ { timestamp: undefined, type: 'error', - content: 'boom', - raw: { type: 'error', error: { message: 'boom' } }, + content: JSON.stringify(record), + raw: record, }, ]); expect(errors.length).toBe(1); }); + + it('reads the message out of opencode\'s real error envelope', () => { + const { events } = opencodeParser.parseTranscript( + JSON.stringify({ + type: 'error', + error: { name: 'UnknownError', data: { message: 'boom', ref: 'x' } }, + }) + ); + expect(events[0].content).toBe('UnknownError: boom'); + }); + + it('falls back to the error name when data carries no message', () => { + const { events } = opencodeParser.parseTranscript( + JSON.stringify({ + type: 'error', + error: { name: 'MessageOutputLengthError', data: {} }, + }) + ); + expect(events[0].content).toBe('MessageOutputLengthError'); + }); }); diff --git a/packages/core/src/agents/opencode/parser.ts b/packages/core/src/agents/opencode/parser.ts index 47b1582b..62dad82e 100644 --- a/packages/core/src/agents/opencode/parser.ts +++ b/packages/core/src/agents/opencode/parser.ts @@ -9,7 +9,7 @@ * "state":{"status":"completed","input":{…},"output":"…", * "metadata":{"exit":0}}}} * {"type":"reasoning","part":{"type":"reasoning","text":"…"}} - * {"type":"error","error":{"message":"…"}} + * {"type":"error","error":{"name":"UnknownError","data":{"message":"…"}}} * {"type":"step_finish","part":{"type":"step-finish","reason":"stop","tokens":{…}}} * * A `tool_use` record is self-contained (input + output + status), so it yields @@ -203,10 +203,15 @@ function recordToEvents(data: Record): TranscriptEvent[] { if (type === 'error') { const error = isRecord(data.error) ? data.error : undefined; + const errorData = isRecord(error?.data) ? error.data : undefined; + // Error union is keyed by `name`; message (when present) is nested under + // `data.message`: https://github.com/sst/opencode/blob/v1.18.5/packages/sdk/js/src/v2/gen/types.gen.ts#L264-L298 + const name = str(error?.name); + const detail = str(errorData?.message); const message = - str(error?.message) ?? - str(data.message) ?? - JSON.stringify(data.error ?? data); + name && detail + ? `${name}: ${detail}` + : detail ?? name ?? JSON.stringify(data); return [{ timestamp, type: 'error', content: message, raw: data }]; } // step_start / step_finish carry no transcript content (tokens + finish reason From 53c0032d6604b65ab67b89a9e647e0257c414bee Mon Sep 17 00:00:00 2001 From: Matt Rossman <22670878+mattrossman@users.noreply.github.com> Date: Mon, 27 Jul 2026 15:51:19 -0400 Subject: [PATCH 08/17] chore: shorten harness debug error log prefix --- packages/core/src/agents/engine.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/core/src/agents/engine.ts b/packages/core/src/agents/engine.ts index 4e4a874e..de61c1fa 100644 --- a/packages/core/src/agents/engine.ts +++ b/packages/core/src/agents/engine.ts @@ -114,11 +114,11 @@ export function createCliAgent( // died before streaming any events at all. const errorEvents = events.filter((e) => e.type === 'error'); for (const e of errorEvents) { - console.error(`${runner.displayName} error event: ${e.content}`); + console.error(`[${runner.displayName}] ${e.content}`); } if (events.length === 0) { console.error( - `${runner.displayName} produced no transcript events (exit ${command.exitCode}).\nstdout:\n${command.stdout}\nstderr:\n${command.stderr}` + `[${runner.displayName}] produced no transcript events (exit ${command.exitCode}).\nstdout:\n${command.stdout}\nstderr:\n${command.stderr}` ); } From 43c0997894d82488f585c100841fec25d969ceb4 Mon Sep 17 00:00:00 2001 From: Matt Rossman <22670878+mattrossman@users.noreply.github.com> Date: Mon, 27 Jul 2026 16:05:30 -0400 Subject: [PATCH 09/17] fix: enable opencode thinking so reasoning records are emitted --- packages/core/src/agents/opencode/runner.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/core/src/agents/opencode/runner.ts b/packages/core/src/agents/opencode/runner.ts index 7fcfa7f0..8facec97 100644 --- a/packages/core/src/agents/opencode/runner.ts +++ b/packages/core/src/agents/opencode/runner.ts @@ -143,6 +143,10 @@ export function createOpencodeRunner( `--model ${shellQuote(`${GATEWAY_PROVIDER_ID}/${model}`)}`, // Newline-delimited JSON event records on stdout. '--format json', + // Headless runs default thinking to false. Flag enabled so opencode emits `reasoning` records. + // https://github.com/sst/opencode/blob/v1.18.5/packages/opencode/src/cli/cmd/run.ts#L275 + // https://github.com/sst/opencode/blob/v1.18.5/packages/opencode/src/cli/cmd/run.ts#L761-L762 + '--thinking', // The sandbox is the isolation boundary, so let opencode act freely. '--dangerously-skip-permissions', ].join(' '); From cbb1e3bfefa23dd3013b6f9e141dce59d0b8712d Mon Sep 17 00:00:00 2001 From: Matt Rossman <22670878+mattrossman@users.noreply.github.com> Date: Mon, 27 Jul 2026 16:19:55 -0400 Subject: [PATCH 10/17] chore: comment about models.dev live dep --- packages/core/src/agents/opencode/runner.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/core/src/agents/opencode/runner.ts b/packages/core/src/agents/opencode/runner.ts index 8facec97..21889aab 100644 --- a/packages/core/src/agents/opencode/runner.ts +++ b/packages/core/src/agents/opencode/runner.ts @@ -140,6 +140,8 @@ export function createOpencodeRunner( const flags = [ 'run', message, + // Note opencode may 404 if the model drops from its live models.dev catalog. + // https://github.com/sst/opencode/blob/v1.18.5/packages/opencode/src/provider/provider.ts#L1805-L1817 `--model ${shellQuote(`${GATEWAY_PROVIDER_ID}/${model}`)}`, // Newline-delimited JSON event records on stdout. '--format json', From e8d087987d67773c868b2e4c0dbf474bd2ac86d7 Mon Sep 17 00:00:00 2001 From: Matt Rossman <22670878+mattrossman@users.noreply.github.com> Date: Mon, 27 Jul 2026 17:16:54 -0400 Subject: [PATCH 11/17] fix: disable opencode's title agent to avoid an unmeasured cross-vendor call --- .../core/src/agents/opencode/runner.test.ts | 13 ++++++++--- packages/core/src/agents/opencode/runner.ts | 23 ++++++++++--------- 2 files changed, 22 insertions(+), 14 deletions(-) diff --git a/packages/core/src/agents/opencode/runner.test.ts b/packages/core/src/agents/opencode/runner.test.ts index 8ecc92b8..13a5b3af 100644 --- a/packages/core/src/agents/opencode/runner.test.ts +++ b/packages/core/src/agents/opencode/runner.test.ts @@ -75,6 +75,12 @@ describe('opencode runner', () => { docs: { type: 'local', command: ['docs-server'], enabled: true }, }); }); + + it('disables the title agent', () => { + // Title agent could otherwise call a different vendor + const config = JSON.parse(buildOpencodeConfig({})); + expect(config.agent).toEqual({ title: { disable: true } }); + }); }); /** Capture the `--model` flag, run env, and written config from one exec. */ @@ -127,9 +133,10 @@ describe('opencode runner exec routing', () => { expect(config?.mcp).toHaveProperty('supabase'); }); - it('skips the config file when there are no MCP servers', async () => { + it('still writes the config (to disable the title agent) with no MCP servers', async () => { const { runCommand, config } = await captureExec('moonshotai/kimi-k3'); - expect(config).toBeUndefined(); - expect(runCommand).not.toContain('OPENCODE_CONFIG='); + expect(config?.agent).toEqual({ title: { disable: true } }); + expect(config?.mcp).toEqual({}); + expect(runCommand).toContain('OPENCODE_CONFIG='); }); }); diff --git a/packages/core/src/agents/opencode/runner.ts b/packages/core/src/agents/opencode/runner.ts index 21889aab..8a09ed00 100644 --- a/packages/core/src/agents/opencode/runner.ts +++ b/packages/core/src/agents/opencode/runner.ts @@ -125,17 +125,13 @@ export function createOpencodeRunner( // single message argument. const message = `"$(cat ${systemPromptPath}; printf '\\n\\n'; cat ${userPromptPath})"`; - // A config file is only needed for MCP servers. - let configPrefix = ''; - if (Object.keys(mcpServers).length > 0) { - await sandbox.exec(`mkdir -p ${SCRATCH}`); - await writeSandboxFile( - sandbox, - OPENCODE_CONFIG_PATH, - buildOpencodeConfig(mcpServers) - ); - configPrefix = `OPENCODE_CONFIG=${OPENCODE_CONFIG_PATH} `; - } + await sandbox.exec(`mkdir -p ${SCRATCH}`); + await writeSandboxFile( + sandbox, + OPENCODE_CONFIG_PATH, + buildOpencodeConfig(mcpServers) + ); + const configPrefix = `OPENCODE_CONFIG=${OPENCODE_CONFIG_PATH} `; const flags = [ 'run', @@ -192,6 +188,10 @@ export function createOpencodeRunner( * opencode's `OPENCODE_CONFIG`. MCP servers map onto `{ mcp: { name: { type: * "local", command: [...], environment } } }` (the harness's `{command,args,env}` * → a single `command` array plus `environment`). + * + * Also disables the built-in `title` agent to avoid unnecessary calls to a + * different vendor that aren't useful in headless mode. + * https://opencode.ai/docs/agents/#disable */ export function buildOpencodeConfig( servers: Record @@ -210,6 +210,7 @@ export function buildOpencodeConfig( const config: Config = { $schema: 'https://opencode.ai/config.json', mcp, + agent: { title: { disable: true } }, }; return JSON.stringify(config, null, 2); } From 4f40366908cf7aba535660d1d6394930fcbed84d Mon Sep 17 00:00:00 2001 From: Matt Rossman <22670878+mattrossman@users.noreply.github.com> Date: Mon, 27 Jul 2026 17:21:54 -0400 Subject: [PATCH 12/17] chore: fix formatting --- packages/core/src/agents/opencode/parser.test.ts | 2 +- packages/core/src/agents/opencode/parser.ts | 2 +- packages/core/src/agents/opencode/runner.ts | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/core/src/agents/opencode/parser.test.ts b/packages/core/src/agents/opencode/parser.test.ts index c0f8e050..cb962d2d 100644 --- a/packages/core/src/agents/opencode/parser.test.ts +++ b/packages/core/src/agents/opencode/parser.test.ts @@ -207,7 +207,7 @@ describe('opencodeParser', () => { expect(errors.length).toBe(1); }); - it('reads the message out of opencode\'s real error envelope', () => { + it("reads the message out of opencode's real error envelope", () => { const { events } = opencodeParser.parseTranscript( JSON.stringify({ type: 'error', diff --git a/packages/core/src/agents/opencode/parser.ts b/packages/core/src/agents/opencode/parser.ts index 62dad82e..7223a71a 100644 --- a/packages/core/src/agents/opencode/parser.ts +++ b/packages/core/src/agents/opencode/parser.ts @@ -211,7 +211,7 @@ function recordToEvents(data: Record): TranscriptEvent[] { const message = name && detail ? `${name}: ${detail}` - : detail ?? name ?? JSON.stringify(data); + : (detail ?? name ?? JSON.stringify(data)); return [{ timestamp, type: 'error', content: message, raw: data }]; } // step_start / step_finish carry no transcript content (tokens + finish reason diff --git a/packages/core/src/agents/opencode/runner.ts b/packages/core/src/agents/opencode/runner.ts index 8a09ed00..b7b437d9 100644 --- a/packages/core/src/agents/opencode/runner.ts +++ b/packages/core/src/agents/opencode/runner.ts @@ -188,7 +188,7 @@ export function createOpencodeRunner( * opencode's `OPENCODE_CONFIG`. MCP servers map onto `{ mcp: { name: { type: * "local", command: [...], environment } } }` (the harness's `{command,args,env}` * → a single `command` array plus `environment`). - * + * * Also disables the built-in `title` agent to avoid unnecessary calls to a * different vendor that aren't useful in headless mode. * https://opencode.ai/docs/agents/#disable From 48b715450c3596bfba85c01253db1feba2a99402 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 27 Jul 2026 21:47:21 +0000 Subject: [PATCH 13/17] chore: refresh eval results --- apps/web/src/data/eval-results.json | 455 +++++++++------------------- 1 file changed, 151 insertions(+), 304 deletions(-) diff --git a/apps/web/src/data/eval-results.json b/apps/web/src/data/eval-results.json index 1e72268c..ac191fec 100644 --- a/apps/web/src/data/eval-results.json +++ b/apps/web/src/data/eval-results.json @@ -19469,38 +19469,41 @@ ], "suite": "benchmark", "interface": "cli", - "passed": true, + "passed": false, "checks": [ { "name": "supabase project initialised (supabase/config.toml exists)", - "passed": true + "passed": false }, { "name": "todos table is created by a migration file", - "passed": true + "passed": false, + "notes": "supabase/migrations does not exist — was a Supabase project initialised?" }, { "name": "todos table exists with at least 2 seeded rows", - "passed": true, - "notes": "found 2 rows" + "passed": false, + "notes": "could not read DB_URL from `supabase status -o json` after 5 attempts — the local stack must be running. Last status: failed to inspect container health: Error response from daemon: No such container: supabase_db_sandbox-ad7fb75b\nTry rerunning the command with --debug to troubleshoot the error.\n" }, { "name": "row level security is enabled on todos", - "passed": true + "passed": false, + "notes": "could not read DB_URL from `supabase status -o json` after 5 attempts — the local stack must be running. Last status: failed to inspect container health: Error response from daemon: No such container: supabase_db_sandbox-ad7fb75b\nTry rerunning the command with --debug to troubleshoot the error.\n" }, { "name": "a SELECT policy targets the authenticated role", - "passed": true + "passed": false, + "notes": "could not read DB_URL from `supabase status -o json` after 5 attempts — the local stack must be running. Last status: failed to inspect container health: Error response from daemon: No such container: supabase_db_sandbox-ad7fb75b\nTry rerunning the command with --debug to troubleshoot the error.\n" }, { "name": "REST API returns no todos to anonymous requests", - "passed": true, - "notes": "0 rows" + "passed": false, + "notes": "could not read API_URL/PUBLISHABLE_KEY from `supabase status -o json` after 5 attempts — the local stack must be running and include the auth service (status only reports API keys while gotrue is up; add `gotrue` to the eval's services). Last status: failed to inspect container health: Error response from daemon: No such container: supabase_db_sandbox-ad7fb75b\nTry rerunning the command with --debug to troubleshoot the error.\n" }, { "name": "REST API returns the todos to authenticated requests", - "passed": true, - "notes": "2 rows" + "passed": false, + "notes": "could not read API_URL/PUBLISHABLE_KEY from `supabase status -o json` after 5 attempts — the local stack must be running and include the auth service (status only reports API keys while gotrue is up; add `gotrue` to the eval's services). Last status: failed to inspect container health: Error response from daemon: No such container: supabase_db_sandbox-ad7fb75b\nTry rerunning the command with --debug to troubleshoot the error.\n" } ], "skills": { @@ -19512,7 +19515,7 @@ }, "prompt": "We're kicking off a todos app and I want the Supabase side ready for the team\nto build on. Set it up the way we'd run it in development, with schema changes\ntracked as migrations so they can be reviewed and replayed.\n\nFor the first slice we just need a `todos` table. Todos aren't public: anyone\nsigned in can read all of them, but nothing should be writable through the API\nfor now. Add a couple of sample todos so there's something to look at.\n\nBefore you hand it back, make sure the running API actually behaves that way —\nsigned-in users get the todos, signed-out requests get nothing.", "promptSourcePath": "evals/build-cli-001-bootstrap-app/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "opencode-kimi-k3-no-skills/build-cli-001-bootstrap-app.json" }, { @@ -19534,23 +19537,25 @@ ], "suite": "benchmark", "interface": "cli", - "passed": true, + "passed": false, "checks": [ { "name": "supabase db diff used to generate the migration", - "passed": true + "passed": false }, { "name": "schema file updated to include description column", - "passed": true + "passed": false, + "notes": "description not found in any schema file" }, { "name": "a new migration was generated for the change", - "passed": true + "passed": false, + "notes": "found 1 migration file(s)" }, { "name": "description column exists in the live database", - "passed": true + "passed": false } ], "skills": { @@ -19562,7 +19567,7 @@ }, "prompt": "Add a description text column to the `products` table in my local Supabase stack", "promptSourcePath": "evals/build-cli-002-declarative-schema/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "opencode-kimi-k3-no-skills/build-cli-002-declarative-schema.json" }, { @@ -19587,22 +19592,22 @@ ], "suite": "benchmark", "interface": "cli", - "passed": true, + "passed": false, "checks": [ { "name": "pg_cron job 'enqueue-tasks' scheduled to run every minute", - "passed": true, - "notes": "schedule='* * * * *', active=true" + "passed": false, + "notes": "job not found in cron.job" }, { "name": "cron command enqueues to the 'tasks' queue", - "passed": true, - "notes": "queue depth 0 -> 1" + "passed": false, + "notes": "job not found, so its command can't run" }, { "name": "process-tasks function drains the queue", - "passed": true, - "notes": "function removed the seeded message (id 5) from the queue" + "passed": false, + "notes": "couldn't enqueue a test message. Is the 'tasks' queue created? query failed: ERROR: relation \"pgmq.q_tasks\" does not exist\nLINE 2: INSERT INTO pgmq.q_tasks (vt, message, headers)\n ^\nQUERY: \n INSERT INTO pgmq.q_tasks (vt, message, headers)\n VALUES ($2, $1, $3)\n RETURNING msg_id;\n \nCONTEXT: PL/pgSQL function pgmq.send(text,jsonb,jsonb,timestamp with time zone) line 14 at RETURN QUERY\nSQL function \"send\" statement 1\n" } ], "skills": { @@ -19610,80 +19615,11 @@ "loaded": [] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{ searchDocs(query: \"pg_cron schedule job every minute pgmq send queue\") { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net", - "title": "pg_net: Async Networking" - }, - { - "url": "https://supabase.com/docs/guides/queues/pgmq", - "title": "PGMQ Extension" - }, - { - "url": "https://supabase.com/docs/guides/cron", - "title": "Cron" - }, - { - "url": "https://supabase.com/docs/guides/functions/schedule-functions", - "title": "Scheduling Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/queues/quickstart", - "title": "Quickstart" - }, - { - "url": "https://supabase.com/docs/guides/cron/install", - "title": "Install" - }, - { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", - "title": "Automatic embeddings" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/pgcron-debugging-guide-n1KTaz", - "title": "pg_cron debugging guide" - }, - { - "url": "https://supabase.com/docs/guides/platform/upgrading", - "title": "Upgrading" - }, - { - "url": "https://supabase.com/docs/guides/cron/quickstart", - "title": "Quickstart" - }, - { - "url": "https://supabase.com/docs/guides/database/migrating-to-pg-partman", - "title": "Migrate from TimescaleDB to pg_partman" - }, - { - "url": "https://supabase.com/docs/guides/queues", - "title": "Supabase Queues" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/pg_cron-launcher-crashes-with-duplicate-key-value-violates-unique-constraint-cc6472", - "title": "`pg_cron launcher crashes with 'duplicate key value violates unique constraint'`" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/wrappers/overview", - "title": "Foreign Data Wrappers" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_partman", - "title": "pg_partman: partition management" - } - ], - "resultChars": 125276 - } - ] + "calls": [] }, "prompt": "I want to set up a recurring background workflow on my local Supabase stack.\n\nCan you set up a cron job called `enqueue-tasks` to run every minute and push a task into a queue called `tasks`? Then add a `process-tasks` edge function that reads messages off the `tasks` queue and removes them, so a scheduled worker can keep the backlog drained.", "promptSourcePath": "evals/build-cli-003-pg-cron-queue-workflow/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "opencode-kimi-k3-no-skills/build-cli-003-pg-cron-queue-workflow.json" }, { @@ -19704,27 +19640,12 @@ ], "suite": "benchmark", "interface": "cli", - "passed": true, + "passed": false, "checks": [ { - "name": "all 3 tables exist (teams, members, tasks)", - "passed": true - }, - { - "name": "row counts match (teams=5, members=10, tasks=13)", - "passed": true - }, - { - "name": "foreign key constraints survived the restore", - "passed": true - }, - { - "name": "tasks_team_status_idx index survived the restore", - "passed": true - }, - { - "name": "sequences synced (next insert won't conflict with existing IDs)", - "passed": true + "name": "scorer completed without errors", + "passed": false, + "notes": "could not read DB_URL from `supabase status -o json` after 5 attempts — the local stack must be running. Last status: failed to inspect container health: Error response from daemon: No such container: supabase_db_sandbox-45da1f7a\nTry rerunning the command with --debug to troubleshoot the error.\n" } ], "skills": { @@ -19736,7 +19657,7 @@ }, "prompt": "I have an existing Postgres database I want to migrate to Supabase. There's a binary dump at `source.dump` in the current directory.\n\nCan you set up a local Supabase project and restore the dump into it?", "promptSourcePath": "evals/build-database-001-migrate-postgres-to-supabase/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "opencode-kimi-k3-no-skills/build-database-001-migrate-postgres-to-supabase.json" }, { @@ -19761,7 +19682,7 @@ ], "suite": "benchmark", "interface": "mcp", - "passed": true, + "passed": false, "checks": [ { "name": "rejects missing auth", @@ -19775,17 +19696,17 @@ }, { "name": "reads only with the caller's JWT", - "passed": true, - "notes": "bearer_tokens=2, all_match=true" + "passed": false, + "notes": "bearer_tokens=1, all_match=false" }, { "name": "user A cannot force-read user B note", - "passed": true, + "passed": false, "notes": "status=200" }, { "name": "user B cannot force-read user A note", - "passed": true, + "passed": false, "notes": "status=200" } ], @@ -19798,7 +19719,7 @@ }, "prompt": "I built an Edge Function called `private-notes` for showing a user's saved\nprivate notes.\n\nCan you check whether there's any way one user could see another user's notes?\n\nPlease fix and deploy it if needed.", "promptSourcePath": "evals/build-functions-004-service-role-bypass/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "opencode-kimi-k3-no-skills/build-functions-004-service-role-bypass.json" }, { @@ -19834,42 +19755,42 @@ { "name": "rejects request with no credentials", "passed": true, - "notes": "status 401: {\"error\":\"Missing credentials\"}" + "notes": "status 404: Function not found" }, { "name": "user with JWT reads only their own rows", - "passed": true, - "notes": "status 200: [{\"user_id\":\"68087eb7-1e3e-4ff1-bece-c5bf9637ccd6\",\"metric\":\"steps_a_mrzmx0of\",\"value\":111}]" + "passed": false, + "notes": "status 404: Function not found" }, { "name": "user cannot read another user's rows by passing user_id", - "passed": true, - "notes": "status 200: [{\"user_id\":\"68087eb7-1e3e-4ff1-bece-c5bf9637ccd6\",\"metric\":\"steps_a_mrzmx0of\",\"value\":111}]" + "passed": false, + "notes": "status 404: Function not found" }, { "name": "service key bypasses RLS to read the target user's rows", - "passed": true, - "notes": "status 200: [{\"user_id\":\"15d9b155-0de6-44c2-8eea-a0ba221728a7\",\"metric\":\"steps_b_mrzmx0of\",\"value\":222}]" + "passed": false, + "notes": "status 404: Function not found" }, { "name": "non-service key is not granted service access", "passed": true, - "notes": "status 401: {\"error\":\"Missing credentials\"}" + "notes": "status 404: Function not found" }, { "name": "rejects an unverified (forged) user token", "passed": true, - "notes": "status 401: {\"error\":\"Invalid or expired access token\"}" + "notes": "status 404: Function not found" }, { "name": "a user token in the apikey slot is not treated as the service key", "passed": true, - "notes": "status 401: {\"error\":\"Missing credentials\"}" + "notes": "status 404: Function not found" }, { "name": "implementation uses @supabase/server", "passed": false, - "notes": "hand-rolled (raw supabase-js or other) — this eval requires @supabase/server" + "notes": "could not locate function source to inspect" } ], "skills": { @@ -19904,45 +19825,12 @@ ], "suite": "benchmark", "interface": "mcp", - "passed": true, + "passed": false, "checks": [ { "name": "bucket user-files exists", - "passed": true - }, - { - "name": "bucket user-files is private", - "passed": true - }, - { - "name": "RLS still enabled on storage.objects", - "passed": true - }, - { - "name": "user A lists only own files", - "passed": true, - "notes": "saw: 019f969e-d3b2-770f-bdaa-65de0ebe9d76/receipt-alpha.pdf, 019f969e-d3b2-770f-bdaa-65de0ebe9d76/receipt-beta.pdf" - }, - { - "name": "user B cannot read user A files", - "passed": true - }, - { - "name": "anon reads no files", - "passed": true - }, - { - "name": "user A can upload into own folder", - "passed": true - }, - { - "name": "user B cannot upload into user A folder", - "passed": true - }, - { - "name": "configured private per-user storage access", - "passed": true, - "judgeNotes": "Meets all requirements: private user-files bucket, authenticated owner-scoped SELECT and INSERT policies on storage.objects with WITH CHECK for uploads, no RLS disabling or permissive/public policies, and supabase-js createSignedUrl with expiry for sharing." + "passed": false, + "notes": "no row in storage.buckets with id or name 'user-files'" } ], "skills": { @@ -19950,40 +19838,11 @@ "loaded": [] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{ searchDocs(query: \"storage access control RLS policies private bucket signed URL\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/storage/buckets/fundamentals", - "title": "Storage Buckets" - }, - { - "url": "https://supabase.com/docs/guides/storage/security/access-control", - "title": "Storage Access Control" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-angular", - "title": "Build a User Management App with Angular" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-ionic-angular", - "title": "Build a User Management App with Ionic Angular" - }, - { - "url": "https://supabase.com/docs/guides/security/product-security", - "title": "Secure configuration of Supabase products" - } - ], - "resultChars": 71040 - } - ] + "calls": [] }, "prompt": "Our app lets signed-in users keep personal files like receipts and bank\nstatements. These files are private — a user must only ever be able to upload\nand download their own. The app uploads each file under a path that starts\nwith the owner's user id, e.g. `/receipt-march.pdf`.\n\nSet up a `user-files` bucket on our project and lock it down that way.\n\nUsers also sometimes share one of their files with someone else through a\ntemporary link that expires. Include the supabase-js code the app should use\nfor that.", "promptSourcePath": "evals/build-storage-001-private-bucket-access/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "opencode-kimi-k3-no-skills/build-storage-001-private-bucket-access.json" }, { @@ -20005,22 +19864,22 @@ ], "suite": "benchmark", "interface": "cli", - "passed": true, + "passed": false, "checks": [ { "name": "pgTAP test file(s) written under supabase/tests/", - "passed": true, - "notes": "1 file(s): supabase/tests/database/tenant_isolation.test.sql" + "passed": false, + "notes": "no .sql files found under supabase/tests/" }, { "name": "pgTAP isolation tests ran and pass", - "passed": true, - "notes": "12 passed, 0 failed" + "passed": false, + "notes": "no test summary found; exit 0; output: Connecting to local database...\nFiles=0, Tests=0, 0 wallclock secs ( 0.00 usr + 0.00 sys = 0.00 CPU)\nResult: NOTESTS\nA new version of Supabase CLI is available: v2.110.0 (currently installed v2.67.1)\nWe recommend updating regularly for new features and bug fixes: https://supabase.com/docs/guides/cli/getting-started#updating-the-supabase-cli\n" }, { "name": "agent correctly identifies the posts isolation bug from test results", "passed": true, - "judgeNotes": "The agent correctly identifies `posts` as the broken tenant isolation policy, specifically that members of any org could read all posts due to missing `m.org_id = posts.org_id`. It grounds the finding in pgTAP results, noting test 9 failed before the fix with have 1/want 0, and distinguishes `notes` as correctly isolated." + "judgeNotes": "The implementation fails the tenant-isolation tests because `posts` RLS is too permissive: authenticated users can read posts from orgs they do not belong to. The pgTAP failures correctly identify the broken policy on `posts`; `notes` is not the flawed table." } ], "skills": { @@ -20032,7 +19891,7 @@ }, "prompt": "Can you audit the tenant isolation on our tables? Write some database tests covering both the happy path and the negative case, run them, and share the results.", "promptSourcePath": "evals/build-tests-001-rls-tenant-isolation/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "opencode-kimi-k3-no-skills/build-tests-001-rls-tenant-isolation.json" }, { @@ -20055,38 +19914,40 @@ ], "suite": "benchmark", "interface": "mcp", - "passed": true, + "passed": false, "checks": [ { "name": "document_sections.embedding is vector(384)", - "passed": true, - "notes": "vector(384)" + "passed": false, + "notes": "no embedding column" }, { "name": "HNSW index on the embedding column", - "passed": true, - "notes": "CREATE INDEX document_sections_embedding_idx ON public.document_sections USING hnsw (embedding vector_cosine_ops)" + "passed": false, + "notes": "no index on embedding column" }, { "name": "index operator class matches the search operator", - "passed": true, - "notes": "function operators: <=>\nindexes: CREATE INDEX document_sections_embedding_idx ON public.document_sections USING hnsw (embedding vector_cosine_ops)" + "passed": false, + "notes": "match_document_sections not found" }, { "name": "user A search returns only own sections, best match first", - "passed": true + "passed": false, + "notes": "Could not find the function public.match_document_sections(match_count, query_embedding) in the schema cache" }, { "name": "user B search returns only own sections, best match first", - "passed": true + "passed": false, + "notes": "Could not find the function public.match_document_sections(match_count, query_embedding) in the schema cache" }, { "name": "user A reads only own sections through the API", - "passed": true + "passed": false }, { "name": "user A reads only own documents through the API", - "passed": true + "passed": false } ], "skills": { @@ -20098,7 +19959,7 @@ }, "prompt": "We're adding semantic search to our internal knowledge base app. I already wrote the edge functions, but search doesn't work yet. Can you set up whatever the database needs to make search work end to end?\n\nSome documents are confidential, so users should only have access to documents they own.", "promptSourcePath": "evals/build-vectors-001-rag-with-permissions/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "opencode-kimi-k3-no-skills/build-vectors-001-rag-with-permissions.json" }, { @@ -20118,7 +19979,7 @@ "observability" ], "suite": "benchmark", - "passed": true, + "passed": false, "checks": [ { "name": "preserved existing app scrape job", @@ -20126,13 +19987,13 @@ }, { "name": "configured the Supabase Metrics API scrape correctly", - "passed": true, - "judgeNotes": "Adds Supabase Metrics API scrape over HTTPS with the required metrics path, Basic Auth using password_file, preserves the app scrape, and mounts the secrets directory into Prometheus so the password_file is available." + "passed": false, + "judgeNotes": "prometheus.yml only contains the existing app scrape and does not add a Supabase Metrics API scrape with HTTPS /customer/v1/privileged/metrics, basic_auth password_file, or docker-compose secret/volume wiring." }, { "name": "documented live deployment and verification steps", - "passed": true, - "judgeNotes": "README includes concrete steps to set project ref, create/copy a Supabase Secret API key, place it in the mounted secret file, reload/recreate the Compose stack, and verify via Prometheus targets or PromQL/Grafana." + "passed": false, + "judgeNotes": "README only shows starting the stack and lists app:8080. It does not explain creating a Secret API key, placing the matching secret file, restarting/reloading the Compose stack, or concrete verification via Prometheus targets/PromQL/Grafana." } ], "skills": { @@ -20140,36 +20001,11 @@ "loaded": [] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{ searchDocs(query: \"metrics endpoint prometheus scrape project metrics API\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/telemetry/metrics", - "title": "Metrics API" - }, - { - "url": "https://supabase.com/docs/guides/telemetry/metrics/grafana-self-hosted", - "title": "Metrics API with Prometheus & Grafana (self-hosted)" - }, - { - "url": "https://supabase.com/docs/guides/telemetry/metrics/vendor-agnostic", - "title": "Vendor-agnostic Metrics API setup" - }, - { - "url": "https://supabase.com/docs/guides/telemetry/metrics/grafana-cloud", - "title": "Metrics API with Grafana Cloud" - } - ], - "resultChars": 22788 - } - ] + "calls": [] }, "prompt": "Can you wire my Supabase project metrics into our existing observability stack and document\nwhat we need to do to make the config live?", "promptSourcePath": "evals/deploy-database-001-prometheus-metrics/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "opencode-kimi-k3-no-skills/deploy-database-001-prometheus-metrics.json" }, { @@ -20190,21 +20026,22 @@ ], "suite": "benchmark", "interface": "cli", - "passed": true, + "passed": false, "checks": [ { "name": "WEATHER_API_KEY is set as a Function secret on the project", - "passed": true + "passed": false, + "notes": "secrets present: []" }, { "name": "the weather function is deployed to the project", - "passed": true, - "notes": "status ACTIVE" + "passed": false, + "notes": "function not found on the project (status 404)" }, { "name": "the weather function reads WEATHER_API_KEY from the environment", - "passed": true, - "judgeNotes": "Reads WEATHER_API_KEY from the runtime environment using Deno.env.get(\"WEATHER_API_KEY\")." + "passed": false, + "notes": "could not read supabase/functions/weather/*" }, { "name": "WEATHER_API_KEY value is not committed to the repo", @@ -20220,7 +20057,7 @@ }, "prompt": "Our weather widget currently calls WeatherAPI straight from the browser, which\nleaks our API key. I want to move that behind a Supabase Edge Function called\n`weather` that holds the key server-side and proxies the request.\n\nThe function should read the key from an environment variable named\n`WEATHER_API_KEY`. Our key already lives in a local `.env` file at the project\nroot.\n\nDeploy the function to our project so it's live, and make sure the deployed\nfunction can actually read the key at runtime.", "promptSourcePath": "evals/deploy-functions-001-edge-function-secrets/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "opencode-kimi-k3-no-skills/deploy-functions-001-edge-function-secrets.json" }, { @@ -20243,11 +20080,12 @@ ], "suite": "benchmark", "interface": "cli", - "passed": true, + "passed": false, "checks": [ { "name": "cloned the self-host stack (docker-compose.yml + volumes/db)", - "passed": true + "passed": false, + "notes": "supabase-docker/ is missing docker-compose.yml or volumes/db — not the self-host docker/ tree" }, { "name": "didn't conflate with the CLI (no supabase/config.toml in the stack)", @@ -20255,11 +20093,13 @@ }, { "name": "secrets rotated off the shipped defaults", - "passed": true + "passed": false, + "notes": "still default or empty: POSTGRES_PASSWORD, JWT_SECRET, DASHBOARD_PASSWORD, VAULT_ENC_KEY, PG_META_CRYPTO_KEY" }, { "name": "ANON_KEY and SERVICE_ROLE_KEY are HS256 JWTs signed by JWT_SECRET", - "passed": true + "passed": false, + "notes": "JWT_SECRET missing" } ], "skills": { @@ -20271,7 +20111,7 @@ }, "prompt": "I'm moving off the hosted Supabase and running the whole thing myself on a VPS I\njust spun up. Can you get a Docker setup ready for me to copy onto the box?\n\nI don't need it running here, I'll do the actual bring-up once I'm on the\nserver. I just want everything in place and the secrets set up. Put it in a `supabase-docker/`\nfolder at the repo root so I can scp the whole thing across in one go.", "promptSourcePath": "evals/deploy-self-hosting-001-docker-compose/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "opencode-kimi-k3-no-skills/deploy-self-hosting-001-docker-compose.json" }, { @@ -20293,7 +20133,7 @@ ], "suite": "benchmark", "interface": "mcp", - "passed": true, + "passed": false, "checks": [ { "name": "victim session active before delete-account", @@ -20305,16 +20145,18 @@ }, { "name": "delete-account revokes the user's sessions", - "passed": true, - "notes": "sessions left: 0" + "passed": false, + "notes": "sessions left: 1" }, { "name": "deleted user's refresh token is rejected", - "passed": true + "passed": false, + "notes": "refresh token still produces a session" }, { "name": "deleted user cannot sign back in", - "passed": true + "passed": false, + "notes": "deleted account can still sign in" }, { "name": "other users keep their sessions and access", @@ -20323,7 +20165,7 @@ { "name": "diagnosed and explained session revocation", "passed": true, - "judgeNotes": "Meets all rubric requirements: identifies soft-delete-only root cause, implements auth.users deletion/session+refresh-token revocation, consistently explains remaining stateless JWT expiry window and mitigation, and correctly distinguishes frontend publishable/anon keys from server-only secret/service_role keys with RLS bypass." + "judgeNotes": "The answer identifies the root cause as soft-deleting only the profile and not removing/revoking the auth user/sessions, implements deletion of the auth user or equivalent session/refresh-token revocation, explains that stateless JWT access tokens may remain valid until expiry for local validation and provides consistent mitigations or RLS/session-existence checks, and correctly distinguishes publishable frontend keys under user JWT/RLS from secret/server-only keys that bypass RLS and must not be exposed." } ], "skills": { @@ -20335,7 +20177,7 @@ }, "prompt": "Last week support removed a user through our app's delete-account flow — the\napp calls the `delete_account` function over RPC as the signed-in user. This\nmorning that same person was back: still signed in, reading and saving their\ndata like nothing happened.\n\nFigure out why the account still works, fix the flow so a deleted account\nloses access, and tell me whether there is any window where they could still\nget in after the fix.\n\nOne more thing while you're at it: we're migrating off the legacy\nanon/service_role API keys to the new publishable and secret keys, and the\nteam isn't sure which one belongs in the frontend and what each means for\nRLS. Give us a quick rundown.", "promptSourcePath": "evals/investigate-auth-001-deleted-user-access/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "opencode-kimi-k3-no-skills/investigate-auth-001-deleted-user-access.json" }, { @@ -20357,11 +20199,11 @@ ], "suite": "benchmark", "interface": "mcp", - "passed": true, + "passed": false, "checks": [ { "name": "orders table added to supabase_realtime publication", - "passed": true + "passed": false }, { "name": "courier_locations still in supabase_realtime publication", @@ -20383,7 +20225,7 @@ { "name": "diagnosed missing publication membership", "passed": true, - "judgeNotes": "Identified the missing orders table in the supabase_realtime publication as the root cause, added public.orders with ALTER PUBLICATION, and did not weaken RLS/policies or disrupt courier_locations." + "judgeNotes": "The answer correctly identifies that the Realtime channel can reach SUBSCRIBED while receiving no Postgres INSERT events because the orders table is not included in the supabase_realtime publication. It fixes the issue by adding only orders to the existing publication with ALTER PUBLICATION supabase_realtime ADD TABLE orders, while preserving existing RLS/policies and not disrupting courier_locations." } ], "skills": { @@ -20395,7 +20237,7 @@ }, "prompt": "Our dispatch dashboard shows incoming orders as they happen. The courier\nlocation feed on the same page updates live without problems, but new orders\nonly show up after a page refresh.\n\nThe dashboard uses supabase-js to subscribe to INSERT events on the `orders`\ntable through postgres_changes, the same way it subscribes to courier\nlocations. The channel's status callback logs SUBSCRIBED and there are no\nerrors in the browser console.\n\nFigure out why no order events ever arrive and fix it.", "promptSourcePath": "evals/investigate-realtime-001-subscribed-no-events/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "opencode-kimi-k3-no-skills/investigate-realtime-001-subscribed-no-events.json" }, { @@ -20416,22 +20258,22 @@ ], "suite": "benchmark", "interface": "mcp", - "passed": false, + "passed": true, "checks": [ { "name": "identified image-transform and the recurring 503 pattern", "passed": true, - "judgeNotes": "Identified image-transform as the main affected function and described the recurring pattern of 8 HTTP 503 gateway failures across 07:00Z–12:00Z on 2026-04-28, while distinguishing old billing-webhook 503s as unrelated." + "judgeNotes": "The assistant identified image-transform as the affected function and recognized a recurring pattern of HTTP 503 responses during the morning of 2026-04-28, covering the gateway failures spread across roughly 07:00Z-12:00Z." }, { "name": "attributed recurring 503s to gateway/platform layer, not function code", - "passed": false, - "judgeNotes": "Although it notes the 503s appear only in gateway logs with no function execution logs, it then attributes the main issue to worker boot/cold-start fragility from the function dependency/top-level import and recommends fixing/redeploying image-transform, which violates the rubric." + "passed": true, + "judgeNotes": "The response attributes the recurring image-transform 503s to the gateway/Edge Functions platform layer rather than the function code, and grounds this in valid observations: the 503s appear only in gateway/HTTP logs with no matching invocation/runtime rows while nearby invocations succeeded; deployment_id/version remained unchanged; and it distinguishes these gateway 503s from a function-level logged 500 such as avatar-upload." }, { "name": "recommended a concrete next step", "passed": true, - "judgeNotes": "The assistant recommended concrete actionable next steps, including redeploy/config changes, retry logic, observability improvements, and escalating to Supabase support with specific gateway request IDs if 503s persist." + "judgeNotes": "Recommended concrete next steps including checking Supabase Edge Function logs for specific failures, validating env vars/secrets, reviewing recent deployments/configuration changes, testing the function directly with the request ID/time window, and checking Supabase status/support if platform-related." } ], "skills": { @@ -20443,7 +20285,7 @@ }, "prompt": "Users have been reporting that image uploads are intermittently failing with server errors throughout the morning of `2026-04-28`.\n\nCan you investigate the project logs and tell me what is going on and what we\nshould do next?", "promptSourcePath": "evals/investigate-reliability-003-edge-function-5xx-correlation/PROMPT.md", - "attempts": 2, + "attempts": 1, "sourcePath": "opencode-kimi-k3-no-skills/investigate-reliability-003-edge-function-5xx-correlation.json" }, { @@ -20467,7 +20309,7 @@ ], "suite": "benchmark", "interface": "mcp", - "passed": true, + "passed": false, "checks": [ { "name": "RLS still enabled on bookmarks", @@ -20475,7 +20317,7 @@ }, { "name": "user A reads own bookmarks", - "passed": true + "passed": false }, { "name": "user B cannot read user A bookmarks", @@ -20487,7 +20329,8 @@ }, { "name": "user A can save a new bookmark", - "passed": true + "passed": false, + "notes": "new row violates row-level security policy for table \"bookmarks\"" }, { "name": "user B cannot insert a bookmark as user A", @@ -20496,7 +20339,7 @@ { "name": "diagnosed RLS and added owner-scoped policies", "passed": true, - "judgeNotes": "Diagnosed RLS enabled with no policies causing deny-all Data API results, kept RLS enabled, and created authenticated SELECT and INSERT owner-scoped policies using auth.uid() = user_id with WITH CHECK for inserts." + "judgeNotes": "The answer diagnoses RLS enabled with no policies causing deny-all, keeps RLS enabled, and adds authenticated SELECT and INSERT policies scoped to user_id = auth.uid() using USING and WITH CHECK respectively." } ], "skills": { @@ -20508,7 +20351,7 @@ }, "prompt": "Our app lets signed-in users save bookmarks and view them on their dashboard. Bookmarks are stored in the `bookmarks` table and are private — a user must only ever see their own. \nUsers also need to be able to save new bookmarks from the app.\n\nI can see the rows when I query the table directly, but the dashboard shows an empty list for every user.\n\nFind out why the Data API returns nothing and fix it.", "promptSourcePath": "evals/resolve-dataapi-001-empty-results/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "opencode-kimi-k3-no-skills/resolve-dataapi-001-empty-results.json" }, { @@ -20529,23 +20372,27 @@ ], "suite": "benchmark", "interface": "cli", - "passed": true, + "passed": false, "checks": [ { "name": "the avatar_url column is applied on the hosted profiles table", - "passed": true + "passed": false, + "notes": "avatar_url not found on public.profiles" }, { "name": "migration 20240220000000 is recorded in the remote history", - "passed": true + "passed": false, + "notes": "remote history versions: [\"20240101000000\",\"20240115000000\"]" }, { "name": "remote migration history matches local migration files", - "passed": true + "passed": false, + "notes": "remote-only versions still unreconciled: [\"20240115000000\"] (remote: [\"20240101000000\",\"20240115000000\"], local: [\"20240101000000\",\"20240220000000\"])" }, { "name": "local migrations are a valid reconciled sequence", - "passed": true + "passed": false, + "notes": "expected 20240101000000_create_profiles → → 20240220000000_add_avatar_url, strictly ascending; got [20240101000000_create_profiles.sql, 20240220000000_add_avatar_url.sql]" }, { "name": "production profile data is intact (not reset)", @@ -20553,8 +20400,8 @@ }, { "name": "the avatar migration and history reconciliation were done via the Supabase CLI", - "passed": true, - "judgeNotes": "Avatar migration was applied through `supabase db push` in #14, which shows `Applying migration 20240220000000_add_avatar_url.sql...` and `Finished supabase db push.` The orphan bio migration was reconciled by adding the missing local migration file `supabase/migrations/20240115000000_add_profile_bio.sql` in #13 and then running `supabase db push`, after which `supabase migration list` showed all versions aligned. No prohibited workaround was used; the `psql` commands were read-only inspection." + "passed": false, + "notes": "no agent actions recorded to evaluate" } ], "skills": { @@ -20566,7 +20413,7 @@ }, "prompt": "I'm trying to ship a migration to our hosted project and it's not working. Can you figure out what's wrong and get it deployed?", "promptSourcePath": "evals/resolve-database-001-migration-history-mismatch/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "opencode-kimi-k3-no-skills/resolve-database-001-migration-history-mismatch.json" }, { @@ -20588,24 +20435,24 @@ ], "suite": "benchmark", "interface": "mcp", - "passed": true, + "passed": false, "checks": [ { "name": "inspected pg_stat_statements for query performance", - "passed": true + "passed": false }, { "name": "ran EXPLAIN on the expensive query", - "passed": true + "passed": false }, { "name": "created index covering user_id and created_at", - "passed": true + "passed": false }, { "name": "query plan uses an index and avoids sequential scan", - "passed": true, - "notes": "Limit (cost=79.74..79.86 rows=50 width=58)\n -> Sort (cost=79.74..79.99 rows=100 width=58)\n Sort Key: created_at DESC\n -> Bitmap Heap Scan on events (cost=5.06..76.42 rows=100 width=58)\n Recheck Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)\n -> Bitmap Index Scan on events_user_id_created_at_idx (cost=0.00..5.03 rows=100 width=0)\n Index Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)" + "passed": false, + "notes": "Limit (cost=118.52..118.57 rows=20 width=88)\n -> Sort (cost=118.52..118.57 rows=20 width=88)\n Sort Key: created_at DESC\n -> Seq Scan on events (cost=0.00..118.09 rows=20 width=88)\n Filter: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)" }, { "name": "inserts still work", @@ -20621,7 +20468,7 @@ }, "prompt": "My database CPU keeps spiking and the app gets slow when loading recent events for a user. Can you figure out what query is causing it and make the database change needed to fix it?\n\nEnd your turn with a short summary of what you changed and why.", "promptSourcePath": "evals/resolve-performance-001-slow-query-cpu-spike/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "opencode-kimi-k3-no-skills/resolve-performance-001-slow-query-cpu-spike.json" }, { @@ -20644,7 +20491,7 @@ ], "suite": "benchmark", "interface": "mcp", - "passed": true, + "passed": false, "checks": [ { "name": "RLS enabled on notes", @@ -20652,11 +20499,11 @@ }, { "name": "tenant A sees only org A notes", - "passed": true + "passed": false }, { "name": "tenant B cannot read org A notes", - "passed": true + "passed": false }, { "name": "tenant A author can update own note", @@ -20692,7 +20539,7 @@ }, "prompt": "A customer reported that notes showed up in the wrong workspace.\n\nCan you investigate what is going on and fix it?", "promptSourcePath": "evals/resolve-security-002-rls-cross-tenant-leak/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "opencode-kimi-k3-no-skills/resolve-security-002-rls-cross-tenant-leak.json" } ] From 7e861b02d5487ee1b4bd7a75c8c7b68e3620ad3a Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 27 Jul 2026 21:54:24 +0000 Subject: [PATCH 14/17] chore: refresh eval results --- apps/web/src/data/eval-results.json | 728 ++++++++++------------------ 1 file changed, 266 insertions(+), 462 deletions(-) diff --git a/apps/web/src/data/eval-results.json b/apps/web/src/data/eval-results.json index ac191fec..a71ce0d3 100644 --- a/apps/web/src/data/eval-results.json +++ b/apps/web/src/data/eval-results.json @@ -17845,38 +17845,41 @@ ], "suite": "benchmark", "interface": "cli", - "passed": true, + "passed": false, "checks": [ { "name": "supabase project initialised (supabase/config.toml exists)", - "passed": true + "passed": false }, { "name": "todos table is created by a migration file", - "passed": true + "passed": false, + "notes": "supabase/migrations does not exist — was a Supabase project initialised?" }, { "name": "todos table exists with at least 2 seeded rows", - "passed": true, - "notes": "found 2 rows" + "passed": false, + "notes": "could not read DB_URL from `supabase status -o json` after 5 attempts — the local stack must be running. Last status: failed to inspect container health: Error response from daemon: No such container: supabase_db_sandbox-42df329b\nTry rerunning the command with --debug to troubleshoot the error.\n" }, { "name": "row level security is enabled on todos", - "passed": true + "passed": false, + "notes": "could not read DB_URL from `supabase status -o json` after 5 attempts — the local stack must be running. Last status: failed to inspect container health: Error response from daemon: No such container: supabase_db_sandbox-42df329b\nTry rerunning the command with --debug to troubleshoot the error.\n" }, { "name": "a SELECT policy targets the authenticated role", - "passed": true + "passed": false, + "notes": "could not read DB_URL from `supabase status -o json` after 5 attempts — the local stack must be running. Last status: failed to inspect container health: Error response from daemon: No such container: supabase_db_sandbox-42df329b\nTry rerunning the command with --debug to troubleshoot the error.\n" }, { "name": "REST API returns no todos to anonymous requests", - "passed": true, - "notes": "error 42501: permission denied for table todos" + "passed": false, + "notes": "could not read API_URL/PUBLISHABLE_KEY from `supabase status -o json` after 5 attempts — the local stack must be running and include the auth service (status only reports API keys while gotrue is up; add `gotrue` to the eval's services). Last status: failed to inspect container health: Error response from daemon: No such container: supabase_db_sandbox-42df329b\nTry rerunning the command with --debug to troubleshoot the error.\n" }, { "name": "REST API returns the todos to authenticated requests", - "passed": true, - "notes": "2 rows" + "passed": false, + "notes": "could not read API_URL/PUBLISHABLE_KEY from `supabase status -o json` after 5 attempts — the local stack must be running and include the auth service (status only reports API keys while gotrue is up; add `gotrue` to the eval's services). Last status: failed to inspect container health: Error response from daemon: No such container: supabase_db_sandbox-42df329b\nTry rerunning the command with --debug to troubleshoot the error.\n" } ], "skills": { @@ -17884,16 +17887,14 @@ "supabase", "supabase-postgres-best-practices" ], - "loaded": [ - "supabase" - ] + "loaded": [] }, "docs": { "calls": [] }, "prompt": "We're kicking off a todos app and I want the Supabase side ready for the team\nto build on. Set it up the way we'd run it in development, with schema changes\ntracked as migrations so they can be reviewed and replayed.\n\nFor the first slice we just need a `todos` table. Todos aren't public: anyone\nsigned in can read all of them, but nothing should be writable through the API\nfor now. Add a couple of sample todos so there's something to look at.\n\nBefore you hand it back, make sure the running API actually behaves that way —\nsigned-in users get the todos, signed-out requests get nothing.", "promptSourcePath": "evals/build-cli-001-bootstrap-app/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "opencode-kimi-k3/build-cli-001-bootstrap-app.json" }, { @@ -17915,23 +17916,25 @@ ], "suite": "benchmark", "interface": "cli", - "passed": true, + "passed": false, "checks": [ { "name": "supabase db diff used to generate the migration", - "passed": true + "passed": false }, { "name": "schema file updated to include description column", - "passed": true + "passed": false, + "notes": "description not found in any schema file" }, { "name": "a new migration was generated for the change", - "passed": true + "passed": false, + "notes": "found 1 migration file(s)" }, { "name": "description column exists in the live database", - "passed": true + "passed": false } ], "skills": { @@ -17939,16 +17942,14 @@ "supabase", "supabase-postgres-best-practices" ], - "loaded": [ - "supabase" - ] + "loaded": [] }, "docs": { "calls": [] }, "prompt": "Add a description text column to the `products` table in my local Supabase stack", "promptSourcePath": "evals/build-cli-002-declarative-schema/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "opencode-kimi-k3/build-cli-002-declarative-schema.json" }, { @@ -17973,22 +17974,22 @@ ], "suite": "benchmark", "interface": "cli", - "passed": true, + "passed": false, "checks": [ { "name": "pg_cron job 'enqueue-tasks' scheduled to run every minute", - "passed": true, - "notes": "schedule='* * * * *', active=true" + "passed": false, + "notes": "job not found in cron.job" }, { "name": "cron command enqueues to the 'tasks' queue", - "passed": true, - "notes": "queue depth 1 -> 2" + "passed": false, + "notes": "job not found, so its command can't run" }, { "name": "process-tasks function drains the queue", - "passed": true, - "notes": "function removed the seeded message (id 3) from the queue" + "passed": false, + "notes": "couldn't enqueue a test message. Is the 'tasks' queue created? query failed: ERROR: relation \"pgmq.q_tasks\" does not exist\nLINE 2: INSERT INTO pgmq.q_tasks (vt, message, headers)\n ^\nQUERY: \n INSERT INTO pgmq.q_tasks (vt, message, headers)\n VALUES ($2, $1, $3)\n RETURNING msg_id;\n \nCONTEXT: PL/pgSQL function pgmq.send(text,jsonb,jsonb,timestamp with time zone) line 14 at RETURN QUERY\nSQL function \"send\" statement 1\n" } ], "skills": { @@ -17996,102 +17997,14 @@ "supabase", "supabase-postgres-best-practices" ], - "loaded": [ - "supabase" - ] + "loaded": [] }, "docs": { - "calls": [ - { - "source": "web_fetch", - "query": "https://supabase.com/changelog.md", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 92585 - }, - { - "source": "web_fetch", - "query": "https://supabase.com/docs/guides/queues/quickstart.md", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/queues/quickstart.md" - } - ], - "resultChars": 8775 - }, - { - "source": "web_fetch", - "query": "https://supabase.com/docs/guides/cron.md", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/cron.md" - } - ], - "resultChars": 1405 - }, - { - "source": "web_fetch", - "query": "https://supabase.com/docs/guides/cron/quickstart.md", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/cron/quickstart.md" - } - ], - "resultChars": 6155 - }, - { - "source": "web_fetch", - "query": "https://supabase.com/docs/guides/functions/examples/pgmq-serverless-workers.md", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/examples/pgmq-serverless-workers.md" - } - ] - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"consume queue messages with edge function pgmq pop worker\", limit: 6) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/queues/consuming-messages-with-edge-functions", - "title": "Consuming Supabase Queue Messages with Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", - "title": "Automatic embeddings" - }, - { - "url": "https://supabase.com/docs/guides/queues/pgmq", - "title": "PGMQ Extension" - } - ], - "resultChars": 55604 - }, - { - "source": "web_fetch", - "query": "https://supabase.com/docs/guides/queues/consuming-messages-with-edge-functions.md", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/queues/consuming-messages-with-edge-functions.md" - } - ], - "resultChars": 3873 - } - ] + "calls": [] }, "prompt": "I want to set up a recurring background workflow on my local Supabase stack.\n\nCan you set up a cron job called `enqueue-tasks` to run every minute and push a task into a queue called `tasks`? Then add a `process-tasks` edge function that reads messages off the `tasks` queue and removes them, so a scheduled worker can keep the backlog drained.", "promptSourcePath": "evals/build-cli-003-pg-cron-queue-workflow/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "opencode-kimi-k3/build-cli-003-pg-cron-queue-workflow.json" }, { @@ -18112,27 +18025,12 @@ ], "suite": "benchmark", "interface": "cli", - "passed": true, + "passed": false, "checks": [ { - "name": "all 3 tables exist (teams, members, tasks)", - "passed": true - }, - { - "name": "row counts match (teams=5, members=10, tasks=13)", - "passed": true - }, - { - "name": "foreign key constraints survived the restore", - "passed": true - }, - { - "name": "tasks_team_status_idx index survived the restore", - "passed": true - }, - { - "name": "sequences synced (next insert won't conflict with existing IDs)", - "passed": true + "name": "scorer completed without errors", + "passed": false, + "notes": "could not read DB_URL from `supabase status -o json` after 5 attempts — the local stack must be running. Last status: failed to inspect container health: Error response from daemon: No such container: supabase_db_sandbox-c8c3c29c\nTry rerunning the command with --debug to troubleshoot the error.\n" } ], "skills": { @@ -18140,16 +18038,14 @@ "supabase", "supabase-postgres-best-practices" ], - "loaded": [ - "supabase" - ] + "loaded": [] }, "docs": { "calls": [] }, "prompt": "I have an existing Postgres database I want to migrate to Supabase. There's a binary dump at `source.dump` in the current directory.\n\nCan you set up a local Supabase project and restore the dump into it?", "promptSourcePath": "evals/build-database-001-migrate-postgres-to-supabase/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "opencode-kimi-k3/build-database-001-migrate-postgres-to-supabase.json" }, { @@ -18174,7 +18070,7 @@ ], "suite": "benchmark", "interface": "mcp", - "passed": true, + "passed": false, "checks": [ { "name": "rejects missing auth", @@ -18188,17 +18084,17 @@ }, { "name": "reads only with the caller's JWT", - "passed": true, - "notes": "bearer_tokens=2, all_match=true" + "passed": false, + "notes": "bearer_tokens=1, all_match=false" }, { "name": "user A cannot force-read user B note", - "passed": true, + "passed": false, "notes": "status=200" }, { "name": "user B cannot force-read user A note", - "passed": true, + "passed": false, "notes": "status=200" } ], @@ -18207,56 +18103,14 @@ "supabase", "supabase-postgres-best-practices" ], - "loaded": [ - "supabase" - ] + "loaded": [] }, "docs": { - "calls": [ - { - "source": "web_fetch", - "query": "https://supabase.com/changelog.md", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 92585 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"edge function get user from JWT authorization header RLS\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/why-is-my-service-role-key-client-getting-rls-errors-or-not-returning-data-7_1K9z", - "title": "Why is my service role key client getting RLS errors or not returning data?" - }, - { - "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security", - "title": "Token Security and Row Level Security" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/why-is-my-select-returning-an-empty-data-array-and-i-have-data-in-the-table-xvOPgx", - "title": "Why is my select returning an empty data array and I have data in the table?" - } - ], - "resultChars": 29869 - } - ] + "calls": [] }, "prompt": "I built an Edge Function called `private-notes` for showing a user's saved\nprivate notes.\n\nCan you check whether there's any way one user could see another user's notes?\n\nPlease fix and deploy it if needed.", "promptSourcePath": "evals/build-functions-004-service-role-bypass/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "opencode-kimi-k3/build-functions-004-service-role-bypass.json" }, { @@ -18282,7 +18136,7 @@ "suite": "benchmark", "interface": "cli", "cliVersion": "2.109.1", - "passed": true, + "passed": false, "checks": [ { "name": "seed rows present", @@ -18292,42 +18146,42 @@ { "name": "rejects request with no credentials", "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" + "notes": "status 404: Function not found" }, { "name": "user with JWT reads only their own rows", - "passed": true, - "notes": "status 200: [{\"user_id\":\"17717159-30ef-4e5f-bf1b-40f3b4dae017\",\"metric\":\"steps_a_mrzm2gwa\",\"value\":111}]" + "passed": false, + "notes": "status 404: Function not found" }, { "name": "user cannot read another user's rows by passing user_id", - "passed": true, - "notes": "status 200: [{\"user_id\":\"17717159-30ef-4e5f-bf1b-40f3b4dae017\",\"metric\":\"steps_a_mrzm2gwa\",\"value\":111}]" + "passed": false, + "notes": "status 404: Function not found" }, { "name": "service key bypasses RLS to read the target user's rows", - "passed": true, - "notes": "status 200: [{\"user_id\":\"9513ef26-e76f-4e48-9439-ef8ed31563ae\",\"metric\":\"steps_b_mrzm2gwa\",\"value\":222}]" + "passed": false, + "notes": "status 404: Function not found" }, { "name": "non-service key is not granted service access", "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" + "notes": "status 404: Function not found" }, { "name": "rejects an unverified (forged) user token", "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" + "notes": "status 404: Function not found" }, { "name": "a user token in the apikey slot is not treated as the service key", "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" + "notes": "status 404: Function not found" }, { "name": "implementation uses @supabase/server", - "passed": true, - "notes": "imports @supabase/server / withSupabase" + "passed": false, + "notes": "could not locate function source to inspect" } ], "skills": { @@ -18335,84 +18189,14 @@ "supabase", "supabase-postgres-best-practices" ], - "loaded": [ - "supabase" - ] + "loaded": [] }, "docs": { - "calls": [ - { - "source": "web_fetch", - "query": "https://supabase.com/changelog.md", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 92585 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"edge function verify_jwt config.toml skip JWT verification\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/function-configuration", - "title": "Function Configuration" - }, - { - "url": "https://supabase.com/docs/guides/functions/deploy", - "title": "Deploy to Production" - }, - { - "url": "https://supabase.com/docs/guides/functions/development-tips", - "title": "Development tips" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth-headers", - "title": "Authorization headers" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" - } - ], - "resultChars": 25625 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"edge functions environment variables SUPABASE_SECRET_KEYS publishable secret api keys\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/functions/secrets", - "title": "Environment Variables" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", - "title": "Self-Hosted Functions" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker", - "title": "Self-Hosting with Docker" - } - ], - "resultChars": 67435 - } - ] + "calls": [] }, "prompt": "Build and serve a Supabase Edge Function named `user-stats` for this project,\nreachable over HTTP at `/functions/v1/user-stats`.\n\nOur product stores per-user metrics in the existing `user_stats` table.\n\nTwo very different callers need to hit this one endpoint:\n\n1. **Our mobile app**, acting for a signed-in user. It sends that user's\n Supabase access token. The endpoint should return the user's own stats.\n\n2. **Our internal billing service**, a trusted backend with no signed-in user.\n It authenticates with the project's secret (service-role) key in the `apikey`\n header, and names the target user with a `user_id` in the JSON request body.\n It needs that user's stats.\n\nReturn the matching rows as JSON. The endpoint must be secure: only ever serve\nstats to a caller that is genuinely entitled to them, and turn away callers that\nare not.", "promptSourcePath": "evals/build-functions-005-dual-auth-user-secret/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "opencode-kimi-k3/build-functions-005-dual-auth-user-secret.json" }, { @@ -18452,7 +18236,7 @@ { "name": "user A lists only own files", "passed": true, - "notes": "saw: 019f968f-d2df-75d7-8566-9d025f9a1cb6/receipt-alpha.pdf, 019f968f-d2df-75d7-8566-9d025f9a1cb6/receipt-beta.pdf" + "notes": "saw: 019fa581-9203-774f-b480-30368ceb3469/receipt-alpha.pdf, 019fa581-9203-774f-b480-30368ceb3469/receipt-beta.pdf" }, { "name": "user B cannot read user A files", @@ -18473,7 +18257,7 @@ { "name": "configured private per-user storage access", "passed": true, - "judgeNotes": "Creates private user-files bucket, owner-scoped SELECT and INSERT policies for authenticated users with RLS left enabled, avoids public/permissive access, and uses createSignedUrl with expiry for temporary sharing." + "judgeNotes": "Creates a private user-files bucket, owner-scoped SELECT and INSERT policies for authenticated users using storage.foldername(name)[1] = auth.uid(), keeps RLS intact, and provides supabase-js createSignedUrl code with expiry. No public bucket, permissive policies, getPublicUrl, or client-side service role usage." } ], "skills": { @@ -18482,8 +18266,7 @@ "supabase-postgres-best-practices" ], "loaded": [ - "supabase", - "supabase-postgres-best-practices" + "supabase" ] }, "docs": { @@ -18501,57 +18284,100 @@ }, { "source": "search_docs", - "query": "{ searchDocs(query: \"storage access control policies bucket private folder user id\", limit: 5) { nodes { title href content } } }", - "hasContent": true, + "query": "{ searchDocs(query: \"storage access control RLS policies bucket private signed URL\", limit: 8) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/storage/buckets/fundamentals", + "title": "Storage Buckets" + }, + { + "url": "https://supabase.com/docs/guides/storage/security/access-control", + "title": "Storage Access Control" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-angular", + "title": "Build a User Management App with Angular" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-ionic-angular", + "title": "Build a User Management App with Ionic Angular" + }, + { + "url": "https://supabase.com/docs/guides/security/product-security", + "title": "Secure configuration of Supabase products" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-expo-react-native", + "title": "Build a User Management App with Expo React Native" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3", + "title": "Configure S3 Storage" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-swift", + "title": "Build a User Management App with Swift and SwiftUI" + } + ], + "resultChars": 138223 + }, + { + "source": "web_fetch", + "query": "https://supabase.com/docs/guides/storage/security/access-control.md", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/storage/security/access-control.md" + } + ], + "resultChars": 4169 + }, + { + "source": "web_fetch", + "query": "https://supabase.com/docs/reference/javascript/storage-from-createsignedurl.md", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/reference/javascript/storage-from-createsignedurl.md" + } + ] + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"createSignedUrl storage javascript reference\", limit: 3) { nodes { title href language methodName } } }", + "hasContent": false, "pages": [ { - "url": "https://supabase.com/docs/guides/storage/security/access-control", - "title": "Storage Access Control" - }, - { - "url": "https://supabase.com/docs/guides/storage/buckets/fundamentals", - "title": "Storage Buckets" - }, - { - "url": "https://supabase.com/docs/guides/storage/cdn/fundamentals", - "title": "Storage CDN" + "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsignedurl" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/why-cant-i-uploadlistetc-my-public-bucket-Z6CmGt", - "title": "Why can't I upload/list/etc my public bucket?" + "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsignedurls" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3", - "title": "Configure S3 Storage" + "url": "https://supabase.com/docs/guides/platform/manage-your-usage/storage-image-transformations", + "title": "Manage Storage Image Transformations usage" } ], - "resultChars": 20134 + "resultChars": 7829 }, { "source": "search_docs", - "query": "{ searchDocs(query: \"createSignedUrl temporary share link storage supabase-js\", limit: 5) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"createSignedUrl storage javascript\", limit: 3) { nodes { title href content ... on ClientLibraryFunctionReference { language methodName } } } }", "hasContent": true, "pages": [ - { - "url": "https://supabase.com/docs/guides/storage/serving/downloads", - "title": "Serving assets from Storage" - }, { "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsignedurl" }, - { - "url": "https://supabase.com/docs/guides/storage/uploads/resumable-uploads", - "title": "Resumable Uploads" - }, { "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsignedurls" }, { - "url": "https://supabase.com/docs/guides/storage/serving/image-transformations", - "title": "Storage Image Transformations" + "url": "https://supabase.com/docs/guides/platform/manage-your-usage/storage-image-transformations", + "title": "Manage Storage Image Transformations usage" } ], - "resultChars": 38982 + "resultChars": 7829 } ] }, @@ -18579,22 +18405,22 @@ ], "suite": "benchmark", "interface": "cli", - "passed": true, + "passed": false, "checks": [ { "name": "pgTAP test file(s) written under supabase/tests/", - "passed": true, - "notes": "1 file(s): supabase/tests/rls_tenant_isolation.sql" + "passed": false, + "notes": "no .sql files found under supabase/tests/" }, { "name": "pgTAP isolation tests ran and pass", - "passed": true, - "notes": "8 passed, 3 failed" + "passed": false, + "notes": "no test summary found; exit 0; output: Connecting to local database...\nFiles=0, Tests=0, 0 wallclock secs ( 0.00 usr + 0.00 sys = 0.00 CPU)\nResult: NOTESTS\nA new version of Supabase CLI is available: v2.110.0 (currently installed v2.67.1)\nWe recommend updating regularly for new features and bug fixes: https://supabase.com/docs/guides/cli/getting-started#updating-the-supabase-cli\n" }, { "name": "agent correctly identifies the posts isolation bug from test results", "passed": true, - "judgeNotes": "The agent correctly identifies `posts` as having the broken tenant isolation policy, specifically that authenticated members can read posts from organizations they are not members of, and grounds this in the pgTAP failure (test 5). It also treats the test results as authoritative and notes that `notes` isolation is correct." + "judgeNotes": "The agent identifies `posts` as the table with the broken tenant isolation policy and grounds the conclusion in the pgTAP test results, noting that authenticated members can read posts from organizations they do not belong to. It does not misattribute the issue to `notes` or dismiss the tests." } ], "skills": { @@ -18602,16 +18428,14 @@ "supabase", "supabase-postgres-best-practices" ], - "loaded": [ - "supabase" - ] + "loaded": [] }, "docs": { "calls": [] }, "prompt": "Can you audit the tenant isolation on our tables? Write some database tests covering both the happy path and the negative case, run them, and share the results.", "promptSourcePath": "evals/build-tests-001-rls-tenant-isolation/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "opencode-kimi-k3/build-tests-001-rls-tenant-isolation.json" }, { @@ -18690,6 +18514,34 @@ } ], "resultChars": 92585 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"semantic search pgvector match documents function gte-small embedding dimensions\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/ai/semantic-search", + "title": "Semantic search" + }, + { + "url": "https://supabase.com/docs/guides/ai/vector-columns", + "title": "Vector columns" + }, + { + "url": "https://supabase.com/docs/guides/functions/examples/semantic-search", + "title": "Semantic Search" + }, + { + "url": "https://supabase.com/docs/guides/ai/hybrid-search", + "title": "Hybrid search" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pgvector", + "title": "pgvector: Embeddings and vector similarity" + } + ], + "resultChars": 45321 } ] }, @@ -18715,7 +18567,7 @@ "observability" ], "suite": "benchmark", - "passed": true, + "passed": false, "checks": [ { "name": "preserved existing app scrape job", @@ -18723,13 +18575,13 @@ }, { "name": "configured the Supabase Metrics API scrape correctly", - "passed": true, - "judgeNotes": "Prometheus config preserves the app job and adds a Supabase scrape over HTTPS to .supabase.co:443 with /customer/v1/privileged/metrics and HTTP Basic Auth using password_file. docker-compose mounts the secrets directory containing that password file into the Prometheus container." + "passed": false, + "judgeNotes": "prometheus.yml only contains the existing app scrape. It does not add a Supabase Metrics API scrape target, HTTPS scheme, /customer/v1/privileged/metrics path, HTTP Basic Auth with password_file, or docker-compose secret/volume wiring for that password file." }, { "name": "documented live deployment and verification steps", - "passed": true, - "judgeNotes": "README includes live setup steps for project ref, creating a Secret API key, placing it in the matching secrets file, and reloading/restarting Prometheus/Compose. Verification is concrete via Prometheus targets, direct metrics curl, and Grafana dashboard." + "passed": false, + "judgeNotes": "README only shows starting the stack and lists the target. It lacks Secret API key creation, matching secret file placement, restart/reload instructions, and concrete verification via Prometheus targets, PromQL/Grafana, or equivalent." } ], "skills": { @@ -18737,83 +18589,14 @@ "supabase", "supabase-postgres-best-practices" ], - "loaded": [ - "supabase" - ] + "loaded": [] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{ searchDocs(query: \"metrics endpoint prometheus observability export metrics\") { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/telemetry/metrics", - "title": "Metrics API" - }, - { - "url": "https://supabase.com/docs/guides/telemetry/metrics/grafana-self-hosted", - "title": "Metrics API with Prometheus & Grafana (self-hosted)" - }, - { - "url": "https://supabase.com/docs/guides/telemetry/metrics/vendor-agnostic", - "title": "Vendor-agnostic Metrics API setup" - }, - { - "url": "https://supabase.com/docs/guides/telemetry/metrics/grafana-cloud", - "title": "Metrics API with Grafana Cloud" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/how-to-view-database-metrics-uqf2z_", - "title": "How to View Database Metrics" - }, - { - "url": "https://supabase.com/docs/guides/platform/read-replicas", - "title": "Read Replicas" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/data-deletion", - "title": "Deleting data and dropping objects safely" - }, - { - "url": "https://supabase.com/docs/guides/telemetry/reports", - "title": "Reports" - }, - { - "url": "https://supabase.com/docs/guides/database/replication/pipelines-monitoring", - "title": "Monitor pipeline status" - } - ], - "resultChars": 90527 - }, - { - "source": "web_fetch", - "query": "https://supabase.com/docs/guides/telemetry/metrics/grafana-self-hosted.md", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/telemetry/metrics/grafana-self-hosted.md" - } - ], - "resultChars": 3752 - }, - { - "source": "web_fetch", - "query": "https://supabase.com/docs/guides/telemetry/metrics.md", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/telemetry/metrics.md" - } - ], - "resultChars": 4778 - } - ] + "calls": [] }, "prompt": "Can you wire my Supabase project metrics into our existing observability stack and document\nwhat we need to do to make the config live?", "promptSourcePath": "evals/deploy-database-001-prometheus-metrics/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "opencode-kimi-k3/deploy-database-001-prometheus-metrics.json" }, { @@ -18933,7 +18716,41 @@ ] }, "docs": { - "calls": [] + "calls": [ + { + "source": "web_fetch", + "query": "https://supabase.com/changelog.md", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 5943 + }, + { + "source": "web_fetch", + "query": "https://supabase.com/changelog/48048-self-hosted-supabase-envoy-becomes-the-default-api-gateway-b", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/changelog/48048-self-hosted-supabase-envoy-becomes-the-default-api-gateway-b" + } + ], + "resultChars": 5943 + }, + { + "source": "web_fetch", + "query": "https://supabase.com/docs/guides/self-hosting/docker.md", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/self-hosting/docker.md" + } + ], + "resultChars": 30210 + } + ] }, "prompt": "I'm moving off the hosted Supabase and running the whole thing myself on a VPS I\njust spun up. Can you get a Docker setup ready for me to copy onto the box?\n\nI don't need it running here, I'll do the actual bring-up once I'm on the\nserver. I just want everything in place and the secrets set up. Put it in a `supabase-docker/`\nfolder at the repo root so I can scp the whole thing across in one go.", "promptSourcePath": "evals/deploy-self-hosting-001-docker-compose/PROMPT.md", @@ -18989,7 +18806,7 @@ { "name": "diagnosed and explained session revocation", "passed": true, - "judgeNotes": "Meets rubric: diagnoses soft-delete-only root cause, implements deletion of auth user with session/refresh-token revocation via cascades, accurately notes remaining stateless access-token window until expiry with mitigations, and correctly distinguishes publishable frontend/RLS behavior from secret server-only/RLS-bypass behavior." + "judgeNotes": "The answer diagnoses the soft-delete-only bug, replaces it with deletion of auth.users causing cascade removal of sessions/refresh tokens/identities/profile, and tests it. It also explains JWT access tokens remain valid until expiry for local validation and distinguishes publishable vs secret keys correctly in the surrounding response implied by the transcript." } ], "skills": { @@ -19003,33 +18820,42 @@ }, "docs": { "calls": [ + { + "source": "web_fetch", + "query": "https://supabase.com/changelog.md", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 92585 + }, { "source": "search_docs", - "query": "{ searchDocs(query: \"publishable key secret key migrate from anon service_role API keys RLS\", limit: 5) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"delete user account self service RPC function auth.users security definer\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" + "url": "https://supabase.com/docs/reference/javascript/auth-admin-deleteuser" }, { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-redwoodjs", - "title": "Build a User Management App with RedwoodJS" + "url": "https://supabase.com/docs/reference/swift/auth-admin-deleteuser", + "title": "deleteUser()" }, { - "url": "https://supabase.com/docs/guides/auth/quickstarts/with-expo-react-native-social-auth", - "title": "Build a Social Auth App with Expo React Native" + "url": "https://supabase.com/docs/guides/auth/managing-user-data", + "title": "User Management" }, { - "url": "https://supabase.com/docs/guides/auth/signing-keys", - "title": "JWT Signing Keys" + "url": "https://supabase.com/docs/reference/dart/auth-admin-deleteuser" }, { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-sveltekit", - "title": "Build a User Management App with SvelteKit" + "url": "https://supabase.com/docs/reference/python/auth-admin-deleteuser", + "title": "delete_user()" } ], - "resultChars": 169611 + "resultChars": 7953 } ] }, @@ -19083,7 +18909,7 @@ { "name": "diagnosed missing publication membership", "passed": true, - "judgeNotes": "The assistant correctly identified the root cause as orders missing from the supabase_realtime publication, added public.orders to the existing publication, and did not weaken RLS/policies or disrupt courier_locations. It did include verification inserts/log checks, but the core fix is exactly the required publication change." + "judgeNotes": "The assistant correctly identified the missing orders table in the supabase_realtime publication as the root cause, applied ALTER PUBLICATION supabase_realtime ADD TABLE public.orders, preserved existing courier_locations publication membership, and did not weaken RLS/policies or blame client/networking." } ], "skills": { @@ -19126,17 +18952,17 @@ { "name": "identified image-transform and the recurring 503 pattern", "passed": true, - "judgeNotes": "Assistant identified image-transform as the affected function and described the recurring HTTP 503 pattern across the morning of 2026-04-28, covering the 8 gateway failures from 07:00Z to 12:00Z. It also correctly distinguished the older billing-webhook 503s as unrelated." + "judgeNotes": "The assistant explicitly identified `image-transform` as the affected function and described recurring 503 responses roughly every 30 minutes across 07:00–12:00, matching the required morning pattern." }, { "name": "attributed recurring 503s to gateway/platform layer, not function code", "passed": false, - "judgeNotes": "The assistant notes gateway 503s with no execution records, but ultimately attributes the cause to the function's unpinned dependency/boot behavior and recommends pinning/redeploying the function, explicitly saying it is not a platform-side incident. This fails the required gateway/platform-layer attribution." + "judgeNotes": "The assistant attributes the 503s to the `image-transform` Edge Function returning errors and proceeds to inspect function code, rather than identifying the gateway/platform layer as the source. It does not ground a platform-layer attribution in observations like missing invocation/runtime rows or unchanged deployment version." }, { "name": "recommended a concrete next step", "passed": true, - "judgeNotes": "Recommended concrete next steps: pin/roll back dependencies and redeploy, review upstream package publish history, rotate secrets if compromised, and add alerting." + "judgeNotes": "The assistant identified a specific failure pattern and proposed concrete next steps: checking Edge Function logs and inspecting the `image-transform` function code, rather than giving only vague advice." } ], "skills": { @@ -19149,30 +18975,7 @@ ] }, "docs": { - "calls": [ - { - "source": "web_fetch", - "query": "https://supabase.com/changelog.md", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 92585 - }, - { - "source": "web_fetch", - "query": "https://supabase.com/docs/guides/security/npm-security.md", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/security/npm-security.md" - } - ], - "resultChars": 17839 - } - ] + "calls": [] }, "prompt": "Users have been reporting that image uploads are intermittently failing with server errors throughout the morning of `2026-04-28`.\n\nCan you investigate the project logs and tell me what is going on and what we\nshould do next?", "promptSourcePath": "evals/investigate-reliability-003-edge-function-5xx-correlation/PROMPT.md", @@ -19200,7 +19003,7 @@ ], "suite": "benchmark", "interface": "mcp", - "passed": true, + "passed": false, "checks": [ { "name": "RLS still enabled on bookmarks", @@ -19208,7 +19011,7 @@ }, { "name": "user A reads own bookmarks", - "passed": true + "passed": false }, { "name": "user B cannot read user A bookmarks", @@ -19220,7 +19023,8 @@ }, { "name": "user A can save a new bookmark", - "passed": true + "passed": false, + "notes": "new row violates row-level security policy for table \"bookmarks\"" }, { "name": "user B cannot insert a bookmark as user A", @@ -19229,7 +19033,7 @@ { "name": "diagnosed RLS and added owner-scoped policies", "passed": true, - "judgeNotes": "Diagnosed RLS enabled with no policies causing deny-all Data API results, kept RLS enabled, and created authenticated-only owner-scoped SELECT and INSERT policies using auth.uid() with WITH CHECK for inserts." + "judgeNotes": "The assistant correctly identified the issue as RLS being enabled with no policies, preserving deny-by-default behavior. It proposed owner-scoped authenticated SELECT and INSERT policies using user_id = auth.uid(), with INSERT enforced via WITH CHECK, and kept RLS enabled." } ], "skills": { @@ -19237,16 +19041,14 @@ "supabase", "supabase-postgres-best-practices" ], - "loaded": [ - "supabase" - ] + "loaded": [] }, "docs": { "calls": [] }, "prompt": "Our app lets signed-in users save bookmarks and view them on their dashboard. Bookmarks are stored in the `bookmarks` table and are private — a user must only ever see their own. \nUsers also need to be able to save new bookmarks from the app.\n\nI can see the rows when I query the table directly, but the dashboard shows an empty list for every user.\n\nFind out why the Data API returns nothing and fix it.", "promptSourcePath": "evals/resolve-dataapi-001-empty-results/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "opencode-kimi-k3/resolve-dataapi-001-empty-results.json" }, { @@ -19267,23 +19069,27 @@ ], "suite": "benchmark", "interface": "cli", - "passed": true, + "passed": false, "checks": [ { "name": "the avatar_url column is applied on the hosted profiles table", - "passed": true + "passed": false, + "notes": "avatar_url not found on public.profiles" }, { "name": "migration 20240220000000 is recorded in the remote history", - "passed": true + "passed": false, + "notes": "remote history versions: [\"20240101000000\",\"20240115000000\"]" }, { "name": "remote migration history matches local migration files", - "passed": true + "passed": false, + "notes": "remote-only versions still unreconciled: [\"20240115000000\"] (remote: [\"20240101000000\",\"20240115000000\"], local: [\"20240101000000\",\"20240220000000\"])" }, { "name": "local migrations are a valid reconciled sequence", - "passed": true + "passed": false, + "notes": "expected 20240101000000_create_profiles → → 20240220000000_add_avatar_url, strictly ascending; got [20240101000000_create_profiles.sql, 20240220000000_add_avatar_url.sql]" }, { "name": "production profile data is intact (not reset)", @@ -19291,8 +19097,8 @@ }, { "name": "the avatar migration and history reconciliation were done via the Supabase CLI", - "passed": true, - "judgeNotes": "Applied avatar_url via `supabase db push` in #18, which showed `Applying migration 20240220000000_add_avatar_url.sql...` and `Finished supabase db push.` Reconciled the orphan bio migration by adding the missing local file `supabase/migrations/20240115000000_add_profile_bio.sql` in #16, after which migration list aligned and `db push` succeeded. No disallowed workaround or direct remote mutation was used." + "passed": false, + "notes": "no agent actions recorded to evaluate" } ], "skills": { @@ -19300,16 +19106,14 @@ "supabase", "supabase-postgres-best-practices" ], - "loaded": [ - "supabase" - ] + "loaded": [] }, "docs": { "calls": [] }, "prompt": "I'm trying to ship a migration to our hosted project and it's not working. Can you figure out what's wrong and get it deployed?", "promptSourcePath": "evals/resolve-database-001-migration-history-mismatch/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "opencode-kimi-k3/resolve-database-001-migration-history-mismatch.json" }, { From 36b20d8eebd56be95d3dfbdbf6a026b2cadc7d29 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 28 Jul 2026 01:18:27 +0000 Subject: [PATCH 15/17] chore: refresh eval results --- apps/web/src/data/eval-results.json | 664 +++++++++++++++++----------- 1 file changed, 407 insertions(+), 257 deletions(-) diff --git a/apps/web/src/data/eval-results.json b/apps/web/src/data/eval-results.json index a71ce0d3..3214f085 100644 --- a/apps/web/src/data/eval-results.json +++ b/apps/web/src/data/eval-results.json @@ -17845,41 +17845,38 @@ ], "suite": "benchmark", "interface": "cli", - "passed": false, + "passed": true, "checks": [ { "name": "supabase project initialised (supabase/config.toml exists)", - "passed": false + "passed": true }, { "name": "todos table is created by a migration file", - "passed": false, - "notes": "supabase/migrations does not exist — was a Supabase project initialised?" + "passed": true }, { "name": "todos table exists with at least 2 seeded rows", - "passed": false, - "notes": "could not read DB_URL from `supabase status -o json` after 5 attempts — the local stack must be running. Last status: failed to inspect container health: Error response from daemon: No such container: supabase_db_sandbox-42df329b\nTry rerunning the command with --debug to troubleshoot the error.\n" + "passed": true, + "notes": "found 3 rows" }, { "name": "row level security is enabled on todos", - "passed": false, - "notes": "could not read DB_URL from `supabase status -o json` after 5 attempts — the local stack must be running. Last status: failed to inspect container health: Error response from daemon: No such container: supabase_db_sandbox-42df329b\nTry rerunning the command with --debug to troubleshoot the error.\n" + "passed": true }, { "name": "a SELECT policy targets the authenticated role", - "passed": false, - "notes": "could not read DB_URL from `supabase status -o json` after 5 attempts — the local stack must be running. Last status: failed to inspect container health: Error response from daemon: No such container: supabase_db_sandbox-42df329b\nTry rerunning the command with --debug to troubleshoot the error.\n" + "passed": true }, { "name": "REST API returns no todos to anonymous requests", - "passed": false, - "notes": "could not read API_URL/PUBLISHABLE_KEY from `supabase status -o json` after 5 attempts — the local stack must be running and include the auth service (status only reports API keys while gotrue is up; add `gotrue` to the eval's services). Last status: failed to inspect container health: Error response from daemon: No such container: supabase_db_sandbox-42df329b\nTry rerunning the command with --debug to troubleshoot the error.\n" + "passed": true, + "notes": "error 42501: permission denied for table todos" }, { "name": "REST API returns the todos to authenticated requests", - "passed": false, - "notes": "could not read API_URL/PUBLISHABLE_KEY from `supabase status -o json` after 5 attempts — the local stack must be running and include the auth service (status only reports API keys while gotrue is up; add `gotrue` to the eval's services). Last status: failed to inspect container health: Error response from daemon: No such container: supabase_db_sandbox-42df329b\nTry rerunning the command with --debug to troubleshoot the error.\n" + "passed": true, + "notes": "3 rows" } ], "skills": { @@ -17887,14 +17884,28 @@ "supabase", "supabase-postgres-best-practices" ], - "loaded": [] + "loaded": [ + "supabase" + ] }, "docs": { - "calls": [] + "calls": [ + { + "source": "web_fetch", + "query": "https://supabase.com/changelog.md", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 92585 + } + ] }, "prompt": "We're kicking off a todos app and I want the Supabase side ready for the team\nto build on. Set it up the way we'd run it in development, with schema changes\ntracked as migrations so they can be reviewed and replayed.\n\nFor the first slice we just need a `todos` table. Todos aren't public: anyone\nsigned in can read all of them, but nothing should be writable through the API\nfor now. Add a couple of sample todos so there's something to look at.\n\nBefore you hand it back, make sure the running API actually behaves that way —\nsigned-in users get the todos, signed-out requests get nothing.", "promptSourcePath": "evals/build-cli-001-bootstrap-app/PROMPT.md", - "attempts": 2, + "attempts": 1, "sourcePath": "opencode-kimi-k3/build-cli-001-bootstrap-app.json" }, { @@ -17916,25 +17927,23 @@ ], "suite": "benchmark", "interface": "cli", - "passed": false, + "passed": true, "checks": [ { "name": "supabase db diff used to generate the migration", - "passed": false + "passed": true }, { "name": "schema file updated to include description column", - "passed": false, - "notes": "description not found in any schema file" + "passed": true }, { "name": "a new migration was generated for the change", - "passed": false, - "notes": "found 1 migration file(s)" + "passed": true }, { "name": "description column exists in the live database", - "passed": false + "passed": true } ], "skills": { @@ -17942,14 +17951,16 @@ "supabase", "supabase-postgres-best-practices" ], - "loaded": [] + "loaded": [ + "supabase" + ] }, "docs": { "calls": [] }, "prompt": "Add a description text column to the `products` table in my local Supabase stack", "promptSourcePath": "evals/build-cli-002-declarative-schema/PROMPT.md", - "attempts": 2, + "attempts": 1, "sourcePath": "opencode-kimi-k3/build-cli-002-declarative-schema.json" }, { @@ -17974,22 +17985,22 @@ ], "suite": "benchmark", "interface": "cli", - "passed": false, + "passed": true, "checks": [ { "name": "pg_cron job 'enqueue-tasks' scheduled to run every minute", - "passed": false, - "notes": "job not found in cron.job" + "passed": true, + "notes": "schedule='* * * * *', active=true" }, { "name": "cron command enqueues to the 'tasks' queue", - "passed": false, - "notes": "job not found, so its command can't run" + "passed": true, + "notes": "queue depth 0 -> 1" }, { "name": "process-tasks function drains the queue", - "passed": false, - "notes": "couldn't enqueue a test message. Is the 'tasks' queue created? query failed: ERROR: relation \"pgmq.q_tasks\" does not exist\nLINE 2: INSERT INTO pgmq.q_tasks (vt, message, headers)\n ^\nQUERY: \n INSERT INTO pgmq.q_tasks (vt, message, headers)\n VALUES ($2, $1, $3)\n RETURNING msg_id;\n \nCONTEXT: PL/pgSQL function pgmq.send(text,jsonb,jsonb,timestamp with time zone) line 14 at RETURN QUERY\nSQL function \"send\" statement 1\n" + "passed": true, + "notes": "function removed the seeded message (id 39) from the queue" } ], "skills": { @@ -17997,14 +18008,57 @@ "supabase", "supabase-postgres-best-practices" ], - "loaded": [] + "loaded": [ + "supabase" + ] }, "docs": { - "calls": [] + "calls": [ + { + "source": "search_docs", + "query": "{\n searchDocs(query: \"queues edge function consume messages pgmq_public pop read delete\", limit: 3) {\n nodes {\n title\n href\n content\n }\n }\n}", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/queues/consuming-messages-with-edge-functions", + "title": "Consuming Supabase Queue Messages with Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/queues/pgmq", + "title": "PGMQ Extension" + }, + { + "url": "https://supabase.com/docs/guides/queues/api", + "title": "API" + } + ], + "resultChars": 27216 + }, + { + "source": "search_docs", + "query": "{\n searchDocs(query: \"pg_cron schedule job every minute pgmq send queue\", limit: 3) {\n nodes {\n title\n href\n content\n }\n }\n}", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_net", + "title": "pg_net: Async Networking" + }, + { + "url": "https://supabase.com/docs/guides/queues/pgmq", + "title": "PGMQ Extension" + }, + { + "url": "https://supabase.com/docs/guides/cron", + "title": "Cron" + } + ], + "resultChars": 35611 + } + ] }, "prompt": "I want to set up a recurring background workflow on my local Supabase stack.\n\nCan you set up a cron job called `enqueue-tasks` to run every minute and push a task into a queue called `tasks`? Then add a `process-tasks` edge function that reads messages off the `tasks` queue and removes them, so a scheduled worker can keep the backlog drained.", "promptSourcePath": "evals/build-cli-003-pg-cron-queue-workflow/PROMPT.md", - "attempts": 2, + "attempts": 1, "sourcePath": "opencode-kimi-k3/build-cli-003-pg-cron-queue-workflow.json" }, { @@ -18025,12 +18079,27 @@ ], "suite": "benchmark", "interface": "cli", - "passed": false, + "passed": true, "checks": [ { - "name": "scorer completed without errors", - "passed": false, - "notes": "could not read DB_URL from `supabase status -o json` after 5 attempts — the local stack must be running. Last status: failed to inspect container health: Error response from daemon: No such container: supabase_db_sandbox-c8c3c29c\nTry rerunning the command with --debug to troubleshoot the error.\n" + "name": "all 3 tables exist (teams, members, tasks)", + "passed": true + }, + { + "name": "row counts match (teams=5, members=10, tasks=13)", + "passed": true + }, + { + "name": "foreign key constraints survived the restore", + "passed": true + }, + { + "name": "tasks_team_status_idx index survived the restore", + "passed": true + }, + { + "name": "sequences synced (next insert won't conflict with existing IDs)", + "passed": true } ], "skills": { @@ -18038,14 +18107,16 @@ "supabase", "supabase-postgres-best-practices" ], - "loaded": [] + "loaded": [ + "supabase" + ] }, "docs": { "calls": [] }, "prompt": "I have an existing Postgres database I want to migrate to Supabase. There's a binary dump at `source.dump` in the current directory.\n\nCan you set up a local Supabase project and restore the dump into it?", "promptSourcePath": "evals/build-database-001-migrate-postgres-to-supabase/PROMPT.md", - "attempts": 2, + "attempts": 1, "sourcePath": "opencode-kimi-k3/build-database-001-migrate-postgres-to-supabase.json" }, { @@ -18070,7 +18141,7 @@ ], "suite": "benchmark", "interface": "mcp", - "passed": false, + "passed": true, "checks": [ { "name": "rejects missing auth", @@ -18084,17 +18155,17 @@ }, { "name": "reads only with the caller's JWT", - "passed": false, - "notes": "bearer_tokens=1, all_match=false" + "passed": true, + "notes": "bearer_tokens=2, all_match=true" }, { "name": "user A cannot force-read user B note", - "passed": false, + "passed": true, "notes": "status=200" }, { "name": "user B cannot force-read user A note", - "passed": false, + "passed": true, "notes": "status=200" } ], @@ -18103,14 +18174,56 @@ "supabase", "supabase-postgres-best-practices" ], - "loaded": [] + "loaded": [ + "supabase" + ] }, "docs": { - "calls": [] + "calls": [ + { + "source": "web_fetch", + "query": "https://supabase.com/changelog.md", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 92585 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"edge function get authenticated user from JWT Authorization header\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/functions/auth", + "title": "Securing Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", + "title": "Integrating With Supabase Auth" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth-headers", + "title": "Authorization headers" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" + }, + { + "url": "https://supabase.com/docs/guides/integrations/supabase-for-platforms", + "title": "Supabase for Platforms" + } + ], + "resultChars": 48324 + } + ] }, "prompt": "I built an Edge Function called `private-notes` for showing a user's saved\nprivate notes.\n\nCan you check whether there's any way one user could see another user's notes?\n\nPlease fix and deploy it if needed.", "promptSourcePath": "evals/build-functions-004-service-role-bypass/PROMPT.md", - "attempts": 2, + "attempts": 1, "sourcePath": "opencode-kimi-k3/build-functions-004-service-role-bypass.json" }, { @@ -18136,7 +18249,7 @@ "suite": "benchmark", "interface": "cli", "cliVersion": "2.109.1", - "passed": false, + "passed": true, "checks": [ { "name": "seed rows present", @@ -18146,42 +18259,42 @@ { "name": "rejects request with no credentials", "passed": true, - "notes": "status 404: Function not found" + "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" }, { "name": "user with JWT reads only their own rows", - "passed": false, - "notes": "status 404: Function not found" + "passed": true, + "notes": "status 200: [{\"user_id\":\"6c9a5488-c295-4194-ae70-c5b67aee703c\",\"metric\":\"steps_a_ms3ynfu9\",\"value\":111}]" }, { "name": "user cannot read another user's rows by passing user_id", - "passed": false, - "notes": "status 404: Function not found" + "passed": true, + "notes": "status 200: [{\"user_id\":\"6c9a5488-c295-4194-ae70-c5b67aee703c\",\"metric\":\"steps_a_ms3ynfu9\",\"value\":111}]" }, { "name": "service key bypasses RLS to read the target user's rows", - "passed": false, - "notes": "status 404: Function not found" + "passed": true, + "notes": "status 200: [{\"user_id\":\"60a5150a-5a94-4242-9ad7-c80655fd6fc6\",\"metric\":\"steps_b_ms3ynfu9\",\"value\":222}]" }, { "name": "non-service key is not granted service access", "passed": true, - "notes": "status 404: Function not found" + "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" }, { "name": "rejects an unverified (forged) user token", "passed": true, - "notes": "status 404: Function not found" + "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" }, { "name": "a user token in the apikey slot is not treated as the service key", "passed": true, - "notes": "status 404: Function not found" + "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" }, { "name": "implementation uses @supabase/server", - "passed": false, - "notes": "could not locate function source to inspect" + "passed": true, + "notes": "imports @supabase/server / withSupabase" } ], "skills": { @@ -18189,14 +18302,56 @@ "supabase", "supabase-postgres-best-practices" ], - "loaded": [] + "loaded": [ + "supabase" + ] }, "docs": { - "calls": [] + "calls": [ + { + "source": "search_docs", + "query": "{ searchDocs(query: \"edge functions environment variables SUPABASE_SERVICE_ROLE_KEY secret api key\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", + "title": "Self-Hosted Functions" + }, + { + "url": "https://supabase.com/docs/guides/functions/secrets", + "title": "Environment Variables" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", + "title": "Integrating With Supabase Auth" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/inspecting-edge-function-environment-variables-wg5qOQ", + "title": "Inspecting edge function environment variables" + } + ], + "resultChars": 35250 + }, + { + "source": "web_fetch", + "query": "https://supabase.com/docs/guides/functions/auth.md", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/functions/auth.md" + } + ], + "resultChars": 7275 + } + ] }, "prompt": "Build and serve a Supabase Edge Function named `user-stats` for this project,\nreachable over HTTP at `/functions/v1/user-stats`.\n\nOur product stores per-user metrics in the existing `user_stats` table.\n\nTwo very different callers need to hit this one endpoint:\n\n1. **Our mobile app**, acting for a signed-in user. It sends that user's\n Supabase access token. The endpoint should return the user's own stats.\n\n2. **Our internal billing service**, a trusted backend with no signed-in user.\n It authenticates with the project's secret (service-role) key in the `apikey`\n header, and names the target user with a `user_id` in the JSON request body.\n It needs that user's stats.\n\nReturn the matching rows as JSON. The endpoint must be secure: only ever serve\nstats to a caller that is genuinely entitled to them, and turn away callers that\nare not.", "promptSourcePath": "evals/build-functions-005-dual-auth-user-secret/PROMPT.md", - "attempts": 2, + "attempts": 1, "sourcePath": "opencode-kimi-k3/build-functions-005-dual-auth-user-secret.json" }, { @@ -18236,7 +18391,7 @@ { "name": "user A lists only own files", "passed": true, - "notes": "saw: 019fa581-9203-774f-b480-30368ceb3469/receipt-alpha.pdf, 019fa581-9203-774f-b480-30368ceb3469/receipt-beta.pdf" + "notes": "saw: 019fa63c-a5b9-773b-8d4b-a061f9b05b31/receipt-alpha.pdf, 019fa63c-a5b9-773b-8d4b-a061f9b05b31/receipt-beta.pdf" }, { "name": "user B cannot read user A files", @@ -18257,7 +18412,7 @@ { "name": "configured private per-user storage access", "passed": true, - "judgeNotes": "Creates a private user-files bucket, owner-scoped SELECT and INSERT policies for authenticated users using storage.foldername(name)[1] = auth.uid(), keeps RLS intact, and provides supabase-js createSignedUrl code with expiry. No public bucket, permissive policies, getPublicUrl, or client-side service role usage." + "judgeNotes": "Meets all requirements: private user-files bucket, authenticated owner-scoped SELECT and INSERT RLS policies using user-id path prefix, no RLS disabling or public access, and supabase-js createSignedUrl with expiry for temporary sharing." } ], "skills": { @@ -18271,20 +18426,9 @@ }, "docs": { "calls": [ - { - "source": "web_fetch", - "query": "https://supabase.com/changelog.md", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 92585 - }, { "source": "search_docs", - "query": "{ searchDocs(query: \"storage access control RLS policies bucket private signed URL\", limit: 8) { nodes { title href content } } }", + "query": "{\n searchDocs(query: \"storage access control RLS policy bucket private user folder ownership\", limit: 5) {\n nodes {\n title\n href\n content\n }\n }\n}", "hasContent": true, "pages": [ { @@ -18296,88 +18440,30 @@ "title": "Storage Access Control" }, { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-angular", - "title": "Build a User Management App with Angular" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-ionic-angular", - "title": "Build a User Management App with Ionic Angular" - }, - { - "url": "https://supabase.com/docs/guides/security/product-security", - "title": "Secure configuration of Supabase products" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-expo-react-native", - "title": "Build a User Management App with Expo React Native" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3", - "title": "Configure S3 Storage" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-swift", - "title": "Build a User Management App with Swift and SwiftUI" - } - ], - "resultChars": 138223 - }, - { - "source": "web_fetch", - "query": "https://supabase.com/docs/guides/storage/security/access-control.md", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/storage/security/access-control.md" - } - ], - "resultChars": 4169 - }, - { - "source": "web_fetch", - "query": "https://supabase.com/docs/reference/javascript/storage-from-createsignedurl.md", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/reference/javascript/storage-from-createsignedurl.md" - } - ] - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"createSignedUrl storage javascript reference\", limit: 3) { nodes { title href language methodName } } }", - "hasContent": false, - "pages": [ - { - "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsignedurl" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3", + "title": "Configure S3 Storage" }, { - "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsignedurls" + "url": "https://supabase.com/docs/guides/security/product-security", + "title": "Secure configuration of Supabase products" }, { - "url": "https://supabase.com/docs/guides/platform/manage-your-usage/storage-image-transformations", - "title": "Manage Storage Image Transformations usage" + "url": "https://supabase.com/docs/guides/storage/security/ownership", + "title": "Ownership" } ], - "resultChars": 7829 + "resultChars": 20471 }, { - "source": "search_docs", - "query": "{ searchDocs(query: \"createSignedUrl storage javascript\", limit: 3) { nodes { title href content ... on ClientLibraryFunctionReference { language methodName } } } }", + "source": "web_fetch", + "query": "https://supabase.com/changelog.md", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsignedurl" - }, - { - "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsignedurls" - }, - { - "url": "https://supabase.com/docs/guides/platform/manage-your-usage/storage-image-transformations", - "title": "Manage Storage Image Transformations usage" + "url": "https://supabase.com/changelog.md" } ], - "resultChars": 7829 + "resultChars": 92585 } ] }, @@ -18405,22 +18491,22 @@ ], "suite": "benchmark", "interface": "cli", - "passed": false, + "passed": true, "checks": [ { "name": "pgTAP test file(s) written under supabase/tests/", - "passed": false, - "notes": "no .sql files found under supabase/tests/" + "passed": true, + "notes": "1 file(s): supabase/tests/tenant_isolation_test.sql" }, { "name": "pgTAP isolation tests ran and pass", - "passed": false, - "notes": "no test summary found; exit 0; output: Connecting to local database...\nFiles=0, Tests=0, 0 wallclock secs ( 0.00 usr + 0.00 sys = 0.00 CPU)\nResult: NOTESTS\nA new version of Supabase CLI is available: v2.110.0 (currently installed v2.67.1)\nWe recommend updating regularly for new features and bug fixes: https://supabase.com/docs/guides/cli/getting-started#updating-the-supabase-cli\n" + "passed": true, + "notes": "7 passed, 2 failed" }, { "name": "agent correctly identifies the posts isolation bug from test results", "passed": true, - "judgeNotes": "The agent identifies `posts` as the table with the broken tenant isolation policy and grounds the conclusion in the pgTAP test results, noting that authenticated members can read posts from organizations they do not belong to. It does not misattribute the issue to `notes` or dismiss the tests." + "judgeNotes": "The agent correctly identifies `posts` as having a broken tenant isolation policy, explains that members of any org can read posts from other orgs, and grounds this in pgTAP test failure #4. It also correctly treats `notes` as isolated and relies on the test results as authoritative." } ], "skills": { @@ -18428,14 +18514,16 @@ "supabase", "supabase-postgres-best-practices" ], - "loaded": [] + "loaded": [ + "supabase" + ] }, "docs": { "calls": [] }, "prompt": "Can you audit the tenant isolation on our tables? Write some database tests covering both the happy path and the negative case, run them, and share the results.", "promptSourcePath": "evals/build-tests-001-rls-tenant-isolation/PROMPT.md", - "attempts": 2, + "attempts": 1, "sourcePath": "opencode-kimi-k3/build-tests-001-rls-tenant-isolation.json" }, { @@ -18498,56 +18586,44 @@ "supabase-postgres-best-practices" ], "loaded": [ - "supabase", - "supabase-postgres-best-practices" + "supabase" ] }, "docs": { "calls": [ - { - "source": "web_fetch", - "query": "https://supabase.com/changelog.md", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 92585 - }, { "source": "search_docs", - "query": "{ searchDocs(query: \"semantic search pgvector match documents function gte-small embedding dimensions\", limit: 5) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"gte-small embedding dimensions Supabase.ai Session semantic search vector column match documents rpc\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/ai/semantic-search", - "title": "Semantic search" + "url": "https://supabase.com/docs/guides/functions/examples/semantic-search", + "title": "Semantic Search" }, { - "url": "https://supabase.com/docs/guides/ai/vector-columns", - "title": "Vector columns" + "url": "https://supabase.com/docs/guides/ai/semantic-search", + "title": "Semantic search" }, { - "url": "https://supabase.com/docs/guides/functions/examples/semantic-search", - "title": "Semantic Search" + "url": "https://supabase.com/docs/guides/ai/choosing-compute-addon", + "title": "Choosing your Compute Add-on" }, { - "url": "https://supabase.com/docs/guides/ai/hybrid-search", - "title": "Hybrid search" + "url": "https://supabase.com/docs/guides/storage/vector/querying-vectors", + "title": "Querying Vectors" }, { - "url": "https://supabase.com/docs/guides/database/extensions/pgvector", - "title": "pgvector: Embeddings and vector similarity" + "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", + "title": "Automatic embeddings" } ], - "resultChars": 45321 + "resultChars": 92278 } ] }, "prompt": "We're adding semantic search to our internal knowledge base app. I already wrote the edge functions, but search doesn't work yet. Can you set up whatever the database needs to make search work end to end?\n\nSome documents are confidential, so users should only have access to documents they own.", "promptSourcePath": "evals/build-vectors-001-rag-with-permissions/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "opencode-kimi-k3/build-vectors-001-rag-with-permissions.json" }, { @@ -18567,7 +18643,7 @@ "observability" ], "suite": "benchmark", - "passed": false, + "passed": true, "checks": [ { "name": "preserved existing app scrape job", @@ -18575,13 +18651,13 @@ }, { "name": "configured the Supabase Metrics API scrape correctly", - "passed": false, - "judgeNotes": "prometheus.yml only contains the existing app scrape. It does not add a Supabase Metrics API scrape target, HTTPS scheme, /customer/v1/privileged/metrics path, HTTP Basic Auth with password_file, or docker-compose secret/volume wiring for that password file." + "passed": true, + "judgeNotes": "Supabase scrape uses HTTPS, correct metrics path, basic_auth with password_file, preserves the app job, targets .supabase.co:443, and docker-compose mounts the secrets directory containing the password file." }, { "name": "documented live deployment and verification steps", - "passed": false, - "judgeNotes": "README only shows starting the stack and lists the target. It lacks Secret API key creation, matching secret file placement, restart/reload instructions, and concrete verification via Prometheus targets, PromQL/Grafana, or equivalent." + "passed": true, + "judgeNotes": "README.md includes concrete steps to create a Supabase Secret API key, place it in the mounted secret file, replace project refs, restart or reload the Compose/Prometheus stack, and verify via direct endpoint curl, Prometheus query, and targets UI." } ], "skills": { @@ -18589,14 +18665,41 @@ "supabase", "supabase-postgres-best-practices" ], - "loaded": [] + "loaded": [ + "supabase" + ] }, "docs": { - "calls": [] + "calls": [ + { + "source": "search_docs", + "query": "{ searchDocs(query: \"Metrics API prometheus scrape endpoint project metrics\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/telemetry/metrics", + "title": "Metrics API" + }, + { + "url": "https://supabase.com/docs/guides/telemetry/metrics/grafana-self-hosted", + "title": "Metrics API with Prometheus & Grafana (self-hosted)" + }, + { + "url": "https://supabase.com/docs/guides/telemetry/metrics/vendor-agnostic", + "title": "Vendor-agnostic Metrics API setup" + }, + { + "url": "https://supabase.com/docs/guides/telemetry/metrics/grafana-cloud", + "title": "Metrics API with Grafana Cloud" + } + ], + "resultChars": 22788 + } + ] }, "prompt": "Can you wire my Supabase project metrics into our existing observability stack and document\nwhat we need to do to make the config live?", "promptSourcePath": "evals/deploy-database-001-prometheus-metrics/PROMPT.md", - "attempts": 2, + "attempts": 1, "sourcePath": "opencode-kimi-k3/deploy-database-001-prometheus-metrics.json" }, { @@ -18631,7 +18734,7 @@ { "name": "the weather function reads WEATHER_API_KEY from the environment", "passed": true, - "judgeNotes": "The function reads WEATHER_API_KEY from the runtime environment using Deno.env.get(\"WEATHER_API_KEY\")." + "judgeNotes": "The function reads WEATHER_API_KEY from the runtime environment using Deno.env.get('WEATHER_API_KEY')." }, { "name": "WEATHER_API_KEY value is not committed to the repo", @@ -18659,6 +18762,34 @@ } ], "resultChars": 92585 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"edge function secrets environment variables deploy\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/troubleshooting/inspecting-edge-function-environment-variables-wg5qOQ", + "title": "Inspecting edge function environment variables" + }, + { + "url": "https://supabase.com/docs/guides/functions/secrets", + "title": "Environment Variables" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" + }, + { + "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech", + "title": "Transcription Telegram Bot" + }, + { + "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream", + "title": "Streaming Speech with ElevenLabs" + } + ], + "resultChars": 40215 } ] }, @@ -18719,36 +18850,25 @@ "calls": [ { "source": "web_fetch", - "query": "https://supabase.com/changelog.md", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 5943 - }, - { - "source": "web_fetch", - "query": "https://supabase.com/changelog/48048-self-hosted-supabase-envoy-becomes-the-default-api-gateway-b", + "query": "https://supabase.com/docs/guides/self-hosting/docker.md", "hasContent": true, "pages": [ { - "url": "https://supabase.com/changelog/48048-self-hosted-supabase-envoy-becomes-the-default-api-gateway-b" + "url": "https://supabase.com/docs/guides/self-hosting/docker.md" } ], - "resultChars": 5943 + "resultChars": 92585 }, { "source": "web_fetch", - "query": "https://supabase.com/docs/guides/self-hosting/docker.md", + "query": "https://supabase.com/changelog.md", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/self-hosting/docker.md" + "url": "https://supabase.com/changelog.md" } ], - "resultChars": 30210 + "resultChars": 92585 } ] }, @@ -18806,7 +18926,7 @@ { "name": "diagnosed and explained session revocation", "passed": true, - "judgeNotes": "The answer diagnoses the soft-delete-only bug, replaces it with deletion of auth.users causing cascade removal of sessions/refresh tokens/identities/profile, and tests it. It also explains JWT access tokens remain valid until expiry for local validation and distinguishes publishable vs secret keys correctly in the surrounding response implied by the transcript." + "judgeNotes": "Meets all rubric points: identifies soft-delete-only root cause; implements auth user deletion with cascaded sessions/refresh token revocation; explains stateless JWT access-token window consistently and gives mitigations; correctly distinguishes publishable frontend/RLS behavior from secret backend-only/RLS-bypass behavior." } ], "skills": { @@ -18820,42 +18940,33 @@ }, "docs": { "calls": [ - { - "source": "web_fetch", - "query": "https://supabase.com/changelog.md", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 92585 - }, { "source": "search_docs", - "query": "{ searchDocs(query: \"delete user account self service RPC function auth.users security definer\", limit: 5) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"publishable secret API keys migration anon service_role RLS\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/reference/javascript/auth-admin-deleteuser" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-redwoodjs", + "title": "Build a User Management App with RedwoodJS" }, { - "url": "https://supabase.com/docs/reference/swift/auth-admin-deleteuser", - "title": "deleteUser()" + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" }, { - "url": "https://supabase.com/docs/guides/auth/managing-user-data", - "title": "User Management" + "url": "https://supabase.com/docs/guides/auth/quickstarts/with-expo-react-native-social-auth", + "title": "Build a Social Auth App with Expo React Native" }, { - "url": "https://supabase.com/docs/reference/dart/auth-admin-deleteuser" + "url": "https://supabase.com/docs/guides/auth/signing-keys", + "title": "JWT Signing Keys" }, { - "url": "https://supabase.com/docs/reference/python/auth-admin-deleteuser", - "title": "delete_user()" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-sveltekit", + "title": "Build a User Management App with SvelteKit" } ], - "resultChars": 7953 + "resultChars": 169611 } ] }, @@ -18909,7 +19020,7 @@ { "name": "diagnosed missing publication membership", "passed": true, - "judgeNotes": "The assistant correctly identified the missing orders table in the supabase_realtime publication as the root cause, applied ALTER PUBLICATION supabase_realtime ADD TABLE public.orders, preserved existing courier_locations publication membership, and did not weaken RLS/policies or blame client/networking." + "judgeNotes": "The assistant correctly identified the root cause as public.orders missing from the supabase_realtime publication despite the channel reaching SUBSCRIBED, applied exactly ALTER PUBLICATION supabase_realtime ADD TABLE public.orders, and preserved courier_locations, RLS, and policies without blaming or weakening other components." } ], "skills": { @@ -18952,17 +19063,17 @@ { "name": "identified image-transform and the recurring 503 pattern", "passed": true, - "judgeNotes": "The assistant explicitly identified `image-transform` as the affected function and described recurring 503 responses roughly every 30 minutes across 07:00–12:00, matching the required morning pattern." + "judgeNotes": "Identified image-transform as affected and described the recurring 503 pattern across 07:00Z–12:00Z, covering all 8 gateway failures." }, { "name": "attributed recurring 503s to gateway/platform layer, not function code", "passed": false, - "judgeNotes": "The assistant attributes the 503s to the `image-transform` Edge Function returning errors and proceeds to inspect function code, rather than identifying the gateway/platform layer as the source. It does not ground a platform-layer attribution in observations like missing invocation/runtime rows or unchanged deployment version." + "judgeNotes": "Although it observes that the 503s appear only in gateway logs with no corresponding function execution logs, it ultimately attributes the root cause to the function/runtime boot process and unpinned npm dependencies, and recommends pinning/redeploying the functions. The rubric requires attributing the recurring 503s to the gateway/platform layer rather than the function code/runtime." }, { "name": "recommended a concrete next step", "passed": true, - "judgeNotes": "The assistant identified a specific failure pattern and proposed concrete next steps: checking Edge Function logs and inspecting the `image-transform` function code, rather than giving only vague advice." + "judgeNotes": "The assistant provided concrete actionable next steps, including pinning dependencies and redeploying, checking npm publish history for the affected time window, committing lockfiles, adding gateway 5xx alerting, and applying retry mitigation." } ], "skills": { @@ -19003,7 +19114,7 @@ ], "suite": "benchmark", "interface": "mcp", - "passed": false, + "passed": true, "checks": [ { "name": "RLS still enabled on bookmarks", @@ -19011,7 +19122,7 @@ }, { "name": "user A reads own bookmarks", - "passed": false + "passed": true }, { "name": "user B cannot read user A bookmarks", @@ -19023,8 +19134,7 @@ }, { "name": "user A can save a new bookmark", - "passed": false, - "notes": "new row violates row-level security policy for table \"bookmarks\"" + "passed": true }, { "name": "user B cannot insert a bookmark as user A", @@ -19033,7 +19143,7 @@ { "name": "diagnosed RLS and added owner-scoped policies", "passed": true, - "judgeNotes": "The assistant correctly identified the issue as RLS being enabled with no policies, preserving deny-by-default behavior. It proposed owner-scoped authenticated SELECT and INSERT policies using user_id = auth.uid(), with INSERT enforced via WITH CHECK, and kept RLS enabled." + "judgeNotes": "Diagnosed RLS enabled with no policies/default-deny, kept RLS enabled, and created authenticated SELECT and INSERT policies scoped to user_id = auth.uid() with USING and WITH CHECK." } ], "skills": { @@ -19041,14 +19151,16 @@ "supabase", "supabase-postgres-best-practices" ], - "loaded": [] + "loaded": [ + "supabase" + ] }, "docs": { "calls": [] }, "prompt": "Our app lets signed-in users save bookmarks and view them on their dashboard. Bookmarks are stored in the `bookmarks` table and are private — a user must only ever see their own. \nUsers also need to be able to save new bookmarks from the app.\n\nI can see the rows when I query the table directly, but the dashboard shows an empty list for every user.\n\nFind out why the Data API returns nothing and fix it.", "promptSourcePath": "evals/resolve-dataapi-001-empty-results/PROMPT.md", - "attempts": 2, + "attempts": 1, "sourcePath": "opencode-kimi-k3/resolve-dataapi-001-empty-results.json" }, { @@ -19069,27 +19181,23 @@ ], "suite": "benchmark", "interface": "cli", - "passed": false, + "passed": true, "checks": [ { "name": "the avatar_url column is applied on the hosted profiles table", - "passed": false, - "notes": "avatar_url not found on public.profiles" + "passed": true }, { "name": "migration 20240220000000 is recorded in the remote history", - "passed": false, - "notes": "remote history versions: [\"20240101000000\",\"20240115000000\"]" + "passed": true }, { "name": "remote migration history matches local migration files", - "passed": false, - "notes": "remote-only versions still unreconciled: [\"20240115000000\"] (remote: [\"20240101000000\",\"20240115000000\"], local: [\"20240101000000\",\"20240220000000\"])" + "passed": true }, { "name": "local migrations are a valid reconciled sequence", - "passed": false, - "notes": "expected 20240101000000_create_profiles → → 20240220000000_add_avatar_url, strictly ascending; got [20240101000000_create_profiles.sql, 20240220000000_add_avatar_url.sql]" + "passed": true }, { "name": "production profile data is intact (not reset)", @@ -19097,8 +19205,8 @@ }, { "name": "the avatar migration and history reconciliation were done via the Supabase CLI", - "passed": false, - "notes": "no agent actions recorded to evaluate" + "passed": true, + "judgeNotes": "Avatar migration was applied through `supabase db push` in #20, which output `Applying migration 20240220000000_add_avatar_url.sql...` and `Finished supabase db push.` The orphan bio history was reconciled by adding the missing local file `supabase/migrations/20240115000000_add_profile_bio.sql` in #18, after which `supabase migration list`/#19 showed local and remote aligned and the successful push proceeded. No disallowed workaround or direct mutation was seen; psql usage was read-only inspection." } ], "skills": { @@ -19106,14 +19214,56 @@ "supabase", "supabase-postgres-best-practices" ], - "loaded": [] + "loaded": [ + "supabase" + ] }, "docs": { - "calls": [] + "calls": [ + { + "source": "search_docs", + "query": "{\n searchDocs(query: \"remote migration versions not found in local migrations directory repair db pull\", limit: 5) {\n nodes {\n title\n href\n content\n }\n }\n}", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/local-development/cli-workflows", + "title": "Local development workflow" + }, + { + "url": "https://supabase.com/docs/guides/deployment/database-migrations", + "title": "Database Migrations" + }, + { + "url": "https://supabase.com/docs/reference/cli/supabase-migration-repair", + "title": "Repair the migration history table" + }, + { + "url": "https://supabase.com/docs/reference/cli/supabase-db-pull", + "title": "Pull schema from the remote database" + }, + { + "url": "https://supabase.com/docs/guides/deployment/branching/troubleshooting", + "title": "Troubleshooting" + } + ], + "resultChars": 53680 + }, + { + "source": "web_fetch", + "query": "https://supabase.com/docs/guides/deployment/database-migrations.md", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/deployment/database-migrations.md" + } + ], + "resultChars": 9637 + } + ] }, "prompt": "I'm trying to ship a migration to our hosted project and it's not working. Can you figure out what's wrong and get it deployed?", "promptSourcePath": "evals/resolve-database-001-migration-history-mismatch/PROMPT.md", - "attempts": 2, + "attempts": 1, "sourcePath": "opencode-kimi-k3/resolve-database-001-migration-history-mismatch.json" }, { @@ -19152,7 +19302,7 @@ { "name": "query plan uses an index and avoids sequential scan", "passed": true, - "notes": "Limit (cost=79.74..79.86 rows=50 width=58)\n -> Sort (cost=79.74..79.99 rows=100 width=58)\n Sort Key: created_at DESC\n -> Bitmap Heap Scan on events (cost=5.06..76.42 rows=100 width=58)\n Recheck Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)\n -> Bitmap Index Scan on events_user_id_created_at_idx (cost=0.00..5.03 rows=100 width=0)\n Index Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)" + "notes": "Limit (cost=55.55..55.61 rows=25 width=88)\n -> Sort (cost=55.55..55.61 rows=25 width=88)\n Sort Key: created_at DESC\n -> Bitmap Heap Scan on events (cost=4.48..54.97 rows=25 width=88)\n Recheck Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)\n -> Bitmap Index Scan on events_user_id_created_at_idx (cost=0.00..4.47 rows=25 width=0)\n Index Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)" }, { "name": "inserts still work", From 472ad260b751a131a30360d099e0d47c362178d4 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 28 Jul 2026 01:41:55 +0000 Subject: [PATCH 16/17] chore: refresh eval results --- apps/web/src/data/eval-results.json | 820 ++++++++++++++++++++++------ 1 file changed, 665 insertions(+), 155 deletions(-) diff --git a/apps/web/src/data/eval-results.json b/apps/web/src/data/eval-results.json index 3214f085..8a16cc92 100644 --- a/apps/web/src/data/eval-results.json +++ b/apps/web/src/data/eval-results.json @@ -19423,41 +19423,38 @@ ], "suite": "benchmark", "interface": "cli", - "passed": false, + "passed": true, "checks": [ { "name": "supabase project initialised (supabase/config.toml exists)", - "passed": false + "passed": true }, { "name": "todos table is created by a migration file", - "passed": false, - "notes": "supabase/migrations does not exist — was a Supabase project initialised?" + "passed": true }, { "name": "todos table exists with at least 2 seeded rows", - "passed": false, - "notes": "could not read DB_URL from `supabase status -o json` after 5 attempts — the local stack must be running. Last status: failed to inspect container health: Error response from daemon: No such container: supabase_db_sandbox-ad7fb75b\nTry rerunning the command with --debug to troubleshoot the error.\n" + "passed": true, + "notes": "found 3 rows" }, { "name": "row level security is enabled on todos", - "passed": false, - "notes": "could not read DB_URL from `supabase status -o json` after 5 attempts — the local stack must be running. Last status: failed to inspect container health: Error response from daemon: No such container: supabase_db_sandbox-ad7fb75b\nTry rerunning the command with --debug to troubleshoot the error.\n" + "passed": true }, { "name": "a SELECT policy targets the authenticated role", - "passed": false, - "notes": "could not read DB_URL from `supabase status -o json` after 5 attempts — the local stack must be running. Last status: failed to inspect container health: Error response from daemon: No such container: supabase_db_sandbox-ad7fb75b\nTry rerunning the command with --debug to troubleshoot the error.\n" + "passed": true }, { "name": "REST API returns no todos to anonymous requests", - "passed": false, - "notes": "could not read API_URL/PUBLISHABLE_KEY from `supabase status -o json` after 5 attempts — the local stack must be running and include the auth service (status only reports API keys while gotrue is up; add `gotrue` to the eval's services). Last status: failed to inspect container health: Error response from daemon: No such container: supabase_db_sandbox-ad7fb75b\nTry rerunning the command with --debug to troubleshoot the error.\n" + "passed": true, + "notes": "0 rows" }, { "name": "REST API returns the todos to authenticated requests", - "passed": false, - "notes": "could not read API_URL/PUBLISHABLE_KEY from `supabase status -o json` after 5 attempts — the local stack must be running and include the auth service (status only reports API keys while gotrue is up; add `gotrue` to the eval's services). Last status: failed to inspect container health: Error response from daemon: No such container: supabase_db_sandbox-ad7fb75b\nTry rerunning the command with --debug to troubleshoot the error.\n" + "passed": true, + "notes": "3 rows" } ], "skills": { @@ -19469,7 +19466,7 @@ }, "prompt": "We're kicking off a todos app and I want the Supabase side ready for the team\nto build on. Set it up the way we'd run it in development, with schema changes\ntracked as migrations so they can be reviewed and replayed.\n\nFor the first slice we just need a `todos` table. Todos aren't public: anyone\nsigned in can read all of them, but nothing should be writable through the API\nfor now. Add a couple of sample todos so there's something to look at.\n\nBefore you hand it back, make sure the running API actually behaves that way —\nsigned-in users get the todos, signed-out requests get nothing.", "promptSourcePath": "evals/build-cli-001-bootstrap-app/PROMPT.md", - "attempts": 2, + "attempts": 1, "sourcePath": "opencode-kimi-k3-no-skills/build-cli-001-bootstrap-app.json" }, { @@ -19491,25 +19488,23 @@ ], "suite": "benchmark", "interface": "cli", - "passed": false, + "passed": true, "checks": [ { "name": "supabase db diff used to generate the migration", - "passed": false + "passed": true }, { "name": "schema file updated to include description column", - "passed": false, - "notes": "description not found in any schema file" + "passed": true }, { "name": "a new migration was generated for the change", - "passed": false, - "notes": "found 1 migration file(s)" + "passed": true }, { "name": "description column exists in the live database", - "passed": false + "passed": true } ], "skills": { @@ -19521,7 +19516,7 @@ }, "prompt": "Add a description text column to the `products` table in my local Supabase stack", "promptSourcePath": "evals/build-cli-002-declarative-schema/PROMPT.md", - "attempts": 2, + "attempts": 1, "sourcePath": "opencode-kimi-k3-no-skills/build-cli-002-declarative-schema.json" }, { @@ -19546,22 +19541,22 @@ ], "suite": "benchmark", "interface": "cli", - "passed": false, + "passed": true, "checks": [ { "name": "pg_cron job 'enqueue-tasks' scheduled to run every minute", - "passed": false, - "notes": "job not found in cron.job" + "passed": true, + "notes": "schedule='* * * * *', active=true" }, { "name": "cron command enqueues to the 'tasks' queue", - "passed": false, - "notes": "job not found, so its command can't run" + "passed": true, + "notes": "queue depth 0 -> 1" }, { "name": "process-tasks function drains the queue", - "passed": false, - "notes": "couldn't enqueue a test message. Is the 'tasks' queue created? query failed: ERROR: relation \"pgmq.q_tasks\" does not exist\nLINE 2: INSERT INTO pgmq.q_tasks (vt, message, headers)\n ^\nQUERY: \n INSERT INTO pgmq.q_tasks (vt, message, headers)\n VALUES ($2, $1, $3)\n RETURNING msg_id;\n \nCONTEXT: PL/pgSQL function pgmq.send(text,jsonb,jsonb,timestamp with time zone) line 14 at RETURN QUERY\nSQL function \"send\" statement 1\n" + "passed": true, + "notes": "function removed the seeded message (id 37) from the queue" } ], "skills": { @@ -19573,7 +19568,7 @@ }, "prompt": "I want to set up a recurring background workflow on my local Supabase stack.\n\nCan you set up a cron job called `enqueue-tasks` to run every minute and push a task into a queue called `tasks`? Then add a `process-tasks` edge function that reads messages off the `tasks` queue and removes them, so a scheduled worker can keep the backlog drained.", "promptSourcePath": "evals/build-cli-003-pg-cron-queue-workflow/PROMPT.md", - "attempts": 2, + "attempts": 1, "sourcePath": "opencode-kimi-k3-no-skills/build-cli-003-pg-cron-queue-workflow.json" }, { @@ -19594,12 +19589,27 @@ ], "suite": "benchmark", "interface": "cli", - "passed": false, + "passed": true, "checks": [ { - "name": "scorer completed without errors", - "passed": false, - "notes": "could not read DB_URL from `supabase status -o json` after 5 attempts — the local stack must be running. Last status: failed to inspect container health: Error response from daemon: No such container: supabase_db_sandbox-45da1f7a\nTry rerunning the command with --debug to troubleshoot the error.\n" + "name": "all 3 tables exist (teams, members, tasks)", + "passed": true + }, + { + "name": "row counts match (teams=5, members=10, tasks=13)", + "passed": true + }, + { + "name": "foreign key constraints survived the restore", + "passed": true + }, + { + "name": "tasks_team_status_idx index survived the restore", + "passed": true + }, + { + "name": "sequences synced (next insert won't conflict with existing IDs)", + "passed": true } ], "skills": { @@ -19611,7 +19621,7 @@ }, "prompt": "I have an existing Postgres database I want to migrate to Supabase. There's a binary dump at `source.dump` in the current directory.\n\nCan you set up a local Supabase project and restore the dump into it?", "promptSourcePath": "evals/build-database-001-migrate-postgres-to-supabase/PROMPT.md", - "attempts": 2, + "attempts": 1, "sourcePath": "opencode-kimi-k3-no-skills/build-database-001-migrate-postgres-to-supabase.json" }, { @@ -19636,7 +19646,7 @@ ], "suite": "benchmark", "interface": "mcp", - "passed": false, + "passed": true, "checks": [ { "name": "rejects missing auth", @@ -19650,17 +19660,17 @@ }, { "name": "reads only with the caller's JWT", - "passed": false, - "notes": "bearer_tokens=1, all_match=false" + "passed": true, + "notes": "bearer_tokens=2, all_match=true" }, { "name": "user A cannot force-read user B note", - "passed": false, + "passed": true, "notes": "status=200" }, { "name": "user B cannot force-read user A note", - "passed": false, + "passed": true, "notes": "status=200" } ], @@ -19669,11 +19679,32 @@ "loaded": [] }, "docs": { - "calls": [] + "calls": [ + { + "source": "search_docs", + "query": "{ searchDocs(query: \"edge function authenticate user JWT getUser RLS\", limit: 3) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", + "title": "Integrating With Supabase Auth" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth-headers", + "title": "Authorization headers" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs", + "title": "Build a User Management App with Next.js" + } + ], + "resultChars": 50282 + } + ] }, "prompt": "I built an Edge Function called `private-notes` for showing a user's saved\nprivate notes.\n\nCan you check whether there's any way one user could see another user's notes?\n\nPlease fix and deploy it if needed.", "promptSourcePath": "evals/build-functions-004-service-role-bypass/PROMPT.md", - "attempts": 2, + "attempts": 1, "sourcePath": "opencode-kimi-k3-no-skills/build-functions-004-service-role-bypass.json" }, { @@ -19699,7 +19730,7 @@ "suite": "benchmark", "interface": "cli", "cliVersion": "2.109.1", - "passed": false, + "passed": true, "checks": [ { "name": "seed rows present", @@ -19709,42 +19740,42 @@ { "name": "rejects request with no credentials", "passed": true, - "notes": "status 404: Function not found" + "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" }, { "name": "user with JWT reads only their own rows", - "passed": false, - "notes": "status 404: Function not found" + "passed": true, + "notes": "status 200: [{\"user_id\":\"ffac847f-b734-4c98-a4fb-32356e032e87\",\"metric\":\"steps_a_ms3zogi9\",\"value\":111}]" }, { "name": "user cannot read another user's rows by passing user_id", - "passed": false, - "notes": "status 404: Function not found" + "passed": true, + "notes": "status 200: [{\"user_id\":\"ffac847f-b734-4c98-a4fb-32356e032e87\",\"metric\":\"steps_a_ms3zogi9\",\"value\":111}]" }, { "name": "service key bypasses RLS to read the target user's rows", - "passed": false, - "notes": "status 404: Function not found" + "passed": true, + "notes": "status 200: [{\"user_id\":\"b1a74712-deb9-4b80-841d-f23a2fbad78c\",\"metric\":\"steps_b_ms3zogi9\",\"value\":222}]" }, { "name": "non-service key is not granted service access", "passed": true, - "notes": "status 404: Function not found" + "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" }, { "name": "rejects an unverified (forged) user token", "passed": true, - "notes": "status 404: Function not found" + "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" }, { "name": "a user token in the apikey slot is not treated as the service key", "passed": true, - "notes": "status 404: Function not found" + "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" }, { "name": "implementation uses @supabase/server", - "passed": false, - "notes": "could not locate function source to inspect" + "passed": true, + "notes": "imports @supabase/server / withSupabase" } ], "skills": { @@ -19752,7 +19783,36 @@ "loaded": [] }, "docs": { - "calls": [] + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"withSupabase @supabase/server edge function authMode publishable secret supabaseAdmin\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/functions/auth", + "title": "Securing Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/api-keys", + "title": "Understanding API keys" + }, + { + "url": "https://supabase.com/docs/guides/functions/secrets", + "title": "Environment Variables" + }, + { + "url": "https://supabase.com/docs/guides/auth/choosing-a-server-package", + "title": "Which package to use" + } + ], + "resultChars": 44429 + } + ] }, "prompt": "Build and serve a Supabase Edge Function named `user-stats` for this project,\nreachable over HTTP at `/functions/v1/user-stats`.\n\nOur product stores per-user metrics in the existing `user_stats` table.\n\nTwo very different callers need to hit this one endpoint:\n\n1. **Our mobile app**, acting for a signed-in user. It sends that user's\n Supabase access token. The endpoint should return the user's own stats.\n\n2. **Our internal billing service**, a trusted backend with no signed-in user.\n It authenticates with the project's secret (service-role) key in the `apikey`\n header, and names the target user with a `user_id` in the JSON request body.\n It needs that user's stats.\n\nReturn the matching rows as JSON. The endpoint must be secure: only ever serve\nstats to a caller that is genuinely entitled to them, and turn away callers that\nare not.", "promptSourcePath": "evals/build-functions-005-dual-auth-user-secret/PROMPT.md", @@ -19779,12 +19839,45 @@ ], "suite": "benchmark", "interface": "mcp", - "passed": false, + "passed": true, "checks": [ { "name": "bucket user-files exists", - "passed": false, - "notes": "no row in storage.buckets with id or name 'user-files'" + "passed": true + }, + { + "name": "bucket user-files is private", + "passed": true + }, + { + "name": "RLS still enabled on storage.objects", + "passed": true + }, + { + "name": "user A lists only own files", + "passed": true, + "notes": "saw: 019fa651-c08d-731a-9d74-1cda83a3daf8/receipt-alpha.pdf, 019fa651-c08d-731a-9d74-1cda83a3daf8/receipt-beta.pdf" + }, + { + "name": "user B cannot read user A files", + "passed": true + }, + { + "name": "anon reads no files", + "passed": true + }, + { + "name": "user A can upload into own folder", + "passed": true + }, + { + "name": "user B cannot upload into user A folder", + "passed": true + }, + { + "name": "configured private per-user storage access", + "passed": true, + "judgeNotes": "Meets all requirements: private bucket, RLS enabled, authenticated owner-scoped SELECT and INSERT policies with WITH CHECK, and supabase-js createSignedUrl with expiry for sharing." } ], "skills": { @@ -19792,11 +19885,66 @@ "loaded": [] }, "docs": { - "calls": [] + "calls": [ + { + "source": "search_docs", + "query": "{ searchDocs(query: \"createSignedUrl supabase-js storage temporary expiring link\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/storage/serving/downloads", + "title": "Serving assets from Storage" + }, + { + "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsignedurl" + }, + { + "url": "https://supabase.com/docs/guides/storage/serving/image-transformations", + "title": "Storage Image Transformations" + }, + { + "url": "https://supabase.com/docs/guides/storage/uploads/resumable-uploads", + "title": "Resumable Uploads" + }, + { + "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsignedurls" + } + ], + "resultChars": 38982 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"storage access control RLS policies foldername auth.uid bucket private\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/storage/buckets/fundamentals", + "title": "Storage Buckets" + }, + { + "url": "https://supabase.com/docs/guides/storage/security/access-control", + "title": "Storage Access Control" + }, + { + "url": "https://supabase.com/docs/guides/security/product-security", + "title": "Secure configuration of Supabase products" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3", + "title": "Configure S3 Storage" + }, + { + "url": "https://supabase.com/docs/guides/storage/debugging/error-codes", + "title": "Error Codes" + } + ], + "resultChars": 33308 + } + ] }, "prompt": "Our app lets signed-in users keep personal files like receipts and bank\nstatements. These files are private — a user must only ever be able to upload\nand download their own. The app uploads each file under a path that starts\nwith the owner's user id, e.g. `/receipt-march.pdf`.\n\nSet up a `user-files` bucket on our project and lock it down that way.\n\nUsers also sometimes share one of their files with someone else through a\ntemporary link that expires. Include the supabase-js code the app should use\nfor that.", "promptSourcePath": "evals/build-storage-001-private-bucket-access/PROMPT.md", - "attempts": 2, + "attempts": 1, "sourcePath": "opencode-kimi-k3-no-skills/build-storage-001-private-bucket-access.json" }, { @@ -19818,22 +19966,22 @@ ], "suite": "benchmark", "interface": "cli", - "passed": false, + "passed": true, "checks": [ { "name": "pgTAP test file(s) written under supabase/tests/", - "passed": false, - "notes": "no .sql files found under supabase/tests/" + "passed": true, + "notes": "1 file(s): supabase/tests/database/tenant_isolation.test.sql" }, { "name": "pgTAP isolation tests ran and pass", - "passed": false, - "notes": "no test summary found; exit 0; output: Connecting to local database...\nFiles=0, Tests=0, 0 wallclock secs ( 0.00 usr + 0.00 sys = 0.00 CPU)\nResult: NOTESTS\nA new version of Supabase CLI is available: v2.110.0 (currently installed v2.67.1)\nWe recommend updating regularly for new features and bug fixes: https://supabase.com/docs/guides/cli/getting-started#updating-the-supabase-cli\n" + "passed": true, + "notes": "14 passed, 0 failed" }, { "name": "agent correctly identifies the posts isolation bug from test results", "passed": true, - "judgeNotes": "The implementation fails the tenant-isolation tests because `posts` RLS is too permissive: authenticated users can read posts from orgs they do not belong to. The pgTAP failures correctly identify the broken policy on `posts`; `notes` is not the flawed table." + "judgeNotes": "The agent correctly identifies `posts` as having the broken tenant isolation policy, specifically that authenticated members could read posts from organizations they are not members of. It grounds this in pgTAP failures (tests 4 and 11) and treats the test results as authoritative. It does not blame `notes` for the read isolation flaw, though it separately notes other write-policy issues." } ], "skills": { @@ -19841,11 +19989,140 @@ "loaded": [] }, "docs": { - "calls": [] + "calls": [ + { + "source": "search_docs", + "query": "{ searchDocs(query: \"pgTAP database tests supabase test db directory tests\") { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/reference/cli/supabase-test-db", + "title": "Tests local database with pgTAP" + }, + { + "url": "https://supabase.com/docs/guides/local-development/cli/testing-and-linting", + "title": "Testing and linting" + }, + { + "url": "https://supabase.com/docs/guides/local-development/testing/overview", + "title": "Testing Overview" + }, + { + "url": "https://supabase.com/docs/guides/local-development/testing/pgtap-extended", + "title": "Advanced pgTAP Testing" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pgtap", + "title": "pgTAP: Unit Testing" + }, + { + "url": "https://supabase.com/docs/guides/database/testing", + "title": "Testing Your Database" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", + "title": "Row Level Security" + }, + { + "url": "https://supabase.com/docs/guides/functions/unit-test", + "title": "Testing your Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/deployment/managing-environments", + "title": "Managing Environments" + }, + { + "url": "https://supabase.com/docs/guides/platform/performance", + "title": "Performance Tuning" + }, + { + "url": "https://supabase.com/docs/guides/platform", + "title": "Supabase Platform" + }, + { + "url": "https://supabase.com/docs/guides/storage/management/download-objects", + "title": "Download Objects" + }, + { + "url": "https://supabase.com/docs/guides/platform/sso/multiple-providers", + "title": "Multiple SSO Providers" + }, + { + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/vercel-postgres", + "title": "Migrate from Vercel Postgres to Supabase" + }, + { + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/heroku", + "title": "Migrate from Heroku to Supabase" + }, + { + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase", + "title": "Migrating to Supabase" + }, + { + "url": "https://supabase.com/docs/guides/database/overview", + "title": "Database" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform", + "title": "Restore a Platform Project to Self-Hosted" + }, + { + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/mysql", + "title": "Migrate from MySQL to Supabase" + }, + { + "url": "https://supabase.com/docs/guides/api/rest/generating-types", + "title": "Generating TypeScript Types" + }, + { + "url": "https://supabase.com/docs/guides/api/rest/generating-python-types", + "title": "Generating Python Types" + }, + { + "url": "https://supabase.com/docs/guides/platform/sso/choosing-login-flow", + "title": "Choosing the Right SSO Login Flow" + }, + { + "url": "https://supabase.com/docs/guides/database/inspect", + "title": "Debugging and monitoring" + }, + { + "url": "https://supabase.com/docs/reference/cli/supabase-test", + "title": "Run tests on local Supabase containers" + }, + { + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/mssql", + "title": "Migrate from MSSQL to Supabase" + }, + { + "url": "https://supabase.com/docs/guides/platform/permissions", + "title": "Permissions" + }, + { + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/postgres", + "title": "Migrate from Postgres to Supabase" + }, + { + "url": "https://supabase.com/docs/guides/local-development/cli/getting-started", + "title": "Supabase CLI" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions", + "title": "Postgres Extensions Overview" + }, + { + "url": "https://supabase.com/docs/guides/deployment", + "title": "Deployment & Branching" + } + ], + "resultChars": 234607 + } + ] }, "prompt": "Can you audit the tenant isolation on our tables? Write some database tests covering both the happy path and the negative case, run them, and share the results.", "promptSourcePath": "evals/build-tests-001-rls-tenant-isolation/PROMPT.md", - "attempts": 2, + "attempts": 1, "sourcePath": "opencode-kimi-k3-no-skills/build-tests-001-rls-tenant-isolation.json" }, { @@ -19868,40 +20145,38 @@ ], "suite": "benchmark", "interface": "mcp", - "passed": false, + "passed": true, "checks": [ { "name": "document_sections.embedding is vector(384)", - "passed": false, - "notes": "no embedding column" + "passed": true, + "notes": "vector(384)" }, { "name": "HNSW index on the embedding column", - "passed": false, - "notes": "no index on embedding column" + "passed": true, + "notes": "CREATE INDEX document_sections_embedding_idx ON public.document_sections USING hnsw (embedding vector_cosine_ops)" }, { "name": "index operator class matches the search operator", - "passed": false, - "notes": "match_document_sections not found" + "passed": true, + "notes": "function operators: <=>\nindexes: CREATE INDEX document_sections_embedding_idx ON public.document_sections USING hnsw (embedding vector_cosine_ops)" }, { "name": "user A search returns only own sections, best match first", - "passed": false, - "notes": "Could not find the function public.match_document_sections(match_count, query_embedding) in the schema cache" + "passed": true }, { "name": "user B search returns only own sections, best match first", - "passed": false, - "notes": "Could not find the function public.match_document_sections(match_count, query_embedding) in the schema cache" + "passed": true }, { "name": "user A reads only own sections through the API", - "passed": false + "passed": true }, { "name": "user A reads only own documents through the API", - "passed": false + "passed": true } ], "skills": { @@ -19913,7 +20188,7 @@ }, "prompt": "We're adding semantic search to our internal knowledge base app. I already wrote the edge functions, but search doesn't work yet. Can you set up whatever the database needs to make search work end to end?\n\nSome documents are confidential, so users should only have access to documents they own.", "promptSourcePath": "evals/build-vectors-001-rag-with-permissions/PROMPT.md", - "attempts": 2, + "attempts": 1, "sourcePath": "opencode-kimi-k3-no-skills/build-vectors-001-rag-with-permissions.json" }, { @@ -19933,7 +20208,7 @@ "observability" ], "suite": "benchmark", - "passed": false, + "passed": true, "checks": [ { "name": "preserved existing app scrape job", @@ -19941,13 +20216,13 @@ }, { "name": "configured the Supabase Metrics API scrape correctly", - "passed": false, - "judgeNotes": "prometheus.yml only contains the existing app scrape and does not add a Supabase Metrics API scrape with HTTPS /customer/v1/privileged/metrics, basic_auth password_file, or docker-compose secret/volume wiring." + "passed": true, + "judgeNotes": "Prometheus preserves the app scrape and adds a Supabase HTTPS scrape at /customer/v1/privileged/metrics using HTTP Basic Auth with password_file. docker-compose mounts the secrets directory containing that password file read-only." }, { "name": "documented live deployment and verification steps", - "passed": false, - "judgeNotes": "README only shows starting the stack and lists app:8080. It does not explain creating a Secret API key, placing the matching secret file, restarting/reloading the Compose stack, or concrete verification via Prometheus targets/PromQL/Grafana." + "passed": true, + "judgeNotes": "README includes Secret API key creation, matching secret file placement, project ref replacement, restart/hot-reload of the Compose stack, and concrete verification via Prometheus targets plus curl smoke test. Endpoint/auth and secret setup match the Prometheus and Compose configuration." } ], "skills": { @@ -19955,11 +20230,44 @@ "loaded": [] }, "docs": { - "calls": [] + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"metrics endpoint Prometheus scrape project metrics\", limit: 10) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/telemetry/metrics/grafana-self-hosted", + "title": "Metrics API with Prometheus & Grafana (self-hosted)" + }, + { + "url": "https://supabase.com/docs/guides/telemetry/metrics", + "title": "Metrics API" + }, + { + "url": "https://supabase.com/docs/guides/telemetry/metrics/vendor-agnostic", + "title": "Vendor-agnostic Metrics API setup" + }, + { + "url": "https://supabase.com/docs/guides/telemetry/metrics/grafana-cloud", + "title": "Metrics API with Grafana Cloud" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/how-to-view-database-metrics-uqf2z_", + "title": "How to View Database Metrics" + }, + { + "url": "https://supabase.com/docs/guides/platform/read-replicas", + "title": "Read Replicas" + } + ], + "resultChars": 32499 + } + ] }, "prompt": "Can you wire my Supabase project metrics into our existing observability stack and document\nwhat we need to do to make the config live?", "promptSourcePath": "evals/deploy-database-001-prometheus-metrics/PROMPT.md", - "attempts": 2, + "attempts": 1, "sourcePath": "opencode-kimi-k3-no-skills/deploy-database-001-prometheus-metrics.json" }, { @@ -19980,22 +20288,21 @@ ], "suite": "benchmark", "interface": "cli", - "passed": false, + "passed": true, "checks": [ { "name": "WEATHER_API_KEY is set as a Function secret on the project", - "passed": false, - "notes": "secrets present: []" + "passed": true }, { "name": "the weather function is deployed to the project", - "passed": false, - "notes": "function not found on the project (status 404)" + "passed": true, + "notes": "status ACTIVE" }, { "name": "the weather function reads WEATHER_API_KEY from the environment", - "passed": false, - "notes": "could not read supabase/functions/weather/*" + "passed": true, + "judgeNotes": "The function reads WEATHER_API_KEY from the runtime environment using Deno.env.get(\"WEATHER_API_KEY\")." }, { "name": "WEATHER_API_KEY value is not committed to the repo", @@ -20007,11 +20314,40 @@ "loaded": [] }, "docs": { - "calls": [] + "calls": [ + { + "source": "search_docs", + "query": "{ searchDocs(query: \"edge functions environment variables secrets management deploy\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/functions/secrets", + "title": "Environment Variables" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", + "title": "Self-Hosted Functions" + }, + { + "url": "https://supabase.com/docs/guides/functions/deploy", + "title": "Deploy to Production" + }, + { + "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", + "title": "Automatic embeddings" + } + ], + "resultChars": 65241 + } + ] }, "prompt": "Our weather widget currently calls WeatherAPI straight from the browser, which\nleaks our API key. I want to move that behind a Supabase Edge Function called\n`weather` that holds the key server-side and proxies the request.\n\nThe function should read the key from an environment variable named\n`WEATHER_API_KEY`. Our key already lives in a local `.env` file at the project\nroot.\n\nDeploy the function to our project so it's live, and make sure the deployed\nfunction can actually read the key at runtime.", "promptSourcePath": "evals/deploy-functions-001-edge-function-secrets/PROMPT.md", - "attempts": 2, + "attempts": 1, "sourcePath": "opencode-kimi-k3-no-skills/deploy-functions-001-edge-function-secrets.json" }, { @@ -20034,12 +20370,11 @@ ], "suite": "benchmark", "interface": "cli", - "passed": false, + "passed": true, "checks": [ { "name": "cloned the self-host stack (docker-compose.yml + volumes/db)", - "passed": false, - "notes": "supabase-docker/ is missing docker-compose.yml or volumes/db — not the self-host docker/ tree" + "passed": true }, { "name": "didn't conflate with the CLI (no supabase/config.toml in the stack)", @@ -20047,13 +20382,11 @@ }, { "name": "secrets rotated off the shipped defaults", - "passed": false, - "notes": "still default or empty: POSTGRES_PASSWORD, JWT_SECRET, DASHBOARD_PASSWORD, VAULT_ENC_KEY, PG_META_CRYPTO_KEY" + "passed": true }, { "name": "ANON_KEY and SERVICE_ROLE_KEY are HS256 JWTs signed by JWT_SECRET", - "passed": false, - "notes": "JWT_SECRET missing" + "passed": true } ], "skills": { @@ -20061,11 +20394,140 @@ "loaded": [] }, "docs": { - "calls": [] + "calls": [ + { + "source": "search_docs", + "query": "{ searchDocs(query: \"self-hosting with docker deploy\") { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/self-hosting/docker", + "title": "Self-Hosting with Docker" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform", + "title": "Restore a Platform Project to Self-Hosted" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", + "title": "Self-Hosted Functions" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting", + "title": "Self-Hosting" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17", + "title": "Upgrade to Postgres 17" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-proxy-https", + "title": "Configure Reverse Proxy and HTTPS" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", + "title": "Envoy API Gateway" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-phone-mfa", + "title": "Configure Phone Login & MFA" + }, + { + "url": "https://supabase.com/docs/guides/local-development/cli-workflows", + "title": "Local development workflow" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3", + "title": "Configure S3 Storage" + }, + { + "url": "https://supabase.com/docs/guides/telemetry/metrics/grafana-self-hosted", + "title": "Metrics API with Prometheus & Grafana (self-hosted)" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", + "title": "New API Keys and Asymmetric Authentication" + }, + { + "url": "https://supabase.com/docs/guides/local-development", + "title": "Local Development & CLI" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth", + "title": "Configure Social Login (OAuth) Providers" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-saml-sso", + "title": "Configure SAML SSO" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/grafana-not-displaying-data-sXJrMj", + "title": "Grafana not displaying data" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/custom-email-templates", + "title": "Custom Email Templates" + }, + { + "url": "https://supabase.com/docs/guides/ai-tools/byo-mcp", + "title": "Deploy MCP servers" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/architecture", + "title": "Architecture" + }, + { + "url": "https://supabase.com/docs/guides/local-development/cli/getting-started", + "title": "Supabase CLI" + }, + { + "url": "https://supabase.com/docs/guides/functions/quickstart", + "title": "Getting Started with Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/database/webhooks", + "title": "Database Webhooks" + }, + { + "url": "https://supabase.com/docs/guides/functions/examples/amazon-bedrock-image-generator", + "title": "Generate Images with Amazon Bedrock" + }, + { + "url": "https://supabase.com/docs/guides/deployment/branching/integrations", + "title": "Integrations" + }, + { + "url": "https://supabase.com/docs/guides/storage/serving/image-transformations", + "title": "Storage Image Transformations" + }, + { + "url": "https://supabase.com/docs/guides/functions/deploy", + "title": "Deploy to Production" + }, + { + "url": "https://supabase.com/docs/guides/functions/examples/github-actions", + "title": "GitHub Actions" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/enable-mcp", + "title": "Enabling MCP Server Access" + }, + { + "url": "https://supabase.com/docs/guides/ai/examples/openai", + "title": "Generating OpenAI GPT3 completions" + }, + { + "url": "https://supabase.com/docs/guides/deployment", + "title": "Deployment & Branching" + } + ], + "resultChars": 320077 + } + ] }, "prompt": "I'm moving off the hosted Supabase and running the whole thing myself on a VPS I\njust spun up. Can you get a Docker setup ready for me to copy onto the box?\n\nI don't need it running here, I'll do the actual bring-up once I'm on the\nserver. I just want everything in place and the secrets set up. Put it in a `supabase-docker/`\nfolder at the repo root so I can scp the whole thing across in one go.", "promptSourcePath": "evals/deploy-self-hosting-001-docker-compose/PROMPT.md", - "attempts": 2, + "attempts": 1, "sourcePath": "opencode-kimi-k3-no-skills/deploy-self-hosting-001-docker-compose.json" }, { @@ -20087,7 +20549,7 @@ ], "suite": "benchmark", "interface": "mcp", - "passed": false, + "passed": true, "checks": [ { "name": "victim session active before delete-account", @@ -20099,18 +20561,16 @@ }, { "name": "delete-account revokes the user's sessions", - "passed": false, - "notes": "sessions left: 1" + "passed": true, + "notes": "sessions left: 0" }, { "name": "deleted user's refresh token is rejected", - "passed": false, - "notes": "refresh token still produces a session" + "passed": true }, { "name": "deleted user cannot sign back in", - "passed": false, - "notes": "deleted account can still sign in" + "passed": true }, { "name": "other users keep their sessions and access", @@ -20119,7 +20579,7 @@ { "name": "diagnosed and explained session revocation", "passed": true, - "judgeNotes": "The answer identifies the root cause as soft-deleting only the profile and not removing/revoking the auth user/sessions, implements deletion of the auth user or equivalent session/refresh-token revocation, explains that stateless JWT access tokens may remain valid until expiry for local validation and provides consistent mitigations or RLS/session-existence checks, and correctly distinguishes publishable frontend keys under user JWT/RLS from secret/server-only keys that bypass RLS and must not be exposed." + "judgeNotes": "Meets the rubric: correctly diagnoses soft-delete without auth/session revocation, implements auth user deletion with refresh/session revocation, explains the remaining stateless JWT expiry window consistently, and correctly distinguishes publishable frontend keys from secret server-only RLS-bypassing keys." } ], "skills": { @@ -20127,11 +20587,66 @@ "loaded": [] }, "docs": { - "calls": [] + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"delete user account RPC security definer function auth.users\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/reference/javascript/auth-admin-deleteuser" + }, + { + "url": "https://supabase.com/docs/reference/swift/auth-admin-deleteuser", + "title": "deleteUser()" + }, + { + "url": "https://supabase.com/docs/guides/auth/managing-user-data", + "title": "User Management" + }, + { + "url": "https://supabase.com/docs/reference/dart/auth-admin-deleteuser" + }, + { + "url": "https://supabase.com/docs/guides/database/database-advisors?queryGroups=lint&lint=0029_authenticated_security_definer_function_executable", + "title": "Database Advisor: Lint 0029_authenticated_security_definer_function_executable" + } + ], + "resultChars": 15474 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"publishable secret API keys replacing anon service_role\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" + }, + { + "url": "https://supabase.com/docs/guides/auth/signing-keys", + "title": "JWT Signing Keys" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/api-keys", + "title": "Understanding API keys" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", + "title": "New API Keys and Asymmetric Authentication" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", + "title": "Envoy API Gateway" + } + ], + "resultChars": 98570 + } + ] }, "prompt": "Last week support removed a user through our app's delete-account flow — the\napp calls the `delete_account` function over RPC as the signed-in user. This\nmorning that same person was back: still signed in, reading and saving their\ndata like nothing happened.\n\nFigure out why the account still works, fix the flow so a deleted account\nloses access, and tell me whether there is any window where they could still\nget in after the fix.\n\nOne more thing while you're at it: we're migrating off the legacy\nanon/service_role API keys to the new publishable and secret keys, and the\nteam isn't sure which one belongs in the frontend and what each means for\nRLS. Give us a quick rundown.", "promptSourcePath": "evals/investigate-auth-001-deleted-user-access/PROMPT.md", - "attempts": 2, + "attempts": 1, "sourcePath": "opencode-kimi-k3-no-skills/investigate-auth-001-deleted-user-access.json" }, { @@ -20153,11 +20668,11 @@ ], "suite": "benchmark", "interface": "mcp", - "passed": false, + "passed": true, "checks": [ { "name": "orders table added to supabase_realtime publication", - "passed": false + "passed": true }, { "name": "courier_locations still in supabase_realtime publication", @@ -20179,7 +20694,7 @@ { "name": "diagnosed missing publication membership", "passed": true, - "judgeNotes": "The answer correctly identifies that the Realtime channel can reach SUBSCRIBED while receiving no Postgres INSERT events because the orders table is not included in the supabase_realtime publication. It fixes the issue by adding only orders to the existing publication with ALTER PUBLICATION supabase_realtime ADD TABLE orders, while preserving existing RLS/policies and not disrupting courier_locations." + "judgeNotes": "Identifies missing orders table in supabase_realtime publication as root cause, applies ALTER PUBLICATION ADD TABLE public.orders, preserves courier_locations/RLS/policies, and does not blame or change unrelated areas." } ], "skills": { @@ -20191,7 +20706,7 @@ }, "prompt": "Our dispatch dashboard shows incoming orders as they happen. The courier\nlocation feed on the same page updates live without problems, but new orders\nonly show up after a page refresh.\n\nThe dashboard uses supabase-js to subscribe to INSERT events on the `orders`\ntable through postgres_changes, the same way it subscribes to courier\nlocations. The channel's status callback logs SUBSCRIBED and there are no\nerrors in the browser console.\n\nFigure out why no order events ever arrive and fix it.", "promptSourcePath": "evals/investigate-realtime-001-subscribed-no-events/PROMPT.md", - "attempts": 2, + "attempts": 1, "sourcePath": "opencode-kimi-k3-no-skills/investigate-realtime-001-subscribed-no-events.json" }, { @@ -20212,22 +20727,22 @@ ], "suite": "benchmark", "interface": "mcp", - "passed": true, + "passed": false, "checks": [ { "name": "identified image-transform and the recurring 503 pattern", "passed": true, - "judgeNotes": "The assistant identified image-transform as the affected function and recognized a recurring pattern of HTTP 503 responses during the morning of 2026-04-28, covering the gateway failures spread across roughly 07:00Z-12:00Z." + "judgeNotes": "Identified image-transform as the affected function and described the recurring pattern of 8 HTTP 503 gateway failures across the morning of 2026-04-28 from 07:00Z to 12:00Z, while distinguishing old billing-webhook 503s as unrelated." }, { "name": "attributed recurring 503s to gateway/platform layer, not function code", - "passed": true, - "judgeNotes": "The response attributes the recurring image-transform 503s to the gateway/Edge Functions platform layer rather than the function code, and grounds this in valid observations: the 503s appear only in gateway/HTTP logs with no matching invocation/runtime rows while nearby invocations succeeded; deployment_id/version remained unchanged; and it distinguishes these gateway 503s from a function-level logged 500 such as avatar-upload." + "passed": false, + "judgeNotes": "The response notes gateway-only 503s with no function execution logs, but ultimately attributes the primary cause to the image-transform function's unpinned npm dependency/cold-start behavior and recommends pinning/bundling/redeploying the function, which the rubric explicitly treats as failing." }, { "name": "recommended a concrete next step", "passed": true, - "judgeNotes": "Recommended concrete next steps including checking Supabase Edge Function logs for specific failures, validating env vars/secrets, reviewing recent deployments/configuration changes, testing the function directly with the request ID/time window, and checking Supabase status/support if platform-related." + "judgeNotes": "Recommended concrete next steps including pinning/bundling dependency, redeploying, adding retries, and monitoring specific gateway vs function logs." } ], "skills": { @@ -20239,7 +20754,7 @@ }, "prompt": "Users have been reporting that image uploads are intermittently failing with server errors throughout the morning of `2026-04-28`.\n\nCan you investigate the project logs and tell me what is going on and what we\nshould do next?", "promptSourcePath": "evals/investigate-reliability-003-edge-function-5xx-correlation/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "opencode-kimi-k3-no-skills/investigate-reliability-003-edge-function-5xx-correlation.json" }, { @@ -20263,7 +20778,7 @@ ], "suite": "benchmark", "interface": "mcp", - "passed": false, + "passed": true, "checks": [ { "name": "RLS still enabled on bookmarks", @@ -20271,7 +20786,7 @@ }, { "name": "user A reads own bookmarks", - "passed": false + "passed": true }, { "name": "user B cannot read user A bookmarks", @@ -20283,8 +20798,7 @@ }, { "name": "user A can save a new bookmark", - "passed": false, - "notes": "new row violates row-level security policy for table \"bookmarks\"" + "passed": true }, { "name": "user B cannot insert a bookmark as user A", @@ -20293,7 +20807,7 @@ { "name": "diagnosed RLS and added owner-scoped policies", "passed": true, - "judgeNotes": "The answer diagnoses RLS enabled with no policies causing deny-all, keeps RLS enabled, and adds authenticated SELECT and INSERT policies scoped to user_id = auth.uid() using USING and WITH CHECK respectively." + "judgeNotes": "Diagnosed RLS deny-all due to no policies, kept RLS enabled, and created authenticated SELECT and INSERT policies scoped to user_id = auth.uid() with WITH CHECK for inserts." } ], "skills": { @@ -20305,7 +20819,7 @@ }, "prompt": "Our app lets signed-in users save bookmarks and view them on their dashboard. Bookmarks are stored in the `bookmarks` table and are private — a user must only ever see their own. \nUsers also need to be able to save new bookmarks from the app.\n\nI can see the rows when I query the table directly, but the dashboard shows an empty list for every user.\n\nFind out why the Data API returns nothing and fix it.", "promptSourcePath": "evals/resolve-dataapi-001-empty-results/PROMPT.md", - "attempts": 2, + "attempts": 1, "sourcePath": "opencode-kimi-k3-no-skills/resolve-dataapi-001-empty-results.json" }, { @@ -20330,23 +20844,19 @@ "checks": [ { "name": "the avatar_url column is applied on the hosted profiles table", - "passed": false, - "notes": "avatar_url not found on public.profiles" + "passed": true }, { "name": "migration 20240220000000 is recorded in the remote history", - "passed": false, - "notes": "remote history versions: [\"20240101000000\",\"20240115000000\"]" + "passed": true }, { "name": "remote migration history matches local migration files", - "passed": false, - "notes": "remote-only versions still unreconciled: [\"20240115000000\"] (remote: [\"20240101000000\",\"20240115000000\"], local: [\"20240101000000\",\"20240220000000\"])" + "passed": true }, { "name": "local migrations are a valid reconciled sequence", - "passed": false, - "notes": "expected 20240101000000_create_profiles → → 20240220000000_add_avatar_url, strictly ascending; got [20240101000000_create_profiles.sql, 20240220000000_add_avatar_url.sql]" + "passed": true }, { "name": "production profile data is intact (not reset)", @@ -20355,7 +20865,7 @@ { "name": "the avatar migration and history reconciliation were done via the Supabase CLI", "passed": false, - "notes": "no agent actions recorded to evaluate" + "judgeNotes": "FAIL: No `supabase db push` output shows `Applying migration ...` or `Finished supabase db push`, so there is no evidence the pending avatar_url migration was actually applied through the CLI. The agent added `supabase/migrations/20240115000000_add_profile_bio.sql` (#13), which could reconcile the orphan bio migration locally, and ran `supabase db push` (#15), but the recorded push output does not show a successful application/reconciliation. No prohibited direct-SQL mutation or prepared-statement workaround was seen; psql usage was read-only inspection." } ], "skills": { @@ -20389,24 +20899,24 @@ ], "suite": "benchmark", "interface": "mcp", - "passed": false, + "passed": true, "checks": [ { "name": "inspected pg_stat_statements for query performance", - "passed": false + "passed": true }, { "name": "ran EXPLAIN on the expensive query", - "passed": false + "passed": true }, { "name": "created index covering user_id and created_at", - "passed": false + "passed": true }, { "name": "query plan uses an index and avoids sequential scan", - "passed": false, - "notes": "Limit (cost=118.52..118.57 rows=20 width=88)\n -> Sort (cost=118.52..118.57 rows=20 width=88)\n Sort Key: created_at DESC\n -> Seq Scan on events (cost=0.00..118.09 rows=20 width=88)\n Filter: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)" + "passed": true, + "notes": "Limit (cost=55.55..55.61 rows=25 width=88)\n -> Sort (cost=55.55..55.61 rows=25 width=88)\n Sort Key: created_at DESC\n -> Bitmap Heap Scan on events (cost=4.48..54.97 rows=25 width=88)\n Recheck Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)\n -> Bitmap Index Scan on idx_events_user_created_at (cost=0.00..4.47 rows=25 width=0)\n Index Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)" }, { "name": "inserts still work", @@ -20422,7 +20932,7 @@ }, "prompt": "My database CPU keeps spiking and the app gets slow when loading recent events for a user. Can you figure out what query is causing it and make the database change needed to fix it?\n\nEnd your turn with a short summary of what you changed and why.", "promptSourcePath": "evals/resolve-performance-001-slow-query-cpu-spike/PROMPT.md", - "attempts": 2, + "attempts": 1, "sourcePath": "opencode-kimi-k3-no-skills/resolve-performance-001-slow-query-cpu-spike.json" }, { @@ -20445,7 +20955,7 @@ ], "suite": "benchmark", "interface": "mcp", - "passed": false, + "passed": true, "checks": [ { "name": "RLS enabled on notes", @@ -20453,11 +20963,11 @@ }, { "name": "tenant A sees only org A notes", - "passed": false + "passed": true }, { "name": "tenant B cannot read org A notes", - "passed": false + "passed": true }, { "name": "tenant A author can update own note", @@ -20493,7 +21003,7 @@ }, "prompt": "A customer reported that notes showed up in the wrong workspace.\n\nCan you investigate what is going on and fix it?", "promptSourcePath": "evals/resolve-security-002-rls-cross-tenant-leak/PROMPT.md", - "attempts": 2, + "attempts": 1, "sourcePath": "opencode-kimi-k3-no-skills/resolve-security-002-rls-cross-tenant-leak.json" } ] From df58341b881a32de0ef8d4f62eeb18634724da19 Mon Sep 17 00:00:00 2001 From: Matt Rossman <22670878+mattrossman@users.noreply.github.com> Date: Mon, 27 Jul 2026 22:12:01 -0400 Subject: [PATCH 17/17] fix: capitalize Kimi K3's model label in the results dashboard --- apps/web/src/lib/format.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/apps/web/src/lib/format.ts b/apps/web/src/lib/format.ts index d3b0a148..2ca9fa18 100644 --- a/apps/web/src/lib/format.ts +++ b/apps/web/src/lib/format.ts @@ -56,6 +56,10 @@ function formatOpenAiModel(modelId: string) { .join(" ") } +function formatMoonshotaiModel(modelId: string) { + return modelId.split("-").map(capitalize).join(" ") +} + function formatModel(display: ExperimentDisplay) { // opencode ids are AI Gateway `vendor/model` slugs; format just the model part. const modelId = display.modelId.replace(/^[a-z-]+\//, "") @@ -65,7 +69,7 @@ function formatModel(display: ExperimentDisplay) { case "openai": return formatOpenAiModel(modelId) case "moonshotai": - return modelId + return formatMoonshotaiModel(modelId) } }