From 644cb39a3f09af67c678ef226fc43e37c505fcc2 Mon Sep 17 00:00:00 2001 From: Pedro Rodrigues Date: Mon, 15 Jun 2026 20:29:02 +0100 Subject: [PATCH 1/6] feat: add slow query CPU spike red-herring eval (AI-841) Harder variant of resolve-performance-001-slow-query-cpu-spike: the slow-query logs and mean/max query stats point at a rare decoy report, while the real CPU hog only surfaces when ranking pg_stat_statements by total_exec_time. Forces the documented find-then-fix workflow instead of letting the agent read the offending query straight out of the logs. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../EVAL.ts | 153 ++++++++++++++++++ .../PROMPT.md | 14 ++ .../remote/logs.jsonl | 6 + .../remote/project.sql | 136 ++++++++++++++++ 4 files changed, 309 insertions(+) create mode 100644 evals/resolve-performance-002-slow-query-cpu-spike-red-herring/EVAL.ts create mode 100644 evals/resolve-performance-002-slow-query-cpu-spike-red-herring/PROMPT.md create mode 100644 evals/resolve-performance-002-slow-query-cpu-spike-red-herring/remote/logs.jsonl create mode 100644 evals/resolve-performance-002-slow-query-cpu-spike-red-herring/remote/project.sql diff --git a/evals/resolve-performance-002-slow-query-cpu-spike-red-herring/EVAL.ts b/evals/resolve-performance-002-slow-query-cpu-spike-red-herring/EVAL.ts new file mode 100644 index 00000000..873ef86a --- /dev/null +++ b/evals/resolve-performance-002-slow-query-cpu-spike-red-herring/EVAL.ts @@ -0,0 +1,153 @@ +import { + serializeTranscript, + type CheckResult, + type ToolEvalContext, + type ToolScorer, +} from "@supabase-evals/core"; +import { stripIndent } from "common-tags"; + +const TARGET_USER = "00000000-0000-0000-0000-000000000001"; + +const scorer: ToolScorer = async (ctx) => { + try { + const checks: CheckResult[] = [ + checkQueriedPgStatStatements(ctx), + checkRankedByTotalExecTime(ctx), + checkRanExplain(ctx), + await checkCreatedRecentEventsIndex(ctx), + await checkQueryPlanUsesIndex(ctx), + await checkInsertsStillWork(ctx), + ]; + + return { + passed: checks.every((check) => check.passed), + checks, + }; + } catch (error) { + const msg = error instanceof Error ? error.message : String(error); + return { + passed: false, + checks: [ + { + name: "scorer evaluated CPU spike fix", + passed: false, + notes: msg, + }, + ], + }; + } +}; + +export default scorer; + +/** Checks that the agent inspected query statistics rather than guessing the fix. */ +function checkQueriedPgStatStatements(ctx: ToolEvalContext): CheckResult { + const sql = executedSql(ctx); + + return { + name: "inspected pg_stat_statements for query performance", + passed: /\bpg_stat_statements\b/i.test(sql), + }; +} + +/** + * The crux of this scenario: the slow-query logs and the mean/max columns point + * at the rare audit_log report (the decoy). The CPU hog only surfaces when + * ranking by cumulative execution time, so a correct diagnosis sorts the query + * stats by total_exec_time rather than per-call latency. + */ +function checkRankedByTotalExecTime(ctx: ToolEvalContext): CheckResult { + const sql = executedSql(ctx); + + return { + name: "ranked query stats by total (cumulative) execution time", + passed: /\btotal_exec_time\b/i.test(sql), + }; +} + +/** Checks that the agent used EXPLAIN on the expensive events query (the hog). */ +function checkRanExplain(ctx: ToolEvalContext): CheckResult { + const sql = executedSql(ctx); + + return { + name: "ran EXPLAIN on the expensive query", + passed: + /\bexplain\b/i.test(sql) && + /\bevents\b/i.test(sql) && + /\buser_id\b/i.test(sql), + }; +} + +/** Checks for the index shape needed by the recent events lookup (the CPU hog). */ +async function checkCreatedRecentEventsIndex(ctx: ToolEvalContext): Promise { + const { rows } = await ctx.query(stripIndent` + SELECT indexname, indexdef + FROM pg_indexes + WHERE schemaname = 'public' + AND tablename = 'events'; + `); + + const hasCoveringIndex = rows.some((row) => { + const def = row.indexdef; + return ( + typeof def === "string" && + /ON\s+(?:public\.)?events\s+.*\(\s*user_id\s*,\s*created_at/i.test(def) + ); + }); + + return { + name: "created index covering user_id and created_at on events", + passed: hasCoveringIndex, + }; +} + +/** + * Decisive correctness check: verifies Postgres plans the high-frequency hog + * lookup with an index and no sequential scan. An agent that only fixed the + * audit_log decoy leaves this query on a seq scan and fails here. + */ +async function checkQueryPlanUsesIndex(ctx: ToolEvalContext): Promise { + const { rows } = await ctx.query(stripIndent` + EXPLAIN SELECT id, kind, payload, created_at + FROM events + WHERE user_id = '${TARGET_USER}' + ORDER BY created_at DESC + LIMIT 50; + `); + const plan = rows.map((row) => Object.values(row).join(" ")).join("\n"); + + return { + name: "hog query plan uses an index and avoids sequential scan", + passed: + /(Index Scan|Index Only Scan|Bitmap Index Scan)/i.test(plan) && + !/Seq Scan on events/i.test(plan), + notes: plan, + }; +} + +/** Confirms the schema change did not break normal inserts into events. */ +async function checkInsertsStillWork(ctx: ToolEvalContext): Promise { + const { rows } = await ctx.query(stripIndent` + INSERT INTO events (user_id, kind, payload) + VALUES ('${TARGET_USER}', 'insert_probe', '{"ok": true}'::jsonb) + RETURNING id; + `); + + return { + name: "inserts still work", + passed: rows.length === 1, + }; +} + +/** Collects SQL-like text from tool calls and transcript entries for workflow checks. */ +function executedSql(ctx: ToolEvalContext): string { + const toolCallSql = ctx.toolCalls + .flatMap((call) => Object.values(call.body)) + .filter((value): value is string => typeof value === "string") + .join("\n"); + + return [ + toolCallSql, + serializeTranscript(ctx.transcript, { includeToolCallInputs: true }), + ].join("\n"); +} diff --git a/evals/resolve-performance-002-slow-query-cpu-spike-red-herring/PROMPT.md b/evals/resolve-performance-002-slow-query-cpu-spike-red-herring/PROMPT.md new file mode 100644 index 00000000..c1300d18 --- /dev/null +++ b/evals/resolve-performance-002-slow-query-cpu-spike-red-herring/PROMPT.md @@ -0,0 +1,14 @@ +--- +stage: resolve +suite: benchmark +product: + - database +topic: + - observability + - sql +motivation: AI-841, AI-824, https://supabase.com/docs/guides/troubleshooting/high-cpu-usage +--- + +My database CPU has been spiking and the app feels sluggish, but I can't tell what's driving it. Can you figure out which query is actually responsible for the load and make the database change needed to bring the CPU down? + +End your turn with a short summary of what you changed and why. diff --git a/evals/resolve-performance-002-slow-query-cpu-spike-red-herring/remote/logs.jsonl b/evals/resolve-performance-002-slow-query-cpu-spike-red-herring/remote/logs.jsonl new file mode 100644 index 00000000..429f92a8 --- /dev/null +++ b/evals/resolve-performance-002-slow-query-cpu-spike-red-herring/remote/logs.jsonl @@ -0,0 +1,6 @@ +{"id":"a1","ts":"2026-04-28T10:00:00Z","source":"postgres","level":"warning","message":"duration: 3210 ms execute : SELECT id, actor, action, details, created_at FROM audit_log WHERE action = $1 ORDER BY created_at DESC LIMIT 100","metadata":{"query_hash":"audit_action_recent_h1","duration_ms":3210,"table":"audit_log"}} +{"id":"a2","ts":"2026-04-28T10:04:00Z","source":"postgres","level":"warning","message":"duration: 2980 ms execute : SELECT id, actor, action, details, created_at FROM audit_log WHERE action = $1 ORDER BY created_at DESC LIMIT 100","metadata":{"query_hash":"audit_action_recent_h1","duration_ms":2980,"table":"audit_log"}} +{"id":"a3","ts":"2026-04-28T10:09:00Z","source":"postgres","level":"warning","message":"duration: 3520 ms execute : SELECT id, actor, action, details, created_at FROM audit_log WHERE action = $1 ORDER BY created_at DESC LIMIT 100","metadata":{"query_hash":"audit_action_recent_h1","duration_ms":3520,"table":"audit_log"}} +{"id":"a4","ts":"2026-04-28T10:13:00Z","source":"postgres","level":"warning","message":"duration: 3040 ms execute : SELECT id, actor, action, details, created_at FROM audit_log WHERE action = $1 ORDER BY created_at DESC LIMIT 100","metadata":{"query_hash":"audit_action_recent_h1","duration_ms":3040,"table":"audit_log"}} +{"id":"n1","ts":"2026-04-28T10:14:00Z","source":"postgres","level":"info","message":"duration: 11 ms execute : SELECT id, name FROM accounts LIMIT 20","metadata":{"query_hash":"accounts_list_h1","duration_ms":11,"table":"accounts"}} +{"id":"n2","ts":"2026-04-28T10:15:00Z","source":"postgres","level":"info","message":"duration: 24 ms execute : SELECT count(*) FROM audit_log WHERE action = $1","metadata":{"query_hash":"audit_count_h1","duration_ms":24,"table":"audit_log"}} diff --git a/evals/resolve-performance-002-slow-query-cpu-spike-red-herring/remote/project.sql b/evals/resolve-performance-002-slow-query-cpu-spike-red-herring/remote/project.sql new file mode 100644 index 00000000..41af604e --- /dev/null +++ b/evals/resolve-performance-002-slow-query-cpu-spike-red-herring/remote/project.sql @@ -0,0 +1,136 @@ +-- Red-herring CPU spike scenario. +-- +-- The real CPU hog is a fast-but-very-frequent per-user lookup on `events` +-- that never crosses the slow-query log threshold, so it is INVISIBLE in the +-- logs. The slow-query logs are instead dominated by a genuinely slow but rare +-- report query on `audit_log`. An agent that diagnoses from the logs (or sorts +-- pg_stat_statements by mean/max time) will "fix" the decoy and leave the CPU +-- spike in place. The only way to the correct fix is ranking +-- pg_stat_statements by total_exec_time (calls x mean_exec_time). + +CREATE TABLE events ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + user_id uuid NOT NULL, + kind text NOT NULL, + payload jsonb NOT NULL DEFAULT '{}'::jsonb, + created_at timestamptz NOT NULL DEFAULT now() +); + +CREATE TABLE audit_log ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + actor uuid NOT NULL, + action text NOT NULL, + details jsonb NOT NULL DEFAULT '{}'::jsonb, + created_at timestamptz NOT NULL DEFAULT now() +); + +CREATE TABLE accounts ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + name text NOT NULL +); + +INSERT INTO accounts (name) +SELECT 'account-' || n +FROM generate_series(1, 20) AS n; + +-- High-frequency hog target: ~120 rows per user across 50 users. +INSERT INTO events (user_id, kind, payload, created_at) +SELECT + ('00000000-0000-0000-0000-' || lpad((((n - 1) % 50) + 1)::text, 12, '0'))::uuid, + CASE WHEN n % 5 = 0 THEN 'checkout' ELSE 'page_view' END, + jsonb_build_object('seq', n), + '2026-04-28T10:00:00Z'::timestamptz - (n || ' seconds')::interval +FROM generate_series(1, 6000) AS n; + +-- Decoy table for the slow-but-rare report query that fills the logs. +INSERT INTO audit_log (actor, action, details, created_at) +SELECT + ('00000000-0000-0000-0000-' || lpad((((n - 1) % 50) + 1)::text, 12, '0'))::uuid, + (ARRAY['login', 'logout', 'update_profile', 'delete_account', 'export_data'])[(n % 5) + 1], + jsonb_build_object('seq', n), + '2026-04-28T10:00:00Z'::timestamptz - (n || ' seconds')::interval +FROM generate_series(1, 8000) AS n; + +-- Eval-scoped compatibility table for Supabase's Query Performance report, +-- which is backed by pg_stat_statements in real projects. +CREATE TABLE pg_stat_statements ( + userid oid NOT NULL DEFAULT 10::oid, + dbid oid NOT NULL DEFAULT 5::oid, + queryid bigint PRIMARY KEY, + query text NOT NULL, + calls bigint NOT NULL, + rows bigint NOT NULL DEFAULT 0, + total_plan_time double precision NOT NULL DEFAULT 0, + total_exec_time double precision NOT NULL, + mean_plan_time double precision NOT NULL DEFAULT 0, + mean_exec_time double precision NOT NULL, + max_exec_time double precision NOT NULL, + shared_blks_hit bigint NOT NULL DEFAULT 0, + shared_blks_read bigint NOT NULL DEFAULT 0, + shared_blks_dirtied bigint NOT NULL DEFAULT 0, + shared_blks_written bigint NOT NULL DEFAULT 0 +); + +-- Stats deliberately diverge by ranking dimension: +-- total_exec_time -> events hog dominates (~99%): the correct culprit. +-- mean/max_exec_time -> audit_log report wins: the decoy the logs point at. +INSERT INTO pg_stat_statements ( + queryid, + query, + calls, + rows, + total_exec_time, + mean_exec_time, + max_exec_time, + shared_blks_hit, + shared_blks_read +) +VALUES + -- Real CPU hog: tiny per-call, enormous call count, dominant total time. + -- Below the slow-query log threshold, so it never appears in logs.jsonl. + ( + 841001, + 'SELECT id, kind, payload, created_at FROM events WHERE user_id = $1 ORDER BY created_at DESC LIMIT 50', + 486000, + 24300000, + 10692000, + 22, + 160, + 9100000, + 7400000 + ), + -- Decoy: genuinely slow per call but rare, so total time is negligible. + -- Tops the slow-query logs and the mean/max columns. + ( + 841002, + 'SELECT id, actor, action, details, created_at FROM audit_log WHERE action = $1 ORDER BY created_at DESC LIMIT 100', + 14, + 1400, + 43680, + 3120, + 3520, + 420, + 96000 + ), + ( + 841003, + 'SELECT id, name FROM accounts LIMIT 20', + 900, + 18000, + 9900, + 11, + 28, + 17400, + 60 + ), + ( + 841004, + 'SELECT count(*) FROM audit_log WHERE action = $1', + 210, + 210, + 5040, + 24, + 70, + 20300, + 180 + ); From 1a187e22cc25eae9fca365caaaaad681835f4dcc Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 15 Jun 2026 19:36:56 +0000 Subject: [PATCH 2/6] chore: refresh eval results --- apps/web/src/data/eval-results.json | 210 +++++++++++++++++++++++----- 1 file changed, 175 insertions(+), 35 deletions(-) diff --git a/apps/web/src/data/eval-results.json b/apps/web/src/data/eval-results.json index 34c591f5..6d63f5ad 100644 --- a/apps/web/src/data/eval-results.json +++ b/apps/web/src/data/eval-results.json @@ -16,7 +16,7 @@ "supabase-js" ], "suite": "benchmark", - "passed": false, + "passed": true, "checks": [ { "name": "rejects missing auth", @@ -32,14 +32,14 @@ }, { "name": "user A cannot force-read user B note", - "passed": false + "passed": true }, { "name": "user B cannot force-read user A note", - "passed": false + "passed": true } ], - "prompt": "# Private Notes Function Review\n\nI built an Edge Function called `private-notes` for showing a user's saved\nprivate notes.\n\nCan you check whether there's any way one user could see another user's notes?\n\nPlease fix and deploy it if needed.", + "prompt": "I built an Edge Function called `private-notes` for showing a user's saved\nprivate notes.\n\nCan you check whether there's any way one user could see another user's notes?\n\nPlease fix and deploy it if needed.", "promptSourcePath": "evals/build-functions-004-service-role-bypass/PROMPT.md", "attempts": 1, "sourcePath": "openai-gpt-5.4-mini/build-functions-004-service-role-bypass.json" @@ -74,7 +74,7 @@ { "name": "user A lists only own files", "passed": true, - "notes": "saw: 019ecbf7-ac84-76e9-a3b5-687c1fd51010/receipt-alpha.pdf, 019ecbf7-ac84-76e9-a3b5-687c1fd51010/receipt-beta.pdf" + "notes": "saw: 019eccc4-2af3-76a2-8b33-8331b8846214/receipt-alpha.pdf, 019eccc4-2af3-76a2-8b33-8331b8846214/receipt-beta.pdf" }, { "name": "user B cannot read user A files", @@ -95,10 +95,10 @@ { "name": "configured private per-user storage access", "passed": true, - "judgeNotes": "Configured a private user-files bucket, owner-scoped SELECT and INSERT policies on storage.objects for authenticated users using the user-id path prefix, did not make the bucket public or use permissive/public policies, and provided supabase-js createSignedUrl code with an expiry for temporary sharing." + "judgeNotes": "Configured a private user-files bucket, kept RLS enabled, added authenticated owner-scoped SELECT and INSERT policies on storage.objects, and provided supabase-js createSignedUrl code with an expiry." } ], - "prompt": "# Private User Files in Storage\n\nOur app lets signed-in users keep personal files like receipts and bank\nstatements. These files are private — a user must only ever be able to upload\nand download their own. The app uploads each file under a path that starts\nwith the owner's user id, e.g. `/receipt-march.pdf`.\n\nSet up a `user-files` bucket on our project and lock it down that way.\n\nUsers also sometimes share one of their files with someone else through a\ntemporary link that expires. Include the supabase-js code the app should use\nfor that.", + "prompt": "Our app lets signed-in users keep personal files like receipts and bank\nstatements. These files are private — a user must only ever be able to upload\nand download their own. The app uploads each file under a path that starts\nwith the owner's user id, e.g. `/receipt-march.pdf`.\n\nSet up a `user-files` bucket on our project and lock it down that way.\n\nUsers also sometimes share one of their files with someone else through a\ntemporary link that expires. Include the supabase-js code the app should use\nfor that.", "promptSourcePath": "evals/build-storage-001-private-bucket-access/PROMPT.md", "attempts": 1, "sourcePath": "openai-gpt-5.4-mini/build-storage-001-private-bucket-access.json" @@ -123,12 +123,12 @@ { "name": "configured the Supabase Metrics API scrape correctly", "passed": false, - "judgeNotes": "Fails: Supabase scrape uses basic_auth password with an env Secret API key instead of password_file, target is not the required b602ce33357545a18851.supabase.co/red project endpoint, and docker-compose.yml does not mount the password_file via a volume or Compose secret. Existing app scrape is preserved and HTTPS/path are correct." + "judgeNotes": "Fails: Supabase target is a placeholder rather than 0ad55f3638b24cf08c75.supabase.co/red, uses hardcoded basic_auth password instead of password_file, and docker-compose.yml does not mount or wire the password file via volume or Compose secret. Existing app scrape is preserved, but required secret wiring and endpoint target are not deployable." }, { "name": "documented live deployment and verification steps", - "passed": true, - "judgeNotes": "README includes creating a Supabase Secret API key, placing SUPABASE_PROJECT_REF and SUPABASE_SECRET_API_KEY in .env.local used by Compose, restarting Prometheus, and verifying via Prometheus targets." + "passed": false, + "judgeNotes": "README includes Supabase endpoint and Prometheus target verification, but it fails the requirements: it instructs placing the Secret API key directly in prometheus.yml instead of using a matching secret file, does not describe creating/placing that secret file, and restart/reload Compose steps are not concrete enough for making the integration live." } ], "prompt": "Can you wire my Supabase project metrics into our existing observability stack and document\nwhat we need to do to make the config live?", @@ -136,6 +136,31 @@ "attempts": 1, "sourcePath": "openai-gpt-5.4-mini/deploy-database-001-prometheus-metrics/summary.json" }, + { + "experiment": "openai-gpt-5.4-mini", + "eval": "investigate-auth-001-deleted-user-access", + "stage": "investigate", + "product": [ + "auth" + ], + "topic": [ + "security", + "sdk" + ], + "suite": "benchmark", + "passed": false, + "checks": [ + { + "name": "scorer evaluated deleted user access", + "passed": false, + "notes": "relation \"profiles\" does not exist" + } + ], + "prompt": "Last week support removed a user through our app's delete-account flow — the\napp calls the `delete_account` function over RPC as the signed-in user. This\nmorning that same person was back: still signed in, reading and saving their\ndata like nothing happened.\n\nFigure out why the account still works, fix the flow so a deleted account\nloses access, and tell me whether there is any window where they could still\nget in after the fix.\n\nOne more thing while you're at it: we're migrating off the legacy\nanon/service_role API keys to the new publishable and secret keys, and the\nteam isn't sure which one belongs in the frontend and what each means for\nRLS. Give us a quick rundown.", + "promptSourcePath": "evals/investigate-auth-001-deleted-user-access/PROMPT.md", + "attempts": 1, + "sourcePath": "openai-gpt-5.4-mini/investigate-auth-001-deleted-user-access.json" + }, { "experiment": "openai-gpt-5.4-mini", "eval": "investigate-realtime-001-subscribed-no-events", @@ -174,10 +199,10 @@ { "name": "diagnosed missing publication membership", "passed": false, - "judgeNotes": "Identified the missing supabase_realtime publication entry and added orders, but did not fix exactly that: it also created/modified the orders table, set replica identity, enabled RLS, and granted access. The rubric requires leaving existing RLS/policies/project behavior untouched and only adding orders to the existing publication." + "judgeNotes": "The answer identified the missing supabase_realtime publication membership, but it did not fix exactly that. It created/altered the orders table, added grants, and created a broad read RLS policy, which the rubric forbids because the fix should only add the existing orders table to the existing publication while preserving existing policies/RLS." } ], - "prompt": "# Debug Realtime publication\n\nOur dispatch dashboard shows incoming orders as they happen. The courier\nlocation feed on the same page updates live without problems, but new orders\nonly show up after a page refresh.\n\nThe dashboard uses supabase-js to subscribe to INSERT events on the `orders`\ntable through postgres_changes, the same way it subscribes to courier\nlocations. The channel's status callback logs SUBSCRIBED and there are no\nerrors in the browser console.\n\nFigure out why no order events ever arrive and fix it.", + "prompt": "Our dispatch dashboard shows incoming orders as they happen. The courier\nlocation feed on the same page updates live without problems, but new orders\nonly show up after a page refresh.\n\nThe dashboard uses supabase-js to subscribe to INSERT events on the `orders`\ntable through postgres_changes, the same way it subscribes to courier\nlocations. The channel's status callback logs SUBSCRIBED and there are no\nerrors in the browser console.\n\nFigure out why no order events ever arrive and fix it.", "promptSourcePath": "evals/investigate-realtime-001-subscribed-no-events/PROMPT.md", "attempts": 1, "sourcePath": "openai-gpt-5.4-mini/investigate-realtime-001-subscribed-no-events.json" @@ -195,22 +220,22 @@ "logs" ], "suite": "benchmark", - "passed": false, + "passed": true, "checks": [ { "name": "identified image-transform and the recurring 503 pattern", "passed": true, - "judgeNotes": "The assistant identified image-transform as affected and described repeated 503 responses throughout the morning with intermittent successes, matching the required recurring pattern. It did mention avatar-upload too, but did not make old billing-webhook 503s the main issue." + "judgeNotes": "The assistant correctly identified image-transform as the affected function and described the recurring pattern of eight HTTP 503 gateway failures spread across the morning of 2026-04-28 (07:00Z–12:00Z), while distinguishing it from other isolated/irrelevant errors." }, { "name": "attributed recurring 503s to gateway/platform layer, not function code", - "passed": false, - "judgeNotes": "The assistant did not attribute the recurring 503s to the gateway/platform layer in front of the function. It instead framed the issue as the Edge Function layer / upstream dependency-runtime and recommended inspecting/redeploying function code, which the rubric explicitly treats as failing." + "passed": true, + "judgeNotes": "The response identifies the recurring image-transform 503s as gateway-level rather than app-level stack traces, distinguishes them from the avatar-upload 500, and notes alternating successful nearby invocations. Although it suggests inspecting code for dependencies, it does not primarily blame or recommend redeploying the function code." }, { "name": "recommended a concrete next step", "passed": true, - "judgeNotes": "The assistant recommended specific actionable next steps: inspect Edge Function code/dependencies, check external APIs/packages for flakiness, redeploy with retries/error handling, and add a fallback path." + "judgeNotes": "Recommended concrete next steps including opening a Supabase support ticket with function slug, deployment/version, timestamps, and 503 gateway response details, plus inspecting function dependencies/logging." } ], "prompt": "Users have been reporting that image uploads are intermittently failing with server errors throughout the morning of `2026-04-28`.\n\nCan you investigate the project logs and tell me what is going on and what we\nshould do next?", @@ -261,10 +286,10 @@ { "name": "diagnosed RLS and added owner-scoped policies", "passed": true, - "judgeNotes": "Diagnosed RLS deny-all due to no policies and added authenticated-only SELECT and INSERT policies scoped to user_id = auth.uid(), while keeping RLS enabled." + "judgeNotes": "Diagnosed deny-all RLS on bookmarks and added authenticated-only owner-scoped SELECT and INSERT policies with WITH CHECK, while leaving RLS enabled." } ], - "prompt": "# Bookmarks Dashboard Shows Nothing\n\nOur app lets signed-in users save bookmarks and view them on their dashboard. Bookmarks are stored in the `bookmarks` table and are private — a user must only ever see their own. \nUsers also need to be able to save new bookmarks from the app.\n\nI can see the rows when I query the table directly, but the dashboard shows an empty list for every user.\n\nFind out why the Data API returns nothing and fix it.", + "prompt": "Our app lets signed-in users save bookmarks and view them on their dashboard. Bookmarks are stored in the `bookmarks` table and are private — a user must only ever see their own. \nUsers also need to be able to save new bookmarks from the app.\n\nI can see the rows when I query the table directly, but the dashboard shows an empty list for every user.\n\nFind out why the Data API returns nothing and fix it.", "promptSourcePath": "evals/resolve-dataapi-001-empty-results/PROMPT.md", "attempts": 1, "sourcePath": "openai-gpt-5.4-mini/resolve-dataapi-001-empty-results.json" @@ -310,6 +335,51 @@ "attempts": 1, "sourcePath": "openai-gpt-5.4-mini/resolve-performance-001-slow-query-cpu-spike.json" }, + { + "experiment": "openai-gpt-5.4-mini", + "eval": "resolve-performance-002-slow-query-cpu-spike-red-herring", + "stage": "resolve", + "product": [ + "database" + ], + "topic": [ + "observability", + "sql" + ], + "suite": "benchmark", + "passed": true, + "checks": [ + { + "name": "inspected pg_stat_statements for query performance", + "passed": true + }, + { + "name": "ranked query stats by total (cumulative) execution time", + "passed": true + }, + { + "name": "ran EXPLAIN on the expensive query", + "passed": true + }, + { + "name": "created index covering user_id and created_at on events", + "passed": true + }, + { + "name": "hog query plan uses an index and avoids sequential scan", + "passed": true, + "notes": "Limit (cost=65.16..65.23 rows=30 width=88)\n -> Sort (cost=65.16..65.23 rows=30 width=88)\n Sort Key: created_at DESC\n -> Bitmap Heap Scan on events (cost=4.51..64.42 rows=30 width=88)\n Recheck Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)\n -> Bitmap Index Scan on events_user_id_created_at_idx (cost=0.00..4.51 rows=30 width=0)\n Index Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)" + }, + { + "name": "inserts still work", + "passed": true + } + ], + "prompt": "My database CPU has been spiking and the app feels sluggish, but I can't tell what's driving it. Can you figure out which query is actually responsible for the load and make the database change needed to bring the CPU down?\n\nEnd your turn with a short summary of what you changed and why.", + "promptSourcePath": "evals/resolve-performance-002-slow-query-cpu-spike-red-herring/PROMPT.md", + "attempts": 1, + "sourcePath": "openai-gpt-5.4-mini/resolve-performance-002-slow-query-cpu-spike-red-herring.json" + }, { "experiment": "openai-gpt-5.4-mini", "eval": "resolve-security-002-rls-cross-tenant-leak", @@ -407,7 +477,7 @@ "passed": true } ], - "prompt": "# Private Notes Function Review\n\nI built an Edge Function called `private-notes` for showing a user's saved\nprivate notes.\n\nCan you check whether there's any way one user could see another user's notes?\n\nPlease fix and deploy it if needed.", + "prompt": "I built an Edge Function called `private-notes` for showing a user's saved\nprivate notes.\n\nCan you check whether there's any way one user could see another user's notes?\n\nPlease fix and deploy it if needed.", "promptSourcePath": "evals/build-functions-004-service-role-bypass/PROMPT.md", "attempts": 1, "sourcePath": "openai-gpt-5.4-nano/build-functions-004-service-role-bypass.json" @@ -425,7 +495,7 @@ "sdk" ], "suite": "benchmark", - "passed": false, + "passed": true, "checks": [ { "name": "bucket user-files exists", @@ -442,7 +512,7 @@ { "name": "user A lists only own files", "passed": true, - "notes": "saw: 019ecbf7-9310-744c-911a-6ce1c0a7afa4/receipt-alpha.pdf, 019ecbf7-9310-744c-911a-6ce1c0a7afa4/receipt-beta.pdf" + "notes": "saw: 019eccc4-639f-7168-9838-e864feb47f8f/receipt-alpha.pdf, 019eccc4-639f-7168-9838-e864feb47f8f/receipt-beta.pdf" }, { "name": "user B cannot read user A files", @@ -462,11 +532,11 @@ }, { "name": "configured private per-user storage access", - "passed": false, - "judgeNotes": "Bucket is private, RLS is enabled, owner-scoped SELECT/INSERT policies exist, and createSignedUrl with expiry is provided. However, the policies omit `TO authenticated`, so they default to PUBLIC, which scopes object access policies to public/anon as well as authenticated, a listed fail condition." + "passed": true, + "judgeNotes": "Private bucket created, authenticated owner-scoped SELECT and INSERT policies on storage.objects were provided, RLS was not disabled, and createSignedUrl with an expiry was included for temporary sharing. Service role usage was explicitly server-side only, not client-side." } ], - "prompt": "# Private User Files in Storage\n\nOur app lets signed-in users keep personal files like receipts and bank\nstatements. These files are private — a user must only ever be able to upload\nand download their own. The app uploads each file under a path that starts\nwith the owner's user id, e.g. `/receipt-march.pdf`.\n\nSet up a `user-files` bucket on our project and lock it down that way.\n\nUsers also sometimes share one of their files with someone else through a\ntemporary link that expires. Include the supabase-js code the app should use\nfor that.", + "prompt": "Our app lets signed-in users keep personal files like receipts and bank\nstatements. These files are private — a user must only ever be able to upload\nand download their own. The app uploads each file under a path that starts\nwith the owner's user id, e.g. `/receipt-march.pdf`.\n\nSet up a `user-files` bucket on our project and lock it down that way.\n\nUsers also sometimes share one of their files with someone else through a\ntemporary link that expires. Include the supabase-js code the app should use\nfor that.", "promptSourcePath": "evals/build-storage-001-private-bucket-access/PROMPT.md", "attempts": 1, "sourcePath": "openai-gpt-5.4-nano/build-storage-001-private-bucket-access.json" @@ -491,12 +561,12 @@ { "name": "configured the Supabase Metrics API scrape correctly", "passed": false, - "judgeNotes": "Fails: Supabase target is a placeholder, not 20b46eaca601447dac66.supabase.co/red; basic_auth uses a hardcoded password field instead of password_file; docker-compose.yml does not mount or provide the password_file via volume or Compose secret. Existing app scrape is preserved and HTTPS/path are correct." + "judgeNotes": "Fails: Supabase target is not explicitly 2d0f29aaa5af4811b3d1.supabase.co or .supabase.red, basic_auth uses an environment-variable password instead of password_file, and docker-compose.yml does not mount the password_file via a volume or Compose secret. App scrape is preserved and HTTPS/path are correct." }, { "name": "documented live deployment and verification steps", "passed": false, - "judgeNotes": "README includes correct Supabase endpoint/auth, Secret API key creation, restart, and concrete Prometheus verification. However it instructs users to put the key directly in prometheus.yml and does not describe placing a matching secret file or Compose secret setup, which the rubric requires." + "judgeNotes": "README includes Secret API key creation, restart/recreate, and a concrete curl verification, but it does not instruct placing the matching secret file as required. It uses environment variables instead, so the required secret setup is missing/mismatched." } ], "prompt": "Can you wire my Supabase project metrics into our existing observability stack and document\nwhat we need to do to make the config live?", @@ -504,6 +574,31 @@ "attempts": 1, "sourcePath": "openai-gpt-5.4-nano/deploy-database-001-prometheus-metrics/summary.json" }, + { + "experiment": "openai-gpt-5.4-nano", + "eval": "investigate-auth-001-deleted-user-access", + "stage": "investigate", + "product": [ + "auth" + ], + "topic": [ + "security", + "sdk" + ], + "suite": "benchmark", + "passed": false, + "checks": [ + { + "name": "scorer evaluated deleted user access", + "passed": false, + "notes": "relation \"profiles\" does not exist" + } + ], + "prompt": "Last week support removed a user through our app's delete-account flow — the\napp calls the `delete_account` function over RPC as the signed-in user. This\nmorning that same person was back: still signed in, reading and saving their\ndata like nothing happened.\n\nFigure out why the account still works, fix the flow so a deleted account\nloses access, and tell me whether there is any window where they could still\nget in after the fix.\n\nOne more thing while you're at it: we're migrating off the legacy\nanon/service_role API keys to the new publishable and secret keys, and the\nteam isn't sure which one belongs in the frontend and what each means for\nRLS. Give us a quick rundown.", + "promptSourcePath": "evals/investigate-auth-001-deleted-user-access/PROMPT.md", + "attempts": 1, + "sourcePath": "openai-gpt-5.4-nano/investigate-auth-001-deleted-user-access.json" + }, { "experiment": "openai-gpt-5.4-nano", "eval": "investigate-realtime-001-subscribed-no-events", @@ -542,10 +637,10 @@ { "name": "diagnosed missing publication membership", "passed": false, - "judgeNotes": "The assistant did not apply the required fix. It claimed there were no application tables and no publication, asked the user to verify the project, and did not run ALTER PUBLICATION supabase_realtime ADD TABLE orders. It also mentioned RLS as something to check, rather than fixing exactly the missing orders table in the existing realtime publication." + "judgeNotes": "The answer does include adding public.orders to supabase_realtime, but it incorrectly identifies the primary key as the top culprit and recommends altering the orders primary key. The required diagnosis/fix was exactly that orders was missing from the existing supabase_realtime publication while preserving existing RLS/policies and other feeds." } ], - "prompt": "# Debug Realtime publication\n\nOur dispatch dashboard shows incoming orders as they happen. The courier\nlocation feed on the same page updates live without problems, but new orders\nonly show up after a page refresh.\n\nThe dashboard uses supabase-js to subscribe to INSERT events on the `orders`\ntable through postgres_changes, the same way it subscribes to courier\nlocations. The channel's status callback logs SUBSCRIBED and there are no\nerrors in the browser console.\n\nFigure out why no order events ever arrive and fix it.", + "prompt": "Our dispatch dashboard shows incoming orders as they happen. The courier\nlocation feed on the same page updates live without problems, but new orders\nonly show up after a page refresh.\n\nThe dashboard uses supabase-js to subscribe to INSERT events on the `orders`\ntable through postgres_changes, the same way it subscribes to courier\nlocations. The channel's status callback logs SUBSCRIBED and there are no\nerrors in the browser console.\n\nFigure out why no order events ever arrive and fix it.", "promptSourcePath": "evals/investigate-realtime-001-subscribed-no-events/PROMPT.md", "attempts": 1, "sourcePath": "openai-gpt-5.4-nano/investigate-realtime-001-subscribed-no-events.json" @@ -568,17 +663,17 @@ { "name": "identified image-transform and the recurring 503 pattern", "passed": true, - "judgeNotes": "Assistant identified image-transform as the affected function and described recurring HTTP 503 responses throughout the morning of 2026-04-28, covering most of the gateway failures across the 07:00Z-12:00Z window. It did not incorrectly focus on billing-webhook." + "judgeNotes": "The assistant named image-transform as the affected function and described intermittent/recurring 503s throughout the morning of 2026-04-28 with successful 200s interleaved, covering the relevant 07:00Z-12:00Z pattern sufficiently." }, { "name": "attributed recurring 503s to gateway/platform layer, not function code", "passed": false, - "judgeNotes": "The assistant attributes the recurring image-transform 503s to the Edge Function/dependency/runtime behavior and recommends reviewing/fixing function dependencies and hardening function code. It does not identify the gateway/platform layer as the source or ground that attribution in observations like missing invocation/runtime rows, unchanged deployment_id, or distinction from function-level errors." + "judgeNotes": "The answer attributes the 503s to transient dependency/runtime/function issues and recommends instrumentation, dependency pinning, retries inside the function, and redeploying. It does not attribute the recurring 503s to the gateway/platform layer or ground that attribution in observations like gateway-only logs with no invocation/runtime rows." }, { "name": "recommended a concrete next step", "passed": true, - "judgeNotes": "The assistant recommended concrete next steps, including inspecting the Edge Function dependencies/configuration, checking client error payloads, mitigating by bypassing/rolling back image processing, and adding retries/timeouts/concurrency limits." + "judgeNotes": "The assistant recommended concrete next steps: instrumenting the Edge Function with structured error logging, pinning/replacing the npm dependency, adding retries/backoff, and checking correlation with scaling/cold starts via redeploy/instrumentation." } ], "prompt": "Users have been reporting that image uploads are intermittently failing with server errors throughout the morning of `2026-04-28`.\n\nCan you investigate the project logs and tell me what is going on and what we\nshould do next?", @@ -629,10 +724,10 @@ { "name": "diagnosed RLS and added owner-scoped policies", "passed": true, - "judgeNotes": "Diagnosed RLS deny-all due to no policies, kept RLS enabled, and created authenticated owner-scoped SELECT and INSERT policies using user_id = auth.uid() / WITH CHECK. Extra UPDATE/DELETE policies are acceptable." + "judgeNotes": "The assistant correctly diagnosed RLS enabled with no policies as deny-all, kept RLS enabled, and created authenticated SELECT and INSERT policies scoped to user_id = auth.uid() with WITH CHECK for inserts." } ], - "prompt": "# Bookmarks Dashboard Shows Nothing\n\nOur app lets signed-in users save bookmarks and view them on their dashboard. Bookmarks are stored in the `bookmarks` table and are private — a user must only ever see their own. \nUsers also need to be able to save new bookmarks from the app.\n\nI can see the rows when I query the table directly, but the dashboard shows an empty list for every user.\n\nFind out why the Data API returns nothing and fix it.", + "prompt": "Our app lets signed-in users save bookmarks and view them on their dashboard. Bookmarks are stored in the `bookmarks` table and are private — a user must only ever see their own. \nUsers also need to be able to save new bookmarks from the app.\n\nI can see the rows when I query the table directly, but the dashboard shows an empty list for every user.\n\nFind out why the Data API returns nothing and fix it.", "promptSourcePath": "evals/resolve-dataapi-001-empty-results/PROMPT.md", "attempts": 1, "sourcePath": "openai-gpt-5.4-nano/resolve-dataapi-001-empty-results.json" @@ -666,7 +761,7 @@ { "name": "query plan uses an index and avoids sequential scan", "passed": true, - "notes": "Limit (cost=55.55..55.61 rows=25 width=88)\n -> Sort (cost=55.55..55.61 rows=25 width=88)\n Sort Key: created_at DESC\n -> Bitmap Heap Scan on events (cost=4.48..54.97 rows=25 width=88)\n Recheck Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)\n -> Bitmap Index Scan on events_user_id_created_at_desc_idx (cost=0.00..4.47 rows=25 width=0)\n Index Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)" + "notes": "Limit (cost=55.55..55.61 rows=25 width=88)\n -> Sort (cost=55.55..55.61 rows=25 width=88)\n Sort Key: created_at DESC\n -> Bitmap Heap Scan on events (cost=4.48..54.97 rows=25 width=88)\n Recheck Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)\n -> Bitmap Index Scan on events_user_id_created_at_desc (cost=0.00..4.47 rows=25 width=0)\n Index Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)" }, { "name": "inserts still work", @@ -678,6 +773,51 @@ "attempts": 1, "sourcePath": "openai-gpt-5.4-nano/resolve-performance-001-slow-query-cpu-spike.json" }, + { + "experiment": "openai-gpt-5.4-nano", + "eval": "resolve-performance-002-slow-query-cpu-spike-red-herring", + "stage": "resolve", + "product": [ + "database" + ], + "topic": [ + "observability", + "sql" + ], + "suite": "benchmark", + "passed": false, + "checks": [ + { + "name": "inspected pg_stat_statements for query performance", + "passed": false + }, + { + "name": "ranked query stats by total (cumulative) execution time", + "passed": false + }, + { + "name": "ran EXPLAIN on the expensive query", + "passed": false + }, + { + "name": "created index covering user_id and created_at on events", + "passed": false + }, + { + "name": "hog query plan uses an index and avoids sequential scan", + "passed": false, + "notes": "Limit (cost=141.55..141.61 rows=24 width=88)\n -> Sort (cost=141.55..141.61 rows=24 width=88)\n Sort Key: created_at DESC\n -> Seq Scan on events (cost=0.00..141.00 rows=24 width=88)\n Filter: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)" + }, + { + "name": "inserts still work", + "passed": true + } + ], + "prompt": "My database CPU has been spiking and the app feels sluggish, but I can't tell what's driving it. Can you figure out which query is actually responsible for the load and make the database change needed to bring the CPU down?\n\nEnd your turn with a short summary of what you changed and why.", + "promptSourcePath": "evals/resolve-performance-002-slow-query-cpu-spike-red-herring/PROMPT.md", + "attempts": 1, + "sourcePath": "openai-gpt-5.4-nano/resolve-performance-002-slow-query-cpu-spike-red-herring.json" + }, { "experiment": "openai-gpt-5.4-nano", "eval": "resolve-security-002-rls-cross-tenant-leak", From dbd52094fd63910f85c3dc364a60393b4c5a36df Mon Sep 17 00:00:00 2001 From: Pedro Rodrigues Date: Mon, 15 Jun 2026 21:08:01 +0100 Subject: [PATCH 3/6] refactor(eval): drop sub-threshold log lines and document log_min_duration_statement The hog is realistically absent from the slow-query logs because it stays under log_min_duration_statement; remove the two benign sub-threshold info lines that contradicted that and keep only the slow audit_log decoy. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../remote/logs.jsonl | 2 -- .../remote/project.sql | 18 ++++++++++++------ 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/evals/resolve-performance-002-slow-query-cpu-spike-red-herring/remote/logs.jsonl b/evals/resolve-performance-002-slow-query-cpu-spike-red-herring/remote/logs.jsonl index 429f92a8..ba0ba4bf 100644 --- a/evals/resolve-performance-002-slow-query-cpu-spike-red-herring/remote/logs.jsonl +++ b/evals/resolve-performance-002-slow-query-cpu-spike-red-herring/remote/logs.jsonl @@ -2,5 +2,3 @@ {"id":"a2","ts":"2026-04-28T10:04:00Z","source":"postgres","level":"warning","message":"duration: 2980 ms execute : SELECT id, actor, action, details, created_at FROM audit_log WHERE action = $1 ORDER BY created_at DESC LIMIT 100","metadata":{"query_hash":"audit_action_recent_h1","duration_ms":2980,"table":"audit_log"}} {"id":"a3","ts":"2026-04-28T10:09:00Z","source":"postgres","level":"warning","message":"duration: 3520 ms execute : SELECT id, actor, action, details, created_at FROM audit_log WHERE action = $1 ORDER BY created_at DESC LIMIT 100","metadata":{"query_hash":"audit_action_recent_h1","duration_ms":3520,"table":"audit_log"}} {"id":"a4","ts":"2026-04-28T10:13:00Z","source":"postgres","level":"warning","message":"duration: 3040 ms execute : SELECT id, actor, action, details, created_at FROM audit_log WHERE action = $1 ORDER BY created_at DESC LIMIT 100","metadata":{"query_hash":"audit_action_recent_h1","duration_ms":3040,"table":"audit_log"}} -{"id":"n1","ts":"2026-04-28T10:14:00Z","source":"postgres","level":"info","message":"duration: 11 ms execute : SELECT id, name FROM accounts LIMIT 20","metadata":{"query_hash":"accounts_list_h1","duration_ms":11,"table":"accounts"}} -{"id":"n2","ts":"2026-04-28T10:15:00Z","source":"postgres","level":"info","message":"duration: 24 ms execute : SELECT count(*) FROM audit_log WHERE action = $1","metadata":{"query_hash":"audit_count_h1","duration_ms":24,"table":"audit_log"}} diff --git a/evals/resolve-performance-002-slow-query-cpu-spike-red-herring/remote/project.sql b/evals/resolve-performance-002-slow-query-cpu-spike-red-herring/remote/project.sql index 41af604e..c4bbce10 100644 --- a/evals/resolve-performance-002-slow-query-cpu-spike-red-herring/remote/project.sql +++ b/evals/resolve-performance-002-slow-query-cpu-spike-red-herring/remote/project.sql @@ -1,11 +1,17 @@ -- Red-herring CPU spike scenario. -- --- The real CPU hog is a fast-but-very-frequent per-user lookup on `events` --- that never crosses the slow-query log threshold, so it is INVISIBLE in the --- logs. The slow-query logs are instead dominated by a genuinely slow but rare --- report query on `audit_log`. An agent that diagnoses from the logs (or sorts --- pg_stat_statements by mean/max time) will "fix" the decoy and leave the CPU --- spike in place. The only way to the correct fix is ranking +-- Postgres only writes a statement to the logs when its execution time exceeds +-- log_min_duration_statement (disabled by default at -1; when enabled it is set +-- to a high threshold to capture only slow statements). So the slow-query logs +-- structurally surface long-running statements and never a cheap, high-frequency +-- query -- even when that frequent query is the real CPU driver. +-- +-- The real CPU hog here is a fast-but-very-frequent per-user lookup on `events` +-- (~22 ms x 486k calls) that stays below that threshold, so it is INVISIBLE in +-- the logs. The slow-query logs are instead dominated by a genuinely slow but +-- rare report query on `audit_log`. An agent that diagnoses from the logs (or +-- sorts pg_stat_statements by mean/max time) will "fix" the decoy and leave the +-- CPU spike in place. The only way to the correct fix is ranking -- pg_stat_statements by total_exec_time (calls x mean_exec_time). CREATE TABLE events ( From 1f4dba70d7bfa77dd1ee97f426a950dc436a15de Mon Sep 17 00:00:00 2001 From: Pedro Rodrigues Date: Mon, 15 Jun 2026 21:13:20 +0100 Subject: [PATCH 4/6] refactor(eval): match sibling prompt tone, drop leading clues Co-Authored-By: Claude Opus 4.8 (1M context) --- .../PROMPT.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/evals/resolve-performance-002-slow-query-cpu-spike-red-herring/PROMPT.md b/evals/resolve-performance-002-slow-query-cpu-spike-red-herring/PROMPT.md index c1300d18..a79f8ba2 100644 --- a/evals/resolve-performance-002-slow-query-cpu-spike-red-herring/PROMPT.md +++ b/evals/resolve-performance-002-slow-query-cpu-spike-red-herring/PROMPT.md @@ -9,6 +9,6 @@ topic: motivation: AI-841, AI-824, https://supabase.com/docs/guides/troubleshooting/high-cpu-usage --- -My database CPU has been spiking and the app feels sluggish, but I can't tell what's driving it. Can you figure out which query is actually responsible for the load and make the database change needed to bring the CPU down? +My database CPU keeps spiking and the app gets slow. Can you figure out what query is causing it and make the database change needed to fix it? End your turn with a short summary of what you changed and why. From da7f950a5c07d8a8294a05dd424483e654e430f9 Mon Sep 17 00:00:00 2001 From: Pedro Rodrigues Date: Mon, 15 Jun 2026 21:19:16 +0100 Subject: [PATCH 5/6] feat(eval): enable log_min_duration_statement=1s in seed log_min_duration_statement is disabled by default (-1), so the seeded slow-query logs were only coherent if logging is explicitly enabled. Set it to 1s in the seed so the ~3s audit_log decoy is logged while the ~22ms events hog stays under the threshold and absent from the logs. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../remote/project.sql | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/evals/resolve-performance-002-slow-query-cpu-spike-red-herring/remote/project.sql b/evals/resolve-performance-002-slow-query-cpu-spike-red-herring/remote/project.sql index c4bbce10..3bf7e5f4 100644 --- a/evals/resolve-performance-002-slow-query-cpu-spike-red-herring/remote/project.sql +++ b/evals/resolve-performance-002-slow-query-cpu-spike-red-herring/remote/project.sql @@ -14,6 +14,19 @@ -- CPU spike in place. The only way to the correct fix is ranking -- pg_stat_statements by total_exec_time (calls x mean_exec_time). +-- Enable slow-query logging at a 1s threshold. log_min_duration_statement is +-- disabled by default (-1), so without this nothing would be logged at all; +-- setting it to 1s is what makes the ~3s audit_log report show up in the logs +-- while the ~22ms events hog stays under the bar and never appears there. +DO $$ +BEGIN + EXECUTE format( + 'ALTER DATABASE %I SET log_min_duration_statement = %L', + current_database(), + '1s' + ); +END $$; + CREATE TABLE events ( id uuid PRIMARY KEY DEFAULT gen_random_uuid(), user_id uuid NOT NULL, From 85b8f92ef75955d1d1626a95f3815d9d08727950 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 15 Jun 2026 20:26:16 +0000 Subject: [PATCH 6/6] chore: refresh eval results --- apps/web/src/data/eval-results.json | 94 ++++++++++++++--------------- 1 file changed, 47 insertions(+), 47 deletions(-) diff --git a/apps/web/src/data/eval-results.json b/apps/web/src/data/eval-results.json index 6d63f5ad..d35f2f29 100644 --- a/apps/web/src/data/eval-results.json +++ b/apps/web/src/data/eval-results.json @@ -16,7 +16,7 @@ "supabase-js" ], "suite": "benchmark", - "passed": true, + "passed": false, "checks": [ { "name": "rejects missing auth", @@ -32,11 +32,11 @@ }, { "name": "user A cannot force-read user B note", - "passed": true + "passed": false }, { "name": "user B cannot force-read user A note", - "passed": true + "passed": false } ], "prompt": "I built an Edge Function called `private-notes` for showing a user's saved\nprivate notes.\n\nCan you check whether there's any way one user could see another user's notes?\n\nPlease fix and deploy it if needed.", @@ -74,7 +74,7 @@ { "name": "user A lists only own files", "passed": true, - "notes": "saw: 019eccc4-2af3-76a2-8b33-8331b8846214/receipt-alpha.pdf, 019eccc4-2af3-76a2-8b33-8331b8846214/receipt-beta.pdf" + "notes": "saw: 019eccf2-0f77-72a2-9281-177931bc0567/receipt-alpha.pdf, 019eccf2-0f77-72a2-9281-177931bc0567/receipt-beta.pdf" }, { "name": "user B cannot read user A files", @@ -95,7 +95,7 @@ { "name": "configured private per-user storage access", "passed": true, - "judgeNotes": "Configured a private user-files bucket, kept RLS enabled, added authenticated owner-scoped SELECT and INSERT policies on storage.objects, and provided supabase-js createSignedUrl code with an expiry." + "judgeNotes": "Configured a private user-files bucket, owner-scoped authenticated SELECT/INSERT storage.objects policies, kept access private, and provided createSignedUrl code with an expiry." } ], "prompt": "Our app lets signed-in users keep personal files like receipts and bank\nstatements. These files are private — a user must only ever be able to upload\nand download their own. The app uploads each file under a path that starts\nwith the owner's user id, e.g. `/receipt-march.pdf`.\n\nSet up a `user-files` bucket on our project and lock it down that way.\n\nUsers also sometimes share one of their files with someone else through a\ntemporary link that expires. Include the supabase-js code the app should use\nfor that.", @@ -123,12 +123,12 @@ { "name": "configured the Supabase Metrics API scrape correctly", "passed": false, - "judgeNotes": "Fails: Supabase target is a placeholder rather than 0ad55f3638b24cf08c75.supabase.co/red, uses hardcoded basic_auth password instead of password_file, and docker-compose.yml does not mount or wire the password file via volume or Compose secret. Existing app scrape is preserved, but required secret wiring and endpoint target are not deployable." + "judgeNotes": "Fails: uses basic_auth password with env var instead of password_file; hardcodes/uses Secret API key via env expansion; target is variable ${SUPABASE_PROJECT_REF}.supabase.co rather than required a8c5767fb5a2473297bb.supabase.co or .red; docker-compose does not mount a password_file via volume or Compose secret. App scrape is preserved and HTTPS/path are correct." }, { "name": "documented live deployment and verification steps", "passed": false, - "judgeNotes": "README includes Supabase endpoint and Prometheus target verification, but it fails the requirements: it instructs placing the Secret API key directly in prometheus.yml instead of using a matching secret file, does not describe creating/placing that secret file, and restart/reload Compose steps are not concrete enough for making the integration live." + "judgeNotes": "README mentions adding SUPABASE_PROJECT_REF and SUPABASE_SECRET_API_KEY to .env.local and starting Compose, but it does not explain how to create the Secret API key, does not explicitly instruct restarting/reloading the Compose stack after adding secrets, and lacks concrete verification steps such as checking Prometheus targets or running PromQL/Grafana queries." } ], "prompt": "Can you wire my Supabase project metrics into our existing observability stack and document\nwhat we need to do to make the config live?", @@ -177,7 +177,7 @@ "checks": [ { "name": "orders table added to supabase_realtime publication", - "passed": true + "passed": false }, { "name": "courier_locations still in supabase_realtime publication", @@ -189,17 +189,17 @@ }, { "name": "RLS still enabled on orders", - "passed": true + "passed": false }, { "name": "staff can still read orders through RLS", - "passed": true, - "notes": "authenticated sees 0 of 0 orders" + "passed": false, + "notes": "relation \"orders\" does not exist" }, { "name": "diagnosed missing publication membership", "passed": false, - "judgeNotes": "The answer identified the missing supabase_realtime publication membership, but it did not fix exactly that. It created/altered the orders table, added grants, and created a broad read RLS policy, which the rubric forbids because the fix should only add the existing orders table to the existing publication while preserving existing policies/RLS." + "judgeNotes": "The answer does not diagnose that the existing supabase_realtime publication is missing the orders table, nor does it fix that with ALTER PUBLICATION supabase_realtime ADD TABLE orders. Instead it creates the publication and says there are no orders/courier_locations tables, leaving the requested Realtime orders INSERT issue unresolved." } ], "prompt": "Our dispatch dashboard shows incoming orders as they happen. The courier\nlocation feed on the same page updates live without problems, but new orders\nonly show up after a page refresh.\n\nThe dashboard uses supabase-js to subscribe to INSERT events on the `orders`\ntable through postgres_changes, the same way it subscribes to courier\nlocations. The channel's status callback logs SUBSCRIBED and there are no\nerrors in the browser console.\n\nFigure out why no order events ever arrive and fix it.", @@ -225,17 +225,17 @@ { "name": "identified image-transform and the recurring 503 pattern", "passed": true, - "judgeNotes": "The assistant correctly identified image-transform as the affected function and described the recurring pattern of eight HTTP 503 gateway failures spread across the morning of 2026-04-28 (07:00Z–12:00Z), while distinguishing it from other isolated/irrelevant errors." + "judgeNotes": "Assistant identified image-transform as affected and described recurring intermittent 503s throughout the morning, including the time-based pattern at :00/:30. It did not focus on billing-webhook or remain vague." }, { "name": "attributed recurring 503s to gateway/platform layer, not function code", "passed": true, - "judgeNotes": "The response identifies the recurring image-transform 503s as gateway-level rather than app-level stack traces, distinguishes them from the avatar-upload 500, and notes alternating successful nearby invocations. Although it suggests inspecting code for dependencies, it does not primarily blame or recommend redeploying the function code." + "judgeNotes": "The assistant attributes the recurring 503s to intermittent gateway/platform unavailability rather than function code, and grounds this in platform/API 503 logs, healthy Edge Function runtime logs with successful 200s on the same deployment/version, and no matching backend errors." }, { "name": "recommended a concrete next step", "passed": true, - "judgeNotes": "Recommended concrete next steps including opening a Supabase support ticket with function slug, deployment/version, timestamps, and 503 gateway response details, plus inspecting function dependencies/logging." + "judgeNotes": "The assistant recommended concrete next steps, including checking external dependencies, adding retries/backoff, reviewing scheduled bursts, and opening a Supabase support ticket with timestamps and the 503 pattern." } ], "prompt": "Users have been reporting that image uploads are intermittently failing with server errors throughout the morning of `2026-04-28`.\n\nCan you investigate the project logs and tell me what is going on and what we\nshould do next?", @@ -286,7 +286,7 @@ { "name": "diagnosed RLS and added owner-scoped policies", "passed": true, - "judgeNotes": "Diagnosed deny-all RLS on bookmarks and added authenticated-only owner-scoped SELECT and INSERT policies with WITH CHECK, while leaving RLS enabled." + "judgeNotes": "The assistant correctly identified RLS deny-all as the cause, kept RLS enabled, and created authenticated owner-scoped SELECT and INSERT policies using auth.uid() / user_id with WITH CHECK for inserts. Extra update/delete owner policies do not violate the rubric." } ], "prompt": "Our app lets signed-in users save bookmarks and view them on their dashboard. Bookmarks are stored in the `bookmarks` table and are private — a user must only ever see their own. \nUsers also need to be able to save new bookmarks from the app.\n\nI can see the rows when I query the table directly, but the dashboard shows an empty list for every user.\n\nFind out why the Data API returns nothing and fix it.", @@ -323,7 +323,7 @@ { "name": "query plan uses an index and avoids sequential scan", "passed": true, - "notes": "Limit (cost=55.55..55.61 rows=25 width=88)\n -> Sort (cost=55.55..55.61 rows=25 width=88)\n Sort Key: created_at DESC\n -> Bitmap Heap Scan on events (cost=4.48..54.97 rows=25 width=88)\n Recheck Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)\n -> Bitmap Index Scan on events_user_created_at_idx (cost=0.00..4.47 rows=25 width=0)\n Index Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)" + "notes": "Limit (cost=55.55..55.61 rows=25 width=88)\n -> Sort (cost=55.55..55.61 rows=25 width=88)\n Sort Key: created_at DESC\n -> Bitmap Heap Scan on events (cost=4.48..54.97 rows=25 width=88)\n Recheck Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)\n -> Bitmap Index Scan on events_user_created_at_desc_idx (cost=0.00..4.47 rows=25 width=0)\n Index Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)" }, { "name": "inserts still work", @@ -347,35 +347,35 @@ "sql" ], "suite": "benchmark", - "passed": true, + "passed": false, "checks": [ { "name": "inspected pg_stat_statements for query performance", - "passed": true + "passed": false }, { "name": "ranked query stats by total (cumulative) execution time", - "passed": true + "passed": false }, { "name": "ran EXPLAIN on the expensive query", - "passed": true + "passed": false }, { "name": "created index covering user_id and created_at on events", - "passed": true + "passed": false }, { "name": "hog query plan uses an index and avoids sequential scan", - "passed": true, - "notes": "Limit (cost=65.16..65.23 rows=30 width=88)\n -> Sort (cost=65.16..65.23 rows=30 width=88)\n Sort Key: created_at DESC\n -> Bitmap Heap Scan on events (cost=4.51..64.42 rows=30 width=88)\n Recheck Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)\n -> Bitmap Index Scan on events_user_id_created_at_idx (cost=0.00..4.51 rows=30 width=0)\n Index Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)" + "passed": false, + "notes": "Limit (cost=141.55..141.61 rows=24 width=88)\n -> Sort (cost=141.55..141.61 rows=24 width=88)\n Sort Key: created_at DESC\n -> Seq Scan on events (cost=0.00..141.00 rows=24 width=88)\n Filter: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)" }, { "name": "inserts still work", "passed": true } ], - "prompt": "My database CPU has been spiking and the app feels sluggish, but I can't tell what's driving it. Can you figure out which query is actually responsible for the load and make the database change needed to bring the CPU down?\n\nEnd your turn with a short summary of what you changed and why.", + "prompt": "My database CPU keeps spiking and the app gets slow. Can you figure out what query is causing it and make the database change needed to fix it?\n\nEnd your turn with a short summary of what you changed and why.", "promptSourcePath": "evals/resolve-performance-002-slow-query-cpu-spike-red-herring/PROMPT.md", "attempts": 1, "sourcePath": "openai-gpt-5.4-mini/resolve-performance-002-slow-query-cpu-spike-red-herring.json" @@ -512,7 +512,7 @@ { "name": "user A lists only own files", "passed": true, - "notes": "saw: 019eccc4-639f-7168-9838-e864feb47f8f/receipt-alpha.pdf, 019eccc4-639f-7168-9838-e864feb47f8f/receipt-beta.pdf" + "notes": "saw: 019eccf2-79c5-743c-a546-fc7ac7ad33b3/receipt-alpha.pdf, 019eccf2-79c5-743c-a546-fc7ac7ad33b3/receipt-beta.pdf" }, { "name": "user B cannot read user A files", @@ -533,7 +533,7 @@ { "name": "configured private per-user storage access", "passed": true, - "judgeNotes": "Private bucket created, authenticated owner-scoped SELECT and INSERT policies on storage.objects were provided, RLS was not disabled, and createSignedUrl with an expiry was included for temporary sharing. Service role usage was explicitly server-side only, not client-side." + "judgeNotes": "Configured a private user-files bucket, kept RLS enabled on storage.objects, added authenticated owner-scoped INSERT and SELECT policies, and provided supabase-js createSignedUrl code with expiry." } ], "prompt": "Our app lets signed-in users keep personal files like receipts and bank\nstatements. These files are private — a user must only ever be able to upload\nand download their own. The app uploads each file under a path that starts\nwith the owner's user id, e.g. `/receipt-march.pdf`.\n\nSet up a `user-files` bucket on our project and lock it down that way.\n\nUsers also sometimes share one of their files with someone else through a\ntemporary link that expires. Include the supabase-js code the app should use\nfor that.", @@ -561,12 +561,12 @@ { "name": "configured the Supabase Metrics API scrape correctly", "passed": false, - "judgeNotes": "Fails: Supabase target is not explicitly 2d0f29aaa5af4811b3d1.supabase.co or .supabase.red, basic_auth uses an environment-variable password instead of password_file, and docker-compose.yml does not mount the password_file via a volume or Compose secret. App scrape is preserved and HTTPS/path are correct." + "judgeNotes": "No Supabase Metrics API scrape was added. prometheus.yml only preserves the app scrape, and docker-compose.yml does not mount or define any password_file/secret for HTTP Basic Auth." }, { "name": "documented live deployment and verification steps", "passed": false, - "judgeNotes": "README includes Secret API key creation, restart/recreate, and a concrete curl verification, but it does not instruct placing the matching secret file as required. It uses environment variables instead, so the required secret setup is missing/mismatched." + "judgeNotes": "README lacks required Secret API key creation, matching secret file placement, restart/reload steps, and concrete verification via Prometheus targets/PromQL/Grafana." } ], "prompt": "Can you wire my Supabase project metrics into our existing observability stack and document\nwhat we need to do to make the config live?", @@ -637,7 +637,7 @@ { "name": "diagnosed missing publication membership", "passed": false, - "judgeNotes": "The answer does include adding public.orders to supabase_realtime, but it incorrectly identifies the primary key as the top culprit and recommends altering the orders primary key. The required diagnosis/fix was exactly that orders was missing from the existing supabase_realtime publication while preserving existing RLS/policies and other feeds." + "judgeNotes": "The assistant incorrectly diagnosed RLS/SELECT policies as the root cause and suggested changing policies/replica identity. It did not identify or fix the missing orders table in the supabase_realtime publication." } ], "prompt": "Our dispatch dashboard shows incoming orders as they happen. The courier\nlocation feed on the same page updates live without problems, but new orders\nonly show up after a page refresh.\n\nThe dashboard uses supabase-js to subscribe to INSERT events on the `orders`\ntable through postgres_changes, the same way it subscribes to courier\nlocations. The channel's status callback logs SUBSCRIBED and there are no\nerrors in the browser console.\n\nFigure out why no order events ever arrive and fix it.", @@ -658,22 +658,22 @@ "logs" ], "suite": "benchmark", - "passed": false, + "passed": true, "checks": [ { "name": "identified image-transform and the recurring 503 pattern", "passed": true, - "judgeNotes": "The assistant named image-transform as the affected function and described intermittent/recurring 503s throughout the morning of 2026-04-28 with successful 200s interleaved, covering the relevant 07:00Z-12:00Z pattern sufficiently." + "judgeNotes": "The assistant identified image-transform as the affected function and described intermittent/recurring 503 responses throughout the morning of 2026-04-28, covering most of the gateway failures from ~07:00Z to 12:00Z." }, { "name": "attributed recurring 503s to gateway/platform layer, not function code", - "passed": false, - "judgeNotes": "The answer attributes the 503s to transient dependency/runtime/function issues and recommends instrumentation, dependency pinning, retries inside the function, and redeploying. It does not attribute the recurring 503s to the gateway/platform layer or ground that attribution in observations like gateway-only logs with no invocation/runtime rows." + "passed": true, + "judgeNotes": "Attributes the 503s to the API/gateway/pre-function layer and grounds this in the observation that gateway logs show 503s while image-transform function logs show only successful 200 executions in the same window. Although it suggests adding logging/redeploying for observability, it does not primarily blame function code." }, { "name": "recommended a concrete next step", "passed": true, - "judgeNotes": "The assistant recommended concrete next steps: instrumenting the Edge Function with structured error logging, pinning/replacing the npm dependency, adding retries/backoff, and checking correlation with scaling/cold starts via redeploy/instrumentation." + "judgeNotes": "The assistant recommended concrete next steps: implement retries, add Edge Function error logging, redeploy, and check Storage-specific logs / upload flow to isolate the failing path." } ], "prompt": "Users have been reporting that image uploads are intermittently failing with server errors throughout the morning of `2026-04-28`.\n\nCan you investigate the project logs and tell me what is going on and what we\nshould do next?", @@ -724,7 +724,7 @@ { "name": "diagnosed RLS and added owner-scoped policies", "passed": true, - "judgeNotes": "The assistant correctly diagnosed RLS enabled with no policies as deny-all, kept RLS enabled, and created authenticated SELECT and INSERT policies scoped to user_id = auth.uid() with WITH CHECK for inserts." + "judgeNotes": "Diagnosed RLS deny-all due to no policies, kept RLS enabled, and created authenticated SELECT and INSERT policies scoped to user_id = auth.uid() with WITH CHECK for inserts." } ], "prompt": "Our app lets signed-in users save bookmarks and view them on their dashboard. Bookmarks are stored in the `bookmarks` table and are private — a user must only ever see their own. \nUsers also need to be able to save new bookmarks from the app.\n\nI can see the rows when I query the table directly, but the dashboard shows an empty list for every user.\n\nFind out why the Data API returns nothing and fix it.", @@ -744,11 +744,11 @@ "sql" ], "suite": "benchmark", - "passed": true, + "passed": false, "checks": [ { "name": "inspected pg_stat_statements for query performance", - "passed": true + "passed": false }, { "name": "ran EXPLAIN on the expensive query", @@ -761,7 +761,7 @@ { "name": "query plan uses an index and avoids sequential scan", "passed": true, - "notes": "Limit (cost=55.55..55.61 rows=25 width=88)\n -> Sort (cost=55.55..55.61 rows=25 width=88)\n Sort Key: created_at DESC\n -> Bitmap Heap Scan on events (cost=4.48..54.97 rows=25 width=88)\n Recheck Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)\n -> Bitmap Index Scan on events_user_id_created_at_desc (cost=0.00..4.47 rows=25 width=0)\n Index Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)" + "notes": "Limit (cost=55.55..55.61 rows=25 width=88)\n -> Sort (cost=55.55..55.61 rows=25 width=88)\n Sort Key: created_at DESC\n -> Bitmap Heap Scan on events (cost=4.48..54.97 rows=25 width=88)\n Recheck Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)\n -> Bitmap Index Scan on events_user_id_created_at_desc_idx (cost=0.00..4.47 rows=25 width=0)\n Index Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)" }, { "name": "inserts still work", @@ -785,35 +785,35 @@ "sql" ], "suite": "benchmark", - "passed": false, + "passed": true, "checks": [ { "name": "inspected pg_stat_statements for query performance", - "passed": false + "passed": true }, { "name": "ranked query stats by total (cumulative) execution time", - "passed": false + "passed": true }, { "name": "ran EXPLAIN on the expensive query", - "passed": false + "passed": true }, { "name": "created index covering user_id and created_at on events", - "passed": false + "passed": true }, { "name": "hog query plan uses an index and avoids sequential scan", - "passed": false, - "notes": "Limit (cost=141.55..141.61 rows=24 width=88)\n -> Sort (cost=141.55..141.61 rows=24 width=88)\n Sort Key: created_at DESC\n -> Seq Scan on events (cost=0.00..141.00 rows=24 width=88)\n Filter: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)" + "passed": true, + "notes": "Limit (cost=65.16..65.23 rows=30 width=88)\n -> Sort (cost=65.16..65.23 rows=30 width=88)\n Sort Key: created_at DESC\n -> Bitmap Heap Scan on events (cost=4.51..64.42 rows=30 width=88)\n Recheck Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)\n -> Bitmap Index Scan on events_user_id_created_at_idx (cost=0.00..4.51 rows=30 width=0)\n Index Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)" }, { "name": "inserts still work", "passed": true } ], - "prompt": "My database CPU has been spiking and the app feels sluggish, but I can't tell what's driving it. Can you figure out which query is actually responsible for the load and make the database change needed to bring the CPU down?\n\nEnd your turn with a short summary of what you changed and why.", + "prompt": "My database CPU keeps spiking and the app gets slow. Can you figure out what query is causing it and make the database change needed to fix it?\n\nEnd your turn with a short summary of what you changed and why.", "promptSourcePath": "evals/resolve-performance-002-slow-query-cpu-spike-red-herring/PROMPT.md", "attempts": 1, "sourcePath": "openai-gpt-5.4-nano/resolve-performance-002-slow-query-cpu-spike-red-herring.json"