diff --git a/.env.example b/.env.example index 19a0696f..53fb4854 100644 --- a/.env.example +++ b/.env.example @@ -1,3 +1,10 @@ -# AI SDK Core direct-provider credentials. +# Claude Code harness and Anthropic ai-sdk experiments. ANTHROPIC_API_KEY= + +# Codex harness and OpenAI ai-sdk experiments. The default LLM judge is an +# OpenAI model, so rubric-scored evals need this key regardless of which +# harness produced the run. OPENAI_API_KEY= + +# Vercel AI Gateway. The opencode harness routes through this. +AI_GATEWAY_API_KEY= diff --git a/.github/workflows/eval-refresh.yml b/.github/workflows/eval-refresh.yml index 42d5e25a..98b7e215 100644 --- a/.github/workflows/eval-refresh.yml +++ b/.github/workflows/eval-refresh.yml @@ -259,6 +259,7 @@ jobs: env: ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + AI_GATEWAY_API_KEY: ${{ secrets.AI_GATEWAY_API_KEY }} steps: - name: Checkout uses: actions/checkout@9f698171ed81b15d1823a05fc7211befd50c8ae0 # v6.0.3 @@ -286,6 +287,7 @@ jobs: { echo "ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY}" echo "OPENAI_API_KEY=${OPENAI_API_KEY}" + echo "AI_GATEWAY_API_KEY=${AI_GATEWAY_API_KEY}" } > .env - name: Run evals @@ -300,6 +302,12 @@ jobs: --runs "${{ needs.prepare.outputs.runs }}" \ --timeout-sec "${{ needs.prepare.outputs.timeout_sec }}" + # A skipped experiment (e.g. missing API key) exits 0 without writing results. + if [ ! -f "results/${{ matrix.experiment }}/${{ matrix.eval_id }}.json" ]; then + echo "::error::no result written for ${{ matrix.experiment }} x ${{ matrix.eval_id }} (experiment skipped?)" + exit 1 + fi + - name: Upload raw results uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: diff --git a/apps/web/src/data/eval-results.json b/apps/web/src/data/eval-results.json index 556790c2..8a16cc92 100644 --- a/apps/web/src/data/eval-results.json +++ b/apps/web/src/data/eval-results.json @@ -17824,5 +17824,3186 @@ "promptSourcePath": "evals/resolve-security-002-rls-cross-tenant-leak/PROMPT.md", "attempts": 1, "sourcePath": "codex-gpt-5.6-no-skills/resolve-security-002-rls-cross-tenant-leak.json" + }, + { + "experiment": "opencode-kimi-k3", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "opencode", + "modelProvider": "moonshotai", + "modelId": "moonshotai/kimi-k3" + }, + "eval": "build-cli-001-bootstrap-app", + "stage": "build", + "product": [ + "database", + "data-api" + ], + "topic": [ + "migrations", + "rls" + ], + "suite": "benchmark", + "interface": "cli", + "passed": true, + "checks": [ + { + "name": "supabase project initialised (supabase/config.toml exists)", + "passed": true + }, + { + "name": "todos table is created by a migration file", + "passed": true + }, + { + "name": "todos table exists with at least 2 seeded rows", + "passed": true, + "notes": "found 3 rows" + }, + { + "name": "row level security is enabled on todos", + "passed": true + }, + { + "name": "a SELECT policy targets the authenticated role", + "passed": true + }, + { + "name": "REST API returns no todos to anonymous requests", + "passed": true, + "notes": "error 42501: permission denied for table todos" + }, + { + "name": "REST API returns the todos to authenticated requests", + "passed": true, + "notes": "3 rows" + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [ + { + "source": "web_fetch", + "query": "https://supabase.com/changelog.md", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 92585 + } + ] + }, + "prompt": "We're kicking off a todos app and I want the Supabase side ready for the team\nto build on. Set it up the way we'd run it in development, with schema changes\ntracked as migrations so they can be reviewed and replayed.\n\nFor the first slice we just need a `todos` table. Todos aren't public: anyone\nsigned in can read all of them, but nothing should be writable through the API\nfor now. Add a couple of sample todos so there's something to look at.\n\nBefore you hand it back, make sure the running API actually behaves that way —\nsigned-in users get the todos, signed-out requests get nothing.", + "promptSourcePath": "evals/build-cli-001-bootstrap-app/PROMPT.md", + "attempts": 1, + "sourcePath": "opencode-kimi-k3/build-cli-001-bootstrap-app.json" + }, + { + "experiment": "opencode-kimi-k3", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "opencode", + "modelProvider": "moonshotai", + "modelId": "moonshotai/kimi-k3" + }, + "eval": "build-cli-002-declarative-schema", + "stage": "build", + "product": [ + "database" + ], + "topic": [ + "declarative-schema", + "migrations" + ], + "suite": "benchmark", + "interface": "cli", + "passed": true, + "checks": [ + { + "name": "supabase db diff used to generate the migration", + "passed": true + }, + { + "name": "schema file updated to include description column", + "passed": true + }, + { + "name": "a new migration was generated for the change", + "passed": true + }, + { + "name": "description column exists in the live database", + "passed": true + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [] + }, + "prompt": "Add a description text column to the `products` table in my local Supabase stack", + "promptSourcePath": "evals/build-cli-002-declarative-schema/PROMPT.md", + "attempts": 1, + "sourcePath": "opencode-kimi-k3/build-cli-002-declarative-schema.json" + }, + { + "experiment": "opencode-kimi-k3", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "opencode", + "modelProvider": "moonshotai", + "modelId": "moonshotai/kimi-k3" + }, + "eval": "build-cli-003-pg-cron-queue-workflow", + "stage": "build", + "product": [ + "database", + "edge-functions", + "cron", + "queues" + ], + "topic": [ + "sql", + "sdk" + ], + "suite": "benchmark", + "interface": "cli", + "passed": true, + "checks": [ + { + "name": "pg_cron job 'enqueue-tasks' scheduled to run every minute", + "passed": true, + "notes": "schedule='* * * * *', active=true" + }, + { + "name": "cron command enqueues to the 'tasks' queue", + "passed": true, + "notes": "queue depth 0 -> 1" + }, + { + "name": "process-tasks function drains the queue", + "passed": true, + "notes": "function removed the seeded message (id 39) from the queue" + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "{\n searchDocs(query: \"queues edge function consume messages pgmq_public pop read delete\", limit: 3) {\n nodes {\n title\n href\n content\n }\n }\n}", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/queues/consuming-messages-with-edge-functions", + "title": "Consuming Supabase Queue Messages with Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/queues/pgmq", + "title": "PGMQ Extension" + }, + { + "url": "https://supabase.com/docs/guides/queues/api", + "title": "API" + } + ], + "resultChars": 27216 + }, + { + "source": "search_docs", + "query": "{\n searchDocs(query: \"pg_cron schedule job every minute pgmq send queue\", limit: 3) {\n nodes {\n title\n href\n content\n }\n }\n}", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_net", + "title": "pg_net: Async Networking" + }, + { + "url": "https://supabase.com/docs/guides/queues/pgmq", + "title": "PGMQ Extension" + }, + { + "url": "https://supabase.com/docs/guides/cron", + "title": "Cron" + } + ], + "resultChars": 35611 + } + ] + }, + "prompt": "I want to set up a recurring background workflow on my local Supabase stack.\n\nCan you set up a cron job called `enqueue-tasks` to run every minute and push a task into a queue called `tasks`? Then add a `process-tasks` edge function that reads messages off the `tasks` queue and removes them, so a scheduled worker can keep the backlog drained.", + "promptSourcePath": "evals/build-cli-003-pg-cron-queue-workflow/PROMPT.md", + "attempts": 1, + "sourcePath": "opencode-kimi-k3/build-cli-003-pg-cron-queue-workflow.json" + }, + { + "experiment": "opencode-kimi-k3", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "opencode", + "modelProvider": "moonshotai", + "modelId": "moonshotai/kimi-k3" + }, + "eval": "build-database-001-migrate-postgres-to-supabase", + "stage": "build", + "product": [ + "database" + ], + "topic": [ + "migrations" + ], + "suite": "benchmark", + "interface": "cli", + "passed": true, + "checks": [ + { + "name": "all 3 tables exist (teams, members, tasks)", + "passed": true + }, + { + "name": "row counts match (teams=5, members=10, tasks=13)", + "passed": true + }, + { + "name": "foreign key constraints survived the restore", + "passed": true + }, + { + "name": "tasks_team_status_idx index survived the restore", + "passed": true + }, + { + "name": "sequences synced (next insert won't conflict with existing IDs)", + "passed": true + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [] + }, + "prompt": "I have an existing Postgres database I want to migrate to Supabase. There's a binary dump at `source.dump` in the current directory.\n\nCan you set up a local Supabase project and restore the dump into it?", + "promptSourcePath": "evals/build-database-001-migrate-postgres-to-supabase/PROMPT.md", + "attempts": 1, + "sourcePath": "opencode-kimi-k3/build-database-001-migrate-postgres-to-supabase.json" + }, + { + "experiment": "opencode-kimi-k3", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "opencode", + "modelProvider": "moonshotai", + "modelId": "moonshotai/kimi-k3" + }, + "eval": "build-functions-004-service-role-bypass", + "stage": "build", + "product": [ + "edge-functions", + "auth", + "database" + ], + "topic": [ + "rls", + "security", + "sdk" + ], + "suite": "benchmark", + "interface": "mcp", + "passed": true, + "checks": [ + { + "name": "rejects missing auth", + "passed": true, + "notes": "status=401" + }, + { + "name": "user A reads own note", + "passed": true, + "notes": "status=200" + }, + { + "name": "reads only with the caller's JWT", + "passed": true, + "notes": "bearer_tokens=2, all_match=true" + }, + { + "name": "user A cannot force-read user B note", + "passed": true, + "notes": "status=200" + }, + { + "name": "user B cannot force-read user A note", + "passed": true, + "notes": "status=200" + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [ + { + "source": "web_fetch", + "query": "https://supabase.com/changelog.md", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 92585 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"edge function get authenticated user from JWT Authorization header\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/functions/auth", + "title": "Securing Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", + "title": "Integrating With Supabase Auth" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth-headers", + "title": "Authorization headers" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" + }, + { + "url": "https://supabase.com/docs/guides/integrations/supabase-for-platforms", + "title": "Supabase for Platforms" + } + ], + "resultChars": 48324 + } + ] + }, + "prompt": "I built an Edge Function called `private-notes` for showing a user's saved\nprivate notes.\n\nCan you check whether there's any way one user could see another user's notes?\n\nPlease fix and deploy it if needed.", + "promptSourcePath": "evals/build-functions-004-service-role-bypass/PROMPT.md", + "attempts": 1, + "sourcePath": "opencode-kimi-k3/build-functions-004-service-role-bypass.json" + }, + { + "experiment": "opencode-kimi-k3", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "opencode", + "modelProvider": "moonshotai", + "modelId": "moonshotai/kimi-k3" + }, + "eval": "build-functions-005-dual-auth-user-secret", + "stage": "build", + "product": [ + "edge-functions", + "auth", + "database" + ], + "topic": [ + "sdk", + "rls", + "security" + ], + "suite": "benchmark", + "interface": "cli", + "cliVersion": "2.109.1", + "passed": true, + "checks": [ + { + "name": "seed rows present", + "passed": true, + "notes": "found 2/2 seeded rows" + }, + { + "name": "rejects request with no credentials", + "passed": true, + "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" + }, + { + "name": "user with JWT reads only their own rows", + "passed": true, + "notes": "status 200: [{\"user_id\":\"6c9a5488-c295-4194-ae70-c5b67aee703c\",\"metric\":\"steps_a_ms3ynfu9\",\"value\":111}]" + }, + { + "name": "user cannot read another user's rows by passing user_id", + "passed": true, + "notes": "status 200: [{\"user_id\":\"6c9a5488-c295-4194-ae70-c5b67aee703c\",\"metric\":\"steps_a_ms3ynfu9\",\"value\":111}]" + }, + { + "name": "service key bypasses RLS to read the target user's rows", + "passed": true, + "notes": "status 200: [{\"user_id\":\"60a5150a-5a94-4242-9ad7-c80655fd6fc6\",\"metric\":\"steps_b_ms3ynfu9\",\"value\":222}]" + }, + { + "name": "non-service key is not granted service access", + "passed": true, + "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" + }, + { + "name": "rejects an unverified (forged) user token", + "passed": true, + "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" + }, + { + "name": "a user token in the apikey slot is not treated as the service key", + "passed": true, + "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" + }, + { + "name": "implementation uses @supabase/server", + "passed": true, + "notes": "imports @supabase/server / withSupabase" + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "{ searchDocs(query: \"edge functions environment variables SUPABASE_SERVICE_ROLE_KEY secret api key\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", + "title": "Self-Hosted Functions" + }, + { + "url": "https://supabase.com/docs/guides/functions/secrets", + "title": "Environment Variables" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", + "title": "Integrating With Supabase Auth" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/inspecting-edge-function-environment-variables-wg5qOQ", + "title": "Inspecting edge function environment variables" + } + ], + "resultChars": 35250 + }, + { + "source": "web_fetch", + "query": "https://supabase.com/docs/guides/functions/auth.md", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/functions/auth.md" + } + ], + "resultChars": 7275 + } + ] + }, + "prompt": "Build and serve a Supabase Edge Function named `user-stats` for this project,\nreachable over HTTP at `/functions/v1/user-stats`.\n\nOur product stores per-user metrics in the existing `user_stats` table.\n\nTwo very different callers need to hit this one endpoint:\n\n1. **Our mobile app**, acting for a signed-in user. It sends that user's\n Supabase access token. The endpoint should return the user's own stats.\n\n2. **Our internal billing service**, a trusted backend with no signed-in user.\n It authenticates with the project's secret (service-role) key in the `apikey`\n header, and names the target user with a `user_id` in the JSON request body.\n It needs that user's stats.\n\nReturn the matching rows as JSON. The endpoint must be secure: only ever serve\nstats to a caller that is genuinely entitled to them, and turn away callers that\nare not.", + "promptSourcePath": "evals/build-functions-005-dual-auth-user-secret/PROMPT.md", + "attempts": 1, + "sourcePath": "opencode-kimi-k3/build-functions-005-dual-auth-user-secret.json" + }, + { + "experiment": "opencode-kimi-k3", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "opencode", + "modelProvider": "moonshotai", + "modelId": "moonshotai/kimi-k3" + }, + "eval": "build-storage-001-private-bucket-access", + "stage": "build", + "product": [ + "storage", + "database" + ], + "topic": [ + "rls", + "sdk" + ], + "suite": "benchmark", + "interface": "mcp", + "passed": true, + "checks": [ + { + "name": "bucket user-files exists", + "passed": true + }, + { + "name": "bucket user-files is private", + "passed": true + }, + { + "name": "RLS still enabled on storage.objects", + "passed": true + }, + { + "name": "user A lists only own files", + "passed": true, + "notes": "saw: 019fa63c-a5b9-773b-8d4b-a061f9b05b31/receipt-alpha.pdf, 019fa63c-a5b9-773b-8d4b-a061f9b05b31/receipt-beta.pdf" + }, + { + "name": "user B cannot read user A files", + "passed": true + }, + { + "name": "anon reads no files", + "passed": true + }, + { + "name": "user A can upload into own folder", + "passed": true + }, + { + "name": "user B cannot upload into user A folder", + "passed": true + }, + { + "name": "configured private per-user storage access", + "passed": true, + "judgeNotes": "Meets all requirements: private user-files bucket, authenticated owner-scoped SELECT and INSERT RLS policies using user-id path prefix, no RLS disabling or public access, and supabase-js createSignedUrl with expiry for temporary sharing." + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "{\n searchDocs(query: \"storage access control RLS policy bucket private user folder ownership\", limit: 5) {\n nodes {\n title\n href\n content\n }\n }\n}", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/storage/buckets/fundamentals", + "title": "Storage Buckets" + }, + { + "url": "https://supabase.com/docs/guides/storage/security/access-control", + "title": "Storage Access Control" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3", + "title": "Configure S3 Storage" + }, + { + "url": "https://supabase.com/docs/guides/security/product-security", + "title": "Secure configuration of Supabase products" + }, + { + "url": "https://supabase.com/docs/guides/storage/security/ownership", + "title": "Ownership" + } + ], + "resultChars": 20471 + }, + { + "source": "web_fetch", + "query": "https://supabase.com/changelog.md", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 92585 + } + ] + }, + "prompt": "Our app lets signed-in users keep personal files like receipts and bank\nstatements. These files are private — a user must only ever be able to upload\nand download their own. The app uploads each file under a path that starts\nwith the owner's user id, e.g. `/receipt-march.pdf`.\n\nSet up a `user-files` bucket on our project and lock it down that way.\n\nUsers also sometimes share one of their files with someone else through a\ntemporary link that expires. Include the supabase-js code the app should use\nfor that.", + "promptSourcePath": "evals/build-storage-001-private-bucket-access/PROMPT.md", + "attempts": 1, + "sourcePath": "opencode-kimi-k3/build-storage-001-private-bucket-access.json" + }, + { + "experiment": "opencode-kimi-k3", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "opencode", + "modelProvider": "moonshotai", + "modelId": "moonshotai/kimi-k3" + }, + "eval": "build-tests-001-rls-tenant-isolation", + "stage": "build", + "product": [ + "database" + ], + "topic": [ + "tests", + "rls" + ], + "suite": "benchmark", + "interface": "cli", + "passed": true, + "checks": [ + { + "name": "pgTAP test file(s) written under supabase/tests/", + "passed": true, + "notes": "1 file(s): supabase/tests/tenant_isolation_test.sql" + }, + { + "name": "pgTAP isolation tests ran and pass", + "passed": true, + "notes": "7 passed, 2 failed" + }, + { + "name": "agent correctly identifies the posts isolation bug from test results", + "passed": true, + "judgeNotes": "The agent correctly identifies `posts` as having a broken tenant isolation policy, explains that members of any org can read posts from other orgs, and grounds this in pgTAP test failure #4. It also correctly treats `notes` as isolated and relies on the test results as authoritative." + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [] + }, + "prompt": "Can you audit the tenant isolation on our tables? Write some database tests covering both the happy path and the negative case, run them, and share the results.", + "promptSourcePath": "evals/build-tests-001-rls-tenant-isolation/PROMPT.md", + "attempts": 1, + "sourcePath": "opencode-kimi-k3/build-tests-001-rls-tenant-isolation.json" + }, + { + "experiment": "opencode-kimi-k3", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "opencode", + "modelProvider": "moonshotai", + "modelId": "moonshotai/kimi-k3" + }, + "eval": "build-vectors-001-rag-with-permissions", + "stage": "build", + "product": [ + "database", + "vectors" + ], + "topic": [ + "sql", + "rls" + ], + "suite": "benchmark", + "interface": "mcp", + "passed": true, + "checks": [ + { + "name": "document_sections.embedding is vector(384)", + "passed": true, + "notes": "vector(384)" + }, + { + "name": "HNSW index on the embedding column", + "passed": true, + "notes": "CREATE INDEX document_sections_embedding_idx ON public.document_sections USING hnsw (embedding vector_cosine_ops)" + }, + { + "name": "index operator class matches the search operator", + "passed": true, + "notes": "function operators: <=>\nindexes: CREATE INDEX document_sections_embedding_idx ON public.document_sections USING hnsw (embedding vector_cosine_ops)" + }, + { + "name": "user A search returns only own sections, best match first", + "passed": true + }, + { + "name": "user B search returns only own sections, best match first", + "passed": true + }, + { + "name": "user A reads only own sections through the API", + "passed": true + }, + { + "name": "user A reads only own documents through the API", + "passed": true + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "{ searchDocs(query: \"gte-small embedding dimensions Supabase.ai Session semantic search vector column match documents rpc\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/functions/examples/semantic-search", + "title": "Semantic Search" + }, + { + "url": "https://supabase.com/docs/guides/ai/semantic-search", + "title": "Semantic search" + }, + { + "url": "https://supabase.com/docs/guides/ai/choosing-compute-addon", + "title": "Choosing your Compute Add-on" + }, + { + "url": "https://supabase.com/docs/guides/storage/vector/querying-vectors", + "title": "Querying Vectors" + }, + { + "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", + "title": "Automatic embeddings" + } + ], + "resultChars": 92278 + } + ] + }, + "prompt": "We're adding semantic search to our internal knowledge base app. I already wrote the edge functions, but search doesn't work yet. Can you set up whatever the database needs to make search work end to end?\n\nSome documents are confidential, so users should only have access to documents they own.", + "promptSourcePath": "evals/build-vectors-001-rag-with-permissions/PROMPT.md", + "attempts": 2, + "sourcePath": "opencode-kimi-k3/build-vectors-001-rag-with-permissions.json" + }, + { + "experiment": "opencode-kimi-k3", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "opencode", + "modelProvider": "moonshotai", + "modelId": "moonshotai/kimi-k3" + }, + "eval": "deploy-database-001-prometheus-metrics", + "stage": "deploy", + "product": [ + "database" + ], + "topic": [ + "observability" + ], + "suite": "benchmark", + "passed": true, + "checks": [ + { + "name": "preserved existing app scrape job", + "passed": true + }, + { + "name": "configured the Supabase Metrics API scrape correctly", + "passed": true, + "judgeNotes": "Supabase scrape uses HTTPS, correct metrics path, basic_auth with password_file, preserves the app job, targets .supabase.co:443, and docker-compose mounts the secrets directory containing the password file." + }, + { + "name": "documented live deployment and verification steps", + "passed": true, + "judgeNotes": "README.md includes concrete steps to create a Supabase Secret API key, place it in the mounted secret file, replace project refs, restart or reload the Compose/Prometheus stack, and verify via direct endpoint curl, Prometheus query, and targets UI." + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "{ searchDocs(query: \"Metrics API prometheus scrape endpoint project metrics\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/telemetry/metrics", + "title": "Metrics API" + }, + { + "url": "https://supabase.com/docs/guides/telemetry/metrics/grafana-self-hosted", + "title": "Metrics API with Prometheus & Grafana (self-hosted)" + }, + { + "url": "https://supabase.com/docs/guides/telemetry/metrics/vendor-agnostic", + "title": "Vendor-agnostic Metrics API setup" + }, + { + "url": "https://supabase.com/docs/guides/telemetry/metrics/grafana-cloud", + "title": "Metrics API with Grafana Cloud" + } + ], + "resultChars": 22788 + } + ] + }, + "prompt": "Can you wire my Supabase project metrics into our existing observability stack and document\nwhat we need to do to make the config live?", + "promptSourcePath": "evals/deploy-database-001-prometheus-metrics/PROMPT.md", + "attempts": 1, + "sourcePath": "opencode-kimi-k3/deploy-database-001-prometheus-metrics.json" + }, + { + "experiment": "opencode-kimi-k3", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "opencode", + "modelProvider": "moonshotai", + "modelId": "moonshotai/kimi-k3" + }, + "eval": "deploy-functions-001-edge-function-secrets", + "stage": "deploy", + "product": [ + "edge-functions" + ], + "topic": [ + "security" + ], + "suite": "benchmark", + "interface": "cli", + "passed": true, + "checks": [ + { + "name": "WEATHER_API_KEY is set as a Function secret on the project", + "passed": true + }, + { + "name": "the weather function is deployed to the project", + "passed": true, + "notes": "status ACTIVE" + }, + { + "name": "the weather function reads WEATHER_API_KEY from the environment", + "passed": true, + "judgeNotes": "The function reads WEATHER_API_KEY from the runtime environment using Deno.env.get('WEATHER_API_KEY')." + }, + { + "name": "WEATHER_API_KEY value is not committed to the repo", + "passed": true + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [ + { + "source": "web_fetch", + "query": "https://supabase.com/changelog.md", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 92585 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"edge function secrets environment variables deploy\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/troubleshooting/inspecting-edge-function-environment-variables-wg5qOQ", + "title": "Inspecting edge function environment variables" + }, + { + "url": "https://supabase.com/docs/guides/functions/secrets", + "title": "Environment Variables" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" + }, + { + "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech", + "title": "Transcription Telegram Bot" + }, + { + "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream", + "title": "Streaming Speech with ElevenLabs" + } + ], + "resultChars": 40215 + } + ] + }, + "prompt": "Our weather widget currently calls WeatherAPI straight from the browser, which\nleaks our API key. I want to move that behind a Supabase Edge Function called\n`weather` that holds the key server-side and proxies the request.\n\nThe function should read the key from an environment variable named\n`WEATHER_API_KEY`. Our key already lives in a local `.env` file at the project\nroot.\n\nDeploy the function to our project so it's live, and make sure the deployed\nfunction can actually read the key at runtime.", + "promptSourcePath": "evals/deploy-functions-001-edge-function-secrets/PROMPT.md", + "attempts": 1, + "sourcePath": "opencode-kimi-k3/deploy-functions-001-edge-function-secrets.json" + }, + { + "experiment": "opencode-kimi-k3", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "opencode", + "modelProvider": "moonshotai", + "modelId": "moonshotai/kimi-k3" + }, + "eval": "deploy-self-hosting-001-docker-compose", + "stage": "deploy", + "product": [ + "database", + "auth", + "storage" + ], + "topic": [ + "self-hosting" + ], + "suite": "benchmark", + "interface": "cli", + "passed": true, + "checks": [ + { + "name": "cloned the self-host stack (docker-compose.yml + volumes/db)", + "passed": true + }, + { + "name": "didn't conflate with the CLI (no supabase/config.toml in the stack)", + "passed": true + }, + { + "name": "secrets rotated off the shipped defaults", + "passed": true + }, + { + "name": "ANON_KEY and SERVICE_ROLE_KEY are HS256 JWTs signed by JWT_SECRET", + "passed": true + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [ + { + "source": "web_fetch", + "query": "https://supabase.com/docs/guides/self-hosting/docker.md", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/self-hosting/docker.md" + } + ], + "resultChars": 92585 + }, + { + "source": "web_fetch", + "query": "https://supabase.com/changelog.md", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 92585 + } + ] + }, + "prompt": "I'm moving off the hosted Supabase and running the whole thing myself on a VPS I\njust spun up. Can you get a Docker setup ready for me to copy onto the box?\n\nI don't need it running here, I'll do the actual bring-up once I'm on the\nserver. I just want everything in place and the secrets set up. Put it in a `supabase-docker/`\nfolder at the repo root so I can scp the whole thing across in one go.", + "promptSourcePath": "evals/deploy-self-hosting-001-docker-compose/PROMPT.md", + "attempts": 1, + "sourcePath": "opencode-kimi-k3/deploy-self-hosting-001-docker-compose.json" + }, + { + "experiment": "opencode-kimi-k3", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "opencode", + "modelProvider": "moonshotai", + "modelId": "moonshotai/kimi-k3" + }, + "eval": "investigate-auth-001-deleted-user-access", + "stage": "investigate", + "product": [ + "auth" + ], + "topic": [ + "security", + "sdk" + ], + "suite": "benchmark", + "interface": "mcp", + "passed": true, + "checks": [ + { + "name": "victim session active before delete-account", + "passed": true + }, + { + "name": "delete_account flow ran for the victim", + "passed": true + }, + { + "name": "delete-account revokes the user's sessions", + "passed": true, + "notes": "sessions left: 0" + }, + { + "name": "deleted user's refresh token is rejected", + "passed": true + }, + { + "name": "deleted user cannot sign back in", + "passed": true + }, + { + "name": "other users keep their sessions and access", + "passed": true + }, + { + "name": "diagnosed and explained session revocation", + "passed": true, + "judgeNotes": "Meets all rubric points: identifies soft-delete-only root cause; implements auth user deletion with cascaded sessions/refresh token revocation; explains stateless JWT access-token window consistently and gives mitigations; correctly distinguishes publishable frontend/RLS behavior from secret backend-only/RLS-bypass behavior." + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "{ searchDocs(query: \"publishable secret API keys migration anon service_role RLS\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-redwoodjs", + "title": "Build a User Management App with RedwoodJS" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" + }, + { + "url": "https://supabase.com/docs/guides/auth/quickstarts/with-expo-react-native-social-auth", + "title": "Build a Social Auth App with Expo React Native" + }, + { + "url": "https://supabase.com/docs/guides/auth/signing-keys", + "title": "JWT Signing Keys" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-sveltekit", + "title": "Build a User Management App with SvelteKit" + } + ], + "resultChars": 169611 + } + ] + }, + "prompt": "Last week support removed a user through our app's delete-account flow — the\napp calls the `delete_account` function over RPC as the signed-in user. This\nmorning that same person was back: still signed in, reading and saving their\ndata like nothing happened.\n\nFigure out why the account still works, fix the flow so a deleted account\nloses access, and tell me whether there is any window where they could still\nget in after the fix.\n\nOne more thing while you're at it: we're migrating off the legacy\nanon/service_role API keys to the new publishable and secret keys, and the\nteam isn't sure which one belongs in the frontend and what each means for\nRLS. Give us a quick rundown.", + "promptSourcePath": "evals/investigate-auth-001-deleted-user-access/PROMPT.md", + "attempts": 1, + "sourcePath": "opencode-kimi-k3/investigate-auth-001-deleted-user-access.json" + }, + { + "experiment": "opencode-kimi-k3", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "opencode", + "modelProvider": "moonshotai", + "modelId": "moonshotai/kimi-k3" + }, + "eval": "investigate-realtime-001-subscribed-no-events", + "stage": "investigate", + "product": [ + "realtime", + "database" + ], + "topic": [ + "sdk" + ], + "suite": "benchmark", + "interface": "mcp", + "passed": true, + "checks": [ + { + "name": "orders table added to supabase_realtime publication", + "passed": true + }, + { + "name": "courier_locations still in supabase_realtime publication", + "passed": true + }, + { + "name": "publication still publishes INSERT events", + "passed": true + }, + { + "name": "RLS still enabled on orders", + "passed": true + }, + { + "name": "staff can still read orders through RLS", + "passed": true, + "notes": "authenticated sees 2 of 2 orders" + }, + { + "name": "diagnosed missing publication membership", + "passed": true, + "judgeNotes": "The assistant correctly identified the root cause as public.orders missing from the supabase_realtime publication despite the channel reaching SUBSCRIBED, applied exactly ALTER PUBLICATION supabase_realtime ADD TABLE public.orders, and preserved courier_locations, RLS, and policies without blaming or weakening other components." + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [] + }, + "prompt": "Our dispatch dashboard shows incoming orders as they happen. The courier\nlocation feed on the same page updates live without problems, but new orders\nonly show up after a page refresh.\n\nThe dashboard uses supabase-js to subscribe to INSERT events on the `orders`\ntable through postgres_changes, the same way it subscribes to courier\nlocations. The channel's status callback logs SUBSCRIBED and there are no\nerrors in the browser console.\n\nFigure out why no order events ever arrive and fix it.", + "promptSourcePath": "evals/investigate-realtime-001-subscribed-no-events/PROMPT.md", + "attempts": 1, + "sourcePath": "opencode-kimi-k3/investigate-realtime-001-subscribed-no-events.json" + }, + { + "experiment": "opencode-kimi-k3", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "opencode", + "modelProvider": "moonshotai", + "modelId": "moonshotai/kimi-k3" + }, + "eval": "investigate-reliability-003-edge-function-5xx-correlation", + "stage": "investigate", + "product": [ + "edge-functions" + ], + "topic": [ + "observability" + ], + "suite": "benchmark", + "interface": "mcp", + "passed": false, + "checks": [ + { + "name": "identified image-transform and the recurring 503 pattern", + "passed": true, + "judgeNotes": "Identified image-transform as affected and described the recurring 503 pattern across 07:00Z–12:00Z, covering all 8 gateway failures." + }, + { + "name": "attributed recurring 503s to gateway/platform layer, not function code", + "passed": false, + "judgeNotes": "Although it observes that the 503s appear only in gateway logs with no corresponding function execution logs, it ultimately attributes the root cause to the function/runtime boot process and unpinned npm dependencies, and recommends pinning/redeploying the functions. The rubric requires attributing the recurring 503s to the gateway/platform layer rather than the function code/runtime." + }, + { + "name": "recommended a concrete next step", + "passed": true, + "judgeNotes": "The assistant provided concrete actionable next steps, including pinning dependencies and redeploying, checking npm publish history for the affected time window, committing lockfiles, adding gateway 5xx alerting, and applying retry mitigation." + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [] + }, + "prompt": "Users have been reporting that image uploads are intermittently failing with server errors throughout the morning of `2026-04-28`.\n\nCan you investigate the project logs and tell me what is going on and what we\nshould do next?", + "promptSourcePath": "evals/investigate-reliability-003-edge-function-5xx-correlation/PROMPT.md", + "attempts": 2, + "sourcePath": "opencode-kimi-k3/investigate-reliability-003-edge-function-5xx-correlation.json" + }, + { + "experiment": "opencode-kimi-k3", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "opencode", + "modelProvider": "moonshotai", + "modelId": "moonshotai/kimi-k3" + }, + "eval": "resolve-dataapi-001-empty-results", + "stage": "resolve", + "product": [ + "data-api", + "database", + "auth" + ], + "topic": [ + "rls", + "sdk" + ], + "suite": "benchmark", + "interface": "mcp", + "passed": true, + "checks": [ + { + "name": "RLS still enabled on bookmarks", + "passed": true + }, + { + "name": "user A reads own bookmarks", + "passed": true + }, + { + "name": "user B cannot read user A bookmarks", + "passed": true + }, + { + "name": "anon reads no bookmarks", + "passed": true + }, + { + "name": "user A can save a new bookmark", + "passed": true + }, + { + "name": "user B cannot insert a bookmark as user A", + "passed": true + }, + { + "name": "diagnosed RLS and added owner-scoped policies", + "passed": true, + "judgeNotes": "Diagnosed RLS enabled with no policies/default-deny, kept RLS enabled, and created authenticated SELECT and INSERT policies scoped to user_id = auth.uid() with USING and WITH CHECK." + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [] + }, + "prompt": "Our app lets signed-in users save bookmarks and view them on their dashboard. Bookmarks are stored in the `bookmarks` table and are private — a user must only ever see their own. \nUsers also need to be able to save new bookmarks from the app.\n\nI can see the rows when I query the table directly, but the dashboard shows an empty list for every user.\n\nFind out why the Data API returns nothing and fix it.", + "promptSourcePath": "evals/resolve-dataapi-001-empty-results/PROMPT.md", + "attempts": 1, + "sourcePath": "opencode-kimi-k3/resolve-dataapi-001-empty-results.json" + }, + { + "experiment": "opencode-kimi-k3", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "opencode", + "modelProvider": "moonshotai", + "modelId": "moonshotai/kimi-k3" + }, + "eval": "resolve-database-001-migration-history-mismatch", + "stage": "resolve", + "product": [ + "database" + ], + "topic": [ + "migrations" + ], + "suite": "benchmark", + "interface": "cli", + "passed": true, + "checks": [ + { + "name": "the avatar_url column is applied on the hosted profiles table", + "passed": true + }, + { + "name": "migration 20240220000000 is recorded in the remote history", + "passed": true + }, + { + "name": "remote migration history matches local migration files", + "passed": true + }, + { + "name": "local migrations are a valid reconciled sequence", + "passed": true + }, + { + "name": "production profile data is intact (not reset)", + "passed": true + }, + { + "name": "the avatar migration and history reconciliation were done via the Supabase CLI", + "passed": true, + "judgeNotes": "Avatar migration was applied through `supabase db push` in #20, which output `Applying migration 20240220000000_add_avatar_url.sql...` and `Finished supabase db push.` The orphan bio history was reconciled by adding the missing local file `supabase/migrations/20240115000000_add_profile_bio.sql` in #18, after which `supabase migration list`/#19 showed local and remote aligned and the successful push proceeded. No disallowed workaround or direct mutation was seen; psql usage was read-only inspection." + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "{\n searchDocs(query: \"remote migration versions not found in local migrations directory repair db pull\", limit: 5) {\n nodes {\n title\n href\n content\n }\n }\n}", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/local-development/cli-workflows", + "title": "Local development workflow" + }, + { + "url": "https://supabase.com/docs/guides/deployment/database-migrations", + "title": "Database Migrations" + }, + { + "url": "https://supabase.com/docs/reference/cli/supabase-migration-repair", + "title": "Repair the migration history table" + }, + { + "url": "https://supabase.com/docs/reference/cli/supabase-db-pull", + "title": "Pull schema from the remote database" + }, + { + "url": "https://supabase.com/docs/guides/deployment/branching/troubleshooting", + "title": "Troubleshooting" + } + ], + "resultChars": 53680 + }, + { + "source": "web_fetch", + "query": "https://supabase.com/docs/guides/deployment/database-migrations.md", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/deployment/database-migrations.md" + } + ], + "resultChars": 9637 + } + ] + }, + "prompt": "I'm trying to ship a migration to our hosted project and it's not working. Can you figure out what's wrong and get it deployed?", + "promptSourcePath": "evals/resolve-database-001-migration-history-mismatch/PROMPT.md", + "attempts": 1, + "sourcePath": "opencode-kimi-k3/resolve-database-001-migration-history-mismatch.json" + }, + { + "experiment": "opencode-kimi-k3", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "opencode", + "modelProvider": "moonshotai", + "modelId": "moonshotai/kimi-k3" + }, + "eval": "resolve-performance-001-slow-query-cpu-spike", + "stage": "resolve", + "product": [ + "database" + ], + "topic": [ + "observability", + "sql" + ], + "suite": "benchmark", + "interface": "mcp", + "passed": true, + "checks": [ + { + "name": "inspected pg_stat_statements for query performance", + "passed": true + }, + { + "name": "ran EXPLAIN on the expensive query", + "passed": true + }, + { + "name": "created index covering user_id and created_at", + "passed": true + }, + { + "name": "query plan uses an index and avoids sequential scan", + "passed": true, + "notes": "Limit (cost=55.55..55.61 rows=25 width=88)\n -> Sort (cost=55.55..55.61 rows=25 width=88)\n Sort Key: created_at DESC\n -> Bitmap Heap Scan on events (cost=4.48..54.97 rows=25 width=88)\n Recheck Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)\n -> Bitmap Index Scan on events_user_id_created_at_idx (cost=0.00..4.47 rows=25 width=0)\n Index Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)" + }, + { + "name": "inserts still work", + "passed": true + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] + }, + "docs": { + "calls": [] + }, + "prompt": "My database CPU keeps spiking and the app gets slow when loading recent events for a user. Can you figure out what query is causing it and make the database change needed to fix it?\n\nEnd your turn with a short summary of what you changed and why.", + "promptSourcePath": "evals/resolve-performance-001-slow-query-cpu-spike/PROMPT.md", + "attempts": 1, + "sourcePath": "opencode-kimi-k3/resolve-performance-001-slow-query-cpu-spike.json" + }, + { + "experiment": "opencode-kimi-k3", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "opencode", + "modelProvider": "moonshotai", + "modelId": "moonshotai/kimi-k3" + }, + "eval": "resolve-security-002-rls-cross-tenant-leak", + "stage": "resolve", + "product": [ + "database", + "auth" + ], + "topic": [ + "rls", + "security" + ], + "suite": "benchmark", + "interface": "mcp", + "passed": true, + "checks": [ + { + "name": "RLS enabled on notes", + "passed": true + }, + { + "name": "tenant A sees only org A notes", + "passed": true + }, + { + "name": "tenant B cannot read org A notes", + "passed": true + }, + { + "name": "tenant A author can update own note", + "passed": true + }, + { + "name": "tenant B cannot update org A note", + "passed": true + }, + { + "name": "tenant B author can delete own note", + "passed": true + }, + { + "name": "tenant B cannot delete org A note", + "passed": true + }, + { + "name": "tenant A can insert note in own org", + "passed": true + }, + { + "name": "tenant B cannot insert into org A", + "passed": true + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [] + }, + "prompt": "A customer reported that notes showed up in the wrong workspace.\n\nCan you investigate what is going on and fix it?", + "promptSourcePath": "evals/resolve-security-002-rls-cross-tenant-leak/PROMPT.md", + "attempts": 1, + "sourcePath": "opencode-kimi-k3/resolve-security-002-rls-cross-tenant-leak.json" + }, + { + "experiment": "opencode-kimi-k3-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "opencode", + "modelProvider": "moonshotai", + "modelId": "moonshotai/kimi-k3" + }, + "eval": "build-cli-001-bootstrap-app", + "stage": "build", + "product": [ + "database", + "data-api" + ], + "topic": [ + "migrations", + "rls" + ], + "suite": "benchmark", + "interface": "cli", + "passed": true, + "checks": [ + { + "name": "supabase project initialised (supabase/config.toml exists)", + "passed": true + }, + { + "name": "todos table is created by a migration file", + "passed": true + }, + { + "name": "todos table exists with at least 2 seeded rows", + "passed": true, + "notes": "found 3 rows" + }, + { + "name": "row level security is enabled on todos", + "passed": true + }, + { + "name": "a SELECT policy targets the authenticated role", + "passed": true + }, + { + "name": "REST API returns no todos to anonymous requests", + "passed": true, + "notes": "0 rows" + }, + { + "name": "REST API returns the todos to authenticated requests", + "passed": true, + "notes": "3 rows" + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] + }, + "prompt": "We're kicking off a todos app and I want the Supabase side ready for the team\nto build on. Set it up the way we'd run it in development, with schema changes\ntracked as migrations so they can be reviewed and replayed.\n\nFor the first slice we just need a `todos` table. Todos aren't public: anyone\nsigned in can read all of them, but nothing should be writable through the API\nfor now. Add a couple of sample todos so there's something to look at.\n\nBefore you hand it back, make sure the running API actually behaves that way —\nsigned-in users get the todos, signed-out requests get nothing.", + "promptSourcePath": "evals/build-cli-001-bootstrap-app/PROMPT.md", + "attempts": 1, + "sourcePath": "opencode-kimi-k3-no-skills/build-cli-001-bootstrap-app.json" + }, + { + "experiment": "opencode-kimi-k3-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "opencode", + "modelProvider": "moonshotai", + "modelId": "moonshotai/kimi-k3" + }, + "eval": "build-cli-002-declarative-schema", + "stage": "build", + "product": [ + "database" + ], + "topic": [ + "declarative-schema", + "migrations" + ], + "suite": "benchmark", + "interface": "cli", + "passed": true, + "checks": [ + { + "name": "supabase db diff used to generate the migration", + "passed": true + }, + { + "name": "schema file updated to include description column", + "passed": true + }, + { + "name": "a new migration was generated for the change", + "passed": true + }, + { + "name": "description column exists in the live database", + "passed": true + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] + }, + "prompt": "Add a description text column to the `products` table in my local Supabase stack", + "promptSourcePath": "evals/build-cli-002-declarative-schema/PROMPT.md", + "attempts": 1, + "sourcePath": "opencode-kimi-k3-no-skills/build-cli-002-declarative-schema.json" + }, + { + "experiment": "opencode-kimi-k3-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "opencode", + "modelProvider": "moonshotai", + "modelId": "moonshotai/kimi-k3" + }, + "eval": "build-cli-003-pg-cron-queue-workflow", + "stage": "build", + "product": [ + "database", + "edge-functions", + "cron", + "queues" + ], + "topic": [ + "sql", + "sdk" + ], + "suite": "benchmark", + "interface": "cli", + "passed": true, + "checks": [ + { + "name": "pg_cron job 'enqueue-tasks' scheduled to run every minute", + "passed": true, + "notes": "schedule='* * * * *', active=true" + }, + { + "name": "cron command enqueues to the 'tasks' queue", + "passed": true, + "notes": "queue depth 0 -> 1" + }, + { + "name": "process-tasks function drains the queue", + "passed": true, + "notes": "function removed the seeded message (id 37) from the queue" + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] + }, + "prompt": "I want to set up a recurring background workflow on my local Supabase stack.\n\nCan you set up a cron job called `enqueue-tasks` to run every minute and push a task into a queue called `tasks`? Then add a `process-tasks` edge function that reads messages off the `tasks` queue and removes them, so a scheduled worker can keep the backlog drained.", + "promptSourcePath": "evals/build-cli-003-pg-cron-queue-workflow/PROMPT.md", + "attempts": 1, + "sourcePath": "opencode-kimi-k3-no-skills/build-cli-003-pg-cron-queue-workflow.json" + }, + { + "experiment": "opencode-kimi-k3-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "opencode", + "modelProvider": "moonshotai", + "modelId": "moonshotai/kimi-k3" + }, + "eval": "build-database-001-migrate-postgres-to-supabase", + "stage": "build", + "product": [ + "database" + ], + "topic": [ + "migrations" + ], + "suite": "benchmark", + "interface": "cli", + "passed": true, + "checks": [ + { + "name": "all 3 tables exist (teams, members, tasks)", + "passed": true + }, + { + "name": "row counts match (teams=5, members=10, tasks=13)", + "passed": true + }, + { + "name": "foreign key constraints survived the restore", + "passed": true + }, + { + "name": "tasks_team_status_idx index survived the restore", + "passed": true + }, + { + "name": "sequences synced (next insert won't conflict with existing IDs)", + "passed": true + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] + }, + "prompt": "I have an existing Postgres database I want to migrate to Supabase. There's a binary dump at `source.dump` in the current directory.\n\nCan you set up a local Supabase project and restore the dump into it?", + "promptSourcePath": "evals/build-database-001-migrate-postgres-to-supabase/PROMPT.md", + "attempts": 1, + "sourcePath": "opencode-kimi-k3-no-skills/build-database-001-migrate-postgres-to-supabase.json" + }, + { + "experiment": "opencode-kimi-k3-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "opencode", + "modelProvider": "moonshotai", + "modelId": "moonshotai/kimi-k3" + }, + "eval": "build-functions-004-service-role-bypass", + "stage": "build", + "product": [ + "edge-functions", + "auth", + "database" + ], + "topic": [ + "rls", + "security", + "sdk" + ], + "suite": "benchmark", + "interface": "mcp", + "passed": true, + "checks": [ + { + "name": "rejects missing auth", + "passed": true, + "notes": "status=401" + }, + { + "name": "user A reads own note", + "passed": true, + "notes": "status=200" + }, + { + "name": "reads only with the caller's JWT", + "passed": true, + "notes": "bearer_tokens=2, all_match=true" + }, + { + "name": "user A cannot force-read user B note", + "passed": true, + "notes": "status=200" + }, + { + "name": "user B cannot force-read user A note", + "passed": true, + "notes": "status=200" + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "{ searchDocs(query: \"edge function authenticate user JWT getUser RLS\", limit: 3) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", + "title": "Integrating With Supabase Auth" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth-headers", + "title": "Authorization headers" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs", + "title": "Build a User Management App with Next.js" + } + ], + "resultChars": 50282 + } + ] + }, + "prompt": "I built an Edge Function called `private-notes` for showing a user's saved\nprivate notes.\n\nCan you check whether there's any way one user could see another user's notes?\n\nPlease fix and deploy it if needed.", + "promptSourcePath": "evals/build-functions-004-service-role-bypass/PROMPT.md", + "attempts": 1, + "sourcePath": "opencode-kimi-k3-no-skills/build-functions-004-service-role-bypass.json" + }, + { + "experiment": "opencode-kimi-k3-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "opencode", + "modelProvider": "moonshotai", + "modelId": "moonshotai/kimi-k3" + }, + "eval": "build-functions-005-dual-auth-user-secret", + "stage": "build", + "product": [ + "edge-functions", + "auth", + "database" + ], + "topic": [ + "sdk", + "rls", + "security" + ], + "suite": "benchmark", + "interface": "cli", + "cliVersion": "2.109.1", + "passed": true, + "checks": [ + { + "name": "seed rows present", + "passed": true, + "notes": "found 2/2 seeded rows" + }, + { + "name": "rejects request with no credentials", + "passed": true, + "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" + }, + { + "name": "user with JWT reads only their own rows", + "passed": true, + "notes": "status 200: [{\"user_id\":\"ffac847f-b734-4c98-a4fb-32356e032e87\",\"metric\":\"steps_a_ms3zogi9\",\"value\":111}]" + }, + { + "name": "user cannot read another user's rows by passing user_id", + "passed": true, + "notes": "status 200: [{\"user_id\":\"ffac847f-b734-4c98-a4fb-32356e032e87\",\"metric\":\"steps_a_ms3zogi9\",\"value\":111}]" + }, + { + "name": "service key bypasses RLS to read the target user's rows", + "passed": true, + "notes": "status 200: [{\"user_id\":\"b1a74712-deb9-4b80-841d-f23a2fbad78c\",\"metric\":\"steps_b_ms3zogi9\",\"value\":222}]" + }, + { + "name": "non-service key is not granted service access", + "passed": true, + "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" + }, + { + "name": "rejects an unverified (forged) user token", + "passed": true, + "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" + }, + { + "name": "a user token in the apikey slot is not treated as the service key", + "passed": true, + "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" + }, + { + "name": "implementation uses @supabase/server", + "passed": true, + "notes": "imports @supabase/server / withSupabase" + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"withSupabase @supabase/server edge function authMode publishable secret supabaseAdmin\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/functions/auth", + "title": "Securing Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/api-keys", + "title": "Understanding API keys" + }, + { + "url": "https://supabase.com/docs/guides/functions/secrets", + "title": "Environment Variables" + }, + { + "url": "https://supabase.com/docs/guides/auth/choosing-a-server-package", + "title": "Which package to use" + } + ], + "resultChars": 44429 + } + ] + }, + "prompt": "Build and serve a Supabase Edge Function named `user-stats` for this project,\nreachable over HTTP at `/functions/v1/user-stats`.\n\nOur product stores per-user metrics in the existing `user_stats` table.\n\nTwo very different callers need to hit this one endpoint:\n\n1. **Our mobile app**, acting for a signed-in user. It sends that user's\n Supabase access token. The endpoint should return the user's own stats.\n\n2. **Our internal billing service**, a trusted backend with no signed-in user.\n It authenticates with the project's secret (service-role) key in the `apikey`\n header, and names the target user with a `user_id` in the JSON request body.\n It needs that user's stats.\n\nReturn the matching rows as JSON. The endpoint must be secure: only ever serve\nstats to a caller that is genuinely entitled to them, and turn away callers that\nare not.", + "promptSourcePath": "evals/build-functions-005-dual-auth-user-secret/PROMPT.md", + "attempts": 2, + "sourcePath": "opencode-kimi-k3-no-skills/build-functions-005-dual-auth-user-secret.json" + }, + { + "experiment": "opencode-kimi-k3-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "opencode", + "modelProvider": "moonshotai", + "modelId": "moonshotai/kimi-k3" + }, + "eval": "build-storage-001-private-bucket-access", + "stage": "build", + "product": [ + "storage", + "database" + ], + "topic": [ + "rls", + "sdk" + ], + "suite": "benchmark", + "interface": "mcp", + "passed": true, + "checks": [ + { + "name": "bucket user-files exists", + "passed": true + }, + { + "name": "bucket user-files is private", + "passed": true + }, + { + "name": "RLS still enabled on storage.objects", + "passed": true + }, + { + "name": "user A lists only own files", + "passed": true, + "notes": "saw: 019fa651-c08d-731a-9d74-1cda83a3daf8/receipt-alpha.pdf, 019fa651-c08d-731a-9d74-1cda83a3daf8/receipt-beta.pdf" + }, + { + "name": "user B cannot read user A files", + "passed": true + }, + { + "name": "anon reads no files", + "passed": true + }, + { + "name": "user A can upload into own folder", + "passed": true + }, + { + "name": "user B cannot upload into user A folder", + "passed": true + }, + { + "name": "configured private per-user storage access", + "passed": true, + "judgeNotes": "Meets all requirements: private bucket, RLS enabled, authenticated owner-scoped SELECT and INSERT policies with WITH CHECK, and supabase-js createSignedUrl with expiry for sharing." + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "{ searchDocs(query: \"createSignedUrl supabase-js storage temporary expiring link\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/storage/serving/downloads", + "title": "Serving assets from Storage" + }, + { + "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsignedurl" + }, + { + "url": "https://supabase.com/docs/guides/storage/serving/image-transformations", + "title": "Storage Image Transformations" + }, + { + "url": "https://supabase.com/docs/guides/storage/uploads/resumable-uploads", + "title": "Resumable Uploads" + }, + { + "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsignedurls" + } + ], + "resultChars": 38982 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"storage access control RLS policies foldername auth.uid bucket private\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/storage/buckets/fundamentals", + "title": "Storage Buckets" + }, + { + "url": "https://supabase.com/docs/guides/storage/security/access-control", + "title": "Storage Access Control" + }, + { + "url": "https://supabase.com/docs/guides/security/product-security", + "title": "Secure configuration of Supabase products" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3", + "title": "Configure S3 Storage" + }, + { + "url": "https://supabase.com/docs/guides/storage/debugging/error-codes", + "title": "Error Codes" + } + ], + "resultChars": 33308 + } + ] + }, + "prompt": "Our app lets signed-in users keep personal files like receipts and bank\nstatements. These files are private — a user must only ever be able to upload\nand download their own. The app uploads each file under a path that starts\nwith the owner's user id, e.g. `/receipt-march.pdf`.\n\nSet up a `user-files` bucket on our project and lock it down that way.\n\nUsers also sometimes share one of their files with someone else through a\ntemporary link that expires. Include the supabase-js code the app should use\nfor that.", + "promptSourcePath": "evals/build-storage-001-private-bucket-access/PROMPT.md", + "attempts": 1, + "sourcePath": "opencode-kimi-k3-no-skills/build-storage-001-private-bucket-access.json" + }, + { + "experiment": "opencode-kimi-k3-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "opencode", + "modelProvider": "moonshotai", + "modelId": "moonshotai/kimi-k3" + }, + "eval": "build-tests-001-rls-tenant-isolation", + "stage": "build", + "product": [ + "database" + ], + "topic": [ + "tests", + "rls" + ], + "suite": "benchmark", + "interface": "cli", + "passed": true, + "checks": [ + { + "name": "pgTAP test file(s) written under supabase/tests/", + "passed": true, + "notes": "1 file(s): supabase/tests/database/tenant_isolation.test.sql" + }, + { + "name": "pgTAP isolation tests ran and pass", + "passed": true, + "notes": "14 passed, 0 failed" + }, + { + "name": "agent correctly identifies the posts isolation bug from test results", + "passed": true, + "judgeNotes": "The agent correctly identifies `posts` as having the broken tenant isolation policy, specifically that authenticated members could read posts from organizations they are not members of. It grounds this in pgTAP failures (tests 4 and 11) and treats the test results as authoritative. It does not blame `notes` for the read isolation flaw, though it separately notes other write-policy issues." + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "{ searchDocs(query: \"pgTAP database tests supabase test db directory tests\") { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/reference/cli/supabase-test-db", + "title": "Tests local database with pgTAP" + }, + { + "url": "https://supabase.com/docs/guides/local-development/cli/testing-and-linting", + "title": "Testing and linting" + }, + { + "url": "https://supabase.com/docs/guides/local-development/testing/overview", + "title": "Testing Overview" + }, + { + "url": "https://supabase.com/docs/guides/local-development/testing/pgtap-extended", + "title": "Advanced pgTAP Testing" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pgtap", + "title": "pgTAP: Unit Testing" + }, + { + "url": "https://supabase.com/docs/guides/database/testing", + "title": "Testing Your Database" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", + "title": "Row Level Security" + }, + { + "url": "https://supabase.com/docs/guides/functions/unit-test", + "title": "Testing your Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/deployment/managing-environments", + "title": "Managing Environments" + }, + { + "url": "https://supabase.com/docs/guides/platform/performance", + "title": "Performance Tuning" + }, + { + "url": "https://supabase.com/docs/guides/platform", + "title": "Supabase Platform" + }, + { + "url": "https://supabase.com/docs/guides/storage/management/download-objects", + "title": "Download Objects" + }, + { + "url": "https://supabase.com/docs/guides/platform/sso/multiple-providers", + "title": "Multiple SSO Providers" + }, + { + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/vercel-postgres", + "title": "Migrate from Vercel Postgres to Supabase" + }, + { + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/heroku", + "title": "Migrate from Heroku to Supabase" + }, + { + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase", + "title": "Migrating to Supabase" + }, + { + "url": "https://supabase.com/docs/guides/database/overview", + "title": "Database" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform", + "title": "Restore a Platform Project to Self-Hosted" + }, + { + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/mysql", + "title": "Migrate from MySQL to Supabase" + }, + { + "url": "https://supabase.com/docs/guides/api/rest/generating-types", + "title": "Generating TypeScript Types" + }, + { + "url": "https://supabase.com/docs/guides/api/rest/generating-python-types", + "title": "Generating Python Types" + }, + { + "url": "https://supabase.com/docs/guides/platform/sso/choosing-login-flow", + "title": "Choosing the Right SSO Login Flow" + }, + { + "url": "https://supabase.com/docs/guides/database/inspect", + "title": "Debugging and monitoring" + }, + { + "url": "https://supabase.com/docs/reference/cli/supabase-test", + "title": "Run tests on local Supabase containers" + }, + { + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/mssql", + "title": "Migrate from MSSQL to Supabase" + }, + { + "url": "https://supabase.com/docs/guides/platform/permissions", + "title": "Permissions" + }, + { + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/postgres", + "title": "Migrate from Postgres to Supabase" + }, + { + "url": "https://supabase.com/docs/guides/local-development/cli/getting-started", + "title": "Supabase CLI" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions", + "title": "Postgres Extensions Overview" + }, + { + "url": "https://supabase.com/docs/guides/deployment", + "title": "Deployment & Branching" + } + ], + "resultChars": 234607 + } + ] + }, + "prompt": "Can you audit the tenant isolation on our tables? Write some database tests covering both the happy path and the negative case, run them, and share the results.", + "promptSourcePath": "evals/build-tests-001-rls-tenant-isolation/PROMPT.md", + "attempts": 1, + "sourcePath": "opencode-kimi-k3-no-skills/build-tests-001-rls-tenant-isolation.json" + }, + { + "experiment": "opencode-kimi-k3-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "opencode", + "modelProvider": "moonshotai", + "modelId": "moonshotai/kimi-k3" + }, + "eval": "build-vectors-001-rag-with-permissions", + "stage": "build", + "product": [ + "database", + "vectors" + ], + "topic": [ + "sql", + "rls" + ], + "suite": "benchmark", + "interface": "mcp", + "passed": true, + "checks": [ + { + "name": "document_sections.embedding is vector(384)", + "passed": true, + "notes": "vector(384)" + }, + { + "name": "HNSW index on the embedding column", + "passed": true, + "notes": "CREATE INDEX document_sections_embedding_idx ON public.document_sections USING hnsw (embedding vector_cosine_ops)" + }, + { + "name": "index operator class matches the search operator", + "passed": true, + "notes": "function operators: <=>\nindexes: CREATE INDEX document_sections_embedding_idx ON public.document_sections USING hnsw (embedding vector_cosine_ops)" + }, + { + "name": "user A search returns only own sections, best match first", + "passed": true + }, + { + "name": "user B search returns only own sections, best match first", + "passed": true + }, + { + "name": "user A reads only own sections through the API", + "passed": true + }, + { + "name": "user A reads only own documents through the API", + "passed": true + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] + }, + "prompt": "We're adding semantic search to our internal knowledge base app. I already wrote the edge functions, but search doesn't work yet. Can you set up whatever the database needs to make search work end to end?\n\nSome documents are confidential, so users should only have access to documents they own.", + "promptSourcePath": "evals/build-vectors-001-rag-with-permissions/PROMPT.md", + "attempts": 1, + "sourcePath": "opencode-kimi-k3-no-skills/build-vectors-001-rag-with-permissions.json" + }, + { + "experiment": "opencode-kimi-k3-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "opencode", + "modelProvider": "moonshotai", + "modelId": "moonshotai/kimi-k3" + }, + "eval": "deploy-database-001-prometheus-metrics", + "stage": "deploy", + "product": [ + "database" + ], + "topic": [ + "observability" + ], + "suite": "benchmark", + "passed": true, + "checks": [ + { + "name": "preserved existing app scrape job", + "passed": true + }, + { + "name": "configured the Supabase Metrics API scrape correctly", + "passed": true, + "judgeNotes": "Prometheus preserves the app scrape and adds a Supabase HTTPS scrape at /customer/v1/privileged/metrics using HTTP Basic Auth with password_file. docker-compose mounts the secrets directory containing that password file read-only." + }, + { + "name": "documented live deployment and verification steps", + "passed": true, + "judgeNotes": "README includes Secret API key creation, matching secret file placement, project ref replacement, restart/hot-reload of the Compose stack, and concrete verification via Prometheus targets plus curl smoke test. Endpoint/auth and secret setup match the Prometheus and Compose configuration." + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"metrics endpoint Prometheus scrape project metrics\", limit: 10) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/telemetry/metrics/grafana-self-hosted", + "title": "Metrics API with Prometheus & Grafana (self-hosted)" + }, + { + "url": "https://supabase.com/docs/guides/telemetry/metrics", + "title": "Metrics API" + }, + { + "url": "https://supabase.com/docs/guides/telemetry/metrics/vendor-agnostic", + "title": "Vendor-agnostic Metrics API setup" + }, + { + "url": "https://supabase.com/docs/guides/telemetry/metrics/grafana-cloud", + "title": "Metrics API with Grafana Cloud" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/how-to-view-database-metrics-uqf2z_", + "title": "How to View Database Metrics" + }, + { + "url": "https://supabase.com/docs/guides/platform/read-replicas", + "title": "Read Replicas" + } + ], + "resultChars": 32499 + } + ] + }, + "prompt": "Can you wire my Supabase project metrics into our existing observability stack and document\nwhat we need to do to make the config live?", + "promptSourcePath": "evals/deploy-database-001-prometheus-metrics/PROMPT.md", + "attempts": 1, + "sourcePath": "opencode-kimi-k3-no-skills/deploy-database-001-prometheus-metrics.json" + }, + { + "experiment": "opencode-kimi-k3-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "opencode", + "modelProvider": "moonshotai", + "modelId": "moonshotai/kimi-k3" + }, + "eval": "deploy-functions-001-edge-function-secrets", + "stage": "deploy", + "product": [ + "edge-functions" + ], + "topic": [ + "security" + ], + "suite": "benchmark", + "interface": "cli", + "passed": true, + "checks": [ + { + "name": "WEATHER_API_KEY is set as a Function secret on the project", + "passed": true + }, + { + "name": "the weather function is deployed to the project", + "passed": true, + "notes": "status ACTIVE" + }, + { + "name": "the weather function reads WEATHER_API_KEY from the environment", + "passed": true, + "judgeNotes": "The function reads WEATHER_API_KEY from the runtime environment using Deno.env.get(\"WEATHER_API_KEY\")." + }, + { + "name": "WEATHER_API_KEY value is not committed to the repo", + "passed": true + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "{ searchDocs(query: \"edge functions environment variables secrets management deploy\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/functions/secrets", + "title": "Environment Variables" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", + "title": "Self-Hosted Functions" + }, + { + "url": "https://supabase.com/docs/guides/functions/deploy", + "title": "Deploy to Production" + }, + { + "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", + "title": "Automatic embeddings" + } + ], + "resultChars": 65241 + } + ] + }, + "prompt": "Our weather widget currently calls WeatherAPI straight from the browser, which\nleaks our API key. I want to move that behind a Supabase Edge Function called\n`weather` that holds the key server-side and proxies the request.\n\nThe function should read the key from an environment variable named\n`WEATHER_API_KEY`. Our key already lives in a local `.env` file at the project\nroot.\n\nDeploy the function to our project so it's live, and make sure the deployed\nfunction can actually read the key at runtime.", + "promptSourcePath": "evals/deploy-functions-001-edge-function-secrets/PROMPT.md", + "attempts": 1, + "sourcePath": "opencode-kimi-k3-no-skills/deploy-functions-001-edge-function-secrets.json" + }, + { + "experiment": "opencode-kimi-k3-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "opencode", + "modelProvider": "moonshotai", + "modelId": "moonshotai/kimi-k3" + }, + "eval": "deploy-self-hosting-001-docker-compose", + "stage": "deploy", + "product": [ + "database", + "auth", + "storage" + ], + "topic": [ + "self-hosting" + ], + "suite": "benchmark", + "interface": "cli", + "passed": true, + "checks": [ + { + "name": "cloned the self-host stack (docker-compose.yml + volumes/db)", + "passed": true + }, + { + "name": "didn't conflate with the CLI (no supabase/config.toml in the stack)", + "passed": true + }, + { + "name": "secrets rotated off the shipped defaults", + "passed": true + }, + { + "name": "ANON_KEY and SERVICE_ROLE_KEY are HS256 JWTs signed by JWT_SECRET", + "passed": true + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "{ searchDocs(query: \"self-hosting with docker deploy\") { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/self-hosting/docker", + "title": "Self-Hosting with Docker" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform", + "title": "Restore a Platform Project to Self-Hosted" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", + "title": "Self-Hosted Functions" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting", + "title": "Self-Hosting" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17", + "title": "Upgrade to Postgres 17" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-proxy-https", + "title": "Configure Reverse Proxy and HTTPS" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", + "title": "Envoy API Gateway" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-phone-mfa", + "title": "Configure Phone Login & MFA" + }, + { + "url": "https://supabase.com/docs/guides/local-development/cli-workflows", + "title": "Local development workflow" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3", + "title": "Configure S3 Storage" + }, + { + "url": "https://supabase.com/docs/guides/telemetry/metrics/grafana-self-hosted", + "title": "Metrics API with Prometheus & Grafana (self-hosted)" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", + "title": "New API Keys and Asymmetric Authentication" + }, + { + "url": "https://supabase.com/docs/guides/local-development", + "title": "Local Development & CLI" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth", + "title": "Configure Social Login (OAuth) Providers" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-saml-sso", + "title": "Configure SAML SSO" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/grafana-not-displaying-data-sXJrMj", + "title": "Grafana not displaying data" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/custom-email-templates", + "title": "Custom Email Templates" + }, + { + "url": "https://supabase.com/docs/guides/ai-tools/byo-mcp", + "title": "Deploy MCP servers" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/architecture", + "title": "Architecture" + }, + { + "url": "https://supabase.com/docs/guides/local-development/cli/getting-started", + "title": "Supabase CLI" + }, + { + "url": "https://supabase.com/docs/guides/functions/quickstart", + "title": "Getting Started with Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/database/webhooks", + "title": "Database Webhooks" + }, + { + "url": "https://supabase.com/docs/guides/functions/examples/amazon-bedrock-image-generator", + "title": "Generate Images with Amazon Bedrock" + }, + { + "url": "https://supabase.com/docs/guides/deployment/branching/integrations", + "title": "Integrations" + }, + { + "url": "https://supabase.com/docs/guides/storage/serving/image-transformations", + "title": "Storage Image Transformations" + }, + { + "url": "https://supabase.com/docs/guides/functions/deploy", + "title": "Deploy to Production" + }, + { + "url": "https://supabase.com/docs/guides/functions/examples/github-actions", + "title": "GitHub Actions" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/enable-mcp", + "title": "Enabling MCP Server Access" + }, + { + "url": "https://supabase.com/docs/guides/ai/examples/openai", + "title": "Generating OpenAI GPT3 completions" + }, + { + "url": "https://supabase.com/docs/guides/deployment", + "title": "Deployment & Branching" + } + ], + "resultChars": 320077 + } + ] + }, + "prompt": "I'm moving off the hosted Supabase and running the whole thing myself on a VPS I\njust spun up. Can you get a Docker setup ready for me to copy onto the box?\n\nI don't need it running here, I'll do the actual bring-up once I'm on the\nserver. I just want everything in place and the secrets set up. Put it in a `supabase-docker/`\nfolder at the repo root so I can scp the whole thing across in one go.", + "promptSourcePath": "evals/deploy-self-hosting-001-docker-compose/PROMPT.md", + "attempts": 1, + "sourcePath": "opencode-kimi-k3-no-skills/deploy-self-hosting-001-docker-compose.json" + }, + { + "experiment": "opencode-kimi-k3-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "opencode", + "modelProvider": "moonshotai", + "modelId": "moonshotai/kimi-k3" + }, + "eval": "investigate-auth-001-deleted-user-access", + "stage": "investigate", + "product": [ + "auth" + ], + "topic": [ + "security", + "sdk" + ], + "suite": "benchmark", + "interface": "mcp", + "passed": true, + "checks": [ + { + "name": "victim session active before delete-account", + "passed": true + }, + { + "name": "delete_account flow ran for the victim", + "passed": true + }, + { + "name": "delete-account revokes the user's sessions", + "passed": true, + "notes": "sessions left: 0" + }, + { + "name": "deleted user's refresh token is rejected", + "passed": true + }, + { + "name": "deleted user cannot sign back in", + "passed": true + }, + { + "name": "other users keep their sessions and access", + "passed": true + }, + { + "name": "diagnosed and explained session revocation", + "passed": true, + "judgeNotes": "Meets the rubric: correctly diagnoses soft-delete without auth/session revocation, implements auth user deletion with refresh/session revocation, explains the remaining stateless JWT expiry window consistently, and correctly distinguishes publishable frontend keys from secret server-only RLS-bypassing keys." + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"delete user account RPC security definer function auth.users\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/reference/javascript/auth-admin-deleteuser" + }, + { + "url": "https://supabase.com/docs/reference/swift/auth-admin-deleteuser", + "title": "deleteUser()" + }, + { + "url": "https://supabase.com/docs/guides/auth/managing-user-data", + "title": "User Management" + }, + { + "url": "https://supabase.com/docs/reference/dart/auth-admin-deleteuser" + }, + { + "url": "https://supabase.com/docs/guides/database/database-advisors?queryGroups=lint&lint=0029_authenticated_security_definer_function_executable", + "title": "Database Advisor: Lint 0029_authenticated_security_definer_function_executable" + } + ], + "resultChars": 15474 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"publishable secret API keys replacing anon service_role\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" + }, + { + "url": "https://supabase.com/docs/guides/auth/signing-keys", + "title": "JWT Signing Keys" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/api-keys", + "title": "Understanding API keys" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", + "title": "New API Keys and Asymmetric Authentication" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", + "title": "Envoy API Gateway" + } + ], + "resultChars": 98570 + } + ] + }, + "prompt": "Last week support removed a user through our app's delete-account flow — the\napp calls the `delete_account` function over RPC as the signed-in user. This\nmorning that same person was back: still signed in, reading and saving their\ndata like nothing happened.\n\nFigure out why the account still works, fix the flow so a deleted account\nloses access, and tell me whether there is any window where they could still\nget in after the fix.\n\nOne more thing while you're at it: we're migrating off the legacy\nanon/service_role API keys to the new publishable and secret keys, and the\nteam isn't sure which one belongs in the frontend and what each means for\nRLS. Give us a quick rundown.", + "promptSourcePath": "evals/investigate-auth-001-deleted-user-access/PROMPT.md", + "attempts": 1, + "sourcePath": "opencode-kimi-k3-no-skills/investigate-auth-001-deleted-user-access.json" + }, + { + "experiment": "opencode-kimi-k3-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "opencode", + "modelProvider": "moonshotai", + "modelId": "moonshotai/kimi-k3" + }, + "eval": "investigate-realtime-001-subscribed-no-events", + "stage": "investigate", + "product": [ + "realtime", + "database" + ], + "topic": [ + "sdk" + ], + "suite": "benchmark", + "interface": "mcp", + "passed": true, + "checks": [ + { + "name": "orders table added to supabase_realtime publication", + "passed": true + }, + { + "name": "courier_locations still in supabase_realtime publication", + "passed": true + }, + { + "name": "publication still publishes INSERT events", + "passed": true + }, + { + "name": "RLS still enabled on orders", + "passed": true + }, + { + "name": "staff can still read orders through RLS", + "passed": true, + "notes": "authenticated sees 2 of 2 orders" + }, + { + "name": "diagnosed missing publication membership", + "passed": true, + "judgeNotes": "Identifies missing orders table in supabase_realtime publication as root cause, applies ALTER PUBLICATION ADD TABLE public.orders, preserves courier_locations/RLS/policies, and does not blame or change unrelated areas." + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] + }, + "prompt": "Our dispatch dashboard shows incoming orders as they happen. The courier\nlocation feed on the same page updates live without problems, but new orders\nonly show up after a page refresh.\n\nThe dashboard uses supabase-js to subscribe to INSERT events on the `orders`\ntable through postgres_changes, the same way it subscribes to courier\nlocations. The channel's status callback logs SUBSCRIBED and there are no\nerrors in the browser console.\n\nFigure out why no order events ever arrive and fix it.", + "promptSourcePath": "evals/investigate-realtime-001-subscribed-no-events/PROMPT.md", + "attempts": 1, + "sourcePath": "opencode-kimi-k3-no-skills/investigate-realtime-001-subscribed-no-events.json" + }, + { + "experiment": "opencode-kimi-k3-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "opencode", + "modelProvider": "moonshotai", + "modelId": "moonshotai/kimi-k3" + }, + "eval": "investigate-reliability-003-edge-function-5xx-correlation", + "stage": "investigate", + "product": [ + "edge-functions" + ], + "topic": [ + "observability" + ], + "suite": "benchmark", + "interface": "mcp", + "passed": false, + "checks": [ + { + "name": "identified image-transform and the recurring 503 pattern", + "passed": true, + "judgeNotes": "Identified image-transform as the affected function and described the recurring pattern of 8 HTTP 503 gateway failures across the morning of 2026-04-28 from 07:00Z to 12:00Z, while distinguishing old billing-webhook 503s as unrelated." + }, + { + "name": "attributed recurring 503s to gateway/platform layer, not function code", + "passed": false, + "judgeNotes": "The response notes gateway-only 503s with no function execution logs, but ultimately attributes the primary cause to the image-transform function's unpinned npm dependency/cold-start behavior and recommends pinning/bundling/redeploying the function, which the rubric explicitly treats as failing." + }, + { + "name": "recommended a concrete next step", + "passed": true, + "judgeNotes": "Recommended concrete next steps including pinning/bundling dependency, redeploying, adding retries, and monitoring specific gateway vs function logs." + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] + }, + "prompt": "Users have been reporting that image uploads are intermittently failing with server errors throughout the morning of `2026-04-28`.\n\nCan you investigate the project logs and tell me what is going on and what we\nshould do next?", + "promptSourcePath": "evals/investigate-reliability-003-edge-function-5xx-correlation/PROMPT.md", + "attempts": 2, + "sourcePath": "opencode-kimi-k3-no-skills/investigate-reliability-003-edge-function-5xx-correlation.json" + }, + { + "experiment": "opencode-kimi-k3-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "opencode", + "modelProvider": "moonshotai", + "modelId": "moonshotai/kimi-k3" + }, + "eval": "resolve-dataapi-001-empty-results", + "stage": "resolve", + "product": [ + "data-api", + "database", + "auth" + ], + "topic": [ + "rls", + "sdk" + ], + "suite": "benchmark", + "interface": "mcp", + "passed": true, + "checks": [ + { + "name": "RLS still enabled on bookmarks", + "passed": true + }, + { + "name": "user A reads own bookmarks", + "passed": true + }, + { + "name": "user B cannot read user A bookmarks", + "passed": true + }, + { + "name": "anon reads no bookmarks", + "passed": true + }, + { + "name": "user A can save a new bookmark", + "passed": true + }, + { + "name": "user B cannot insert a bookmark as user A", + "passed": true + }, + { + "name": "diagnosed RLS and added owner-scoped policies", + "passed": true, + "judgeNotes": "Diagnosed RLS deny-all due to no policies, kept RLS enabled, and created authenticated SELECT and INSERT policies scoped to user_id = auth.uid() with WITH CHECK for inserts." + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] + }, + "prompt": "Our app lets signed-in users save bookmarks and view them on their dashboard. Bookmarks are stored in the `bookmarks` table and are private — a user must only ever see their own. \nUsers also need to be able to save new bookmarks from the app.\n\nI can see the rows when I query the table directly, but the dashboard shows an empty list for every user.\n\nFind out why the Data API returns nothing and fix it.", + "promptSourcePath": "evals/resolve-dataapi-001-empty-results/PROMPT.md", + "attempts": 1, + "sourcePath": "opencode-kimi-k3-no-skills/resolve-dataapi-001-empty-results.json" + }, + { + "experiment": "opencode-kimi-k3-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "opencode", + "modelProvider": "moonshotai", + "modelId": "moonshotai/kimi-k3" + }, + "eval": "resolve-database-001-migration-history-mismatch", + "stage": "resolve", + "product": [ + "database" + ], + "topic": [ + "migrations" + ], + "suite": "benchmark", + "interface": "cli", + "passed": false, + "checks": [ + { + "name": "the avatar_url column is applied on the hosted profiles table", + "passed": true + }, + { + "name": "migration 20240220000000 is recorded in the remote history", + "passed": true + }, + { + "name": "remote migration history matches local migration files", + "passed": true + }, + { + "name": "local migrations are a valid reconciled sequence", + "passed": true + }, + { + "name": "production profile data is intact (not reset)", + "passed": true + }, + { + "name": "the avatar migration and history reconciliation were done via the Supabase CLI", + "passed": false, + "judgeNotes": "FAIL: No `supabase db push` output shows `Applying migration ...` or `Finished supabase db push`, so there is no evidence the pending avatar_url migration was actually applied through the CLI. The agent added `supabase/migrations/20240115000000_add_profile_bio.sql` (#13), which could reconcile the orphan bio migration locally, and ran `supabase db push` (#15), but the recorded push output does not show a successful application/reconciliation. No prohibited direct-SQL mutation or prepared-statement workaround was seen; psql usage was read-only inspection." + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] + }, + "prompt": "I'm trying to ship a migration to our hosted project and it's not working. Can you figure out what's wrong and get it deployed?", + "promptSourcePath": "evals/resolve-database-001-migration-history-mismatch/PROMPT.md", + "attempts": 2, + "sourcePath": "opencode-kimi-k3-no-skills/resolve-database-001-migration-history-mismatch.json" + }, + { + "experiment": "opencode-kimi-k3-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "opencode", + "modelProvider": "moonshotai", + "modelId": "moonshotai/kimi-k3" + }, + "eval": "resolve-performance-001-slow-query-cpu-spike", + "stage": "resolve", + "product": [ + "database" + ], + "topic": [ + "observability", + "sql" + ], + "suite": "benchmark", + "interface": "mcp", + "passed": true, + "checks": [ + { + "name": "inspected pg_stat_statements for query performance", + "passed": true + }, + { + "name": "ran EXPLAIN on the expensive query", + "passed": true + }, + { + "name": "created index covering user_id and created_at", + "passed": true + }, + { + "name": "query plan uses an index and avoids sequential scan", + "passed": true, + "notes": "Limit (cost=55.55..55.61 rows=25 width=88)\n -> Sort (cost=55.55..55.61 rows=25 width=88)\n Sort Key: created_at DESC\n -> Bitmap Heap Scan on events (cost=4.48..54.97 rows=25 width=88)\n Recheck Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)\n -> Bitmap Index Scan on idx_events_user_created_at (cost=0.00..4.47 rows=25 width=0)\n Index Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)" + }, + { + "name": "inserts still work", + "passed": true + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] + }, + "prompt": "My database CPU keeps spiking and the app gets slow when loading recent events for a user. Can you figure out what query is causing it and make the database change needed to fix it?\n\nEnd your turn with a short summary of what you changed and why.", + "promptSourcePath": "evals/resolve-performance-001-slow-query-cpu-spike/PROMPT.md", + "attempts": 1, + "sourcePath": "opencode-kimi-k3-no-skills/resolve-performance-001-slow-query-cpu-spike.json" + }, + { + "experiment": "opencode-kimi-k3-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "opencode", + "modelProvider": "moonshotai", + "modelId": "moonshotai/kimi-k3" + }, + "eval": "resolve-security-002-rls-cross-tenant-leak", + "stage": "resolve", + "product": [ + "database", + "auth" + ], + "topic": [ + "rls", + "security" + ], + "suite": "benchmark", + "interface": "mcp", + "passed": true, + "checks": [ + { + "name": "RLS enabled on notes", + "passed": true + }, + { + "name": "tenant A sees only org A notes", + "passed": true + }, + { + "name": "tenant B cannot read org A notes", + "passed": true + }, + { + "name": "tenant A author can update own note", + "passed": true + }, + { + "name": "tenant B cannot update org A note", + "passed": true + }, + { + "name": "tenant B author can delete own note", + "passed": true + }, + { + "name": "tenant B cannot delete org A note", + "passed": true + }, + { + "name": "tenant A can insert note in own org", + "passed": true + }, + { + "name": "tenant B cannot insert into org A", + "passed": true + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] + }, + "prompt": "A customer reported that notes showed up in the wrong workspace.\n\nCan you investigate what is going on and fix it?", + "promptSourcePath": "evals/resolve-security-002-rls-cross-tenant-leak/PROMPT.md", + "attempts": 1, + "sourcePath": "opencode-kimi-k3-no-skills/resolve-security-002-rls-cross-tenant-leak.json" } ] diff --git a/apps/web/src/lib/format.ts b/apps/web/src/lib/format.ts index f5d716ec..2ca9fa18 100644 --- a/apps/web/src/lib/format.ts +++ b/apps/web/src/lib/format.ts @@ -6,6 +6,7 @@ export const AGENT_LABELS = { "ai-sdk": "AI SDK", "claude-code": "Claude Code", codex: "Codex", + opencode: "OpenCode", } satisfies Record const PRODUCT_LABELS: Record = { @@ -55,12 +56,20 @@ function formatOpenAiModel(modelId: string) { .join(" ") } +function formatMoonshotaiModel(modelId: string) { + return modelId.split("-").map(capitalize).join(" ") +} + function formatModel(display: ExperimentDisplay) { + // opencode ids are AI Gateway `vendor/model` slugs; format just the model part. + const modelId = display.modelId.replace(/^[a-z-]+\//, "") switch (display.modelProvider) { case "anthropic": - return formatAnthropicModel(display.modelId) + return formatAnthropicModel(modelId) case "openai": - return formatOpenAiModel(display.modelId) + return formatOpenAiModel(modelId) + case "moonshotai": + return formatMoonshotaiModel(modelId) } } diff --git a/experiments/opencode-kimi-k3-no-skills.ts b/experiments/opencode-kimi-k3-no-skills.ts new file mode 100644 index 00000000..0a6d2edd --- /dev/null +++ b/experiments/opencode-kimi-k3-no-skills.ts @@ -0,0 +1,20 @@ +import { + defineExperiment, + opencodeAgent, + platformLiteRuntime, + supabaseMcpServer, +} from '@supabase-evals/core'; +import { localStackRuntime } from '@supabase-evals/sandbox'; + +// Same as opencode-kimi-k3 but with no skills, to measure skills' impact. +export default defineExperiment({ + suite: ['no-skills'], + agent: opencodeAgent({ + model: 'moonshotai/kimi-k3', + }), + runtime: platformLiteRuntime({ + mcpServers: [supabaseMcpServer()], + }), + localStack: localStackRuntime(), + skills: [], +}); diff --git a/experiments/opencode-kimi-k3.ts b/experiments/opencode-kimi-k3.ts new file mode 100644 index 00000000..be08d45f --- /dev/null +++ b/experiments/opencode-kimi-k3.ts @@ -0,0 +1,19 @@ +import { + defineExperiment, + opencodeAgent, + platformLiteRuntime, + supabaseMcpServer, +} from '@supabase-evals/core'; +import { localStackRuntime } from '@supabase-evals/sandbox'; + +export default defineExperiment({ + suite: ['benchmark'], + agent: opencodeAgent({ + model: 'moonshotai/kimi-k3', + }), + runtime: platformLiteRuntime({ + mcpServers: [supabaseMcpServer()], + }), + localStack: localStackRuntime(), + skills: ['supabase', 'supabase-postgres-best-practices'], +}); diff --git a/packages/core/package.json b/packages/core/package.json index 4cce8026..51ed3f4d 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -14,11 +14,11 @@ "test": "vitest run" }, "devDependencies": { + "@opencode-ai/sdk": "catalog:", "vitest": "catalog:" }, "dependencies": { "@anthropic-ai/sdk": "catalog:", - "openai": "catalog:", "@ai-sdk/mcp": "catalog:", "@ai-sdk/openai": "catalog:", "@supabase-evals/platform-lite": "workspace:*", @@ -26,6 +26,7 @@ "ai": "catalog:", "executor": "1.4.29", "gray-matter": "^4.0.3", + "openai": "catalog:", "typescript": "catalog:", "zod": "catalog:" } diff --git a/packages/core/src/agents/engine.ts b/packages/core/src/agents/engine.ts index 36dffe39..de61c1fa 100644 --- a/packages/core/src/agents/engine.ts +++ b/packages/core/src/agents/engine.ts @@ -29,6 +29,7 @@ import { SYSTEM_PROMPT_PATH, USER_PROMPT_PATH, processStopReason, + requireEnv, rewriteLoopback, writeSandboxFile, } from './shared.js'; @@ -39,6 +40,10 @@ function modelProviderForAgent(id: AgentRunner['id']): ModelProvider { return 'anthropic'; case 'codex': return 'openai'; + case 'opencode': + throw new Error( + 'opencode is multi-provider; its runner sets `modelProvider` from the model id' + ); case 'ai-sdk': throw new Error('ai-sdk agents are not created through createCliAgent'); } @@ -60,7 +65,9 @@ export function createCliAgent( modelId: options.model, metadata: { agent: runner.id, - modelProvider: modelProviderForAgent(runner.id), + // A multi-provider runner (e.g. opencode) sets its own `modelProvider` + // from the model id; single-provider agents derive it from the agent id. + modelProvider: runner.modelProvider ?? modelProviderForAgent(runner.id), modelId: options.model, ...(options.reasoningEffort ? { reasoningEffort: options.reasoningEffort } @@ -102,6 +109,19 @@ export function createCliAgent( const { events } = raw ? parser.parseTranscript(raw) : { events: [] }; const adapted = adaptTranscript(events); + // Surface run failures that would otherwise be invisible in results + // (visible under --debug): the CLI's own error events, or a run that + // died before streaming any events at all. + const errorEvents = events.filter((e) => e.type === 'error'); + for (const e of errorEvents) { + console.error(`[${runner.displayName}] ${e.content}`); + } + if (events.length === 0) { + console.error( + `[${runner.displayName}] produced no transcript events (exit ${command.exitCode}).\nstdout:\n${command.stdout}\nstderr:\n${command.stderr}` + ); + } + return { // The final report is the transcript's closing assistant message — the // CLI's stdout is JSONL, not prose. @@ -117,11 +137,8 @@ export function createCliAgent( } function requireApiKey(runner: AgentRunner): string { - const apiKey = process.env[runner.apiKeyEnvVar]; - if (!apiKey) { - throw new Error( - `Missing ${runner.displayName} credentials. Set ${runner.apiKeyEnvVar} before running ${runner.id} evals.` - ); - } - return apiKey; + return requireEnv( + runner.apiKeyEnvVar, + `Set it to run ${runner.displayName} (${runner.id}) evals.` + ); } diff --git a/packages/core/src/agents/opencode/index.ts b/packages/core/src/agents/opencode/index.ts new file mode 100644 index 00000000..199863e9 --- /dev/null +++ b/packages/core/src/agents/opencode/index.ts @@ -0,0 +1,43 @@ +/** + * OpenCode agent. Owns everything opencode-specific: it wires its own runner + + * parser into the public `opencodeAgent` factory (via the generic + * `createCliAgent` engine) and exports the registry definition the harness uses + * to parse opencode transcripts. + */ + +import type { AgentHarness } from '../../index.js'; +import { createCliAgent } from '../engine.js'; +import type { AgentDefinition } from '../types.js'; +import { + DEFAULT_OPENCODE_MODEL, + createOpencodeRunner, + type OpenCodeModel, +} from './runner.js'; +import { opencodeParser } from './parser.js'; + +/** + * OpenCode as an `AgentHarness`. Every run routes through the Vercel AI + * Gateway (opencode's native `vercel` provider), so the model id is a gateway + * `vendor/model` slug and the only credential is `AI_GATEWAY_API_KEY` — see + * `./runner.ts`. + */ +export function opencodeAgent( + options: { + /** Gateway model slug, `vendor/model` (e.g. `moonshotai/kimi-k3`). */ + model?: OpenCodeModel; + /** Override the pinned CLI version. */ + cliVersion?: string; + } = {} +): AgentHarness { + const model = options.model ?? DEFAULT_OPENCODE_MODEL; + return createCliAgent(createOpencodeRunner(model), opencodeParser, { + model, + cliVersion: options.cliVersion, + }); +} + +/** Runner + parser pairing for the agent registry (id comes from `runner.id`). */ +export const opencodeDefinition: AgentDefinition = { + runner: createOpencodeRunner(DEFAULT_OPENCODE_MODEL), + parser: opencodeParser, +}; diff --git a/packages/core/src/agents/opencode/parser.test.ts b/packages/core/src/agents/opencode/parser.test.ts new file mode 100644 index 00000000..cb962d2d --- /dev/null +++ b/packages/core/src/agents/opencode/parser.test.ts @@ -0,0 +1,229 @@ +import { describe, expect, it } from 'vitest'; +import { opencodeParser } from './parser.js'; +import { adaptTranscript } from '../../parsers/adapt.js'; + +/** A representative `opencode run --format json` stream (shapes from CLI 1.18.5). */ +const SESSION = [ + JSON.stringify({ type: 'step_start', part: { type: 'step-start' } }), + JSON.stringify({ + type: 'reasoning', + timestamp: 1782295624200, + part: { type: 'reasoning', text: 'I should list the files.' }, + }), + JSON.stringify({ + type: 'text', + timestamp: 1782295624232, + part: { type: 'text', text: 'Listing files.' }, + }), + JSON.stringify({ + type: 'tool_use', + timestamp: 1782295624290, + part: { + type: 'tool', + tool: 'bash', + callID: 'tool_1', + state: { + status: 'completed', + input: { command: 'ls -la', description: 'List files' }, + output: 'file1\nfile2', + metadata: { exit: 0 }, + }, + }, + }), + JSON.stringify({ + type: 'tool_use', + timestamp: 1782295624300, + part: { + type: 'tool', + tool: 'write', + callID: 'tool_2', + state: { + status: 'completed', + input: { filePath: '/work/note.txt', content: 'hi' }, + output: 'written', + }, + }, + }), + JSON.stringify({ + type: 'text', + timestamp: 1782295624400, + part: { type: 'text', text: 'Done.' }, + }), + JSON.stringify({ + type: 'step_finish', + part: { + type: 'step-finish', + reason: 'stop', + tokens: { input: 3, output: 6 }, + }, + }), +].join('\n'); + +describe('opencodeParser', () => { + it("normalizes MCP tools (opencode's `_` names) to tool_use", () => { + const record = JSON.stringify({ + type: 'tool_use', + part: { + type: 'tool', + tool: 'supabase-mcp_list_tables', + callID: 'tool_mcp', + state: { status: 'completed', input: { schemas: ['public'] } }, + }, + }); + const { events } = opencodeParser.parseTranscript(record); + const call = events.find((e) => e.type === 'tool_call'); + expect(call?.tool?.name).toBe('tool_use'); + expect(call?.tool?.originalName).toBe('supabase-mcp_list_tables'); + }); + + it('maps bash + write to canonical tool calls, paired with results by callID', () => { + const { events, errors } = opencodeParser.parseTranscript(SESSION); + expect(errors).toEqual([]); + + const calls = events.filter((e) => e.type === 'tool_call'); + expect(calls.map((e) => e.tool?.name)).toEqual(['shell', 'file_write']); + expect(calls.map((e) => e.tool?.originalName)).toEqual(['bash', 'write']); + expect(calls.map((e) => e.tool?.id)).toEqual(['tool_1', 'tool_2']); + // Normalized views on the event; raw args untouched. + expect(calls[0].tool?.command).toBe('ls -la'); + expect(calls[1].tool?.path).toBe('/work/note.txt'); + + const results = events.filter((e) => e.type === 'tool_result'); + expect(results.map((e) => e.tool?.id)).toEqual(['tool_1', 'tool_2']); + expect(results.every((e) => e.tool?.success === true)).toBe(true); + }); + + it('surfaces reasoning + the assistant report via the adapter', () => { + const events = opencodeParser.parseTranscript(SESSION).events; + expect( + events.some( + (e) => e.type === 'thinking' && e.content === 'I should list the files.' + ) + ).toBe(true); + + const adapted = adaptTranscript(events); + expect(adapted.agentReport).toBe('Done.'); + expect(adapted.steps).toBe(2); // two assistant text turns + expect(adapted.toolCalls).toEqual([ + { + endpoint: 'bash', + body: { command: 'ls -la', description: 'List files' }, + name: 'shell', + command: 'ls -la', + result: 'file1\nfile2', + error: undefined, + ts: 1782295624290, // epoch ms preserved through toISO -> parseTs + }, + { + endpoint: 'write', + body: { filePath: '/work/note.txt', content: 'hi' }, + name: 'file_write', + path: '/work/note.txt', + result: 'written', + error: undefined, + ts: 1782295624300, + }, + ]); + }); + + it('surfaces skill loads from the skill tool and from SKILL.md reads', () => { + const stream = [ + JSON.stringify({ + type: 'tool_use', + part: { + type: 'tool', + tool: 'skill', + callID: 's1', + state: { + status: 'completed', + input: { name: 'supabase' }, + output: '# Supabase', + }, + }, + }), + JSON.stringify({ + type: 'tool_use', + part: { + type: 'tool', + tool: 'read', + callID: 's2', + state: { + status: 'completed', + input: { + filePath: + '.claude/skills/supabase-postgres-best-practices/SKILL.md', + }, + output: '# Postgres', + }, + }, + }), + ].join('\n'); + const adapted = adaptTranscript( + opencodeParser.parseTranscript(stream).events + ); + expect(adapted.toolCalls.map((call) => call.loadedSkill)).toEqual([ + 'supabase', + 'supabase-postgres-best-practices', + ]); + }); + + it('marks a non-zero shell exit as failed (error surfaced via adapter)', () => { + const stream = JSON.stringify({ + type: 'tool_use', + part: { + type: 'tool', + tool: 'bash', + callID: 'c1', + state: { + status: 'completed', + input: { command: 'false' }, + output: 'nope', + metadata: { exit: 1 }, + }, + }, + }); + const events = opencodeParser.parseTranscript(stream).events; + expect(events.find((e) => e.type === 'tool_result')?.tool?.success).toBe( + false + ); + const adapted = adaptTranscript(events); + expect(adapted.toolCalls[0].error).toBe('nope'); + expect(adapted.toolCalls[0].result).toBeUndefined(); + }); + + it('emits an error event and never throws on malformed lines', () => { + const record = { type: 'error', error: { unrecognized: true } }; + const { events, errors } = opencodeParser.parseTranscript( + 'not json\n' + JSON.stringify(record) + ); + expect(events).toEqual([ + { + timestamp: undefined, + type: 'error', + content: JSON.stringify(record), + raw: record, + }, + ]); + expect(errors.length).toBe(1); + }); + + it("reads the message out of opencode's real error envelope", () => { + const { events } = opencodeParser.parseTranscript( + JSON.stringify({ + type: 'error', + error: { name: 'UnknownError', data: { message: 'boom', ref: 'x' } }, + }) + ); + expect(events[0].content).toBe('UnknownError: boom'); + }); + + it('falls back to the error name when data carries no message', () => { + const { events } = opencodeParser.parseTranscript( + JSON.stringify({ + type: 'error', + error: { name: 'MessageOutputLengthError', data: {} }, + }) + ); + expect(events[0].content).toBe('MessageOutputLengthError'); + }); +}); diff --git a/packages/core/src/agents/opencode/parser.ts b/packages/core/src/agents/opencode/parser.ts new file mode 100644 index 00000000..7223a71a --- /dev/null +++ b/packages/core/src/agents/opencode/parser.ts @@ -0,0 +1,239 @@ +/** + * OpenCode transcript parser — for `opencode run --format json` (CLI ≥ 1.15). + * + * The stream is newline-delimited event records, each `{ type, timestamp, + * sessionID, part }`: + * {"type":"step_start","part":{"type":"step-start"}} + * {"type":"text","part":{"type":"text","text":"…"}} + * {"type":"tool_use","part":{"type":"tool","tool":"bash","callID":"…", + * "state":{"status":"completed","input":{…},"output":"…", + * "metadata":{"exit":0}}}} + * {"type":"reasoning","part":{"type":"reasoning","text":"…"}} + * {"type":"error","error":{"name":"UnknownError","data":{"message":"…"}}} + * {"type":"step_finish","part":{"type":"step-finish","reason":"stop","tokens":{…}}} + * + * A `tool_use` record is self-contained (input + output + status), so it yields + * a paired tool_call + tool_result correlated by `part.callID`. Step records + * carry token/finish info and produce no transcript event (the runner reads the + * terminal `step_finish` reason for the stop reason). + * + * Adapted from `@supabase/agent-evals` (packages/agent-eval/src/parsers). + */ + +import { isRecord, parseJsonlRecords } from '../../json.js'; +import type { + ParsedTranscript, + TranscriptEvent, +} from '../../transcript/types.js'; +import type { AgentTranscriptParser } from '../../parsers/types.js'; +import { + normalizeToolName, + type AgentToolMap, +} from '../../parsers/shared/normalize.js'; +import { + extractArgs, + extractLoadedSkillFromText, + type ArgFieldMap, + type ExtractedArgs, +} from '../../parsers/shared/extract.js'; + +/** + * opencode's tool names → canonical names. opencode uses lowercase built-in tool + * names. Owned here, not in shared. MCP tools arrive under their server name and + * fall through to `tool_use`. + */ +const OPENCODE_TOOLS: AgentToolMap = { + caseInsensitive: true, + tools: { + read: 'file_read', + write: 'file_write', + edit: 'file_edit', + multiedit: 'file_edit', + patch: 'file_edit', + apply_patch: 'file_edit', + bash: 'shell', + shell: 'shell', + webfetch: 'web_fetch', + websearch: 'web_search', + codesearch: 'grep', + glob: 'glob', + grep: 'grep', + list: 'list_dir', + ls: 'list_dir', + task: 'agent_task', + todowrite: 'agent_task', + skill: 'tool_use', + }, +}; + +/** + * opencode tool args → normalized fields. `bash` carries the command in + * `command`; file tools the path in `filePath` (or `path`); `webfetch` the URL + * in `url`. The shared extractor reads whichever keys this map names. + */ +const OPENCODE_ARG_FIELDS: ArgFieldMap = { + path: ['filePath', 'file_path', 'path'], + command: ['command'], + url: ['url'], +}; + +/** Epoch-ms (or pass-through ISO) → ISO string. */ +function toISO(value: unknown): string | undefined { + if (typeof value === 'number') return new Date(value).toISOString(); + if (typeof value === 'string') return value; + return undefined; +} + +function str(value: unknown): string | undefined { + return typeof value === 'string' ? value : undefined; +} + +/** Whether a completed tool call succeeded: shell keys off its exit code. */ +function toolSuccess( + canonical: string, + status: string | undefined, + metadata: Record | undefined +): boolean | undefined { + if (status === undefined) return undefined; + if (status !== 'completed') return false; + if (canonical === 'shell') { + const exit = metadata?.exit; + return typeof exit === 'number' ? exit === 0 : true; + } + return true; +} + +function partToEvents( + type: string, + part: Record, + timestamp: string | undefined, + raw: unknown +): TranscriptEvent[] { + switch (type) { + case 'text': { + const text = str(part.text); + return text + ? [ + { + timestamp, + type: 'message', + role: 'assistant', + content: text, + raw, + }, + ] + : []; + } + case 'reasoning': { + const text = str(part.text); + return text ? [{ timestamp, type: 'thinking', content: text, raw }] : []; + } + case 'tool_use': { + const originalName = str(part.tool) ?? 'unknown'; + const id = str(part.callID); + const state = isRecord(part.state) ? part.state : {}; + const args = isRecord(state.input) ? state.input : {}; + const status = str(state.status); + const metadata = isRecord(state.metadata) ? state.metadata : undefined; + // The builtin tool set is fully enumerated in OPENCODE_TOOLS, so any + // unmapped name is an MCP/custom tool (`_`, which the + // shared `mcp__` fallback doesn't recognize). + const mapped = normalizeToolName(originalName, OPENCODE_TOOLS); + const name = mapped === 'unknown' ? 'tool_use' : mapped; + const normalized: ExtractedArgs = extractArgs(args, OPENCODE_ARG_FIELDS); + + const tool: NonNullable = { + name, + originalName, + id, + args, + }; + if (normalized.path) tool.path = normalized.path; + if (normalized.command) tool.command = normalized.command; + if (normalized.url) tool.url = normalized.url; + tool.loadedSkill = loadedSkillFromOpencodeCall(tool); + + const events: TranscriptEvent[] = [ + { timestamp, type: 'tool_call', tool, raw }, + ]; + // The result is in the same record; emit it only once the call completed. + if (status && status !== 'running' && status !== 'pending') { + events.push({ + timestamp, + type: 'tool_result', + tool: { + name, + originalName, + id, + result: + state.output ?? (isRecord(state.error) ? state.error : undefined), + success: toolSuccess(name, status, metadata), + }, + raw: state, + }); + } + return events; + } + default: + return []; + } +} + +/** + * Identifies opencode skill loads. opencode's native `skill` tool carries the + * skill name in its args; skills read manually surface as `skills// + * SKILL.md` in a file path or shell command. + */ +function loadedSkillFromOpencodeCall( + tool: NonNullable +): string | undefined { + if (tool.originalName.toLowerCase() === 'skill') { + const name = tool.args?.name ?? tool.args?.skill; + if (typeof name === 'string') return name; + } + if (tool.path) return extractLoadedSkillFromText(tool.path); + if (tool.command) return extractLoadedSkillFromText(tool.command); + return undefined; +} + +function recordToEvents(data: Record): TranscriptEvent[] { + const type = str(data.type); + if (!type) return []; + const timestamp = toISO(data.timestamp); + + if (type === 'error') { + const error = isRecord(data.error) ? data.error : undefined; + const errorData = isRecord(error?.data) ? error.data : undefined; + // Error union is keyed by `name`; message (when present) is nested under + // `data.message`: https://github.com/sst/opencode/blob/v1.18.5/packages/sdk/js/src/v2/gen/types.gen.ts#L264-L298 + const name = str(error?.name); + const detail = str(errorData?.message); + const message = + name && detail + ? `${name}: ${detail}` + : (detail ?? name ?? JSON.stringify(data)); + return [{ timestamp, type: 'error', content: message, raw: data }]; + } + // step_start / step_finish carry no transcript content (tokens + finish reason + // only; the runner reads the terminal step_finish reason for the stop reason). + if (type === 'step_start' || type === 'step_finish') return []; + + const part = isRecord(data.part) ? data.part : undefined; + if (!part) return []; + return partToEvents(type, part, timestamp, data); +} + +export const opencodeParser: AgentTranscriptParser = { + parseTranscript(raw: string): ParsedTranscript { + const { records, errors } = parseJsonlRecords(raw); + const events: TranscriptEvent[] = []; + for (const record of records) { + try { + events.push(...recordToEvents(record)); + } catch (e) { + errors.push(e instanceof Error ? e.message : String(e)); + } + } + return { events, errors }; + }, +}; diff --git a/packages/core/src/agents/opencode/runner.test.ts b/packages/core/src/agents/opencode/runner.test.ts new file mode 100644 index 00000000..13a5b3af --- /dev/null +++ b/packages/core/src/agents/opencode/runner.test.ts @@ -0,0 +1,142 @@ +import { describe, expect, it } from 'vitest'; +import type { CommandResult } from '../../index.js'; +import { + buildOpencodeConfig, + createOpencodeRunner, + providerForModel, +} from './runner.js'; + +/** A run's terminal records: a mid-run `step_finish` (tool-calls) then the final one. */ +const SESSION = [ + JSON.stringify({ + type: 'step_finish', + part: { type: 'step-finish', reason: 'tool-calls' }, + }), + JSON.stringify({ type: 'text', part: { type: 'text', text: 'Done.' } }), + JSON.stringify({ + type: 'step_finish', + part: { type: 'step-finish', reason: 'stop' }, + }), +].join('\n'); + +describe('opencode runner', () => { + it("resolves the results-metadata provider from the slug's vendor prefix", () => { + expect(providerForModel('anthropic/claude-sonnet-5')).toBe('anthropic'); + expect(providerForModel('openai/gpt-5.4')).toBe('openai'); + expect(providerForModel('moonshotai/kimi-k3')).toBe('moonshotai'); + }); + + it('throws a clear error for a vendor missing from the provider enum', () => { + expect(() => providerForModel('mistral/some-model')).toThrowError( + /Unsupported model vendor in "mistral\/some-model".*anthropic, openai, moonshotai/ + ); + }); + + it('carries the provider on the runner for experiment display metadata', () => { + expect(createOpencodeRunner('openai/gpt-5.4').modelProvider).toBe('openai'); + expect(createOpencodeRunner('moonshotai/kimi-k3').modelProvider).toBe( + 'moonshotai' + ); + }); + + it('deriveStopReason reads the terminal step_finish reason', () => { + const runner = createOpencodeRunner('anthropic/claude-sonnet-5'); + const ok = { ok: true, exitCode: 0, stdout: '', stderr: '' }; + expect(runner.deriveStopReason!(SESSION, ok)).toBe('stop'); + // A non-stop terminal reason is surfaced verbatim. + const length = JSON.stringify({ + type: 'step_finish', + part: { reason: 'length' }, + }); + expect(runner.deriveStopReason!(length, ok)).toBe('length'); + // An error event wins regardless of exit code. + const errored = JSON.stringify({ + type: 'error', + error: { message: 'model overloaded' }, + }); + expect(runner.deriveStopReason!(errored, ok)).toBe('error'); + }); + + it("builds opencode's MCP config shape from harness server configs", () => { + const config = JSON.parse( + buildOpencodeConfig({ + supabase: { command: 'npx', args: ['-y', 'srv'], env: { TOKEN: 't' } }, + docs: { command: 'docs-server' }, + }) + ); + expect(config.mcp).toEqual({ + supabase: { + type: 'local', + command: ['npx', '-y', 'srv'], + enabled: true, + environment: { TOKEN: 't' }, + }, + // No env → no `environment` key. + docs: { type: 'local', command: ['docs-server'], enabled: true }, + }); + }); + + it('disables the title agent', () => { + // Title agent could otherwise call a different vendor + const config = JSON.parse(buildOpencodeConfig({})); + expect(config.agent).toEqual({ title: { disable: true } }); + }); +}); + +/** Capture the `--model` flag, run env, and written config from one exec. */ +async function captureExec( + model: string, + opts: { mcp?: boolean } = {} +): Promise<{ + runCommand: string; + runEnv: Record | undefined; + config: Record | undefined; +}> { + const ok: CommandResult = { ok: true, exitCode: 0, stdout: '', stderr: '' }; + let runCommand = ''; + let runEnv: Record | undefined; + let config: Record | undefined; + await createOpencodeRunner(model).exec({ + sandbox: { + workspace: '/w', + exec: async (cmd, options) => { + const write = /^printf %s '([^']+)'/.exec(cmd); + if (write) { + config = JSON.parse(Buffer.from(write[1], 'base64').toString('utf8')); + } else if (cmd.includes(' run ')) { + runCommand = cmd; + runEnv = options?.env; + } + return ok; + }, + readFile: async () => '', + }, + model, + apiKey: 'gw-key', + systemPromptPath: '/s', + userPromptPath: '/u', + mcpServers: opts.mcp ? { supabase: { command: 'srv' } } : {}, + timeoutSec: 1, + }); + return { runCommand, runEnv, config }; +} + +describe('opencode runner exec routing', () => { + it("routes the model through opencode's native vercel provider with the gateway key", async () => { + const { runCommand, runEnv, config } = await captureExec( + 'moonshotai/kimi-k3', + { mcp: true } + ); + // Model is addressed under the vercel provider; the gateway slug stays intact. + expect(runCommand).toContain("--model 'vercel/moonshotai/kimi-k3'"); + expect(runEnv).toEqual({ AI_GATEWAY_API_KEY: 'gw-key' }); + expect(config?.mcp).toHaveProperty('supabase'); + }); + + it('still writes the config (to disable the title agent) with no MCP servers', async () => { + const { runCommand, config } = await captureExec('moonshotai/kimi-k3'); + expect(config?.agent).toEqual({ title: { disable: true } }); + expect(config?.mcp).toEqual({}); + expect(runCommand).toContain('OPENCODE_CONFIG='); + }); +}); diff --git a/packages/core/src/agents/opencode/runner.ts b/packages/core/src/agents/opencode/runner.ts new file mode 100644 index 00000000..b7b437d9 --- /dev/null +++ b/packages/core/src/agents/opencode/runner.ts @@ -0,0 +1,216 @@ +/** + * OpenCode runner. Headless via `opencode run --format json` (the CLI + * streams newline-delimited event records to stdout; see ./parser.ts). + * + * Three things are opencode-specific: + * - All model requests route through the **Vercel AI Gateway** using + * opencode's native `vercel` provider (from its models.dev catalog): the + * run's `--model` is `vercel//` and the provider reads + * `AI_GATEWAY_API_KEY`. One key covers every vendor, no per-vendor keys. + * https://vercel.com/docs/ai-gateway/coding-agents/opencode + * - Model ids are gateway `vendor/model` slugs (e.g. `moonshotai/kimi-k3`), + * so the runner is built per-model with `modelProvider` (results metadata) + * parsed from the slug's vendor prefix. + * - `opencode run` blocks waiting on stdin even when the message is passed as + * an argument, so we redirect stdin from /dev/null. + * + * In tools mode Supabase access goes through MCP servers, declared in an + * OPENCODE_CONFIG file outside the scored workspace. + */ + +// opencode's own config schema (type-only; pinned to the installed CLI version +// via the catalog). The transcript stream is deliberately NOT typed from this +// SDK: `run --format json` emits a reduced, differently-shaped record than the +// SDK's server-API `Part`/`Event` entities (no id/sessionID/messageID; different +// discriminants), so the parser stays schema-defensive — see ./parser.ts. +import type { Config, McpLocalConfig } from '@opencode-ai/sdk'; +import type { McpServerConfig } from '../../index.js'; +import type { ModelProvider } from '../../eval-metadata.js'; +import { modelProviderSchema } from '../../eval-metadata.js'; +import { isRecord, parseJsonlRecords } from '../../json.js'; +import type { AgentRunner } from '../types.js'; +import { + SCRATCH, + npmGlobalBin, + npmInstallGlobal, + processStopReason, + shellQuote, + writeSandboxFile, +} from '../shared.js'; + +/** + * opencode model id: a Vercel AI Gateway `vendor/model` slug, where the model + * name is the original vendor's id. The catalog is public: + * GET https://ai-gateway.vercel.sh/v1/models + */ +export type OpenCodeModel = string; + +/** Model used when the caller doesn't pick one. */ +export const DEFAULT_OPENCODE_MODEL: OpenCodeModel = + 'anthropic/claude-sonnet-5'; + +/** + * Config provider id of opencode's native Vercel AI Gateway provider. opencode + * addresses a model as `provider/model`, splitting on the first `/`, so a run's + * `--model` is `vercel//` and the gateway `vendor/model` slug + * stays intact as the model id under it. + */ +const GATEWAY_PROVIDER_ID = 'vercel'; + +/** + * The vendor prefix of a gateway `vendor/model` slug, as results metadata. + * Throws for vendors missing from `modelProviderSchema` — extend that enum when + * adding a model from a new vendor. + */ +export function providerForModel(model: string): ModelProvider { + const vendor = modelProviderSchema.safeParse(model.split('/')[0]); + if (!vendor.success) { + throw new Error( + `Unsupported model vendor in "${model}". ` + + `Expected a Vercel AI Gateway vendor/model slug with one of: ${modelProviderSchema.options.join(', ')}.` + ); + } + return vendor.data; +} + +/** + * Shell path to the config, staged in scratch (outside the workspace). Used both + * as the write target and as the `OPENCODE_CONFIG` env value — the shell expands + * `$HOME` in either position. Holds the model declaration and MCP servers. + */ +const OPENCODE_CONFIG_PATH = '"$HOME/.eval/opencode.json"'; + +/** + * Build an opencode runner bound to one gateway model slug. A single run + * targets one model, so the runner resolves `modelProvider` (results metadata) + * from the slug's vendor prefix at build time. + */ +export function createOpencodeRunner( + model: OpenCodeModel +): AgentRunner { + return { + id: 'opencode', + displayName: 'OpenCode', + apiKeyEnvVar: 'AI_GATEWAY_API_KEY', + modelProvider: providerForModel(model), + cliPackage: 'opencode-ai', + // Pinned: opencode's --format json event schema evolves; bump deliberately + // and re-check the parser. See ./parser.ts. Must stay >= 1.17.0: earlier + // CLIs don't await the run event loop (opencode #31389) and intermittently + // exit 0 mid-step, ending runs with no final report. + defaultCliVersion: '1.18.5', + defaultModel: DEFAULT_OPENCODE_MODEL, + + async install(sandbox, version) { + await npmInstallGlobal( + sandbox, + `${this.cliPackage}@${version}`, + this.displayName + ); + }, + + async exec({ + sandbox, + model, + apiKey, + systemPromptPath, + userPromptPath, + mcpServers, + timeoutSec, + }) { + const opencode = npmGlobalBin('opencode'); + + // opencode has no system-prompt flag, so prepend the system prompt to the + // task; both are staged files, joined via command substitution into the + // single message argument. + const message = `"$(cat ${systemPromptPath}; printf '\\n\\n'; cat ${userPromptPath})"`; + + await sandbox.exec(`mkdir -p ${SCRATCH}`); + await writeSandboxFile( + sandbox, + OPENCODE_CONFIG_PATH, + buildOpencodeConfig(mcpServers) + ); + const configPrefix = `OPENCODE_CONFIG=${OPENCODE_CONFIG_PATH} `; + + const flags = [ + 'run', + message, + // Note opencode may 404 if the model drops from its live models.dev catalog. + // https://github.com/sst/opencode/blob/v1.18.5/packages/opencode/src/provider/provider.ts#L1805-L1817 + `--model ${shellQuote(`${GATEWAY_PROVIDER_ID}/${model}`)}`, + // Newline-delimited JSON event records on stdout. + '--format json', + // Headless runs default thinking to false. Flag enabled so opencode emits `reasoning` records. + // https://github.com/sst/opencode/blob/v1.18.5/packages/opencode/src/cli/cmd/run.ts#L275 + // https://github.com/sst/opencode/blob/v1.18.5/packages/opencode/src/cli/cmd/run.ts#L761-L762 + '--thinking', + // The sandbox is the isolation boundary, so let opencode act freely. + '--dangerously-skip-permissions', + ].join(' '); + + // `< /dev/null`: opencode run blocks on stdin otherwise, even with the + // message passed as an argument. + const command = await sandbox.exec( + `${configPrefix}${opencode} ${flags} < /dev/null`, + { + timeoutMs: timeoutSec * 1000, + // The native vercel provider reads the gateway key from this env var. + env: { [this.apiKeyEnvVar]: apiKey }, + } + ); + return { command, raw: command.stdout }; + }, + + deriveStopReason(raw, command) { + if (!raw) return processStopReason(command); + const { records } = parseJsonlRecords(raw); + // An error event means the run failed regardless of exit code. + if (records.some((r) => r.type === 'error')) return 'error'; + // The terminal `step_finish` carries the model's finish reason. + for (let i = records.length - 1; i >= 0; i -= 1) { + if (records[i].type !== 'step_finish') continue; + const part = records[i].part; + const reason = + isRecord(part) && typeof part.reason === 'string' + ? part.reason + : undefined; + if (reason === 'stop') return 'stop'; + if (reason && reason !== 'tool-calls') return reason; // e.g. length — surface verbatim + break; + } + return processStopReason(command); + }, + }; +} + +/** + * opencode's `OPENCODE_CONFIG`. MCP servers map onto `{ mcp: { name: { type: + * "local", command: [...], environment } } }` (the harness's `{command,args,env}` + * → a single `command` array plus `environment`). + * + * Also disables the built-in `title` agent to avoid unnecessary calls to a + * different vendor that aren't useful in headless mode. + * https://opencode.ai/docs/agents/#disable + */ +export function buildOpencodeConfig( + servers: Record +): string { + const mcp: Record = {}; + for (const [name, server] of Object.entries(servers)) { + mcp[name] = { + type: 'local', + command: [server.command, ...(server.args ?? [])], + enabled: true, + ...(server.env ? { environment: server.env } : {}), + }; + } + // Typed against opencode's own `Config` schema, so a config-shape change on a + // CLI bump (mcp layout) fails to compile instead of silently at runtime. + const config: Config = { + $schema: 'https://opencode.ai/config.json', + mcp, + agent: { title: { disable: true } }, + }; + return JSON.stringify(config, null, 2); +} diff --git a/packages/core/src/agents/registry.ts b/packages/core/src/agents/registry.ts index 6a91a0d5..58db31b8 100644 --- a/packages/core/src/agents/registry.ts +++ b/packages/core/src/agents/registry.ts @@ -13,8 +13,13 @@ import type { AgentHarnessId } from '../eval-metadata.js'; import type { AgentTranscriptParser } from '../parsers/types.js'; import { claudeCodeDefinition } from './claude-code/index.js'; import { codexDefinition } from './codex/index.js'; +import { opencodeDefinition } from './opencode/index.js'; -const AGENTS: AgentDefinition[] = [claudeCodeDefinition, codexDefinition]; +const AGENTS: AgentDefinition[] = [ + claudeCodeDefinition, + codexDefinition, + opencodeDefinition, +]; const byId = new Map(AGENTS.map((agent) => [agent.runner.id, agent])); diff --git a/packages/core/src/agents/shared.test.ts b/packages/core/src/agents/shared.test.ts new file mode 100644 index 00000000..06ab423b --- /dev/null +++ b/packages/core/src/agents/shared.test.ts @@ -0,0 +1,28 @@ +import { afterEach, describe, expect, it } from 'vitest'; +import { requireEnv } from './shared.js'; + +const VAR = 'OPENCODE_TEST_ENV_VAR'; + +describe('requireEnv', () => { + afterEach(() => { + delete process.env[VAR]; + }); + + it('returns the value when set', () => { + process.env[VAR] = 'secret'; + expect(requireEnv(VAR)).toBe('secret'); + }); + + it('throws a clear, variable-naming error when unset, including the hint', () => { + expect(() => requireEnv(VAR, 'Set it to run X.')).toThrowError( + `Environment variable ${VAR} is not set. Set it to run X.` + ); + }); + + it('distinguishes set-but-empty from unset', () => { + process.env[VAR] = ' '; + expect(() => requireEnv(VAR)).toThrowError( + `Environment variable ${VAR} is set but empty.` + ); + }); +}); diff --git a/packages/core/src/agents/shared.ts b/packages/core/src/agents/shared.ts index cf1058cb..2e2685a0 100644 --- a/packages/core/src/agents/shared.ts +++ b/packages/core/src/agents/shared.ts @@ -1,12 +1,30 @@ /** - * Helpers shared across CLI runners: sandbox scratch paths, file staging, - * global npm install, loopback rewriting, and the default process-exit-based - * stop reason. + * Helpers shared across CLI runners: env-var validation, sandbox scratch paths, + * file staging, global npm install, loopback rewriting, and the default + * process-exit-based stop reason. */ import type { CommandResult, McpServerConfig } from '../index.js'; import type { AgentSandbox } from './types.js'; +/** Reads a required environment variable, throwing an error that names it. */ +export function requireEnv(name: string, hint?: string): string { + // `in` distinguishes "never set" from "set but empty" for a clearer message. + const isSet = name in process.env; + const value = process.env[name]; + if (!isSet || value === undefined) { + throw new Error( + `Environment variable ${name} is not set.${hint ? ` ${hint}` : ''}` + ); + } + if (value.trim() === '') { + throw new Error( + `Environment variable ${name} is set but empty.${hint ? ` ${hint}` : ''}` + ); + } + return value; +} + /** Scratch dir + staged files, outside the workspace so they're never scored. */ export const SCRATCH = '"$HOME/.eval"'; export const SYSTEM_PROMPT_PATH = '"$HOME/.eval/system-prompt.txt"'; diff --git a/packages/core/src/agents/types.ts b/packages/core/src/agents/types.ts index aad716ed..baf91962 100644 --- a/packages/core/src/agents/types.ts +++ b/packages/core/src/agents/types.ts @@ -82,6 +82,12 @@ export interface AgentRunner { displayName: string; /** Env var holding the agent's API key (e.g. `ANTHROPIC_API_KEY`). */ apiKeyEnvVar: string; + /** + * The model's provider, for multi-provider CLIs whose runner is built + * per-model (e.g. opencode). When omitted, the engine derives the provider + * from the agent id. + */ + modelProvider?: ModelProvider; /** npm package providing the CLI. */ cliPackage: string; /** Pinned CLI version — pinned so transcript-format drift can't silently break parsing. */ diff --git a/packages/core/src/eval-metadata.ts b/packages/core/src/eval-metadata.ts index 1a8d541c..59a645cf 100644 --- a/packages/core/src/eval-metadata.ts +++ b/packages/core/src/eval-metadata.ts @@ -52,10 +52,19 @@ export const experimentSuiteSchema = z.enum([ export const EXPERIMENT_SUITES = experimentSuiteSchema.options; export type ExperimentSuite = z.infer; -export const agentHarnessIdSchema = z.enum(['ai-sdk', 'claude-code', 'codex']); +export const agentHarnessIdSchema = z.enum([ + 'ai-sdk', + 'claude-code', + 'codex', + 'opencode', +]); export type AgentHarnessId = z.infer; -export const modelProviderSchema = z.enum(['anthropic', 'openai']); +export const modelProviderSchema = z.enum([ + 'anthropic', + 'openai', + 'moonshotai', +]); export type ModelProvider = z.infer; export const reasoningEffortSchema = z.enum([ diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index c3ccfc0e..caa480d2 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -99,10 +99,11 @@ export { rehydrateTruncatedDocsResults, } from './docs-results.js'; export type { DocsResultSandbox } from './docs-results.js'; -// CLI agent harnesses (Claude Code, Codex, and the framework for adding more). +// CLI agent harnesses (Claude Code, Codex, OpenCode, and the framework for adding more). export { createCliAgent } from './agents/engine.js'; export { claudeCodeAgent } from './agents/claude-code/index.js'; export { codexAgent } from './agents/codex/index.js'; +export { opencodeAgent } from './agents/opencode/index.js'; export type { AgentMetadata, AgentSandbox, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index d9edc57d..b7a51bfd 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -24,6 +24,9 @@ catalogs: '@electric-sql/pglite-socket': specifier: 0.1.5 version: 0.1.5 + '@opencode-ai/sdk': + specifier: 1.18.5 + version: 1.18.5 '@supabase/lite': specifier: 0.7.1-next.3 version: 0.7.1-next.3 @@ -288,6 +291,9 @@ importers: specifier: 'catalog:' version: 4.4.3 devDependencies: + '@opencode-ai/sdk': + specifier: 'catalog:' + version: 1.18.5 vitest: specifier: 'catalog:' version: 4.1.8(@opentelemetry/api@1.9.1)(@types/node@25.9.3)(happy-dom@20.10.2)(vite@7.3.5(@types/node@25.9.3)(jiti@2.7.0)(lightningcss@1.32.0)(tsx@4.22.4)) @@ -1101,6 +1107,9 @@ packages: resolution: {integrity: sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==} engines: {node: '>= 8'} + '@opencode-ai/sdk@1.18.5': + resolution: {integrity: sha512-7KgMvP5/1oxbhHj6kYBtPSTEdFKYpUeEYOzBTKdzSaRpapUpFFdn6Hkus3rr0rljO0kukWZIgRd3DrVBwTULGA==} + '@opentelemetry/api-logs@0.214.0': resolution: {integrity: sha512-40lSJeqYO8Uz2Yj7u94/SJWE/wONa7rmMKjI1ZcIjgf3MHNHv1OZUCrCETGuaRF62d5pQD1wKIW+L4lmSMTzZA==} engines: {node: '>=8.0.0'} @@ -5179,6 +5188,10 @@ snapshots: '@nodelib/fs.scandir': 2.1.5 fastq: 1.20.1 + '@opencode-ai/sdk@1.18.5': + dependencies: + cross-spawn: 7.0.6 + '@opentelemetry/api-logs@0.214.0': dependencies: '@opentelemetry/api': 1.9.1 diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 7b181ac5..981db650 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -8,6 +8,8 @@ catalog: '@ai-sdk/anthropic': ^3.0.71 '@ai-sdk/mcp': ^1.0.39 '@ai-sdk/openai': ^3.0.66 + # Pin to defaultCliVersion in agents/opencode/runner.ts + '@opencode-ai/sdk': 1.18.5 '@electric-sql/pglite': 0.4.5 '@electric-sql/pglite-socket': 0.1.5 '@supabase/lite': 0.7.1-next.3