feat(eval): Kimi K3 through OpenCode via the Vercel AI Gateway

.env.example

@@ -2,8 +2,14 @@
 ANTHROPIC_API_KEY=
 OPENAI_API_KEY=
 
+# Opencode Gemini API key
+GOOGLE_GENERATIVE_AI_API_KEY=
+
+# Opencode Moonshot (Kimi) API key
+MOONSHOT_API_KEY=
+
 # Vercel AI Gateway — one key for every vendor. Direct keys above stay the
 # default; set RUN_THROUGH_GATEWAY=true (the eval-refresh workflow's
 # run_through_gateway input) to route the whole run through the gateway.
 AI_GATEWAY_API_KEY=
-RUN_THROUGH_GATEWAY=
+RUN_THROUGH_GATEWAY=

apps/web/src/App.tsx

@@ -198,6 +198,7 @@ const AGENT_LABELS = {
   "ai-sdk": "AI SDK",
   "claude-code": "Claude Code",
   codex: "Codex",
+  opencode: "OpenCode",
 } satisfies Record<ExperimentDisplay["agent"], string>
 
 const EXPERIMENT_SUITES = ["benchmark", "no-skills"] as const
@@ -313,13 +314,17 @@ function formatOpenAiModel(modelId: string) {
 }
 
 function formatModel(display: ExperimentDisplay) {
-  // AI Gateway model ids are `vendor/model` slugs; format just the model part.
+  // opencode ids are `provider/model` and AI Gateway ids are `vendor/model`
+  // slugs; either way, format just the model part.
   const modelId = display.modelId.replace(/^[a-z-]+\//, "")
   switch (display.modelProvider) {
     case "anthropic":
       return formatAnthropicModel(modelId)
     case "openai":
       return formatOpenAiModel(modelId)
+    case "google":
+    case "moonshotai":
+      return modelId
   }
 }
 

experiments/opencode-claude-sonnet-5.ts

@@ -0,0 +1,22 @@
+import {
+  defineExperiment,
+  opencodeAgent,
+  platformLiteRuntime,
+  supabaseMcpServer,
+} from '@supabase-evals/core';
+import { localStackRuntime } from '@supabase-evals/sandbox';
+
+// OpenCode is a CLI agent driving Claude Sonnet 5. Like Claude Code / Codex it
+// runs in both modes: `runtime` supplies the MCP servers for tools-mode evals
+// (written into opencode's config) and `localStack` drives local-stack evals.
+// Which mode an eval uses is a property of the eval, not the agent.
+export default defineExperiment({
+  agent: opencodeAgent({
+    model: 'anthropic/claude-sonnet-5',
+  }),
+  runtime: platformLiteRuntime({
+    mcpServers: [supabaseMcpServer()],
+  }),
+  localStack: localStackRuntime(),
+  skills: ['supabase', 'supabase-postgres-best-practices'],
+});

experiments/opencode-gemini-flash.ts

@@ -0,0 +1,24 @@
+import {
+  defineExperiment,
+  opencodeAgent,
+  platformLiteRuntime,
+  supabaseMcpServer,
+} from '@supabase-evals/core';
+import { localStackRuntime } from '@supabase-evals/sandbox';
+
+// OpenCode driving Google's latest Gemini Flash (cheapest tier). Runs in both
+// modes (see opencode-claude-sonnet-5.ts); the `google/` prefix selects the
+// GOOGLE_GENERATIVE_AI_API_KEY credential (Google AI Studio, not Vertex).
+// `gemini-flash-latest` tracks the newest Flash — the only Gemini Flash id that
+// the AI-Studio key serves end-to-end (pinned 2.5/3.x-flash ids returned no
+// output via opencode 1.15.7).
+export default defineExperiment({
+  agent: opencodeAgent({
+    model: 'google/gemini-flash-latest',
+  }),
+  runtime: platformLiteRuntime({
+    mcpServers: [supabaseMcpServer()],
+  }),
+  localStack: localStackRuntime(),
+  skills: ['supabase', 'supabase-postgres-best-practices'],
+});

experiments/opencode-gpt-5.4-mini.ts

@@ -0,0 +1,21 @@
+import {
+  defineExperiment,
+  opencodeAgent,
+  platformLiteRuntime,
+  supabaseMcpServer,
+} from '@supabase-evals/core';
+import { localStackRuntime } from '@supabase-evals/sandbox';
+
+// OpenCode driving OpenAI GPT-5.4 mini. Runs in both modes (see opencode-claude-
+// sonnet-5.ts); the `openai/` model prefix selects the OPENAI_API_KEY
+// credential.
+export default defineExperiment({
+  agent: opencodeAgent({
+    model: 'openai/gpt-5.4-mini',
+  }),
+  runtime: platformLiteRuntime({
+    mcpServers: [supabaseMcpServer()],
+  }),
+  localStack: localStackRuntime(),
+  skills: ['supabase', 'supabase-postgres-best-practices'],
+});

experiments/opencode-kimi-k3.ts

@@ -0,0 +1,25 @@
+import {
+  defineExperiment,
+  opencodeAgent,
+  platformLiteRuntime,
+  supabaseMcpServer,
+} from '@supabase-evals/core';
+import { localStackRuntime } from '@supabase-evals/sandbox';
+
+// OpenCode driving Moonshot's Kimi K3 through the Vercel AI Gateway. `gateway:
+// true` writes a gateway provider into opencode's config (see agents/opencode/
+// runner.ts) and routes on the `moonshotai/kimi-k3` catalog slug, so the run
+// needs AI_GATEWAY_API_KEY rather than a direct MOONSHOT_API_KEY. Runs in both
+// modes like the other opencode experiments (see opencode-claude-sonnet-5.ts).
+export default defineExperiment({
+  suite: ['benchmark'],
+  agent: opencodeAgent({
+    model: 'moonshotai/kimi-k3',
+    gateway: true,
+  }),
+  runtime: platformLiteRuntime({
+    mcpServers: [supabaseMcpServer()],
+  }),
+  localStack: localStackRuntime(),
+  skills: ['supabase', 'supabase-postgres-best-practices'],
+});

packages/core/package.json

@@ -14,18 +14,20 @@
     "test": "vitest run"
   },
   "devDependencies": {
+    "@opencode-ai/sdk": "catalog:",
     "vitest": "catalog:"
   },
   "dependencies": {
     "@anthropic-ai/sdk": "catalog:",
-    "openai": "catalog:",
+    "@ai-sdk/google": "catalog:",
     "@ai-sdk/mcp": "catalog:",
     "@ai-sdk/openai": "catalog:",
     "@supabase-evals/platform-lite": "workspace:*",
     "@supabase/supabase-js": "catalog:",
     "ai": "catalog:",
     "executor": "1.4.29",
     "gray-matter": "^4.0.3",
+    "openai": "catalog:",
     "typescript": "catalog:",
     "zod": "catalog:"
   }

packages/core/src/agents/engine.ts

@@ -34,6 +34,7 @@ import {
   SYSTEM_PROMPT_PATH,
   USER_PROMPT_PATH,
   processStopReason,
+  requireEnv,
   rewriteLoopback,
   writeSandboxFile,
 } from './shared.js';
@@ -44,6 +45,10 @@ function modelProviderForAgent(id: AgentRunner['id']): ModelProvider {
       return 'anthropic';
     case 'codex':
       return 'openai';
+    case 'opencode':
+      throw new Error(
+        'opencode is multi-provider; its runner sets `modelProvider` from the model id'
+      );
     case 'ai-sdk':
       throw new Error('ai-sdk agents are not created through createCliAgent');
   }
@@ -72,10 +77,11 @@ export function createCliAgent<M extends string = string>(
     metadata: {
       agent: runner.id,
       // Through the gateway the model may be any vendor's; derive the vendor
-      // from the model slug instead of from the agent.
+      // from the model slug instead of from the agent. On the direct path a
+      // multi-provider runner (e.g. opencode) sets its own `modelProvider`.
       modelProvider: useGateway
         ? gatewayModelProvider(options.model)
-        : modelProviderForAgent(runner.id),
+        : (runner.modelProvider ?? modelProviderForAgent(runner.id)),
       modelId: options.model,
       ...(options.reasoningEffort
         ? { reasoningEffort: options.reasoningEffort }
@@ -134,11 +140,8 @@ export function createCliAgent<M extends string = string>(
 
 function requireApiKey(runner: AgentRunner, gateway = false): string {
   if (gateway) return requireGatewayApiKey(runner.displayName);
-  const apiKey = process.env[runner.apiKeyEnvVar];
-  if (!apiKey) {
-    throw new Error(
-      `Missing ${runner.displayName} credentials. Set ${runner.apiKeyEnvVar} before running ${runner.id} evals.`
-    );
-  }
-  return apiKey;
+  return requireEnv(
+    runner.apiKeyEnvVar,
+    `Set it to run ${runner.displayName} (${runner.id}) evals.`
+  );
 }

packages/core/src/agents/opencode/index.ts

@@ -0,0 +1,49 @@
+/**
+ * OpenCode agent. Owns everything opencode-specific: it wires its own runner +
+ * parser into the public `opencodeAgent` factory (via the generic
+ * `createCliAgent` engine) and exports the registry definition the harness uses
+ * to parse opencode transcripts. Runs in both modes, like Claude Code / Codex.
+ */
+
+import type { AgentHarness } from '../../index.js';
+import { createCliAgent } from '../engine.js';
+import type { AgentDefinition } from '../types.js';
+import {
+  DEFAULT_OPENCODE_MODEL,
+  createOpencodeRunner,
+  type OpenCodeModel,
+} from './runner.js';
+import { opencodeParser } from './parser.js';
+
+/**
+ * OpenCode as an `AgentHarness`. Multi-provider: the `provider/model` id selects
+ * the credential (anthropic / openai / google), so the runner is built per-model
+ * with the matching API-key env var and provider.
+ */
+export function opencodeAgent(
+  options: {
+    /**
+     * opencode model id, `provider/model` (e.g. `openai/gpt-5.4`). With
+     * `gateway`, this is the AI Gateway `vendor/model` slug (e.g.
+     * `moonshotai/kimi-k3`) — see `./runner.ts`.
+     */
+    model?: OpenCodeModel;
+    /** Override the pinned CLI version. */
+    cliVersion?: string;
+    /** Route through the Vercel AI Gateway instead of the vendor's own key. */
+    gateway?: boolean;
+  } = {}
+): AgentHarness {
+  const model = options.model ?? DEFAULT_OPENCODE_MODEL;
+  return createCliAgent(createOpencodeRunner(model), opencodeParser, {
+    model,
+    cliVersion: options.cliVersion,
+    gateway: options.gateway,
+  });
+}
+
+/** Runner + parser pairing for the agent registry (id comes from `runner.id`). */
+export const opencodeDefinition: AgentDefinition = {
+  runner: createOpencodeRunner(DEFAULT_OPENCODE_MODEL),
+  parser: opencodeParser,
+};