feat(eval): add Gemini CLI agent harness

.env.example

@@ -1,3 +1,4 @@
 # AI SDK Core direct-provider credentials.
 ANTHROPIC_API_KEY=
 OPENAI_API_KEY=
+GEMINI_API_KEY=

.github/workflows/eval-refresh.yml

@@ -212,6 +212,7 @@ jobs:
     env:
       ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
       OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+      GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}
     steps:
       - name: Checkout
         uses: actions/checkout@9f698171ed81b15d1823a05fc7211befd50c8ae0 # v6.0.3
@@ -239,6 +240,7 @@ jobs:
           {
             echo "ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY}"
             echo "OPENAI_API_KEY=${OPENAI_API_KEY}"
+            echo "GEMINI_API_KEY=${GEMINI_API_KEY}"
           } > .env
 
       - name: Run evals

apps/web/src/App.tsx

@@ -150,6 +150,7 @@ const AGENT_LABELS = {
   "ai-sdk": "AI SDK",
   "claude-code": "Claude Code",
   codex: "Codex",
+  "gemini-cli": "Gemini CLI",
 } satisfies Record<ExperimentDisplay["agent"], string>
 
 function capitalize(value: string) {
@@ -187,12 +188,32 @@ function formatOpenAiModel(modelId: string) {
     .join(" ")
 }
 
+function formatGoogleModel(modelId: string) {
+  if (!modelId.startsWith("gemini-")) {
+    return modelId
+  }
+
+  // gemini-3.5-flash → Gemini 3.5 Flash; gemini-3.1-pro-preview → Gemini 3.1 Pro Preview.
+  const suffix = modelId.slice("gemini-".length)
+  const match = /^(\d+(?:[.-]\d+)*)(?:-(.+))?$/.exec(suffix)
+  if (!match) {
+    return capitalize(modelId)
+  }
+
+  const [, version, variant] = match
+  return [`Gemini ${version.replaceAll("-", ".")}`, variant?.split("-").map(capitalize).join(" ")]
+    .filter(Boolean)
+    .join(" ")
+}
+
 function formatModel(display: ExperimentDisplay) {
   switch (display.modelProvider) {
     case "anthropic":
       return formatAnthropicModel(display.modelId)
     case "openai":
       return formatOpenAiModel(display.modelId)
+    case "google":
+      return formatGoogleModel(display.modelId)
   }
 }
 

experiments/gemini-cli-3.1-pro.ts

@@ -0,0 +1,28 @@
+import {
+  defineExperiment,
+  geminiCliAgent,
+  platformLiteRuntime,
+  supabaseMcpServer,
+} from "@supabase-evals/core";
+import { localStackRuntime } from "@supabase-evals/sandbox";
+
+// Google's Gemini CLI driving Gemini 3.1 Pro — Google's comparable to GPT-5.5
+// and Opus 4.8. Like Claude Code / Codex it runs in both modes: `runtime`
+// supplies the MCP servers for tools-mode evals (written into
+// ~/.gemini/settings.json) and `localStack` drives local-stack evals. Which
+// mode an eval uses is a property of the eval, not the agent.
+export default defineExperiment({
+  suite: ["benchmark"],
+  agent: geminiCliAgent({
+    model: "gemini-3.1-pro-preview",
+    // gemini-3.1-pro-preview thinks at `high` by default (inherited from
+    // chat-base-3); gemini-cli has no flag to change it. Recorded for parity
+    // with the other high-effort benchmark agents (Opus 4.8).
+    reasoningEffort: "high",
+  }),
+  runtime: platformLiteRuntime({
+    mcpServers: [supabaseMcpServer()],
+  }),
+  localStack: localStackRuntime(),
+  skills: ["supabase", "supabase-postgres-best-practices"],
+});

packages/core/package.json

@@ -19,6 +19,7 @@
   "dependencies": {
     "@anthropic-ai/sdk": "catalog:",
     "openai": "catalog:",
+    "@ai-sdk/google": "catalog:",
     "@ai-sdk/mcp": "catalog:",
     "@ai-sdk/openai": "catalog:",
     "@supabase-evals/platform-lite": "workspace:*",

packages/core/src/agents/engine.ts

@@ -39,6 +39,8 @@ function modelProviderForAgent(id: AgentRunner["id"]): ModelProvider {
       return "anthropic";
     case "codex":
       return "openai";
+    case "gemini-cli":
+      return "google";
     case "ai-sdk":
       throw new Error("ai-sdk agents are not created through createCliAgent");
   }

packages/core/src/agents/gemini-cli/index.ts

@@ -0,0 +1,48 @@
+/**
+ * Gemini CLI agent. Owns everything Gemini-CLI-specific: it wires its own runner
+ * + parser into the public `geminiCliAgent` factory (via the generic
+ * `createCliAgent` engine) and exports the registry definition the harness uses
+ * to parse Gemini CLI transcripts. Runs in both modes, like Claude Code / Codex.
+ *
+ * gemini-cli has no flag to *set* reasoning effort — it's a per-model default
+ * baked into the CLI (e.g. gemini-3.1-pro-preview thinks at `high`). So
+ * `reasoningEffort` here is display-only: it records the model's effort in the
+ * experiment metadata for benchmark parity with Claude Code / Codex, but is not
+ * passed to the CLI (the runner ignores it).
+ */
+
+import type { AgentHarness } from "../../index.js";
+import type { ReasoningEffortLevel } from "../../eval-metadata.js";
+import { createCliAgent } from "../engine.js";
+import type { AgentDefinition } from "../types.js";
+import { geminiCliRunner, type GeminiCliModel } from "./runner.js";
+import { geminiCliParser } from "./parser.js";
+
+/** Google's Gemini CLI as an `AgentHarness`. */
+export function geminiCliAgent(
+  options: {
+    /** Gemini model id (typed from `@ai-sdk/google`; any string accepted). */
+    model?: GeminiCliModel;
+    /**
+     * The model's reasoning effort, for display/benchmark metadata only.
+     * gemini-cli has no flag to change it, so this must match the model's own
+     * default (e.g. `high` for gemini-3.1-pro-preview) — it is recorded, not
+     * applied. `max` is not a Gemini effort level.
+     */
+    reasoningEffort?: Exclude<ReasoningEffortLevel, "max">;
+    /** Override the pinned CLI version. */
+    cliVersion?: string;
+  } = {},
+): AgentHarness {
+  return createCliAgent(geminiCliRunner, geminiCliParser, {
+    model: options.model ?? geminiCliRunner.defaultModel,
+    reasoningEffort: options.reasoningEffort,
+    cliVersion: options.cliVersion,
+  });
+}
+
+/** Runner + parser pairing for the agent registry (id comes from `runner.id`). */
+export const geminiCliDefinition: AgentDefinition = {
+  runner: geminiCliRunner,
+  parser: geminiCliParser,
+};

packages/core/src/agents/gemini-cli/parser.test.ts

@@ -0,0 +1,156 @@
+import { describe, expect, it } from "vitest";
+import { geminiCliParser } from "./parser.js";
+import { buildGeminiSettings, geminiCliRunner } from "./runner.js";
+import { adaptTranscript } from "../../parsers/adapt.js";
+
+/** A representative `gemini --output-format stream-json` stream (shapes from CLI 0.20.2). */
+const SESSION = [
+  JSON.stringify({ type: "init", session_id: "s1", model: "gemini-3.5-flash" }),
+  JSON.stringify({ type: "message", role: "user", content: "Count the rows." }),
+  JSON.stringify({
+    type: "tool_use",
+    timestamp: "2026-06-24T11:14:21.542Z",
+    tool_name: "run_shell_command",
+    tool_id: "run_shell_command-1-abc",
+    parameters: { description: "echo", command: "echo 42" },
+  }),
+  JSON.stringify({
+    type: "tool_result",
+    tool_id: "run_shell_command-1-abc",
+    status: "success",
+    output: "42",
+  }),
+  // Assistant text streamed as two delta chunks — must merge into one message.
+  JSON.stringify({ type: "message", role: "assistant", content: "The number ", delta: true }),
+  JSON.stringify({ type: "message", role: "assistant", content: "is 42.", delta: true }),
+  JSON.stringify({ type: "result", status: "success", stats: { tool_calls: 1 } }),
+].join("\n");
+
+describe("geminiCliParser", () => {
+  it("maps run_shell_command to a canonical shell call, paired by tool_id", () => {
+    const { events, errors } = geminiCliParser.parseTranscript(SESSION);
+    expect(errors).toEqual([]);
+
+    const calls = events.filter((e) => e.type === "tool_call");
+    expect(calls.map((e) => e.tool?.name)).toEqual(["shell"]);
+    expect(calls.map((e) => e.tool?.originalName)).toEqual(["run_shell_command"]);
+    expect(calls[0].tool?.id).toBe("run_shell_command-1-abc");
+    expect(calls[0].tool?.command).toBe("echo 42");
+
+    const results = events.filter((e) => e.type === "tool_result");
+    expect(results[0].tool?.id).toBe("run_shell_command-1-abc");
+    expect(results[0].tool?.success).toBe(true);
+  });
+
+  it("merges streamed assistant deltas into one message + surfaces the report", () => {
+    const events = geminiCliParser.parseTranscript(SESSION).events;
+    const assistantMessages = events.filter(
+      (e) => e.type === "message" && e.role === "assistant",
+    );
+    expect(assistantMessages).toHaveLength(1);
+    expect(assistantMessages[0].content).toBe("The number is 42.");
+
+    const adapted = adaptTranscript(events);
+    expect(adapted.agentReport).toBe("The number is 42.");
+    expect(adapted.steps).toBe(1); // one merged assistant turn, not two deltas
+    expect(adapted.toolCalls).toEqual([
+      {
+        endpoint: "run_shell_command",
+        body: { description: "echo", command: "echo 42" },
+        command: "echo 42",
+        result: "42",
+        error: undefined,
+        ts: Date.parse("2026-06-24T11:14:21.542Z"),
+      },
+    ]);
+  });
+
+  it("marks a failed tool_result via status and never throws on malformed lines", () => {
+    const stream = [
+      "not json",
+      JSON.stringify({ type: "tool_use", tool_name: "run_shell_command", tool_id: "t-1", parameters: { command: "false" } }),
+      JSON.stringify({ type: "tool_result", tool_id: "t-1", status: "error", error: "boom" }),
+    ].join("\n");
+    const { events, errors } = geminiCliParser.parseTranscript(stream);
+    expect(errors.length).toBe(1);
+    const result = events.find((e) => e.type === "tool_result");
+    expect(result?.tool?.success).toBe(false);
+    expect(adaptTranscript(events).toolCalls[0].error).toBe("boom");
+  });
+
+  it("identifies a loaded skill from a SKILL.md read", () => {
+    const stream = JSON.stringify({
+      type: "tool_use",
+      tool_name: "read_file",
+      tool_id: "r-1",
+      parameters: { absolute_path: "/workspace/.claude/skills/supabase/SKILL.md" },
+    });
+    const call = geminiCliParser.parseTranscript(stream).events.find((e) => e.type === "tool_call");
+    expect(call?.tool?.loadedSkill).toBe("supabase");
+    expect(adaptTranscript([call!]).toolCalls[0].loadedSkill).toBe("supabase");
+  });
+
+  it("maps the 0.46+ update_topic built-in to agent_task", () => {
+    const stream = JSON.stringify({
+      type: "tool_use",
+      tool_name: "update_topic",
+      tool_id: "u-1",
+      parameters: { topic: "Counting rows" },
+    });
+    const call = geminiCliParser.parseTranscript(stream).events.find((e) => e.type === "tool_call");
+    expect(call?.tool?.name).toBe("agent_task");
+    expect(call?.tool?.originalName).toBe("update_topic");
+  });
+
+  it("keeps two non-delta assistant messages as separate turns", () => {
+    const stream = [
+      JSON.stringify({ type: "message", role: "assistant", content: "First." }),
+      JSON.stringify({ type: "message", role: "assistant", content: "Second." }),
+    ].join("\n");
+    const events = geminiCliParser.parseTranscript(stream).events;
+    expect(
+      events.filter((e) => e.type === "message" && e.role === "assistant").map((e) => e.content),
+    ).toEqual(["First.", "Second."]);
+  });
+
+  it("emits an error event from an error record", () => {
+    const events = geminiCliParser.parseTranscript(
+      JSON.stringify({ type: "error", error: { message: "quota exceeded" } }),
+    ).events;
+    expect(events).toEqual([
+      { timestamp: undefined, type: "error", content: "quota exceeded", raw: { type: "error", error: { message: "quota exceeded" } } },
+    ]);
+  });
+});
+
+describe("gemini-cli runner", () => {
+  const ok = { ok: true, exitCode: 0, stdout: "", stderr: "" };
+
+  it("deriveStopReason reads the terminal result status", () => {
+    expect(geminiCliRunner.deriveStopReason!(SESSION, ok)).toBe("stop");
+    const errored = JSON.stringify({ type: "result", status: "error" });
+    expect(geminiCliRunner.deriveStopReason!(errored, ok)).toBe("error");
+  });
+
+  it("deriveStopReason falls back to the process result when there's no terminal event", () => {
+    const timedOut = { ok: false, exitCode: 124, stdout: "", stderr: "timed out" };
+    expect(geminiCliRunner.deriveStopReason!("", timedOut)).toBe("timeout");
+  });
+
+  it("reads the key from GEMINI_API_KEY", () => {
+    expect(geminiCliRunner.apiKeyEnvVar).toBe("GEMINI_API_KEY");
+  });
+
+  it("builds gemini-cli's settings.json MCP shape", () => {
+    const settings = JSON.parse(
+      buildGeminiSettings({
+        supabase: { command: "npx", args: ["-y", "srv"], env: { TOKEN: "t" } },
+        docs: { command: "docs-server" },
+      }),
+    );
+    expect(settings.mcpServers).toEqual({
+      supabase: { command: "npx", args: ["-y", "srv"], env: { TOKEN: "t" } },
+      docs: { command: "docs-server" },
+    });
+  });
+});