feat(eval): add Cursor CLI agent harness

.env.example

@@ -1,3 +1,4 @@
 # AI SDK Core direct-provider credentials.
 ANTHROPIC_API_KEY=
 OPENAI_API_KEY=
+CURSOR_API_KEY=

.github/workflows/eval-refresh.yml

@@ -212,6 +212,7 @@ jobs:
     env:
       ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
       OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+      CURSOR_API_KEY: ${{ secrets.CURSOR_API_KEY }}
     steps:
       - name: Checkout
         uses: actions/checkout@9f698171ed81b15d1823a05fc7211befd50c8ae0 # v6.0.3
@@ -239,6 +240,7 @@ jobs:
           {
             echo "ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY}"
             echo "OPENAI_API_KEY=${OPENAI_API_KEY}"
+            echo "CURSOR_API_KEY=${CURSOR_API_KEY}"
           } > .env
 
       - name: Run evals

apps/web/src/App.tsx

@@ -150,6 +150,7 @@ const AGENT_LABELS = {
   "ai-sdk": "AI SDK",
   "claude-code": "Claude Code",
   codex: "Codex",
+  "cursor-cli": "Cursor CLI",
 } satisfies Record<ExperimentDisplay["agent"], string>
 
 function capitalize(value: string) {
@@ -187,12 +188,21 @@ function formatOpenAiModel(modelId: string) {
     .join(" ")
 }
 
+function formatCursorModel(modelId: string) {
+  // composer-2.5 → Composer 2.5; auto → Auto. Split family from version parts.
+  const [family = "", ...rest] = modelId.split("-")
+  if (!family) return modelId
+  return [capitalize(family), rest.join(".")].filter(Boolean).join(" ")
+}
+
 function formatModel(display: ExperimentDisplay) {
   switch (display.modelProvider) {
     case "anthropic":
       return formatAnthropicModel(display.modelId)
     case "openai":
       return formatOpenAiModel(display.modelId)
+    case "cursor":
+      return formatCursorModel(display.modelId)
   }
 }
 

experiments/cursor-cli.ts

@@ -0,0 +1,27 @@
+import {
+  cursorCliAgent,
+  defineExperiment,
+  platformLiteRuntime,
+  supabaseMcpServer,
+} from "@supabase-evals/core";
+import { localStackRuntime } from "@supabase-evals/sandbox";
+
+// Cursor's CLI (`cursor-agent`) driving Composer 2.5 (Cursor's own model, fast
+// mode / Max mode off). Like Claude Code / Codex it runs in both modes:
+// `runtime` supplies the MCP servers for tools-mode evals (written into
+// ~/.cursor/mcp.json) and `localStack` drives local-stack evals. Which mode an
+// eval uses is a property of the eval, not the agent.
+export default defineExperiment({
+  suite: ["benchmark"],
+  agent: cursorCliAgent({
+    model: "composer-2.5",
+    // Max mode is off (the default), so no reasoningEffort is recorded. Cursor
+    // has no low/medium/high dial — set reasoningEffort: "max" only to record
+    // that Max mode is on.
+  }),
+  runtime: platformLiteRuntime({
+    mcpServers: [supabaseMcpServer()],
+  }),
+  localStack: localStackRuntime(),
+  skills: ["supabase", "supabase-postgres-best-practices"],
+});

packages/core/src/agents/claude-code/index.ts

@@ -18,8 +18,11 @@ export function claudeCodeAgent(
   options: {
     /** Anthropic model id (typed from `@anthropic-ai/sdk`). Defaults to Sonnet. */
     model?: AnthropicModel;
-    /** Reasoning effort (`--effort`). Omit to use Claude Code's own default. */
-    reasoningEffort?: ReasoningEffortLevel;
+    /**
+     * Reasoning effort (`--effort`). Omit to use Claude Code's own default.
+     * `max` is excluded — it's Cursor's Max-mode marker, not a Claude effort.
+     */
+    reasoningEffort?: Exclude<ReasoningEffortLevel, "max">;
     /** Override the pinned CLI version. */
     cliVersion?: string;
   } = {},

packages/core/src/agents/codex/index.ts

@@ -17,8 +17,11 @@ export function codexAgent(
   options: {
     /** OpenAI model id (typed from `openai`; any string accepted). */
     model?: CodexModel;
-    /** Reasoning effort (`model_reasoning_effort`). Omit to use Codex's default. */
-    reasoningEffort?: ReasoningEffortLevel;
+    /**
+     * Reasoning effort (`model_reasoning_effort`). Omit to use Codex's default.
+     * `max` is excluded — it's Cursor's Max-mode marker, not a Codex effort.
+     */
+    reasoningEffort?: Exclude<ReasoningEffortLevel, "max">;
     /** Override the pinned CLI version. */
     cliVersion?: string;
   } = {},

packages/core/src/agents/cursor-cli/index.ts

@@ -0,0 +1,46 @@
+/**
+ * Cursor CLI agent. Owns everything Cursor-CLI-specific: it wires its own runner
+ * + parser into the public `cursorCliAgent` factory (via the generic
+ * `createCliAgent` engine) and exports the registry definition the harness uses
+ * to parse Cursor CLI transcripts. Runs in both modes, like Claude Code / Codex.
+ *
+ * Cursor has no low/medium/high effort dial — only "Max mode" (off by default).
+ * So `reasoningEffort` here is display-only: set it to `max` to record that Max
+ * mode is on (for benchmark metadata parity), or omit it for the default fast
+ * mode. It is recorded, not applied — cursor-agent exposes no flag to toggle it.
+ */
+
+import type { AgentHarness } from "../../index.js";
+import type { ReasoningEffortLevel } from "../../eval-metadata.js";
+import { createCliAgent } from "../engine.js";
+import type { AgentDefinition } from "../types.js";
+import { cursorCliRunner, type CursorCliModel } from "./runner.js";
+import { cursorCliParser } from "./parser.js";
+
+/** Cursor's CLI (`cursor-agent`) as an `AgentHarness`. */
+export function cursorCliAgent(
+  options: {
+    /** Cursor model id (e.g. `composer-2.5`). */
+    model?: CursorCliModel;
+    /**
+     * Display-only effort metadata. Cursor has no low/medium/high dial — only
+     * Max mode — so the only settable value is `max` (Max mode on); omit it for
+     * the default fast mode.
+     */
+    reasoningEffort?: Extract<ReasoningEffortLevel, "max">;
+    /** Override the pinned CLI version. */
+    cliVersion?: string;
+  } = {},
+): AgentHarness {
+  return createCliAgent(cursorCliRunner, cursorCliParser, {
+    model: options.model ?? cursorCliRunner.defaultModel,
+    reasoningEffort: options.reasoningEffort,
+    cliVersion: options.cliVersion,
+  });
+}
+
+/** Runner + parser pairing for the agent registry (id comes from `runner.id`). */
+export const cursorCliDefinition: AgentDefinition = {
+  runner: cursorCliRunner,
+  parser: cursorCliParser,
+};

packages/core/src/agents/cursor-cli/parser.test.ts

@@ -0,0 +1,126 @@
+import { describe, expect, it } from "vitest";
+import { cursorCliParser } from "./parser.js";
+import { buildCursorMcpConfig, cursorCliRunner } from "./runner.js";
+import { adaptTranscript } from "../../parsers/adapt.js";
+
+/** A representative `cursor-agent --output-format stream-json` stream. */
+const SESSION = [
+  JSON.stringify({ type: "system", subtype: "init", model: "composer-2.5", session_id: "s1" }),
+  JSON.stringify({
+    type: "user",
+    message: { role: "user", content: [{ type: "text", text: "Count the rows." }] },
+  }),
+  JSON.stringify({
+    type: "tool_call",
+    subtype: "started",
+    call_id: "c1",
+    timestamp_ms: 1_780_000_000_000,
+    tool_call: { shellToolCall: { args: { command: "echo 42" } } },
+  }),
+  JSON.stringify({
+    type: "tool_call",
+    subtype: "completed",
+    call_id: "c1",
+    tool_call: { shellToolCall: { args: { command: "echo 42" }, result: { success: { output: "42" } } } },
+  }),
+  JSON.stringify({
+    type: "assistant",
+    message: { role: "assistant", content: [{ type: "text", text: "The count is 42." }] },
+  }),
+  JSON.stringify({ type: "result", subtype: "success", is_error: false, result: "The count is 42." }),
+].join("\n");
+
+describe("cursorCliParser", () => {
+  it("maps a shellToolCall to a canonical shell call, paired by call_id", () => {
+    const { events, errors } = cursorCliParser.parseTranscript(SESSION);
+    expect(errors).toEqual([]);
+
+    const calls = events.filter((e) => e.type === "tool_call");
+    expect(calls.map((e) => e.tool?.name)).toEqual(["shell"]);
+    expect(calls.map((e) => e.tool?.originalName)).toEqual(["shellToolCall"]);
+    expect(calls[0].tool?.id).toBe("c1");
+    expect(calls[0].tool?.command).toBe("echo 42");
+
+    const results = events.filter((e) => e.type === "tool_result");
+    expect(results[0].tool?.id).toBe("c1");
+    expect(results[0].tool?.success).toBe(true);
+  });
+
+  it("adapts the transcript: report, one assistant step, paired tool result", () => {
+    const { events } = cursorCliParser.parseTranscript(SESSION);
+    const adapted = adaptTranscript(events);
+    expect(adapted.agentReport).toBe("The count is 42.");
+    expect(adapted.steps).toBe(1);
+    expect(adapted.toolCalls).toEqual([
+      {
+        endpoint: "shellToolCall",
+        body: { command: "echo 42" },
+        command: "echo 42",
+        result: { success: { output: "42" } },
+        error: undefined,
+        ts: 1_780_000_000_000,
+      },
+    ]);
+  });
+
+  it("marks a failed tool_result via the error result shape", () => {
+    const stream = [
+      JSON.stringify({ type: "tool_call", subtype: "started", call_id: "t1", tool_call: { shellToolCall: { args: { command: "false" } } } }),
+      JSON.stringify({ type: "tool_call", subtype: "completed", call_id: "t1", tool_call: { shellToolCall: { args: { command: "false" }, result: { error: { message: "boom" } } } } }),
+    ].join("\n");
+    const { events } = cursorCliParser.parseTranscript(stream);
+    const result = events.find((e) => e.type === "tool_result");
+    expect(result?.tool?.success).toBe(false);
+    expect(adaptTranscript(events).toolCalls[0].error).toContain("boom");
+  });
+
+  it("identifies a loaded skill from a SKILL.md read", () => {
+    const stream = JSON.stringify({
+      type: "tool_call",
+      subtype: "started",
+      call_id: "r1",
+      tool_call: { readToolCall: { args: { path: "/workspace/.claude/skills/supabase/SKILL.md" } } },
+    });
+    const call = cursorCliParser.parseTranscript(stream).events.find((e) => e.type === "tool_call");
+    expect(call?.tool?.name).toBe("file_read");
+    expect(call?.tool?.loadedSkill).toBe("supabase");
+    expect(adaptTranscript([call!]).toolCalls[0].loadedSkill).toBe("supabase");
+  });
+
+  it("never throws on malformed lines and records the error", () => {
+    const { errors } = cursorCliParser.parseTranscript("not json\n{}\n");
+    expect(errors.length).toBe(1);
+  });
+});
+
+describe("cursor-cli runner", () => {
+  const ok = { ok: true, exitCode: 0, stdout: "", stderr: "" };
+
+  it("deriveStopReason reads the terminal result status", () => {
+    expect(cursorCliRunner.deriveStopReason!(SESSION, ok)).toBe("stop");
+    const errored = JSON.stringify({ type: "result", subtype: "error", is_error: true });
+    expect(cursorCliRunner.deriveStopReason!(errored, ok)).toBe("error");
+  });
+
+  it("deriveStopReason falls back to the process result when there's no terminal event", () => {
+    const timedOut = { ok: false, exitCode: 124, stdout: "", stderr: "timed out" };
+    expect(cursorCliRunner.deriveStopReason!("", timedOut)).toBe("timeout");
+  });
+
+  it("reads the key from CURSOR_API_KEY", () => {
+    expect(cursorCliRunner.apiKeyEnvVar).toBe("CURSOR_API_KEY");
+  });
+
+  it("builds cursor-agent's mcp.json shape", () => {
+    const config = JSON.parse(
+      buildCursorMcpConfig({
+        supabase: { command: "npx", args: ["-y", "srv"], env: { TOKEN: "t" } },
+        docs: { command: "docs-server" },
+      }),
+    );
+    expect(config.mcpServers).toEqual({
+      supabase: { command: "npx", args: ["-y", "srv"], env: { TOKEN: "t" } },
+      docs: { command: "docs-server" },
+    });
+  });
+});