diff --git a/apps/framework/harness/run-eval.ts b/apps/framework/harness/run-eval.ts index 0f758e53..67724fc1 100644 --- a/apps/framework/harness/run-eval.ts +++ b/apps/framework/harness/run-eval.ts @@ -450,9 +450,6 @@ async function runOne( mcpServers: session.mcpServers, timeoutSec: TIMEOUT_SEC, }); - // Must run before session disposes below, see rehydrateTruncatedDocsResults. - await rehydrateTruncatedDocsResults(session.sandbox, run.toolCalls); - lastToolCalls = run.toolCalls; lastTranscript = run.transcript; lastAgentReport = run.agentReport; @@ -485,6 +482,9 @@ async function runOne( }, }); + // Runs after scoring so the scorer sees what the agent actually saw, not rehydrated content. + await rehydrateTruncatedDocsResults(session.sandbox, run.toolCalls); + if (STOP_ON_PASS && last.passed) { return { ...last, @@ -536,10 +536,6 @@ async function runOne( sandbox: cliSandbox?.sandbox, timeoutSec: TIMEOUT_SEC, }); - // Must run before cliSandbox disposes below, see rehydrateTruncatedDocsResults. - if (cliSandbox) - await rehydrateTruncatedDocsResults(cliSandbox.sandbox, run.toolCalls); - lastToolCalls = run.toolCalls; lastTranscript = run.transcript; lastAgentReport = run.agentReport; @@ -551,6 +547,10 @@ async function runOne( agentReport: run.agentReport, }); + // Runs after scoring so the scorer sees what the agent actually saw, not rehydrated content. + if (cliSandbox) + await rehydrateTruncatedDocsResults(cliSandbox.sandbox, run.toolCalls); + if (STOP_ON_PASS && last.passed) { return { ...last, diff --git a/apps/web/src/components/results/eval-details.tsx b/apps/web/src/components/results/eval-details.tsx index c83e256b..3cffc9c6 100644 --- a/apps/web/src/components/results/eval-details.tsx +++ b/apps/web/src/components/results/eval-details.tsx @@ -2,6 +2,7 @@ import type { ReactNode } from "react" import { CheckIcon, ChevronRightIcon, + CircleHelpIcon, FileTextIcon, SearchIcon, XIcon, @@ -16,6 +17,7 @@ const DOCS_CALL_SOURCE_LABEL: Record = { search_docs: "MCP", web_fetch: "Web Fetch", web_search: "Web Search", + shell_fetch: "Shell", } // Cool color for MCP (our own docs tool), warm for the agent going around it onto the open web. @@ -23,6 +25,7 @@ const DOCS_CALL_SOURCE_CHIP_CLASS: Record = { search_docs: "bg-indigo-500/10 text-indigo-700 dark:text-indigo-400", web_fetch: "bg-amber-500/10 text-amber-700 dark:text-amber-400", web_search: "bg-amber-500/10 text-amber-700 dark:text-amber-400", + shell_fetch: "bg-orange-500/10 text-orange-700 dark:text-orange-400", } const docsSourceChipClassName = @@ -31,9 +34,10 @@ const docsSourceChipClassName = const evalMetaGridClassName = "grid grid-cols-[6.5rem_minmax(0,1fr)] items-start gap-x-4 gap-y-4 text-xs" -/** Search icon for a bare hit, file icon for a call that actually pulled in page text. */ +/** Shows whether page content was read, not read, or cannot be determined. */ function docsCallIcon(call: DocsCall) { - return call.hasContent === false ? SearchIcon : FileTextIcon + if (call.hasContent === undefined) return CircleHelpIcon + return call.hasContent ? FileTextIcon : SearchIcon } /** Pulls the quoted search term out of search_docs's raw GraphQL query for display, else returns the query as-is. */ @@ -120,6 +124,7 @@ function ResultDocsCalls({ calls }: { calls: DocsCall[] }) {
{calls.map((call, index) => { const searchOnly = call.hasContent === false + const contentUnknown = call.hasContent === undefined const Icon = docsCallIcon(call) const queryLabel = docsCallQueryLabel(call) const sizeLabel = docsCallSizeLabel(call) @@ -132,24 +137,29 @@ function ResultDocsCalls({ calls }: { calls: DocsCall[] }) { className="size-4 shrink-0 text-muted-foreground/50 transition-transform group-open:rotate-90" aria-hidden /> - + + + {queryLabel} + {contentUnknown ? ( + Content unknown + ) : null} {sizeLabel ? ( {sizeLabel} diff --git a/packages/core/src/agents/codex/parser.test.ts b/packages/core/src/agents/codex/parser.test.ts index b3fbdeda..ddb52ebd 100644 --- a/packages/core/src/agents/codex/parser.test.ts +++ b/packages/core/src/agents/codex/parser.test.ts @@ -168,6 +168,27 @@ describe('codexParser', () => { expect(result?.tool?.originalName).toBe('search_docs'); }); + it("keeps a web_search item's action, which says what the hosted tool did", () => { + const url = 'https://supabase.com/changelog.md'; + const stream = JSON.stringify({ + type: 'item.completed', + item: { + id: 'ws_0', + type: 'web_search', + query: url, + action: { type: 'open_page', url }, + status: 'completed', + }, + }); + + const adapted = adaptTranscript(codexParser.parseTranscript(stream).events); + expect(adapted.toolCalls[0].name).toBe('web_search'); + expect(adapted.toolCalls[0].body).toEqual({ + query: url, + action: { type: 'open_page', url }, + }); + }); + it('emits an error event for a failed turn', () => { const stream = [ JSON.stringify({ type: 'turn.started' }), diff --git a/packages/core/src/agents/codex/parser.ts b/packages/core/src/agents/codex/parser.ts index 5a5d433f..452bb8c8 100644 --- a/packages/core/src/agents/codex/parser.ts +++ b/packages/core/src/agents/codex/parser.ts @@ -181,10 +181,14 @@ function itemToEvents(item: Record): TranscriptEvent[] { ); } case 'web_search': { + // `action` says what the hosted tool actually did (`search`, + // `open_page`, `find_in_page`). `query` is only its display rendering, + // which collapses a url open and a search for that url into the same + // string, so keep the action itself. return toolCallPair( id, 'web_search', - { query: item.query }, + { query: item.query, action: item.action }, undefined, statusSuccess(item.status) ); diff --git a/packages/core/src/docs-results.test.ts b/packages/core/src/docs-results.test.ts index ed68789d..2ac7d1dd 100644 --- a/packages/core/src/docs-results.test.ts +++ b/packages/core/src/docs-results.test.ts @@ -9,7 +9,9 @@ import type { ToolCallRecord } from './index.js'; function toolCall( endpoint: string, body: Record, - options: Partial> = {} + options: Partial< + Pick + > = {} ): ToolCallRecord { return { endpoint, body, ...options, ts: 0 }; } @@ -328,6 +330,533 @@ describe('buildDocsResult', () => { expect(result.calls[0].hasContent).toBeUndefined(); }); + it('trusts an open_page action over the query string, recording the page as read', () => { + const result = buildDocsResult([ + toolCall( + 'web_search', + { + query: 'https://supabase.com/changelog.md', + action: { + type: 'open_page', + url: 'https://supabase.com/changelog.md', + }, + }, + { name: 'web_search' } + ), + ]); + + expect(result.calls).toEqual([ + { + source: 'web_search', + query: 'https://supabase.com/changelog.md', + hasContent: true, + pages: [{ url: 'https://supabase.com/changelog.md' }], + }, + ]); + }); + + it('records a find_in_page action, whose query rendering the URL-shape fallback never matches', () => { + const url = 'https://supabase.com/docs/guides/database/extensions/pgmq'; + const result = buildDocsResult([ + toolCall( + 'web_search', + { + query: `'send(' in ${url}`, + action: { type: 'find_in_page', url, pattern: 'send(' }, + }, + { name: 'web_search' } + ), + ]); + + expect(result.calls).toEqual([ + { + source: 'web_search', + query: `'send(' in ${url}`, + hasContent: true, + pages: [{ url }], + }, + ]); + }); + + it("leaves a search action's content unknown, and doesn't mistake its url-shaped query for a page read", () => { + const result = buildDocsResult([ + toolCall( + 'web_search', + { + query: 'https://supabase.com/changelog.md', + action: { + type: 'search', + query: 'https://supabase.com/changelog.md', + }, + }, + { name: 'web_search' } + ), + ]); + + // No pages: the action says this was a query, not an open, even though the + // query text is a bare url the fallback would have counted as a read. + expect(result.calls).toEqual([ + { + source: 'web_search', + query: 'https://supabase.com/changelog.md', + pages: [], + }, + ]); + expect(result.calls[0].hasContent).toBeUndefined(); + }); + + it('drops an open_page action pointing somewhere other than supabase.com', () => { + const result = buildDocsResult([ + toolCall( + 'web_search', + { + query: 'https://github.com/pgmq/pgmq', + action: { type: 'open_page', url: 'https://github.com/pgmq/pgmq' }, + }, + { name: 'web_search' } + ), + ]); + + expect(result.calls).toEqual([]); + }); + + it('drops an open_page action on a supabase service subdomain', () => { + const result = buildDocsResult([ + toolCall( + 'web_search', + { + query: 'https://mcp.supabase.com/mcp', + action: { type: 'open_page', url: 'https://mcp.supabase.com/mcp' }, + }, + { name: 'web_search' } + ), + ]); + + expect(result.calls).toEqual([]); + }); + + it('counts a docs url curled from the shell, sized by what the pipe actually returned', () => { + const command = + '/bin/bash -lc "curl -fsSL https://supabase.com/changelog.md | sed -n \'1,160p\'"'; + const result = buildDocsResult([ + toolCall( + 'command_execution', + { command }, + { + name: 'shell', + command, + result: '# Changelog\n\n2026-06-12 breaking-change ...', + } + ), + ]); + + expect(result.calls).toEqual([ + { + source: 'shell_fetch', + query: command, + hasContent: true, + pages: [{ url: 'https://supabase.com/changelog.md' }], + resultChars: '# Changelog\n\n2026-06-12 breaking-change ...'.length, + }, + ]); + }); + + it('keeps a fetch from a mixed shell call without attributing the combined output to it', () => { + const command = + 'curl -fsSL https://supabase.com/changelog.md | rg breaking || true; cat local-notes.md'; + const result = buildDocsResult([ + toolCall( + 'command_execution', + { command }, + { name: 'shell', command, result: 'unrelated local notes' } + ), + ]); + + expect(result.calls).toEqual([ + { + source: 'shell_fetch', + query: command, + pages: [{ url: 'https://supabase.com/changelog.md' }], + resultChars: 'unrelated local notes'.length, + }, + ]); + expect(result.calls[0].hasContent).toBeUndefined(); + }); + + it('finds a curl fetch on its own line in a multi-line script', () => { + const command = + 'echo starting\ncurl -fsSL https://supabase.com/changelog.md'; + const result = buildDocsResult([ + toolCall( + 'command_execution', + { command }, + { name: 'shell', command, result: '# Changelog' } + ), + ]); + + expect(result.calls).toEqual([ + { + source: 'shell_fetch', + query: command, + pages: [{ url: 'https://supabase.com/changelog.md' }], + resultChars: '# Changelog'.length, + }, + ]); + // Result shared with other command(s) means we can't be certain about content attribution. + expect(result.calls[0].hasContent).toBeUndefined(); + }); + + it('records each curl fetch in a multi-line script', () => { + const command = + 'set -e\n' + + "curl -fsSL https://supabase.com/docs/guides/functions/auth.md | sed -n '1,280p'\n" + + "curl -fsSL https://supabase.com/docs/guides/functions/auth-headers.md | sed -n '1,260p'"; + const result = buildDocsResult([ + toolCall( + 'command_execution', + { command }, + { name: 'shell', command, result: '# Auth\n\n# Auth headers' } + ), + ]); + + expect(result.calls).toEqual([ + { + source: 'shell_fetch', + query: command, + pages: [ + { url: 'https://supabase.com/docs/guides/functions/auth.md' }, + { url: 'https://supabase.com/docs/guides/functions/auth-headers.md' }, + ], + resultChars: '# Auth\n\n# Auth headers'.length, + }, + ]); + // Result shared with other command(s) means we can't be certain about content attribution. + expect(result.calls[0].hasContent).toBeUndefined(); + }); + + it('still attributes a fetch pipeline whose only command-list operator is a trailing fallback', () => { + const command = + 'curl -fsSL https://supabase.com/changelog.md | rg breaking || true'; + const result = buildDocsResult([ + toolCall( + 'command_execution', + { command }, + { name: 'shell', command, result: 'breaking change' } + ), + ]); + + expect(result.calls[0].hasContent).toBe(true); + expect(result.calls[0].resultChars).toBe('breaking change'.length); + }); + + it('records every supabase url a single shell fetch pulled down', () => { + const command = + 'wget -qO- https://supabase.com/changelog.md https://supabase.com/docs/guides/auth.md'; + const result = buildDocsResult([ + toolCall( + 'command_execution', + { command }, + { name: 'shell', command, result: 'page 1\npage 2' } + ), + ]); + + expect(result.calls[0].pages).toEqual([ + { url: 'https://supabase.com/changelog.md' }, + { url: 'https://supabase.com/docs/guides/auth.md' }, + ]); + }); + + it('drops an ordinary wget whose page was saved to disk instead of shown to the model', () => { + const command = 'wget https://supabase.com/changelog.md'; + const result = buildDocsResult([ + toolCall( + 'command_execution', + { command }, + { name: 'shell', command, result: 'saved changelog.md' } + ), + ]); + + expect(result.calls).toEqual([]); + }); + + it('counts wget with the long-form stdout destination', () => { + const command = + 'wget --quiet --output-document=- https://supabase.com/changelog.md'; + const result = buildDocsResult([ + toolCall( + 'command_execution', + { command }, + { name: 'shell', command, result: '# Changelog' } + ), + ]); + + expect(result.calls).toHaveLength(1); + }); + + it('drops curl output saved to a file instead of shown to the model', () => { + const command = + 'curl -fsSL -ochangelog.md https://supabase.com/changelog.md'; + const result = buildDocsResult([ + toolCall( + 'command_execution', + { command }, + { name: 'shell', command, result: 'download complete' } + ), + ]); + + expect(result.calls).toEqual([]); + }); + + it('counts a curl fetch with stderr redirected to /dev/null', () => { + const command = + 'curl -fsSL https://supabase.com/changelog.md 2>/dev/null | head -50'; + const result = buildDocsResult([ + toolCall( + 'command_execution', + { command }, + { name: 'shell', command, result: '# Changelog' } + ), + ]); + + expect(result.calls).toHaveLength(1); + }); + + it('counts a curl fetch with stderr merged into stdout via 2>&1', () => { + const command = + 'curl -fsSL https://supabase.com/changelog.md 2>&1 | head -50'; + const result = buildDocsResult([ + toolCall( + 'command_execution', + { command }, + { name: 'shell', command, result: '# Changelog' } + ), + ]); + + expect(result.calls).toHaveLength(1); + }); + + it.each(['>out.txt', '1>out.txt', '&>out.txt'])( + 'drops a curl fetch redirected with %s', + (redirect) => { + const command = `curl -fsSL https://supabase.com/changelog.md ${redirect}`; + const result = buildDocsResult([ + toolCall( + 'command_execution', + { command }, + { name: 'shell', command, result: 'download complete' } + ), + ]); + + expect(result.calls).toEqual([]); + } + ); + + it('drops a curl fetch whose stdout is redirected past a silenced stderr', () => { + const command = + 'curl -fsSL https://supabase.com/changelog.md 2>/dev/null >out.txt'; + const result = buildDocsResult([ + toolCall( + 'command_execution', + { command }, + { name: 'shell', command, result: 'download complete' } + ), + ]); + + expect(result.calls).toEqual([]); + }); + + it('drops a curl fetch with stdout swapped onto stderr via >&2', () => { + const command = 'curl -fsSL https://supabase.com/changelog.md >&2'; + const result = buildDocsResult([ + toolCall( + 'command_execution', + { command }, + { name: 'shell', command, result: 'download complete' } + ), + ]); + + expect(result.calls).toEqual([]); + }); + + it('does not attribute an unrelated url elsewhere in a compound command to curl', () => { + const command = + 'echo https://supabase.com/changelog.md; curl -fsSL https://example.com'; + const result = buildDocsResult([ + toolCall( + 'command_execution', + { command }, + { name: 'shell', command, result: 'example page' } + ), + ]); + + expect(result.calls).toEqual([]); + }); + + it('drops a shell fetch that failed, since a non-zero exit leaves no output to read', () => { + const command = 'curl -fsSL https://supabase.com/changelog.md'; + const result = buildDocsResult([ + toolCall( + 'command_execution', + { command }, + { + name: 'shell', + command, + error: 'curl: (22) The requested URL returned error: 404', + } + ), + ]); + + expect(result.calls).toEqual([]); + }); + + it("marks a bare curl's content as unknown on a real body", () => { + // No -f/--fail, so a 0 exit doesn't prove the body is a real page. + const command = 'curl https://supabase.com/changelog.md'; + const result = buildDocsResult([ + toolCall( + 'command_execution', + { command }, + { name: 'shell', command, result: '# Changelog\n\nreal content' } + ), + ]); + + expect(result.calls).toEqual([ + { + source: 'shell_fetch', + query: command, + pages: [{ url: 'https://supabase.com/changelog.md' }], + resultChars: '# Changelog\n\nreal content'.length, + }, + ]); + expect(result.calls[0].hasContent).toBeUndefined(); + }); + + it("marks a bare curl's content as unknown on an HTTP-error body", () => { + const command = 'curl https://supabase.com/changelog.md'; + const result = buildDocsResult([ + toolCall( + 'command_execution', + { command }, + { name: 'shell', command, result: '404: Not Found' } + ), + ]); + + expect(result.calls).toEqual([ + { + source: 'shell_fetch', + query: command, + pages: [{ url: 'https://supabase.com/changelog.md' }], + resultChars: '404: Not Found'.length, + }, + ]); + expect(result.calls[0].hasContent).toBeUndefined(); + }); + + it("trusts a -f curl's content regardless of body", () => { + // A real failure would already be routed to `error` by --fail. + const command = 'curl -fsSL https://supabase.com/changelog.md'; + const result = buildDocsResult([ + toolCall( + 'command_execution', + { command }, + { + name: 'shell', + command, + result: '500 organizations now use Supabase in production', + } + ), + ]); + + expect(result.calls).toEqual([ + { + source: 'shell_fetch', + query: command, + hasContent: true, + pages: [{ url: 'https://supabase.com/changelog.md' }], + resultChars: '500 organizations now use Supabase in production'.length, + }, + ]); + }); + + it('drops a wget with 404 response', () => { + const command = + 'wget --quiet --output-document=- https://supabase.com/changelog.md'; + const result = buildDocsResult([ + toolCall( + 'command_execution', + { command }, + { + name: 'shell', + command, + error: 'wget: server returned error: HTTP/1.1 404 Not Found', + } + ), + ]); + + expect(result.calls).toEqual([]); + }); + + it('drops a shell fetch of a supabase service endpoint, which is a probe not a read', () => { + const command = 'curl -s https://mcp.supabase.com/mcp'; + const result = buildDocsResult([ + toolCall( + 'command_execution', + { command }, + { name: 'shell', command, result: 'Unauthorized' } + ), + ]); + + expect(result.calls).toEqual([]); + }); + + it('drops a shell fetch from a lookalike hostname', () => { + const command = 'curl https://not-supabase.com/foo'; + const result = buildDocsResult([ + toolCall( + 'command_execution', + { command }, + { name: 'shell', command, result: 'not Supabase docs' } + ), + ]); + + expect(result.calls).toEqual([]); + }); + + it('drops a shell command that only mentions a docs url without fetching it', () => { + const command = + 'echo "see https://supabase.com/docs/guides/auth for details"'; + const result = buildDocsResult([ + toolCall( + 'command_execution', + { command }, + { + name: 'shell', + command, + result: 'see https://supabase.com/docs/guides/auth for details', + } + ), + ]); + + expect(result.calls).toEqual([]); + }); + + it('drops an echo that merely mentions curl rather than invoking it', () => { + const command = 'echo curl https://supabase.com/changelog.md'; + const result = buildDocsResult([ + toolCall( + 'command_execution', + { command }, + { + name: 'shell', + command, + result: 'curl https://supabase.com/changelog.md', + } + ), + ]); + + expect(result.calls).toEqual([]); + }); + it('ignores WebFetch/WebSearch-shaped calls when the parser never normalized a canonical name', () => { const result = buildDocsResult([ toolCall( @@ -471,6 +1000,26 @@ describe('rehydrateTruncatedDocsResults', () => { expect(call.result).toBe(stub); }); + it('rehydrates a shell fetch whose output the CLI truncated to disk', async () => { + const path = '/home/node/.claude/projects/x/tool-results/changelog.txt'; + const command = 'curl -fsSL https://supabase.com/changelog.md'; + const call = toolCall( + 'Bash', + { command }, + { + name: 'shell', + command, + result: `\nOutput too large (50.8KB). Full output saved to: ${path}`, + } + ); + const readFile = vi.fn().mockResolvedValue('# Changelog'); + + await rehydrateTruncatedDocsResults({ readFile }, [call]); + + expect(readFile).toHaveBeenCalledWith(path); + expect(call.result).toBe('# Changelog'); + }); + it("ignores calls that aren't truncated, and calls outside the docs channels", async () => { const readFile = vi.fn(); const calls = [ diff --git a/packages/core/src/docs-results.ts b/packages/core/src/docs-results.ts index 10bdcd9e..b64c4df9 100644 --- a/packages/core/src/docs-results.ts +++ b/packages/core/src/docs-results.ts @@ -1,5 +1,6 @@ import type { DocsCall, DocsCallPage, DocsResult } from './eval-metadata.js'; import type { ToolCallRecord } from './index.js'; +import { isRecord } from './json.js'; /** The subset of AgentSandbox rehydration needs; avoids a hard dependency on its full interface. */ export interface DocsResultSandbox { @@ -24,6 +25,31 @@ const TRUNCATION_APPROX_SIZE_PATTERN = const TITLED_PAGE_PATTERN = /"title":"([^"]*)"\s*,?\s*"(?:href|url)":"([^"]+)"/g; const HREF_PATTERN = /"(?:href|url)":"([^"]+)"/g; +const CURL_PATTERN = /\bcurl\b/; +// curl/wget as a segment's actual command, not merely mentioned somewhere in +// another command's arguments (e.g. `echo curl ...`). Allows leading env +// assignments (`FOO=bar curl ...`). +const FETCHER_COMMAND_PATTERN = /^\s*(?:[A-Za-z_]\w*=\S*\s+)*(curl|wget)\b/; +// A `bash/sh/zsh -c "..."` wrapper, how agents commonly run a whole piped +// command in one shell call. Unwrapped first, so a pipe inside the quotes +// doesn't get split as if it belonged to the outer command. +const SHELL_WRAPPER_PATTERN = + /^\s*(?:\S*\/)?(?:bash|sh|zsh)\s+-\S*c\S*\s+(['"])([\s\S]*)\1\s*$/; +const CURL_NO_BODY_PATTERN = + /(?:^|\s)(?:-[^-\s]*[oOI]\S*|--(?:output|remote-name|head)(?:=\S*)?)(?=\s|$)/; +const WGET_STDOUT_PATTERN = + /(?:^|\s)(?:-[A-Za-z]*O-|-[A-Za-z]*O\s+-|--output-document(?:=|\s+)-)(?=\s|$)/; +const SHELL_SEGMENT_PATTERN = /(?:&&|\|\||[;|]|\r?\n)/; +const SHELL_COMMAND_LIST_PATTERN = /(?:&&|\|\||;|\r?\n)/; +// Matches curl's -f/--fail flag. +const CURL_FAIL_FLAG_PATTERN = + /(?:^|\s)(?:-[A-Za-z]*f[A-Za-z]*|--fail\b)(?=\s|$)/; +// Bare `>`, `1>`, and `&>` redirect stdout away from the pipe. A lone `2>`, +// or `2>&1` which folds stderr back into stdout, leaves the body on stdout. +const STDOUT_REDIRECT_PATTERN = /(?:^|[^2])>/; +// Urls inside a shell command, stopping at the shell metacharacters that can +// legally follow one (`|`, `>`, quotes, backslash-escapes). +const URL_IN_COMMAND_PATTERN = /https?:\/\/[^\s'"`\\;|&>()]+/g; /** The search_docs query arg, flat on `body` or nested under `body.arguments` (Codex's shape). */ function extractGraphqlQuery( @@ -38,16 +64,83 @@ function extractGraphqlQuery( return undefined; } -/** True when a URL string points at any supabase.com host. */ -function isSupabaseUrl(value: string): boolean { +/** + * True for the apex supabase.com host. Docs, changelog, and blog all live + * there. Subdomains like `api.` and `mcp.` are service endpoints, so they + * don't count. + */ +function isSupabaseApexUrl(value: string): boolean { try { - const { hostname } = new URL(value); - return hostname === 'supabase.com' || hostname.endsWith('.supabase.com'); + return new URL(value).hostname === 'supabase.com'; } catch { return false; } } +/** The shell command a call ran, from the parser's normalized view or the raw body. */ +function shellCommand(call: ToolCallRecord): string | undefined { + if (call.name !== 'shell') return undefined; + if (call.command) return call.command; + return typeof call.body.command === 'string' ? call.body.command : undefined; +} + +/** The docs urls a shell command writes to stdout, empty when page text won't reach the model. */ +function shellFetchUrls(command: string | undefined): string[] { + if (!command) return []; + const wrapped = command.match(SHELL_WRAPPER_PATTERN); + if (wrapped) return shellFetchUrls(wrapped[2]); + + const urls: string[] = []; + for (const segment of command.split(SHELL_SEGMENT_PATTERN)) { + const fetcherMatch = segment.match(FETCHER_COMMAND_PATTERN); + if (!fetcherMatch || fetcherMatch.index === undefined) continue; + const fetcher = fetcherMatch[1]; + + const curlToStdout = + fetcher === 'curl' && + !CURL_NO_BODY_PATTERN.test(segment) && + !STDOUT_REDIRECT_PATTERN.test(segment); + const wgetToStdout = + fetcher === 'wget' && + WGET_STDOUT_PATTERN.test(segment) && + !STDOUT_REDIRECT_PATTERN.test(segment); + if (!curlToStdout && !wgetToStdout) continue; + + for (const match of segment + .slice(fetcherMatch.index + fetcherMatch[0].length) + .match(URL_IN_COMMAND_PATTERN) ?? []) { + // Trailing sentence punctuation glues onto a url in prose; a real one + // never ends in a period or comma. + const url = match.replace(/[.,]+$/, ''); + if (isSupabaseApexUrl(url) && !urls.includes(url)) urls.push(url); + } + } + return urls; +} + +/** True when no separate command can contribute to a shell fetch's combined result. */ +function shellFetchOwnsResult(command: string): boolean { + const withoutIgnoredFailure = command.replace( + /\|\|\s*true\s*(?=(?:["'])?\s*$)/, + '' + ); + return !SHELL_COMMAND_LIST_PATTERN.test(withoutIgnoredFailure); +} + +/** + * Codex's `web_search` action, the tool's own statement of what it did. Only + * `type` and `url` matter here: `query` is already carried separately, and no + * other field says anything about which page was read. + */ +function webSearchAction( + body: Record +): { type: string; url?: string } | undefined { + const action = body.action; + if (!isRecord(action) || typeof action.type !== 'string') return undefined; + const url = typeof action.url === 'string' ? action.url : undefined; + return { type: action.type, url }; +} + /** The in-container path a truncated result was persisted to, if `result` is one of the known stub shapes. */ function extractTruncatedResultPath(result: unknown): string | undefined { if (typeof result !== 'string') return undefined; @@ -82,19 +175,19 @@ function isDocsRelatedCall(call: ToolCallRecord): boolean { return ( call.endpoint.endsWith('search_docs') || call.name === 'web_fetch' || - call.name === 'web_search' + call.name === 'web_search' || + shellFetchUrls(shellCommand(call)).length > 0 ); } -/** Fetches a truncated docs call's real result back from disk. Must run before the sandbox disposes, the file lives inside that container. */ +/** Fetches a truncated docs call's real result back from disk, so a real page fetch doesn't get counted as just the tiny truncation stub. Must run before the sandbox disposes, the file lives inside that container. */ export async function rehydrateTruncatedDocsResults( sandbox: DocsResultSandbox, toolCalls: ToolCallRecord[] ): Promise { for (const call of toolCalls) { - if (!isDocsRelatedCall(call)) continue; const path = extractTruncatedResultPath(call.result); - if (!path) continue; + if (!path || !isDocsRelatedCall(call)) continue; try { call.result = await sandbox.readFile(path); } catch {} @@ -118,12 +211,12 @@ function extractPages(result: unknown): DocsCallPage[] { const pages: DocsCallPage[] = []; const seen = new Set(); for (const [, title, url] of text.matchAll(TITLED_PAGE_PATTERN)) { - if (!isSupabaseUrl(url) || seen.has(url)) continue; + if (!isSupabaseApexUrl(url) || seen.has(url)) continue; seen.add(url); pages.push(title ? { url, title } : { url }); } for (const [, url] of text.matchAll(HREF_PATTERN)) { - if (!isSupabaseUrl(url) || seen.has(url)) continue; + if (!isSupabaseApexUrl(url) || seen.has(url)) continue; seen.add(url); pages.push({ url }); } @@ -151,7 +244,7 @@ export function buildDocsResult(toolCalls: ToolCallRecord[]): DocsResult { } if (call.name === 'web_fetch') { - if (!call.url || !isSupabaseUrl(call.url)) continue; + if (!call.url || !isSupabaseApexUrl(call.url)) continue; // WebFetch runs the fetch through an LLM extraction step guided by // `prompt`, so that's the meaningful "ask" here (same role `query` // plays for search_docs), not the url. Url still recorded, in `pages`. @@ -170,10 +263,52 @@ export function buildDocsResult(toolCalls: ToolCallRecord[]): DocsResult { const query = typeof body.query === 'string' ? body.query : undefined; if (!query) continue; - // Codex's web_search doubles as a fetch when the query is a URL. No - // result payload is ever exposed on this tool, so content is unknown, not false. + // Codex states what its hosted search did, so trust that over the shape + // of the query string, which renders a page open and a search for that + // same url identically. + // + // Only `search` actually arrives intact on CLI 0.138: exec re-parses the + // app-server action, which serializes camelCase (`openPage`), into a + // snake_case enum (`open_page`), so both page-reading variants land on + // the catch-all and reach us as `other`. `search` survives because it's + // one word in either casing. The `other` calls fall through to the + // url-shape branch below, same as before. See codex's + // event_processor_with_jsonl_output.rs (the from_value round trip) and + // app-server-protocol v2/item.rs vs protocol/models.rs for the two enums. + const action = webSearchAction(body); + + if (action?.type === 'open_page' || action?.type === 'find_in_page') { + if (!action.url || !isSupabaseApexUrl(action.url)) continue; + calls.push({ + source: 'web_search', + query, + hasContent: true, + pages: [{ url: action.url }], + resultChars: resultCharCount(result), + }); + continue; + } + + if (action?.type === 'search') { + if (!/supabase/i.test(query)) continue; + // No page to attribute: the hits never reach the client. `hasContent` + // stays unknown rather than false, because those hits carry snippet + // text the model may well have read, and we can't see it either way. + // The gain over the url-shape fallback is knowing this wasn't a page + // open even when the query happens to be a bare url. + calls.push({ + source: 'web_search', + query, + pages: [], + resultChars: resultCharCount(result), + }); + continue; + } + + // No action reported (Claude Code, or an action type we don't know): + // fall back to the query's shape, which is all there is to go on. if (URL_PATTERN.test(query)) { - if (!isSupabaseUrl(query)) continue; + if (!isSupabaseApexUrl(query)) continue; calls.push({ source: 'web_search', query, @@ -196,6 +331,30 @@ export function buildDocsResult(toolCalls: ToolCallRecord[]): DocsResult { }); continue; } + + if (call.name === 'shell') { + const command = shellCommand(call); + const urls = shellFetchUrls(command); + if (!command || urls.length === 0) continue; + // A non-zero exit already proves the fetch failed. + if (call.error) continue; + // With no shell output at all, nothing from the fetch reached the model. + if (typeof result !== 'string' || result.length === 0) continue; + // curl without -f/--fail exits 0 on a 4xx/5xx. wget doesn't need the + // check: it already exits non-zero on an HTTP error by default + // (https://www.gnu.org/software/wget/manual/html_node/Exit-Status.html). + const curlExitUnproven = + CURL_PATTERN.test(command) && !CURL_FAIL_FLAG_PATTERN.test(command); + const ownsResult = shellFetchOwnsResult(command); + calls.push({ + source: 'shell_fetch', + query: command, + hasContent: !curlExitUnproven && ownsResult ? true : undefined, + pages: urls.map((url) => ({ url })), + resultChars: resultCharCount(result), + }); + continue; + } } return { calls }; diff --git a/packages/core/src/eval-metadata.ts b/packages/core/src/eval-metadata.ts index 59a645cf..03c8582c 100644 --- a/packages/core/src/eval-metadata.ts +++ b/packages/core/src/eval-metadata.ts @@ -243,10 +243,13 @@ export type SkillResult = z.infer; // tool vocabulary). `web_fetch`/`web_search` are the harness's own normalized // name, so Claude Code's `WebSearch` and Codex's `web_search` (the same // action, spelled differently per harness) collapse into one value here. +// `shell_fetch` is a docs url fetched to shell stdout, which is how an agent +// with no fetch tool of its own reads a page (Codex curls the changelog). export const docsPageSourceSchema = z.enum([ 'search_docs', 'web_fetch', 'web_search', + 'shell_fetch', ]); export type DocsPageSource = z.infer; @@ -261,14 +264,15 @@ export type DocsCallPage = z.infer; export const docsCallSchema = z.object({ source: docsPageSourceSchema, - // Whichever field is the meaningful "ask" for that source: search term, GraphQL query, WebFetch's extraction prompt, or target URL. + // Whichever field is the meaningful "ask" for that source: search term, GraphQL query, WebFetch's extraction prompt, or shell command. query: z.string(), // Whether the call's results included page text, not just a title/url hit. // Known for search_docs (whether the agent's own GraphQL selection asked - // for `content`) and web_fetch (always true, that's what fetching is). - // False for Claude Code's WebSearch (its results never include page text, - // only title/url). Unknown (omitted) for a Codex web_search used as a - // fetch: no result payload is ever exposed on that tool. + // for `content`), web_fetch, and an isolated shell_fetch. False for Claude + // Code's WebSearch, whose results never include page text. Unknown (omitted) + // when the available trace cannot prove either state, including Codex + // web_search, a shell fetch mixed with other commands, and an unflagged + // curl (see docs-results.ts). hasContent: z.boolean().optional(), pages: z.array(docsCallPageSchema), // Size of the result the call actually produced, in characters, an @@ -277,7 +281,8 @@ export const docsCallSchema = z.object({ // rehydrated file when the CLI truncated the result, or parsed out of the // truncation message's own reported size when rehydration wasn't // possible or wasn't attempted (e.g. no sandbox, ai-sdk/Codex which don't - // truncate this way). Omitted only when there's no result at all. + // truncate this way). Omitted when the trace exposes no result, including + // Codex web_search and failed calls whose output is recorded as an error. resultChars: z.number().optional(), }); export type DocsCall = z.infer;