diff --git a/evals/capability/lib.test.ts b/evals/capability/lib.test.ts index eade147de..323ca297f 100644 --- a/evals/capability/lib.test.ts +++ b/evals/capability/lib.test.ts @@ -71,6 +71,7 @@ function sampleResult(over: Partial = {}): CaseResult { repeat: over.repeat ?? 0, behaviors: over.behaviors ?? null, providerFallback: over.providerFallback ?? null, + diagnostics: over.diagnostics ?? null, }; } @@ -755,6 +756,38 @@ describe("parseEvalRunReport", () => { }); }); + test("round-trips diagnostics stamping on a case result", () => { + const report = parseEvalRunReport({ + version: 3, + provider: "xai", + model: "grok", + cases: [ + sampleResult({ + diagnostics: { + codexInstructionsHash: "abc123def456", + advertisedTools: ["read_file", "run_shell"], + reasoningEffort: "high", + }, + }), + ], + }); + expect(report.cases[0]!.diagnostics).toEqual({ + codexInstructionsHash: "abc123def456", + advertisedTools: ["read_file", "run_shell"], + reasoningEffort: "high", + }); + }); + + test("legacy reports with no diagnostics parse to null", () => { + const report = parseEvalRunReport({ + version: 3, + provider: "xai", + model: "grok", + cases: [sampleResult()], + }); + expect(report.cases[0]!.diagnostics).toBeNull(); + }); + test("legacy reports default repeat 0 and null behaviors", () => { const report = parseEvalRunReport({ version: 2, diff --git a/evals/capability/lib.ts b/evals/capability/lib.ts index be7df8d70..8c32dbc47 100644 --- a/evals/capability/lib.ts +++ b/evals/capability/lib.ts @@ -134,9 +134,19 @@ export type CaseResult = { behaviors: BehaviorMetrics | null; /** Set when the resolved provider/model differed from what was requested. */ providerFallback: ProviderFallbackInfo | null; + /** Per-cell diagnostics for debugging eval failures; null when unavailable. */ + diagnostics: EvalDiagnostics | null; textPreview?: string; }; +export type EvalDiagnostics = { + /** Short identity (hash) of the pinned Codex instructions text in use; null for non-Codex providers. */ + codexInstructionsHash: string | null; + /** Built-in tool names advertised to the model for this run. */ + advertisedTools: readonly string[]; + reasoningEffort: string | null; +}; + export type MetricStats = { min: number; median: number; @@ -726,10 +736,23 @@ function parseCaseResult(raw: unknown): CaseResult { : 0, behaviors: parseBehaviorMetrics(raw.behaviors), providerFallback: parseProviderFallback(raw.providerFallback), + diagnostics: parseEvalDiagnostics(raw.diagnostics), ...(typeof raw.textPreview === "string" ? { textPreview: raw.textPreview } : {}), }; } +function parseEvalDiagnostics(raw: unknown): EvalDiagnostics | null { + if (!isRecord(raw)) return null; + if (!Array.isArray(raw.advertisedTools)) return null; + const advertisedTools = raw.advertisedTools.filter((t): t is string => typeof t === "string"); + return { + codexInstructionsHash: + typeof raw.codexInstructionsHash === "string" ? raw.codexInstructionsHash : null, + advertisedTools, + reasoningEffort: typeof raw.reasoningEffort === "string" ? raw.reasoningEffort : null, + }; +} + function parseProviderFallback(raw: unknown): ProviderFallbackInfo | null { if (!isRecord(raw)) return null; const resolvedProvider = raw.resolvedProvider; diff --git a/scripts/eval-capability.test.ts b/scripts/eval-capability.test.ts index 1f8460a54..51581ceaf 100644 --- a/scripts/eval-capability.test.ts +++ b/scripts/eval-capability.test.ts @@ -5,10 +5,32 @@ import { join } from "node:path"; import { execFile } from "node:child_process"; import { promisify } from "node:util"; -import { initEvalGitRepo, mapPool, parseArgs } from "./eval-capability.ts"; +import { initEvalGitRepo, mapPool, parseArgs, buildEvalDiagnostics } from "./eval-capability.ts"; +import type { Config } from "../src/config/index.ts"; const execFileAsync = promisify(execFile); +function sampleConfig(over: Partial = {}): Config { + return { + configured: true, + apiKey: "key", + baseURL: "https://example.test", + model: "gpt-5", + providerName: "openai", + cwd: process.cwd(), + task: "do it", + force: true, + dangerouslySkipPermissions: true, + skipPermissionsFromSettings: false, + auto: false, + command: "exec", + globalSettingsPath: "/dev/null", + providers: [], + sessionId: "sess-1", + ...over, + } as Config; +} + describe("parseArgs", () => { const savedConcurrency = process.env.CORBITS_EVAL_CONCURRENCY; @@ -232,3 +254,30 @@ describe("initEvalGitRepo", () => { } }); }); + +describe("buildEvalDiagnostics", () => { + test("non-Codex provider gets a null instructions hash and the default orchestrator tool list", async () => { + const diagnostics = await buildEvalDiagnostics(sampleConfig({ providerName: "openai" })); + expect(diagnostics.codexInstructionsHash).toBeNull(); + expect(diagnostics.advertisedTools).toContain("read_file"); + expect(diagnostics.advertisedTools).toContain("run_shell"); + expect(diagnostics.reasoningEffort).toBeNull(); + }); + + test("Codex provider gets a non-null instructions hash", async () => { + const diagnostics = await buildEvalDiagnostics(sampleConfig({ providerName: "codex/default" })); + expect(diagnostics.codexInstructionsHash).toMatch(/^[0-9a-f]{12}$/); + }); + + test("echoes back the configured reasoning effort", async () => { + const diagnostics = await buildEvalDiagnostics(sampleConfig({ reasoningEffort: "high" })); + expect(diagnostics.reasoningEffort).toBe("high"); + }); + + test("--director build reports the director's own advertised allowlist", async () => { + const diagnostics = await buildEvalDiagnostics(sampleConfig({ director: "build" })); + expect(diagnostics.advertisedTools).not.toEqual( + (await buildEvalDiagnostics(sampleConfig({}))).advertisedTools, + ); + }); +}); diff --git a/scripts/eval-capability.ts b/scripts/eval-capability.ts index 5605ebaaa..c183e2c5b 100755 --- a/scripts/eval-capability.ts +++ b/scripts/eval-capability.ts @@ -14,9 +14,15 @@ import { tmpdir } from "node:os"; import { join, dirname, resolve } from "node:path"; import { fileURLToPath } from "node:url"; import { spawn } from "node:child_process"; -import { loadConfig } from "../src/config/index.js"; -import { runExec } from "../src/exec/runner.js"; +import { loadConfig, type Config } from "../src/config/index.js"; +import { runExec, resolveExecDirectorOverlay } from "../src/exec/runner.js"; import { SETTINGS_DIR_NAME } from "../src/branding.js"; +import { codexProfileFromProviderName } from "../src/config/codex-providers.js"; +import { codexInstructionsHash } from "../src/auth/codex/instructions.js"; +import { advertisedToolNamesForSessionMode } from "../src/agent/tool-search.js"; +import { detectLanguageServerAvailable } from "../src/agent/lsp-availability.js"; +import { resolveSessionMode } from "../src/config/session-mode.js"; +import { loadLocalSettings, localSettingsPath } from "../src/config/settings.js"; import { loadEvalCases, filterCases, @@ -41,6 +47,7 @@ import { type EvalVariant, type EvalTokenUsage, type ProviderFallbackInfo, + type EvalDiagnostics, } from "../evals/capability/lib.js"; import { deriveBehaviorMetrics, @@ -531,6 +538,34 @@ async function resolveVariantLabels( }; } +/** + * Per-cell diagnostics for debugging eval failures: which Codex instructions + * text was pinned, which built-in tools the model was offered, and the + * requested reasoning effort. Reuses the exec runner's own resolution + * (resolveSessionMode, resolveExecDirectorOverlay) rather than forking the + * logic, so a --director overlay or a non-default session mode here reports + * the same advertised list exec actually runs with. + * + * reasoningEffort echoes the configured value, not the provider's internal + * default when unset — accepted as-is per review. + */ +export async function buildEvalDiagnostics(config: Config): Promise { + const codexProfile = codexProfileFromProviderName(config.providerName); + const localSettings = await loadLocalSettings(localSettingsPath(config.cwd)).catch(() => null); + const sessionMode = resolveSessionMode(config.settings, localSettings) ?? "orchestrator"; + const overlay = resolveExecDirectorOverlay(config.director); + const advertisedTools = + overlay.advertisedAllow ?? + advertisedToolNamesForSessionMode(sessionMode, { + languageServerAvailable: detectLanguageServerAvailable(config.cwd), + }); + return { + codexInstructionsHash: codexProfile !== undefined ? codexInstructionsHash() : null, + advertisedTools, + reasoningEffort: config.reasoningEffort ?? null, + }; +} + function failResult( caseDef: EvalCase, variant: EvalVariant, @@ -568,6 +603,7 @@ function failResult( repeat, behaviors: null, providerFallback: null, + diagnostics: null, ...partial, }; } @@ -633,6 +669,7 @@ async function runCase( ); } + const diagnostics = await buildEvalDiagnostics(config); const agentStarted = Date.now(); // runExec runs the agent in-process (no child, unlike verify.sh below), so // the fixture origin must reach it via process.env directly for the @@ -770,6 +807,7 @@ async function runCase( repeat, behaviors, providerFallback, + diagnostics, textPreview, }; } catch (err) { diff --git a/src/auth/codex/instructions.test.ts b/src/auth/codex/instructions.test.ts new file mode 100644 index 000000000..63a798b79 --- /dev/null +++ b/src/auth/codex/instructions.test.ts @@ -0,0 +1,119 @@ +import { afterEach, beforeEach, describe, expect, mock, test } from "bun:test"; + +// Reused by both the TUI and exec boot paths (src/tui/runner.ts, +// src/exec/runner.ts) to refresh the pinned Codex instructions before first +// Codex inference. This exercises the shared refresh/fallback logic directly, +// with disk I/O faked so tests never touch the real ~/.corbits cache. + +let fakeDisk = new Map(); + +mock.module("node:fs", () => ({ + readFileSync: (path: string) => { + const contents = fakeDisk.get(path); + if (contents === undefined) { + const err = new Error("ENOENT") as NodeJS.ErrnoException; + err.code = "ENOENT"; + throw err; + } + return contents; + }, + writeFileSync: (path: string, contents: string) => { + fakeDisk.set(path, contents); + }, + mkdirSync: () => undefined, +})); + +const { refreshCodexInstructions, codexInstructions, codexInstructionsHash } = await import( + "./instructions.js" +); +const { GPT_5_CODEX_PROMPT } = await import("./prompts/gpt-5-codex.js"); + +const VALID_PROMPT = `You are Codex${"x".repeat(1200)}`; + +function mockFetchSequence(tag: string, promptResponse: () => Response): typeof fetch { + return (async (input: RequestInfo | URL) => { + const url = String(input); + if (url.includes("releases/latest")) { + return new Response(JSON.stringify({ tag_name: tag }), { status: 200 }); + } + return promptResponse(); + }) as typeof fetch; +} + +describe("refreshCodexInstructions", () => { + const originalFetch = global.fetch; + + beforeEach(() => { + fakeDisk = new Map(); + }); + + afterEach(() => { + global.fetch = originalFetch; + }); + + test("bundled copy is used before any refresh", () => { + expect(codexInstructions()).toBe(GPT_5_CODEX_PROMPT); + }); + + test("updates in-memory instructions on a successful fetch", async () => { + global.fetch = mockFetchSequence("rust-v1.2.3", () => new Response(VALID_PROMPT, { status: 200 })); + + await refreshCodexInstructions(); + expect(codexInstructions()).toBe(VALID_PROMPT); + }); + + test("falls back without throwing the run when the release lookup network call fails", async () => { + const before = codexInstructions(); + global.fetch = (() => Promise.reject(new Error("network down"))) as unknown as typeof fetch; + + // The function itself rejects; callers (TUI/exec boot) catch this and + // keep running on cache/bundled instructions — see src/exec/runner.ts and + // src/tui/runner.ts refresh call sites. + await expect(refreshCodexInstructions()).rejects.toThrow("network down"); + expect(codexInstructions()).toBe(before); + }); + + test("falls back without throwing when the prompt fetch returns a non-200", async () => { + const before = codexInstructions(); + global.fetch = mockFetchSequence("rust-v1.2.3", () => new Response("not found", { status: 404 })); + + await expect(refreshCodexInstructions()).rejects.toThrow(/HTTP 404/); + expect(codexInstructions()).toBe(before); + }); + + test("rejects a 200 response whose body is not a valid Codex prompt (CDN error page)", async () => { + const before = codexInstructions(); + global.fetch = mockFetchSequence("rust-v1.2.3", () => new Response("oops", { status: 200 })); + + await expect(refreshCodexInstructions()).rejects.toThrow(/unexpected body/); + expect(codexInstructions()).toBe(before); + }); + + test("rejects within the timeout when a fetch never resolves (hung connection)", async () => { + const before = codexInstructions(); + global.fetch = ((_input: RequestInfo | URL, init?: RequestInit) => { + return new Promise((_resolve, reject) => { + const signal = init?.signal; + if (signal) { + signal.addEventListener("abort", () => reject(signal.reason as Error)); + } + }); + }) as unknown as typeof fetch; + + const started = Date.now(); + await expect(refreshCodexInstructions()).rejects.toBeTruthy(); + expect(Date.now() - started).toBeLessThan(15_000); + expect(codexInstructions()).toBe(before); + }, 20_000); + + test("codexInstructionsHash reflects the currently resolved instructions text", async () => { + const hashBefore = codexInstructionsHash(); + expect(hashBefore).toMatch(/^[0-9a-f]{12}$/); + + const otherPrompt = `You are Codex${"y".repeat(1200)}`; + global.fetch = mockFetchSequence("rust-v1.2.4", () => new Response(otherPrompt, { status: 200 })); + await refreshCodexInstructions(); + + expect(codexInstructionsHash()).not.toBe(hashBefore); + }); +}); diff --git a/src/auth/codex/instructions.ts b/src/auth/codex/instructions.ts index 2b8f385e3..e53c502c2 100644 --- a/src/auth/codex/instructions.ts +++ b/src/auth/codex/instructions.ts @@ -1,4 +1,5 @@ import { readFileSync, writeFileSync, mkdirSync } from "node:fs"; +import { createHash } from "node:crypto"; import { homedir } from "node:os"; import { join } from "node:path"; import { GPT_5_CODEX_PROMPT } from "./prompts/gpt-5-codex.js"; @@ -19,6 +20,10 @@ const PROMPT_PATH = "codex-rs/core/gpt_5_codex_prompt.md"; const PROMPT_SENTINEL = "You are Codex"; const MIN_PROMPT_LENGTH = 1000; +// Bounds both network calls below so a black-holed connection can never hang +// exec boot, which awaits refreshCodexInstructions before first inference. +const CODEX_INSTRUCTIONS_TIMEOUT_MS = 10_000; + export function isValidCodexPrompt(text: string): boolean { return text.length >= MIN_PROMPT_LENGTH && text.startsWith(PROMPT_SENTINEL); } @@ -42,8 +47,19 @@ export function codexInstructions(): string { return instructions; } +/** + * Short identity for the in-use instructions text, for eval/diagnostic + * records — never sent to the model. + */ +export function codexInstructionsHash(): string { + return createHash("sha256").update(codexInstructions()).digest("hex").slice(0, 12); +} + async function latestReleaseTag(): Promise { - const res = await fetch(RELEASES_LATEST, { headers: { accept: "application/vnd.github+json" } }); + const res = await fetch(RELEASES_LATEST, { + headers: { accept: "application/vnd.github+json" }, + signal: AbortSignal.timeout(CODEX_INSTRUCTIONS_TIMEOUT_MS), + }); if (!res.ok) throw new Error(`Codex release lookup failed (HTTP ${String(res.status)}).`); const data = (await res.json()) as { tag_name?: unknown }; if (typeof data.tag_name !== "string") throw new Error("Codex release lookup returned no tag."); @@ -52,7 +68,10 @@ async function latestReleaseTag(): Promise { export async function refreshCodexInstructions(): Promise { const tag = await latestReleaseTag(); - const res = await fetch(`https://raw.githubusercontent.com/openai/codex/${encodeURIComponent(tag)}/${PROMPT_PATH}`); + const res = await fetch( + `https://raw.githubusercontent.com/openai/codex/${encodeURIComponent(tag)}/${PROMPT_PATH}`, + { signal: AbortSignal.timeout(CODEX_INSTRUCTIONS_TIMEOUT_MS) }, + ); if (!res.ok) throw new Error(`Codex prompt fetch failed (HTTP ${String(res.status)}).`); const text = await res.text(); if (!isValidCodexPrompt(text)) throw new Error("Codex prompt fetch returned an unexpected body."); diff --git a/src/exec/runner.ts b/src/exec/runner.ts index e6fb4fec7..692bd6b97 100644 --- a/src/exec/runner.ts +++ b/src/exec/runner.ts @@ -32,6 +32,7 @@ import { DIRECTOR_REGISTRY } from "../agent/directors/registry.js"; import type { DirectorId } from "../agent/directors/types.js"; import { createInferenceDependencies } from "../provider/inference-dependencies.js"; import { getValidCodexToken } from "../auth/codex/session.js"; +import { refreshCodexInstructions } from "../auth/codex/instructions.js"; import { getValidXaiToken } from "../auth/xai/session.js"; import { seedPricingMetadataFromCache } from "../cost/pricing-metadata.js"; import { defaultPricingCachePath } from "../cost/pricing-fetcher.js"; @@ -572,6 +573,18 @@ export async function runExec(config: Config): Promise { liveSources[0] ?? buildInitialSourceFallback(); + // Refresh pinned Codex instructions before first inference, same as the + // TUI path. Best-effort: a network failure falls back to the disk cache + // or bundled copy without failing the run. Exec is one-shot (no long-lived + // session to catch up later), so this is awaited rather than fire-and-forget. + if (initialCodexProfile !== undefined) { + await refreshCodexInstructions().catch((err: unknown) => { + logger.warn("Codex instructions refresh failed: {error}", { + error: formatCaughtError(err), + }); + }); + } + // Refresh OAuth tokens before first inference when starting on codex/xai. if (initialCodexProfile !== undefined) { const { access } = await getValidCodexToken(initialCodexProfile);