Refresh Codex instructions on exec boot and record eval diagnostics

evals/capability/lib.test.ts

@@ -71,6 +71,7 @@ function sampleResult(over: Partial<CaseResult> = {}): CaseResult {
     repeat: over.repeat ?? 0,
     behaviors: over.behaviors ?? null,
     providerFallback: over.providerFallback ?? null,
+    diagnostics: over.diagnostics ?? null,
   };
 }
 
@@ -755,6 +756,38 @@ describe("parseEvalRunReport", () => {
     });
   });
 
+  test("round-trips diagnostics stamping on a case result", () => {
+    const report = parseEvalRunReport({
+      version: 3,
+      provider: "xai",
+      model: "grok",
+      cases: [
+        sampleResult({
+          diagnostics: {
+            codexInstructionsHash: "abc123def456",
+            advertisedTools: ["read_file", "run_shell"],
+            reasoningEffort: "high",
+          },
+        }),
+      ],
+    });
+    expect(report.cases[0]!.diagnostics).toEqual({
+      codexInstructionsHash: "abc123def456",
+      advertisedTools: ["read_file", "run_shell"],
+      reasoningEffort: "high",
+    });
+  });
+
+  test("legacy reports with no diagnostics parse to null", () => {
+    const report = parseEvalRunReport({
+      version: 3,
+      provider: "xai",
+      model: "grok",
+      cases: [sampleResult()],
+    });
+    expect(report.cases[0]!.diagnostics).toBeNull();
+  });
+
   test("legacy reports default repeat 0 and null behaviors", () => {
     const report = parseEvalRunReport({
       version: 2,

evals/capability/lib.ts

@@ -134,9 +134,19 @@ export type CaseResult = {
   behaviors: BehaviorMetrics | null;
   /** Set when the resolved provider/model differed from what was requested. */
   providerFallback: ProviderFallbackInfo | null;
+  /** Per-cell diagnostics for debugging eval failures; null when unavailable. */
+  diagnostics: EvalDiagnostics | null;
   textPreview?: string;
 };
 
+export type EvalDiagnostics = {
+  /** Short identity (hash) of the pinned Codex instructions text in use; null for non-Codex providers. */
+  codexInstructionsHash: string | null;
+  /** Built-in tool names advertised to the model for this run. */
+  advertisedTools: readonly string[];
+  reasoningEffort: string | null;
+};
+
 export type MetricStats = {
   min: number;
   median: number;
@@ -726,10 +736,23 @@ function parseCaseResult(raw: unknown): CaseResult {
         : 0,
     behaviors: parseBehaviorMetrics(raw.behaviors),
     providerFallback: parseProviderFallback(raw.providerFallback),
+    diagnostics: parseEvalDiagnostics(raw.diagnostics),
     ...(typeof raw.textPreview === "string" ? { textPreview: raw.textPreview } : {}),
   };
 }
 
+function parseEvalDiagnostics(raw: unknown): EvalDiagnostics | null {
+  if (!isRecord(raw)) return null;
+  if (!Array.isArray(raw.advertisedTools)) return null;
+  const advertisedTools = raw.advertisedTools.filter((t): t is string => typeof t === "string");
+  return {
+    codexInstructionsHash:
+      typeof raw.codexInstructionsHash === "string" ? raw.codexInstructionsHash : null,
+    advertisedTools,
+    reasoningEffort: typeof raw.reasoningEffort === "string" ? raw.reasoningEffort : null,
+  };
+}
+
 function parseProviderFallback(raw: unknown): ProviderFallbackInfo | null {
   if (!isRecord(raw)) return null;
   const resolvedProvider = raw.resolvedProvider;

scripts/eval-capability.test.ts

@@ -5,10 +5,32 @@ import { join } from "node:path";
 import { execFile } from "node:child_process";
 import { promisify } from "node:util";
 
-import { initEvalGitRepo, mapPool, parseArgs } from "./eval-capability.ts";
+import { initEvalGitRepo, mapPool, parseArgs, buildEvalDiagnostics } from "./eval-capability.ts";
+import type { Config } from "../src/config/index.ts";
 
 const execFileAsync = promisify(execFile);
 
+function sampleConfig(over: Partial<Config> = {}): Config {
+  return {
+    configured: true,
+    apiKey: "key",
+    baseURL: "https://example.test",
+    model: "gpt-5",
+    providerName: "openai",
+    cwd: process.cwd(),
+    task: "do it",
+    force: true,
+    dangerouslySkipPermissions: true,
+    skipPermissionsFromSettings: false,
+    auto: false,
+    command: "exec",
+    globalSettingsPath: "/dev/null",
+    providers: [],
+    sessionId: "sess-1",
+    ...over,
+  } as Config;
+}
+
 describe("parseArgs", () => {
   const savedConcurrency = process.env.CORBITS_EVAL_CONCURRENCY;
 
@@ -232,3 +254,30 @@ describe("initEvalGitRepo", () => {
     }
   });
 });
+
+describe("buildEvalDiagnostics", () => {
+  test("non-Codex provider gets a null instructions hash and the default orchestrator tool list", async () => {
+    const diagnostics = await buildEvalDiagnostics(sampleConfig({ providerName: "openai" }));
+    expect(diagnostics.codexInstructionsHash).toBeNull();
+    expect(diagnostics.advertisedTools).toContain("read_file");
+    expect(diagnostics.advertisedTools).toContain("run_shell");
+    expect(diagnostics.reasoningEffort).toBeNull();
+  });
+
+  test("Codex provider gets a non-null instructions hash", async () => {
+    const diagnostics = await buildEvalDiagnostics(sampleConfig({ providerName: "codex/default" }));
+    expect(diagnostics.codexInstructionsHash).toMatch(/^[0-9a-f]{12}$/);
+  });
+
+  test("echoes back the configured reasoning effort", async () => {
+    const diagnostics = await buildEvalDiagnostics(sampleConfig({ reasoningEffort: "high" }));
+    expect(diagnostics.reasoningEffort).toBe("high");
+  });
+
+  test("--director build reports the director's own advertised allowlist", async () => {
+    const diagnostics = await buildEvalDiagnostics(sampleConfig({ director: "build" }));
+    expect(diagnostics.advertisedTools).not.toEqual(
+      (await buildEvalDiagnostics(sampleConfig({}))).advertisedTools,
+    );
+  });
+});

scripts/eval-capability.ts

@@ -14,9 +14,15 @@ import { tmpdir } from "node:os";
 import { join, dirname, resolve } from "node:path";
 import { fileURLToPath } from "node:url";
 import { spawn } from "node:child_process";
-import { loadConfig } from "../src/config/index.js";
-import { runExec } from "../src/exec/runner.js";
+import { loadConfig, type Config } from "../src/config/index.js";
+import { runExec, resolveExecDirectorOverlay } from "../src/exec/runner.js";
 import { SETTINGS_DIR_NAME } from "../src/branding.js";
+import { codexProfileFromProviderName } from "../src/config/codex-providers.js";
+import { codexInstructionsHash } from "../src/auth/codex/instructions.js";
+import { advertisedToolNamesForSessionMode } from "../src/agent/tool-search.js";
+import { detectLanguageServerAvailable } from "../src/agent/lsp-availability.js";
+import { resolveSessionMode } from "../src/config/session-mode.js";
+import { loadLocalSettings, localSettingsPath } from "../src/config/settings.js";
 import {
   loadEvalCases,
   filterCases,
@@ -41,6 +47,7 @@ import {
   type EvalVariant,
   type EvalTokenUsage,
   type ProviderFallbackInfo,
+  type EvalDiagnostics,
 } from "../evals/capability/lib.js";
 import {
   deriveBehaviorMetrics,
@@ -531,6 +538,34 @@ async function resolveVariantLabels(
   };
 }
 
+/**
+ * Per-cell diagnostics for debugging eval failures: which Codex instructions
+ * text was pinned, which built-in tools the model was offered, and the
+ * requested reasoning effort. Reuses the exec runner's own resolution
+ * (resolveSessionMode, resolveExecDirectorOverlay) rather than forking the
+ * logic, so a --director overlay or a non-default session mode here reports
+ * the same advertised list exec actually runs with.
+ *
+ * reasoningEffort echoes the configured value, not the provider's internal
+ * default when unset — accepted as-is per review.
+ */
+export async function buildEvalDiagnostics(config: Config): Promise<EvalDiagnostics> {
+  const codexProfile = codexProfileFromProviderName(config.providerName);
+  const localSettings = await loadLocalSettings(localSettingsPath(config.cwd)).catch(() => null);
+  const sessionMode = resolveSessionMode(config.settings, localSettings) ?? "orchestrator";
+  const overlay = resolveExecDirectorOverlay(config.director);
+  const advertisedTools =
+    overlay.advertisedAllow ??
+    advertisedToolNamesForSessionMode(sessionMode, {
+      languageServerAvailable: detectLanguageServerAvailable(config.cwd),
+    });
+  return {
+    codexInstructionsHash: codexProfile !== undefined ? codexInstructionsHash() : null,
+    advertisedTools,
+    reasoningEffort: config.reasoningEffort ?? null,
+  };
+}
+
 function failResult(
   caseDef: EvalCase,
   variant: EvalVariant,
@@ -568,6 +603,7 @@ function failResult(
     repeat,
     behaviors: null,
     providerFallback: null,
+    diagnostics: null,
     ...partial,
   };
 }
@@ -633,6 +669,7 @@ async function runCase(
       );
     }
 
+    const diagnostics = await buildEvalDiagnostics(config);
     const agentStarted = Date.now();
     // runExec runs the agent in-process (no child, unlike verify.sh below), so
     // the fixture origin must reach it via process.env directly for the
@@ -770,6 +807,7 @@ async function runCase(
       repeat,
       behaviors,
       providerFallback,
+      diagnostics,
       textPreview,
     };
   } catch (err) {

src/auth/codex/instructions.test.ts

@@ -0,0 +1,119 @@
+import { afterEach, beforeEach, describe, expect, mock, test } from "bun:test";
+
+// Reused by both the TUI and exec boot paths (src/tui/runner.ts,
+// src/exec/runner.ts) to refresh the pinned Codex instructions before first
+// Codex inference. This exercises the shared refresh/fallback logic directly,
+// with disk I/O faked so tests never touch the real ~/.corbits cache.
+
+let fakeDisk = new Map<string, string>();
+
+mock.module("node:fs", () => ({
+  readFileSync: (path: string) => {
+    const contents = fakeDisk.get(path);
+    if (contents === undefined) {
+      const err = new Error("ENOENT") as NodeJS.ErrnoException;
+      err.code = "ENOENT";
+      throw err;
+    }
+    return contents;
+  },
+  writeFileSync: (path: string, contents: string) => {
+    fakeDisk.set(path, contents);
+  },
+  mkdirSync: () => undefined,
+}));
+
+const { refreshCodexInstructions, codexInstructions, codexInstructionsHash } = await import(
+  "./instructions.js"
+);
+const { GPT_5_CODEX_PROMPT } = await import("./prompts/gpt-5-codex.js");
+
+const VALID_PROMPT = `You are Codex${"x".repeat(1200)}`;
+
+function mockFetchSequence(tag: string, promptResponse: () => Response): typeof fetch {
+  return (async (input: RequestInfo | URL) => {
+    const url = String(input);
+    if (url.includes("releases/latest")) {
+      return new Response(JSON.stringify({ tag_name: tag }), { status: 200 });
+    }
+    return promptResponse();
+  }) as typeof fetch;
+}
+
+describe("refreshCodexInstructions", () => {
+  const originalFetch = global.fetch;
+
+  beforeEach(() => {
+    fakeDisk = new Map();
+  });
+
+  afterEach(() => {
+    global.fetch = originalFetch;
+  });
+
+  test("bundled copy is used before any refresh", () => {
+    expect(codexInstructions()).toBe(GPT_5_CODEX_PROMPT);
+  });
+
+  test("updates in-memory instructions on a successful fetch", async () => {
+    global.fetch = mockFetchSequence("rust-v1.2.3", () => new Response(VALID_PROMPT, { status: 200 }));
+
+    await refreshCodexInstructions();
+    expect(codexInstructions()).toBe(VALID_PROMPT);
+  });
+
+  test("falls back without throwing the run when the release lookup network call fails", async () => {
+    const before = codexInstructions();
+    global.fetch = (() => Promise.reject(new Error("network down"))) as unknown as typeof fetch;
+
+    // The function itself rejects; callers (TUI/exec boot) catch this and
+    // keep running on cache/bundled instructions — see src/exec/runner.ts and
+    // src/tui/runner.ts refresh call sites.
+    await expect(refreshCodexInstructions()).rejects.toThrow("network down");
+    expect(codexInstructions()).toBe(before);
+  });
+
+  test("falls back without throwing when the prompt fetch returns a non-200", async () => {
+    const before = codexInstructions();
+    global.fetch = mockFetchSequence("rust-v1.2.3", () => new Response("not found", { status: 404 }));
+
+    await expect(refreshCodexInstructions()).rejects.toThrow(/HTTP 404/);
+    expect(codexInstructions()).toBe(before);
+  });
+
+  test("rejects a 200 response whose body is not a valid Codex prompt (CDN error page)", async () => {
+    const before = codexInstructions();
+    global.fetch = mockFetchSequence("rust-v1.2.3", () => new Response("<html>oops</html>", { status: 200 }));
+
+    await expect(refreshCodexInstructions()).rejects.toThrow(/unexpected body/);
+    expect(codexInstructions()).toBe(before);
+  });
+
+  test("rejects within the timeout when a fetch never resolves (hung connection)", async () => {
+    const before = codexInstructions();
+    global.fetch = ((_input: RequestInfo | URL, init?: RequestInit) => {
+      return new Promise((_resolve, reject) => {
+        const signal = init?.signal;
+        if (signal) {
+          signal.addEventListener("abort", () => reject(signal.reason as Error));
+        }
+      });
+    }) as unknown as typeof fetch;
+
+    const started = Date.now();
+    await expect(refreshCodexInstructions()).rejects.toBeTruthy();
+    expect(Date.now() - started).toBeLessThan(15_000);
+    expect(codexInstructions()).toBe(before);
+  }, 20_000);
+
+  test("codexInstructionsHash reflects the currently resolved instructions text", async () => {
+    const hashBefore = codexInstructionsHash();
+    expect(hashBefore).toMatch(/^[0-9a-f]{12}$/);
+
+    const otherPrompt = `You are Codex${"y".repeat(1200)}`;
+    global.fetch = mockFetchSequence("rust-v1.2.4", () => new Response(otherPrompt, { status: 200 }));
+    await refreshCodexInstructions();
+
+    expect(codexInstructionsHash()).not.toBe(hashBefore);
+  });
+});