From a49c1d13cc679d08976dbdb38122d31a8093a3f8 Mon Sep 17 00:00:00 2001 From: Sawyer Cutler Date: Sun, 23 Aug 2026 16:18:37 -0700 Subject: [PATCH 1/2] Add --effort flag to eval-capability, fix --config auth bypass --effort threads reasoning effort through to the exec runner by writing it into the fixture workdir's local settings, the same path an interactive session uses. Rejects an effort/model pairing the target model does not accept via supportedEfforts, before any inference runs. Matrix cells can carry their own effort (provider:model:effort) alongside the run-level --effort default. Each result row records the effort used. --config was silently dropping the OAuth profile catalog (codex/xai credentials live in home-level auth stores, separate from settings.json), so any codex/xai run through --config reached the provider unauthenticated (HTTP 426/404 before a single turn). Fixed by merging OAuth profiles regardless of --config; only the programmatic globalSettingsPath test override (not a CLI flag) still opts out, for full test isolation. --- evals/capability/lib.test.ts | 36 ++++++++++++++ evals/capability/lib.ts | 33 ++++++++++--- scripts/eval-capability.test.ts | 16 +++++++ scripts/eval-capability.ts | 83 +++++++++++++++++++++++++++++++-- src/config/index.ts | 15 +++--- tests/unit/config.test.ts | 80 ++++++++++++++++++++++++++++--- 6 files changed, 240 insertions(+), 23 deletions(-) diff --git a/evals/capability/lib.test.ts b/evals/capability/lib.test.ts index 8dfa730be..31835e3a0 100644 --- a/evals/capability/lib.test.ts +++ b/evals/capability/lib.test.ts @@ -78,6 +78,7 @@ function sampleResult(over: Partial = {}): CaseResult { behaviors: over.behaviors ?? null, providerFallback: over.providerFallback ?? null, diagnostics: over.diagnostics ?? null, + effort: over.effort ?? null, }; } @@ -327,6 +328,41 @@ describe("parseMatrix", () => { const v = parseMatrix("xai:", { model: "grok-4.5" }); expect(v[0]).toEqual({ id: "xai:grok-4.5", provider: "xai", model: "grok-4.5" }); }); + + test("parses a third colon segment as effort", () => { + const v = parseMatrix("xai/thegreataxios:grok-4.6:xhigh", {}); + expect(v[0]).toEqual({ + id: "xai/thegreataxios:grok-4.6", + provider: "xai/thegreataxios", + model: "grok-4.6", + effort: "xhigh", + }); + }); + + test("labeled cell can also carry an effort segment", () => { + const v = parseMatrix("fast=xai:grok-4.6:high", {}); + expect(v[0]).toEqual({ id: "fast", provider: "xai", model: "grok-4.6", effort: "high" }); + }); + + test("a trailing segment that is not a real effort literal falls through to the model", () => { + // "grok-4.6:not-an-effort" has no valid effort literal in the third slot, + // so the whole thing after the first colon is the model id. + const v = parseMatrix("xai:grok-4.6:not-an-effort", {}); + expect(v[0]!.provider).toBe("xai"); + expect(v[0]!.model).toBe("grok-4.6:not-an-effort"); + expect(v[0]!.effort).toBeUndefined(); + }); + + test("--effort fallback applies to a cell that doesn't specify its own", () => { + const v = parseMatrix("xai:grok-4.6,openai:gpt-5", { effort: "medium" }); + expect(v[0]!.effort).toBe("medium"); + expect(v[1]!.effort).toBe("medium"); + }); + + test("a cell's own effort wins over the --effort fallback", () => { + const v = parseMatrix("xai:grok-4.6:high", { effort: "medium" }); + expect(v[0]!.effort).toBe("high"); + }); }); describe("expandMatrix", () => { diff --git a/evals/capability/lib.ts b/evals/capability/lib.ts index 04099920f..47149a2ca 100644 --- a/evals/capability/lib.ts +++ b/evals/capability/lib.ts @@ -6,6 +6,7 @@ import { readdir, readFile, stat } from "node:fs/promises"; import { join, resolve } from "node:path"; import { runWithEvalHttpEnv, evalHttpEnvGet } from "../../src/tools/eval-http-env.js"; +import { isReasoningEffort, type ReasoningEffort } from "../../src/provider/reasoning-effort.js"; import { isNumericBehaviorMetric, parseBehaviorMetrics, @@ -100,6 +101,8 @@ export interface EvalVariant { id: string; provider?: string; model?: string; + /** Reasoning effort for this cell; overrides the run-level --effort. */ + effort?: ReasoningEffort; } /** @@ -149,6 +152,8 @@ export interface CaseResult { providerFallback: ProviderFallbackInfo | null; /** Per-cell diagnostics for debugging eval failures; null when unavailable. */ diagnostics: EvalDiagnostics | null; + /** Requested reasoning effort for this cell (--effort or matrix cell); null when unset. */ + effort: ReasoningEffort | null; textPreview?: string; } @@ -538,15 +543,17 @@ export function defaultVariantId(provider?: string, model?: string): string { * Parse a matrix string of variants. * Formats (comma-separated cells): * - `provider:model` + * - `provider:model:effort` * - `provider/model` (slash only when no colon) - * - `label=provider:model` - * Empty / omitted → single default variant (caller provider/model flags). + * - `label=provider:model[:effort]` + * Empty / omitted → single default variant (caller provider/model/effort flags). * Each expanded cell must have both provider and model (after applying - * `--provider`/`--model` as cell defaults when a side is omitted). + * `--provider`/`--model` as cell defaults when a side is omitted); effort + * falls back to `--effort` when the cell does not specify its own. */ export function parseMatrix( matrix: string | undefined, - fallback: { provider?: string; model?: string }, + fallback: { provider?: string; model?: string; effort?: ReasoningEffort }, ): EvalVariant[] { if (matrix === undefined || matrix.trim().length === 0) { const id = defaultVariantId(fallback.provider, fallback.model); @@ -555,6 +562,7 @@ export function parseMatrix( id, ...(fallback.provider !== undefined ? { provider: fallback.provider } : {}), ...(fallback.model !== undefined ? { model: fallback.model } : {}), + ...(fallback.effort !== undefined ? { effort: fallback.effort } : {}), }, ]; } @@ -571,7 +579,7 @@ export function parseMatrix( function parseMatrixCell( cell: string, index: number, - fallback: { provider?: string; model?: string }, + fallback: { provider?: string; model?: string; effort?: ReasoningEffort }, ): EvalVariant { let label: string | undefined; let rest = cell; @@ -582,8 +590,17 @@ function parseMatrixCell( } let provider: string | undefined; let model: string | undefined; + let effort: ReasoningEffort | undefined; if (rest.includes(":")) { - const [p, ...mParts] = rest.split(":"); + const parts = rest.split(":"); + // provider:model or provider:model:effort — the last segment is treated + // as effort only when it parses as a real reasoning-effort literal, so a + // model id that happens to contain a colon still falls through cleanly. + if (parts.length >= 3 && isReasoningEffort(parts.at(-1)!.trim())) { + effort = parts.at(-1)!.trim() as ReasoningEffort; + parts.pop(); + } + const [p, ...mParts] = parts; provider = p!.trim() || undefined; model = mParts.join(":").trim() || undefined; } else if (rest.includes("/")) { @@ -597,11 +614,12 @@ function parseMatrixCell( } provider = provider ?? fallback.provider; model = model ?? fallback.model; + effort = effort ?? fallback.effort; if (provider === undefined || model === undefined) { throw new Error(`matrix cell ${index + 1} "${cell}" must specify both provider and model`); } const id = label ?? defaultVariantId(provider, model); - return { id, provider, model }; + return { id, provider, model, ...(effort !== undefined ? { effort } : {}) }; } /** Cartesian product of cases × variants (cases outer for stable progress). */ @@ -744,6 +762,7 @@ function parseCaseResult(raw: unknown): CaseResult { behaviors: parseBehaviorMetrics(raw.behaviors), providerFallback: parseProviderFallback(raw.providerFallback), diagnostics: parseEvalDiagnostics(raw.diagnostics), + effort: isReasoningEffort(raw.effort) ? raw.effort : null, ...(typeof raw.textPreview === "string" ? { textPreview: raw.textPreview } : {}), }; } diff --git a/scripts/eval-capability.test.ts b/scripts/eval-capability.test.ts index 946cca158..ad94e5be3 100644 --- a/scripts/eval-capability.test.ts +++ b/scripts/eval-capability.test.ts @@ -91,6 +91,22 @@ describe("parseArgs", () => { expect(() => parseArgs(["--matrix", ":grok-4.5"])).toThrow(/both provider and model/); }); + test("--effort accepts a canonical literal", () => { + const opts = parseArgs(["--provider", "foo", "--model", "bar", "--effort", "high"]); + expect(opts.effort).toBe("high"); + }); + + test("--effort rejects an unknown literal", () => { + expect(() => parseArgs(["--provider", "foo", "--model", "bar", "--effort", "bogus"])).toThrow( + /--effort must be one of/, + ); + }); + + test("--matrix cell can carry its own effort as a third segment", () => { + const opts = parseArgs(["--matrix", "xai/thegreataxios:grok-4.6:xhigh"]); + expect(opts.matrix).toBe("xai/thegreataxios:grok-4.6:xhigh"); + }); + test("parsed defaults never equal xai/thegreataxios", () => { const help = parseArgs(["--help"]); const pair = parseArgs(["--provider", "foo", "--model", "bar"]); diff --git a/scripts/eval-capability.ts b/scripts/eval-capability.ts index 6bc00dde0..0313179a1 100755 --- a/scripts/eval-capability.ts +++ b/scripts/eval-capability.ts @@ -17,7 +17,16 @@ import { spawn } from "node:child_process"; import { loadConfig, type Config } from "../src/config/index.js"; import { runExec, resolveExecDirectorOverlay } from "../src/exec/runner.js"; import { SETTINGS_DIR_NAME } from "../src/branding.js"; -import { codexProfileFromProviderName } from "../src/config/codex-providers.js"; +import { + codexProfileFromProviderName, + isCodexProviderName, +} from "../src/config/codex-providers.js"; +import { + REASONING_EFFORTS, + isReasoningEffort, + validateEffort, + type ReasoningEffort, +} from "../src/provider/reasoning-effort.js"; import { codexInstructionsHash } from "../src/auth/codex/instructions.js"; import { advertisedToolNamesForSessionMode } from "../src/agent/tool-search.js"; import { detectLanguageServerAvailable } from "../src/agent/lsp-availability.js"; @@ -64,6 +73,8 @@ interface CliOptions { model?: string; /** Comma-separated matrix of provider:model cells. */ matrix?: string; + /** Reasoning effort applied to every variant that does not specify its own. */ + effort?: ReasoningEffort; configPath?: string; outPath?: string; baselinePath?: string; @@ -98,7 +109,11 @@ function printUsage(): void { --case Case id (default: all) --provider Provider name (required except --help, or --matrix with complete cells) --model Model id (required except --help, or --matrix with complete cells) - --matrix Multi-variant: "p1:m1,p2:m2" or "label=p:m,..."; each cell needs both sides + --matrix Multi-variant: "p1:m1,p2:m2" or "label=p:m[:effort],..."; each cell needs + both provider and model; effort is optional per cell + --effort Reasoning effort for variants that don't set their own + (${REASONING_EFFORTS.join("|")}); rejected when the + target model does not support it --config Settings file override --out Write results JSON --baseline Compare to prior results JSON @@ -195,6 +210,14 @@ export function parseArgs(argv: readonly string[]): CliOptions { case "--matrix": opts.matrix = next(); break; + case "--effort": { + const v = next(); + if (!isReasoningEffort(v)) { + throw new Error(`--effort must be one of: ${REASONING_EFFORTS.join(", ")}`); + } + opts.effort = v; + break; + } case "--config": opts.configPath = next(); break; @@ -260,10 +283,15 @@ export function parseArgs(argv: readonly string[]): CliOptions { // exactOptionalPropertyTypes forbids passing an explicit `undefined` for an // optional field, so build the fallback object with the key present only // when the CLI option was actually given. -function providerModelFallback(opts: CliOptions): { provider?: string; model?: string } { +function providerModelFallback(opts: CliOptions): { + provider?: string; + model?: string; + effort?: ReasoningEffort; +} { return { ...(opts.provider !== undefined ? { provider: opts.provider } : {}), ...(opts.model !== undefined ? { model: opts.model } : {}), + ...(opts.effort !== undefined ? { effort: opts.effort } : {}), }; } @@ -548,6 +576,51 @@ async function resolveVariantLabels( }; } +/** + * Fail fast, before any inference runs, when a variant's requested reasoning + * effort is not one the resolved model accepts. Per-model rungs genuinely + * differ (grok-4.6 takes xhigh, grok-composer-2.5-fast does not; the + * gpt-5.6 family also takes max/ultra) — silently running at the provider's + * default instead would poison a matrix without anyone noticing. + */ +async function validateVariantEfforts( + variants: readonly EvalVariant[], + opts: CliOptions, +): Promise { + for (const variant of variants) { + if (variant.effort === undefined) continue; + const labels = await resolveVariantLabels(variant, opts); + const isCodex = isCodexProviderName(labels.provider); + const verdict = validateEffort(labels.model, variant.effort, isCodex); + if (!verdict.ok) { + throw new Error( + `variant "${variant.id}" (${labels.provider}/${labels.model}): ${verdict.error}`, + ); + } + } +} + +/** + * Write the requested reasoning effort into the fixture workdir's local + * settings so the product path (loadConfig -> local settings -> Config) + * picks it up the same way an interactive session would — without ever + * touching the operator's real ~/.corbits/settings.json. + */ +async function applyEvalEffort( + workdir: string, + effort: ReasoningEffort | undefined, +): Promise { + if (effort === undefined) return; + const path = localSettingsPath(workdir); + const existing = await loadLocalSettings(path).catch(() => null); + await mkdir(dirname(path), { recursive: true }); + await writeFile( + path, + `${JSON.stringify({ ...existing, reasoningEffort: effort }, null, 2)}\n`, + "utf8", + ); +} + /** * Per-cell diagnostics for debugging eval failures: which Codex instructions * text was pinned, which built-in tools the model was offered, and the @@ -614,6 +687,7 @@ function failResult( behaviors: null, providerFallback: null, diagnostics: null, + effort: variant.effort ?? null, ...partial, }; } @@ -634,6 +708,7 @@ async function runCase( const prepared = await prepareWorkdir(caseDef); workdir = prepared.workdir; capturePath = prepared.capturePath; + await applyEvalEffort(workdir, variant.effort); console.log( `\n=== ${variant.id} × ${caseDef.id} (${caseDef.tier})` + ` [repeat ${repeat + 1}/${opts.repeats}] — ${caseDef.title}`, @@ -819,6 +894,7 @@ async function runCase( behaviors, providerFallback, diagnostics, + effort: variant.effort ?? null, textPreview, }; } catch (err) { @@ -860,6 +936,7 @@ async function main(): Promise { const all = await loadEvalCases(CASES_ROOT); const selected = filterCases(all, opts.caseSelector); const variants = parseMatrix(opts.matrix, providerModelFallback(opts)); + await validateVariantEfforts(variants, opts); const plan = expandMatrix(selected, variants); console.log( diff --git a/src/config/index.ts b/src/config/index.ts index 7bef84c2b..843ba7797 100644 --- a/src/config/index.ts +++ b/src/config/index.ts @@ -626,12 +626,15 @@ export async function loadConfig( dangerouslySkipPermissions = dangerouslySkipPermissions || settings?.dangerouslySkipPermissions === true; - // OAuth profiles live in home-level auth stores, not in settings files. They - // are merged in only for the real default settings path: an explicit --config - // or test override selects a controlled provider set that should not pull in - // home credentials. Profiles surface as "provider/" so selection and the - // picker treat them like any other provider. - const useOAuthProfiles = configPath === undefined && options.globalSettingsPath === undefined; + // OAuth profiles live in home-level auth stores (~/.corbits/codex-auth.json, + // xai-auth.json), entirely separate from settings.json. --config only + // overrides where provider *definitions* come from, so it must still merge + // in OAuth profiles or every codex/xai OAuth run through --config reaches + // the provider unauthenticated (CL-6973: HTTP 426/404 before a single + // turn). Only the programmatic `globalSettingsPath` test override — never + // exposed as a CLI flag — opts out, for tests that want a fully controlled + // provider set with no home-directory reads at all. + const useOAuthProfiles = options.globalSettingsPath === undefined; const [codexProfiles, xaiProfiles]: [CodexProfile[], XaiProfile[]] = useOAuthProfiles ? await Promise.all([listCodexProfiles(), listXaiProfiles()]) : [[], []]; diff --git a/tests/unit/config.test.ts b/tests/unit/config.test.ts index 2a54d1963..7385600b5 100644 --- a/tests/unit/config.test.ts +++ b/tests/unit/config.test.ts @@ -196,13 +196,14 @@ test("loadSettings cannot silently drop a known optional key", async () => { // settings.json — home-level auth stores are the source of truth, and // loadConfig merges them into the catalog it hands to resolveProvider (see // "OAuth profiles live in home-level auth stores" in src/config/index.ts). -// A caller that wants that merge to happen (the eval runner's per-case -// probe, same as any interactive run) must pass neither --config nor a -// globalSettingsPath override — either one narrows resolution to a -// controlled provider set and skips OAuth entirely. This proves loadConfig -// picks an OAuth-ish catalog provider that exists in no settings file at -// all, using a synthetic profile written straight to the home-level auth -// store loadConfig actually reads. +// --config only overrides where provider *definitions* come from (CL-6973); +// it must still merge in the OAuth catalog, or every codex/xai OAuth run +// through --config reaches the provider unauthenticated. Only the +// programmatic `globalSettingsPath` test override (never exposed as a CLI +// flag) opts out, for full test isolation. This proves loadConfig picks an +// OAuth-ish catalog provider that exists in no settings file at all, using a +// synthetic profile written straight to the home-level auth store loadConfig +// actually reads. test("loadConfig resolves an OAuth-profile provider absent from any settings file", async () => { const fakeHome = await mkdtemp(join(tmpdir(), "ic-unit-config-oauth-home-")); const cwd = await mkdtemp(join(tmpdir(), "ic-unit-config-oauth-cwd-")); @@ -253,3 +254,68 @@ test("loadConfig resolves an OAuth-profile provider absent from any settings fil await rm(cwd, { recursive: true, force: true }); } }); + +// CL-6973: --config passes a settings-file override for provider +// *definitions*, but credentials for OAuth-profile providers (codex/, +// xai/) live in separate home-level auth stores that --config never +// touches. Before this fix, an explicit --config unconditionally suppressed +// the OAuth catalog merge, so any codex/xai run through --config resolved to +// an unauthenticated provider (HTTP 426/404 on the first turn). This proves +// --config composes with auth: the OAuth profile still resolves even though +// a --config file is also given, and the file's own settings still apply. +test("--config composes with OAuth profile auth instead of suppressing it", async () => { + const fakeHome = await mkdtemp(join(tmpdir(), "ic-unit-config-oauth-compose-home-")); + const cwd = await mkdtemp(join(tmpdir(), "ic-unit-config-oauth-compose-cwd-")); + try { + await mkdir(join(fakeHome, ".corbits"), { recursive: true }); + await writeFile( + join(fakeHome, ".corbits", "xai-auth.json"), + JSON.stringify({ + profiles: { + synthetic: { + name: "synthetic", + tokens: { + access: "test-access-token", + refresh: "test-refresh", + expiresAt: Date.now() + 3_600_000, + }, + createdAt: Date.now(), + }, + }, + }), + ); + const configPath = join(cwd, "settings-override.json"); + await writeFile(configPath, JSON.stringify({ providers: {} })); + + await withMockedModuleDuring( + import.meta.resolve("node:os"), + (real: typeof import("node:os")) => ({ ...real, homedir: () => fakeHome }), + async () => { + const { impl } = offlineFetch(); + const config = await loadConfig( + [ + "exec", + "--cwd", + cwd, + "--config", + configPath, + "--provider", + "xai/synthetic", + "do something", + ], + { pricing: { fetchImpl: impl } }, + ); + + expect(config.configured).toBe(true); + if (config.configured) { + expect(config.providerName).toBe("xai/synthetic"); + expect(config.providers.some((p) => p.name === "xai/synthetic")).toBe(true); + expect(config.globalSettingsPath).toBe(configPath); + } + }, + ); + } finally { + await rm(fakeHome, { recursive: true, force: true }); + await rm(cwd, { recursive: true, force: true }); + } +}); From 76f2627e01a11d63ad27b5c67b2d6a42ee14362c Mon Sep 17 00:00:00 2001 From: Sawyer Cutler Date: Sun, 23 Aug 2026 16:28:24 -0700 Subject: [PATCH 2/2] Document --config's OAuth-compose behavior, add wiring-level effort rejection test --config is a general CLI flag, not eval-only: it now composes with home-level codex/xai OAuth credentials instead of excluding them. Document that in docs/PRODUCT.md and docs/IMPLEMENTATION.md (prose and flag table) so an operator relying on --config to pin a single API-key provider on a shared/CI machine knows OAuth sessions in ~/.corbits/codex-auth.json / xai-auth.json now apply too when their config names a codex/* or xai/* provider. Export validateVariantEfforts and add a wiring-level test exercising the full parseArgs -> parseMatrix -> validateVariantEfforts path for an unsupported model/effort matrix cell (grok-composer-2.5-fast:xhigh), plus the matching accept case. --- docs/IMPLEMENTATION.md | 4 +++- docs/PRODUCT.md | 2 +- scripts/eval-capability.test.ts | 38 ++++++++++++++++++++++++++++++++- scripts/eval-capability.ts | 2 +- 4 files changed, 42 insertions(+), 4 deletions(-) diff --git a/docs/IMPLEMENTATION.md b/docs/IMPLEMENTATION.md index 3915c788f..e847f0a36 100644 --- a/docs/IMPLEMENTATION.md +++ b/docs/IMPLEMENTATION.md @@ -283,6 +283,8 @@ OpenAI-compatible `baseURL` values are normalized during provider resolution. A `--config ` replaces the global settings file as the provider source (useful for CI to inject a provider per run). The per-repo `.corbits/settings.json` selection still applies on top of a `--config` source (definitions come from `--config`, selection from the local file; CLI `--provider`/`--model` override both). A provider must be defined in one of these settings files; there is no environment-variable fallback. +`--config` composes with, rather than replaces, the home-level OAuth profile catalog: codex/xai credentials live in `~/.corbits/codex-auth.json` and `xai-auth.json`, entirely separate from settings.json, and are merged into the resolved provider catalog on every run regardless of `--config` (CL-6973). A `--config` file that names a `codex/*` or `xai/*` provider by ID does not by itself grant that provider's credentials — those come from the OAuth store whenever a matching profile exists there, independent of which settings file supplied the provider definitions. The only way to fully exclude the home OAuth catalog is the programmatic `globalSettingsPath` option to `loadConfig`, used by tests for full isolation; it is not exposed as a CLI flag. + ### Profiles (`src/config/profiles.ts`) Profiles supply per-project or named-profile overrides for `model`, `maxTurns`, and `systemPromptExtensions` (the only allowed keys; any other key is rejected on load). @@ -323,7 +325,7 @@ Printed by `corbits --help` / `-h` from `CLI_HELP_TEXT` in `src/config/index.ts` | `resume ` | — | Reopen a specific session | | `resume --pick` / `--list` | — | Interactive session picker | | `--cwd ` | `process.cwd()` | Working directory | -| `--config ` | `~/.corbits/settings.json` | Settings file to use | +| `--config ` | `~/.corbits/settings.json` | Settings file to use for provider definitions; composes with (does not exclude) home-level codex/xai OAuth credentials | | `--provider ` | from settings | Select a configured provider | | `--model ` | provider default | Select a model for the active provider | | `--profile ` | — | Settings profile | diff --git a/docs/PRODUCT.md b/docs/PRODUCT.md index 4e18a6757..f6e0f703c 100644 --- a/docs/PRODUCT.md +++ b/docs/PRODUCT.md @@ -132,7 +132,7 @@ The exact turn thresholds are model-family-dependent (tighter for models with ob ## Configuration -Providers and models are configured in `~/.corbits/settings.json` (holds providers + credentials), with a selection-only per-repo `.corbits/settings.json` override. Select at launch with `--provider` / `--model`, or point at an alternate file with `--config `. Credentials are read only from these settings files — there is no environment-variable override and `.env` files are not loaded, so a stale or exported key can't shadow the configured provider. The agent is denied read access to both settings files. +Providers and models are configured in `~/.corbits/settings.json` (holds providers + credentials), with a selection-only per-repo `.corbits/settings.json` override. Select at launch with `--provider` / `--model`, or point at an alternate file with `--config `. `--config` only overrides where provider _definitions_ come from; it composes with, rather than replaces, credentials for codex/xai OAuth-profile providers, which live in separate home-level auth stores (`~/.corbits/codex-auth.json`, `xai-auth.json`) and are merged into the catalog regardless of `--config`. Credentials are read only from these settings files and the OAuth auth stores — there is no environment-variable override and `.env` files are not loaded, so a stale or exported key can't shadow the configured provider. The agent is denied read access to both settings files. ## Optional Capabilities (plugins) diff --git a/scripts/eval-capability.test.ts b/scripts/eval-capability.test.ts index ad94e5be3..fc071d64f 100644 --- a/scripts/eval-capability.test.ts +++ b/scripts/eval-capability.test.ts @@ -5,7 +5,14 @@ import { join } from "node:path"; import { execFile } from "node:child_process"; import { promisify } from "node:util"; -import { initEvalGitRepo, mapPool, parseArgs, buildEvalDiagnostics } from "./eval-capability.js"; +import { + initEvalGitRepo, + mapPool, + parseArgs, + buildEvalDiagnostics, + validateVariantEfforts, +} from "./eval-capability.js"; +import { parseMatrix } from "../evals/capability/lib.js"; import type { Config } from "../src/config/index.js"; const execFileAsync = promisify(execFile); @@ -174,6 +181,35 @@ describe("parseArgs", () => { }); }); +describe("validateVariantEfforts", () => { + // Wiring-level regression: parseArgs -> parseMatrix -> validateVariantEfforts, + // the same path main() runs before any inference. A matrix cell pairing an + // effort the model does not accept must fail fast, naming the model and its + // accepted levels, rather than silently falling back to the provider default + // and poisoning the matrix. + test("rejects an unsupported model/effort matrix cell before any inference runs", async () => { + const opts = parseArgs(["--matrix", "xai/thegreataxios:grok-composer-2.5-fast:xhigh"]); + const variants = parseMatrix(opts.matrix, { + ...(opts.provider !== undefined ? { provider: opts.provider } : {}), + ...(opts.model !== undefined ? { model: opts.model } : {}), + ...(opts.effort !== undefined ? { effort: opts.effort } : {}), + }); + await expect(validateVariantEfforts(variants, opts)).rejects.toThrow( + /grok-composer-2\.5-fast.*does not support reasoning effort "xhigh".*supported: low, medium, high/s, + ); + }); + + test("accepts a supported model/effort matrix cell", async () => { + const opts = parseArgs(["--matrix", "xai/thegreataxios:grok-4.6:xhigh"]); + const variants = parseMatrix(opts.matrix, { + ...(opts.provider !== undefined ? { provider: opts.provider } : {}), + ...(opts.model !== undefined ? { model: opts.model } : {}), + ...(opts.effort !== undefined ? { effort: opts.effort } : {}), + }); + await expect(validateVariantEfforts(variants, opts)).resolves.toBeUndefined(); + }); +}); + describe("mapPool", () => { test("N overlapping jobs with concurrency N finish in ~one job duration", async () => { const jobMs = 80; diff --git a/scripts/eval-capability.ts b/scripts/eval-capability.ts index 0313179a1..31fa92cd9 100755 --- a/scripts/eval-capability.ts +++ b/scripts/eval-capability.ts @@ -583,7 +583,7 @@ async function resolveVariantLabels( * gpt-5.6 family also takes max/ultra) — silently running at the provider's * default instead would poison a matrix without anyone noticing. */ -async function validateVariantEfforts( +export async function validateVariantEfforts( variants: readonly EvalVariant[], opts: CliOptions, ): Promise {