From 321c481beb25d819d1222514ff5fcb59ac8e71e7 Mon Sep 17 00:00:00 2001 From: Sawyer Cutler Date: Sat, 22 Aug 2026 00:07:37 -0700 Subject: [PATCH] Add an exec flag to run as a chosen director --- CHANGELOG.md | 7 +++ evals/capability/README.md | 5 ++ scripts/eval-capability.test.ts | 17 +++++++ scripts/eval-capability.ts | 11 +++++ src/config.test.ts | 50 +++++++++++++++++++ src/config/index.ts | 24 +++++++++ src/exec/runner.ts | 86 +++++++++++++++++++++++++-------- tests/unit/exec/runner.test.ts | 22 ++++++++- 8 files changed, 202 insertions(+), 20 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4dfd85962..5e905a285 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -45,6 +45,13 @@ parallel copies under `docs/` or `scripts/notes/`. At cut time: rename write-free. Shell file-writes stay denied. Spawn is a judgment call, not a tool ban. +- **Exec and capability evals can run as a chosen primary director.** + `corbits exec --director ` (and eval `--director`) overlays that + package's system prompt and tool allowlist on the product exec path. + Omit / skywalker keep the default Skywalker session. Directors that + cannot spawn (for example implement) do not mount `task`. This is an + exec/eval/CI override, not a TUI or single-agent mode. + ## [0.2.99] - 2026-08-21 Skywalker is the primary orchestrator over a closed director fleet: product write tools stay off the primary, and you cannot spawn Skywalker as a task leaf. Workers are not done until they return the four-heading report. First-party action skills ship as slashes; eval runners require an explicit provider/model pair; the style skill no longer refuses non-git folders. diff --git a/evals/capability/README.md b/evals/capability/README.md index 95ef6cd32..5da2a8a70 100644 --- a/evals/capability/README.md +++ b/evals/capability/README.md @@ -132,6 +132,10 @@ bun run eval:capability -- --provider --model \ bun run eval:capability -- --provider --model --repeats 5 \ --out evals/capability/results/candidate.json \ --baseline evals/capability/results/baseline-0286.json + +# Overlay a closed-fleet director on the product exec path (eval/CI override, +# not single-agent mode). Omit / skywalker keep the default Skywalker session. +bun run eval:capability -- --provider --model --director implement ``` ## Confirmation gate for behavior changes @@ -170,6 +174,7 @@ Flags: | `--repeats ` | Runs per case×variant cell (default `1`; gate runs use `5`, baseline freezes `3`). Results record every repeat plus per-cell aggregates | | `--concurrency ` | Independent case×variant×repeat cells in parallel (default `1`, env `CORBITS_EVAL_CONCURRENCY`). Each cell still uses its own temp workdir. Use `--concurrency 4` (or similar) to run a live matrix faster | | `--dry-run` | Load cases × variants and print plan; no inference. Still requires `--provider`/`--model` or `--matrix` | +| `--director ` | Exec overlay: run the product `corbits exec` path as this closed-fleet director (default: skywalker). Eval/CI override, not single-agent mode. Directors that cannot spawn (for example `implement`) do not mount `task`. | ## Case format diff --git a/scripts/eval-capability.test.ts b/scripts/eval-capability.test.ts index 372b3fc06..47555c88c 100644 --- a/scripts/eval-capability.test.ts +++ b/scripts/eval-capability.test.ts @@ -118,6 +118,22 @@ describe("parseArgs", () => { /CORBITS_EVAL_CONCURRENCY must be a positive integer/, ); }); + + test("--director implement is parsed", () => { + const opts = parseArgs(["--provider", "foo", "--model", "bar", "--director", "implement"]); + expect(opts.director).toBe("implement"); + }); + + test("omitted --director stays undefined", () => { + const opts = parseArgs(["--provider", "foo", "--model", "bar"]); + expect(opts.director).toBeUndefined(); + }); + + test("--director without a value throws", () => { + expect(() => parseArgs(["--provider", "foo", "--model", "bar", "--director"])).toThrow( + "--director requires a value", + ); + }); }); describe("mapPool", () => { @@ -150,6 +166,7 @@ describe("mapPool", () => { test("rejects non-positive concurrency", async () => { await expect(mapPool([1], 0, async (item) => item)).rejects.toThrow(/positive integer/); }); + }); describe("initEvalGitRepo", () => { diff --git a/scripts/eval-capability.ts b/scripts/eval-capability.ts index 663cc2dc5..5605ebaaa 100755 --- a/scripts/eval-capability.ts +++ b/scripts/eval-capability.ts @@ -77,6 +77,11 @@ type CliOptions = { * differs from what was requested, instead of hard-failing. */ allowProviderFallback: boolean; + /** + * Exec overlay: run the product path as this closed-fleet director. + * Eval/CI override, not single-agent mode. Omitted = skywalker default. + */ + director?: string; }; function printUsage(): void { @@ -99,6 +104,8 @@ function printUsage(): void { --dry-run List cases × variants only (still requires --provider/--model or --matrix) --allow-provider-fallback Allow resolved provider/model to differ from what was requested (default: hard-fail) + --director Exec overlay: run as this director (default: skywalker). + Eval/CI override, not single-agent mode -h, --help Show help `); } @@ -230,6 +237,9 @@ export function parseArgs(argv: readonly string[]): CliOptions { case "--allow-provider-fallback": opts.allowProviderFallback = true; break; + case "--director": + opts.director = next(); + break; default: throw new Error(`Unknown argument: ${a}`); } @@ -608,6 +618,7 @@ async function runCase( if (opts.configPath !== undefined) argv.push("--config", opts.configPath); if (opts.skipPermissions) argv.push("--dangerously-skip-permissions"); argv.push("--force"); + if (opts.director !== undefined) argv.push("--director", opts.director); const maxTurns = opts.maxTurnsOverride ?? caseDef.maxTurns ?? null; // maxTurns is a soft post-run budget (case fails if exceeded). It does not diff --git a/src/config.test.ts b/src/config.test.ts index 41efccfe4..b0ef2c75b 100644 --- a/src/config.test.ts +++ b/src/config.test.ts @@ -4,6 +4,7 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { buildBifrostSource, buildOpenAISource, buildXaiSource, buildProviderCatalog, catalogEntryAsProviderSettings, CliHelpError, CLI_HELP_TEXT, KEYLESS_API_KEY, loadConfig, providerCatalogToSettings, runtimeSettingsWithCatalog, SOURCE_MAX_TOKENS } from "./config/index.js"; +import { DIRECTOR_IDS } from "./agent/directors/types.js"; import type { Config, UnconfiguredConfig } from "./config/index.js"; import { mergeProviderIntoSettings, type ResolvedProvider, type Settings } from "./config/settings.js"; import { OPENCODE_GO_BASE_URL } from "../packages/opencode-go/src/index.js"; @@ -183,6 +184,55 @@ describe("loadConfig", () => { } }); + test("parses exec --director implement", async () => { + const cwd = await emptyCwd(); + try { + const globalPath = await writeGlobalSettings(cwd); + const config = await loadConfig(["exec", "--cwd", cwd, "--director", "implement", "ship it"], { + globalSettingsPath: globalPath, + }); + assertConfigured(config); + expect(config.command).toBe("exec"); + expect(config.director).toBe("implement"); + expect(config.task).toBe("ship it"); + } finally { + await rm(cwd, { recursive: true, force: true }); + } + }); + + test("omits director as undefined (skywalker default)", async () => { + const cwd = await emptyCwd(); + try { + const globalPath = await writeGlobalSettings(cwd); + const config = await loadConfig(["exec", "--cwd", cwd, "ship it"], { + globalSettingsPath: globalPath, + }); + assertConfigured(config); + expect(config.command).toBe("exec"); + expect(config.director).toBeUndefined(); + } finally { + await rm(cwd, { recursive: true, force: true }); + } + }); + + test("unknown --director id errors listing DIRECTOR_IDS", async () => { + await expect( + loadConfig(["exec", "--director", "nope", "ship it"], { globalSettingsPath: NO_SETTINGS }), + ).rejects.toThrow(new RegExp(`Unknown director "nope".*${DIRECTOR_IDS.join(", ")}`)); + }); + + test("--director without a value errors", async () => { + await expect(loadConfig(["exec", "--director"], { globalSettingsPath: NO_SETTINGS })).rejects.toThrow( + "--director requires a value", + ); + }); + + test("--director without exec/run is rejected", async () => { + await expect( + loadConfig(["--director", "implement", "ship it"], { globalSettingsPath: NO_SETTINGS }), + ).rejects.toThrow("--director is only available in exec mode"); + }); + test("resume --pick opens the session picker without requiring prior sessions", async () => { const cwd = await emptyCwd(); try { diff --git a/src/config/index.ts b/src/config/index.ts index 56fd40a80..fc4fffe76 100644 --- a/src/config/index.ts +++ b/src/config/index.ts @@ -5,6 +5,8 @@ import { generateSessionId, isSessionId, migrateLegacySessionIfNeeded } from ".. import { loadState } from "../session/state.js"; +import { isDirectorId } from "../agent/directors/registry.js"; +import { DIRECTOR_IDS, type DirectorId } from "../agent/directors/types.js"; import { validateEffort, type ReasoningEffort } from "../provider/reasoning-effort.js"; import { bootstrapPricingMetadata } from "../cost/pricing-metadata.js"; import { defaultPricingCachePath, type PricingFetcherOptions } from "../cost/pricing-fetcher.js"; @@ -278,6 +280,11 @@ export type Config = { force: boolean; dangerouslySkipPermissions: boolean; auto: boolean; + /** + * Exec-only chosen primary director. Omitted = Skywalker (product default). + * `--director` is rejected in TUI mode. + */ + director?: DirectorId; /** * Entry mode. `"tui"` is the interactive Ink shell; `"exec"` is the non-TUI * product agent path (`corbits exec "prompt"`). Same directors/tools/permissions. @@ -343,6 +350,8 @@ export type UnconfiguredConfig = { dangerouslySkipPermissions: boolean; auto: boolean; command: "tui" | "exec"; + /** Exec-only chosen primary. Omitted on the unconfigured path too. */ + director?: DirectorId; // Path where the onboarding flow should write the new settings. globalSettingsPath: string; // The original error message, used for non-TUI (exec) error output. @@ -377,6 +386,7 @@ Flags: --profile settings profile --resume interactive session picker --force override an existing run state + --director exec-only: run as this director (default: skywalker) --dangerously-skip-permissions --auto / --no-auto auto mode on/off --help, -h show this help @@ -474,6 +484,7 @@ export async function loadConfig( // to ask-on-every-write. There is currently no in-session key to toggle auto; // Shift+Tab in the TUI cycles reasoning effort instead. let auto = true; + let director: DirectorId | undefined; let configPath: string | undefined; let provider: string | undefined; let model: string | undefined; @@ -515,6 +526,17 @@ export async function loadConfig( force = true; continue; } + if (arg === "--director") { + const value = requireValue("--director", args[++i]); + if (command !== "exec") { + throw new Error("--director is only available in exec mode"); + } + if (!isDirectorId(value)) { + throw new Error(`Unknown director "${value}". Use one of: ${DIRECTOR_IDS.join(", ")}.`); + } + director = value; + continue; + } if (arg === "--dangerously-skip-permissions") { dangerouslySkipPermissions = true; @@ -633,6 +655,7 @@ export async function loadConfig( dangerouslySkipPermissions, auto, command, + ...(director !== undefined ? { director } : {}), globalSettingsPath: effectiveSettingsPath, providerError: err instanceof Error ? err.message : String(err), // Keep diagnostics even when provider setup fails early so junk local @@ -684,6 +707,7 @@ export async function loadConfig( dangerouslySkipPermissions, auto, command, + ...(director !== undefined ? { director } : {}), globalSettingsPath: effectiveSettingsPath, sessionId, noWorkflow, diff --git a/src/exec/runner.ts b/src/exec/runner.ts index 5597bb483..b4251a87a 100644 --- a/src/exec/runner.ts +++ b/src/exec/runner.ts @@ -27,6 +27,9 @@ import { } from "../config/settings.js"; import { codexProfileFromProviderName } from "../config/codex-providers.js"; import { xaiProfileFromProviderName } from "../config/xai-providers.js"; +import { formatDirectorSystemPrompt } from "../agent/directors/identity.js"; +import { DIRECTOR_REGISTRY } from "../agent/directors/registry.js"; +import type { DirectorId } from "../agent/directors/types.js"; import { createInferenceDependencies } from "../provider/inference-dependencies.js"; import { getValidCodexToken } from "../auth/codex/session.js"; import { getValidXaiToken } from "../auth/xai/session.js"; @@ -111,6 +114,40 @@ export function formatCaughtError(err: unknown): string { return err instanceof Error ? err.message : String(err); } +/** + * Exec-primary director overlay. Omit / skywalker keep the product default + * (`loadSessionChatPrompt` + advertised session tools). Any other closed-fleet + * id uses the package prompt and allowlist. Worker effort/nudge are not applied. + */ +export type ExecDirectorOverlay = { + /** Package system prompt; omitted on the skywalker default path. */ + systemPrompt?: string; + /** `pkg.tools.allow` (task stripped when `maySpawn` is false). */ + advertisedAllow?: readonly string[]; + mountTask: boolean; +}; + +export function resolveExecDirectorOverlay( + director: DirectorId | undefined, +): ExecDirectorOverlay { + if (director === undefined || director === "skywalker") { + return { mountTask: true }; + } + const pkg = DIRECTOR_REGISTRY[director]; + const allow = pkg.tools?.allow; + const advertisedAllow = + allow !== undefined && allow.length > 0 + ? pkg.spawn.maySpawn + ? [...allow] + : allow.filter((name) => name !== "task") + : undefined; + return { + systemPrompt: formatDirectorSystemPrompt(pkg), + ...(advertisedAllow !== undefined ? { advertisedAllow } : {}), + mountTask: pkg.spawn.maySpawn, + }; +} + /** Content-less inbound used after compact so the reactor re-enters (matches TUI). */ export function buildCompactionContinuationMessage(): InboundMessage { return { @@ -348,6 +385,8 @@ export async function runExec(config: Config): Promise { let currentAgent: Agent | null = null; + const overlay = resolveExecDirectorOverlay(config.director); + const agentToolset = await createAgentToolset({ cwd: config.cwd, permissionGate, @@ -384,30 +423,39 @@ export async function runExec(config: Config): Promise { ); return result.kind === "option" && result.index === 0; }, - subAgent: { - provider: () => liveSubAgentProvider.current, - sessions: subAgentSessions, - getWorkdirBase: () => sessionDir(config.cwd, sessionId), - onProgress: () => undefined, - ...(config.settings !== undefined ? { settings: () => config.settings! } : {}), - catalog: () => config.providers, - profiles: () => liveAgentProfiles, - }, + ...(overlay.mountTask + ? { + subAgent: { + provider: () => liveSubAgentProvider.current, + sessions: subAgentSessions, + getWorkdirBase: () => sessionDir(config.cwd, sessionId), + onProgress: () => undefined, + ...(config.settings !== undefined ? { settings: () => config.settings! } : {}), + catalog: () => config.providers, + profiles: () => liveAgentProfiles, + }, + } + : {}), ...(extraToolPlugins.length > 0 ? { extraToolPlugins } : {}), }); toolset = agentToolset; - const { systemPrompt } = await loadSessionChatPrompt({ - cwd: config.cwd, - skillDirs, - ...(config.systemPromptExtensions !== undefined - ? { systemPromptExtensions: config.systemPromptExtensions } - : {}), - sessionMode, - toolAvailability, - }); + const systemPrompt = + overlay.systemPrompt ?? + ( + await loadSessionChatPrompt({ + cwd: config.cwd, + skillDirs, + ...(config.systemPromptExtensions !== undefined + ? { systemPromptExtensions: config.systemPromptExtensions } + : {}), + sessionMode, + toolAvailability, + }) + ).systemPrompt; - const advertisedBuiltInPrefix = advertisedToolNamesForSessionMode(sessionMode, toolAvailability); + const advertisedBuiltInPrefix = + overlay.advertisedAllow ?? advertisedToolNamesForSessionMode(sessionMode, toolAvailability); const activatedToolNames = createActivatedToolTracker(); // Advertise then family-gate wire schemas (kimi gets a non-recursive present). const computeAdvertised = (all: readonly ToolDefinition[]): ToolDefinition[] => diff --git a/tests/unit/exec/runner.test.ts b/tests/unit/exec/runner.test.ts index 00b828579..d45ea99c1 100644 --- a/tests/unit/exec/runner.test.ts +++ b/tests/unit/exec/runner.test.ts @@ -1,6 +1,7 @@ import { describe, expect, test } from "bun:test"; import type { Config } from "../../../src/config/index.js"; -import { formatCaughtError, runExec } from "../../../src/exec/runner.js"; +import { formatCaughtError, resolveExecDirectorOverlay, runExec } from "../../../src/exec/runner.js"; +import { IMPLEMENT_TOOLS } from "../../../src/agent/directors/tool-sets.js"; function bareConfig(task: string): Config { // Minimal unconfigured-shaped object is not enough — runExec only needs @@ -48,3 +49,22 @@ describe("runExec", () => { } }); }); + +describe("resolveExecDirectorOverlay", () => { + test("implement exec primary does not mount task", () => { + const overlay = resolveExecDirectorOverlay("implement"); + expect(overlay.mountTask).toBe(false); + expect(overlay.advertisedAllow).toBeDefined(); + expect(overlay.advertisedAllow).not.toContain("task"); + expect(overlay.advertisedAllow).toEqual([...IMPLEMENT_TOOLS]); + expect(overlay.systemPrompt).toContain("ImplementDirector"); + }); + + test("skywalker default still can mount task", () => { + expect(resolveExecDirectorOverlay(undefined).mountTask).toBe(true); + expect(resolveExecDirectorOverlay(undefined).systemPrompt).toBeUndefined(); + expect(resolveExecDirectorOverlay(undefined).advertisedAllow).toBeUndefined(); + expect(resolveExecDirectorOverlay("skywalker").mountTask).toBe(true); + expect(resolveExecDirectorOverlay("skywalker").systemPrompt).toBeUndefined(); + }); +});