Merge pull request #496 from corbitsdev/cl-6630-exec-and-capability-evals-can-run-as-a-chosen-primary

Add an exec flag to run as a chosen director

Author: TheGreatAxios

CHANGELOG.md

@@ -45,6 +45,13 @@ parallel copies under `docs/` or `scripts/notes/`. At cut time: rename
   write-free. Shell file-writes stay denied. Spawn is a judgment call,
   not a tool ban.
 
+- **Exec and capability evals can run as a chosen primary director.**
+  `corbits exec --director <id>` (and eval `--director`) overlays that
+  package's system prompt and tool allowlist on the product exec path.
+  Omit / skywalker keep the default Skywalker session. Directors that
+  cannot spawn (for example implement) do not mount `task`. This is an
+  exec/eval/CI override, not a TUI or single-agent mode.
+
 ## [0.2.99] - 2026-08-21
 
 Skywalker is the primary orchestrator over a closed director fleet: product write tools stay off the primary, and you cannot spawn Skywalker as a task leaf. Workers are not done until they return the four-heading report. First-party action skills ship as slashes; eval runners require an explicit provider/model pair; the style skill no longer refuses non-git folders.

evals/capability/README.md

@@ -132,6 +132,10 @@ bun run eval:capability -- --provider <name> --model <id> \
 bun run eval:capability -- --provider <name> --model <id> --repeats 5 \
   --out evals/capability/results/candidate.json \
   --baseline evals/capability/results/baseline-0286.json
+
+# Overlay a closed-fleet director on the product exec path (eval/CI override,
+# not single-agent mode). Omit / skywalker keep the default Skywalker session.
+bun run eval:capability -- --provider <name> --model <id> --director implement
 ```
 
 ## Confirmation gate for behavior changes
@@ -170,6 +174,7 @@ Flags:
 | `--repeats <n>` | Runs per case×variant cell (default `1`; gate runs use `5`, baseline freezes `3`). Results record every repeat plus per-cell aggregates |
 | `--concurrency <n>` | Independent case×variant×repeat cells in parallel (default `1`, env `CORBITS_EVAL_CONCURRENCY`). Each cell still uses its own temp workdir. Use `--concurrency 4` (or similar) to run a live matrix faster |
 | `--dry-run` | Load cases × variants and print plan; no inference. Still requires `--provider`/`--model` or `--matrix` |
+| `--director <id>` | Exec overlay: run the product `corbits exec` path as this closed-fleet director (default: skywalker). Eval/CI override, not single-agent mode. Directors that cannot spawn (for example `implement`) do not mount `task`. |
 
 ## Case format
 

scripts/eval-capability.test.ts

@@ -118,6 +118,22 @@ describe("parseArgs", () => {
       /CORBITS_EVAL_CONCURRENCY must be a positive integer/,
     );
   });
+
+  test("--director implement is parsed", () => {
+    const opts = parseArgs(["--provider", "foo", "--model", "bar", "--director", "implement"]);
+    expect(opts.director).toBe("implement");
+  });
+
+  test("omitted --director stays undefined", () => {
+    const opts = parseArgs(["--provider", "foo", "--model", "bar"]);
+    expect(opts.director).toBeUndefined();
+  });
+
+  test("--director without a value throws", () => {
+    expect(() => parseArgs(["--provider", "foo", "--model", "bar", "--director"])).toThrow(
+      "--director requires a value",
+    );
+  });
 });
 
 describe("mapPool", () => {
@@ -150,6 +166,7 @@ describe("mapPool", () => {
   test("rejects non-positive concurrency", async () => {
     await expect(mapPool([1], 0, async (item) => item)).rejects.toThrow(/positive integer/);
   });
+
 });
 
 describe("initEvalGitRepo", () => {

scripts/eval-capability.ts

@@ -77,6 +77,11 @@ type CliOptions = {
    * differs from what was requested, instead of hard-failing.
    */
   allowProviderFallback: boolean;
+  /**
+   * Exec overlay: run the product path as this closed-fleet director.
+   * Eval/CI override, not single-agent mode. Omitted = skywalker default.
+   */
+  director?: string;
 };
 
 function printUsage(): void {
@@ -99,6 +104,8 @@ function printUsage(): void {
   --dry-run             List cases × variants only (still requires --provider/--model or --matrix)
   --allow-provider-fallback  Allow resolved provider/model to differ from
                              what was requested (default: hard-fail)
+  --director <id>       Exec overlay: run as this director (default: skywalker).
+                        Eval/CI override, not single-agent mode
   -h, --help            Show help
 `);
 }
@@ -230,6 +237,9 @@ export function parseArgs(argv: readonly string[]): CliOptions {
       case "--allow-provider-fallback":
         opts.allowProviderFallback = true;
         break;
+      case "--director":
+        opts.director = next();
+        break;
       default:
         throw new Error(`Unknown argument: ${a}`);
     }
@@ -608,6 +618,7 @@ async function runCase(
     if (opts.configPath !== undefined) argv.push("--config", opts.configPath);
     if (opts.skipPermissions) argv.push("--dangerously-skip-permissions");
     argv.push("--force");
+    if (opts.director !== undefined) argv.push("--director", opts.director);
 
     const maxTurns = opts.maxTurnsOverride ?? caseDef.maxTurns ?? null;
     // maxTurns is a soft post-run budget (case fails if exceeded). It does not

src/config.test.ts

@@ -4,6 +4,7 @@ import { tmpdir } from "node:os";
 import { join } from "node:path";
 
 import { buildBifrostSource, buildOpenAISource, buildXaiSource, buildProviderCatalog, catalogEntryAsProviderSettings, CliHelpError, CLI_HELP_TEXT, KEYLESS_API_KEY, loadConfig, providerCatalogToSettings, runtimeSettingsWithCatalog, SOURCE_MAX_TOKENS } from "./config/index.js";
+import { DIRECTOR_IDS } from "./agent/directors/types.js";
 import type { Config, UnconfiguredConfig } from "./config/index.js";
 import { mergeProviderIntoSettings, type ResolvedProvider, type Settings } from "./config/settings.js";
 import { OPENCODE_GO_BASE_URL } from "../packages/opencode-go/src/index.js";
@@ -183,6 +184,55 @@ describe("loadConfig", () => {
     }
   });
 
+  test("parses exec --director implement", async () => {
+    const cwd = await emptyCwd();
+    try {
+      const globalPath = await writeGlobalSettings(cwd);
+      const config = await loadConfig(["exec", "--cwd", cwd, "--director", "implement", "ship it"], {
+        globalSettingsPath: globalPath,
+      });
+      assertConfigured(config);
+      expect(config.command).toBe("exec");
+      expect(config.director).toBe("implement");
+      expect(config.task).toBe("ship it");
+    } finally {
+      await rm(cwd, { recursive: true, force: true });
+    }
+  });
+
+  test("omits director as undefined (skywalker default)", async () => {
+    const cwd = await emptyCwd();
+    try {
+      const globalPath = await writeGlobalSettings(cwd);
+      const config = await loadConfig(["exec", "--cwd", cwd, "ship it"], {
+        globalSettingsPath: globalPath,
+      });
+      assertConfigured(config);
+      expect(config.command).toBe("exec");
+      expect(config.director).toBeUndefined();
+    } finally {
+      await rm(cwd, { recursive: true, force: true });
+    }
+  });
+
+  test("unknown --director id errors listing DIRECTOR_IDS", async () => {
+    await expect(
+      loadConfig(["exec", "--director", "nope", "ship it"], { globalSettingsPath: NO_SETTINGS }),
+    ).rejects.toThrow(new RegExp(`Unknown director "nope".*${DIRECTOR_IDS.join(", ")}`));
+  });
+
+  test("--director without a value errors", async () => {
+    await expect(loadConfig(["exec", "--director"], { globalSettingsPath: NO_SETTINGS })).rejects.toThrow(
+      "--director requires a value",
+    );
+  });
+
+  test("--director without exec/run is rejected", async () => {
+    await expect(
+      loadConfig(["--director", "implement", "ship it"], { globalSettingsPath: NO_SETTINGS }),
+    ).rejects.toThrow("--director is only available in exec mode");
+  });
+
   test("resume --pick opens the session picker without requiring prior sessions", async () => {
     const cwd = await emptyCwd();
     try {

src/config/index.ts

@@ -5,6 +5,8 @@ import { generateSessionId, isSessionId, migrateLegacySessionIfNeeded } from "..
 import { loadState } from "../session/state.js";
 
 
+import { isDirectorId } from "../agent/directors/registry.js";
+import { DIRECTOR_IDS, type DirectorId } from "../agent/directors/types.js";
 import { validateEffort, type ReasoningEffort } from "../provider/reasoning-effort.js";
 import { bootstrapPricingMetadata } from "../cost/pricing-metadata.js";
 import { defaultPricingCachePath, type PricingFetcherOptions } from "../cost/pricing-fetcher.js";
@@ -278,6 +280,11 @@ export type Config = {
   force: boolean;
   dangerouslySkipPermissions: boolean;
   auto: boolean;
+  /**
+   * Exec-only chosen primary director. Omitted = Skywalker (product default).
+   * `--director` is rejected in TUI mode.
+   */
+  director?: DirectorId;
   /**
    * Entry mode. `"tui"` is the interactive Ink shell; `"exec"` is the non-TUI
    * product agent path (`corbits exec "prompt"`). Same directors/tools/permissions.
@@ -343,6 +350,8 @@ export type UnconfiguredConfig = {
   dangerouslySkipPermissions: boolean;
   auto: boolean;
   command: "tui" | "exec";
+  /** Exec-only chosen primary. Omitted on the unconfigured path too. */
+  director?: DirectorId;
   // Path where the onboarding flow should write the new settings.
   globalSettingsPath: string;
   // The original error message, used for non-TUI (exec) error output.
@@ -377,6 +386,7 @@ Flags:
   --profile <name>            settings profile
   --resume                    interactive session picker
   --force                     override an existing run state
+  --director <id>             exec-only: run as this director (default: skywalker)
   --dangerously-skip-permissions
   --auto / --no-auto          auto mode on/off
   --help, -h                  show this help
@@ -474,6 +484,7 @@ export async function loadConfig(
   // to ask-on-every-write. There is currently no in-session key to toggle auto;
   // Shift+Tab in the TUI cycles reasoning effort instead.
   let auto = true;
+  let director: DirectorId | undefined;
   let configPath: string | undefined;
   let provider: string | undefined;
   let model: string | undefined;
@@ -515,6 +526,17 @@ export async function loadConfig(
       force = true;
       continue;
     }
+    if (arg === "--director") {
+      const value = requireValue("--director", args[++i]);
+      if (command !== "exec") {
+        throw new Error("--director is only available in exec mode");
+      }
+      if (!isDirectorId(value)) {
+        throw new Error(`Unknown director "${value}". Use one of: ${DIRECTOR_IDS.join(", ")}.`);
+      }
+      director = value;
+      continue;
+    }
 
     if (arg === "--dangerously-skip-permissions") {
       dangerouslySkipPermissions = true;
@@ -633,6 +655,7 @@ export async function loadConfig(
       dangerouslySkipPermissions,
       auto,
       command,
+      ...(director !== undefined ? { director } : {}),
       globalSettingsPath: effectiveSettingsPath,
       providerError: err instanceof Error ? err.message : String(err),
       // Keep diagnostics even when provider setup fails early so junk local
@@ -684,6 +707,7 @@ export async function loadConfig(
     dangerouslySkipPermissions,
     auto,
     command,
+    ...(director !== undefined ? { director } : {}),
     globalSettingsPath: effectiveSettingsPath,
     sessionId,
     noWorkflow,

src/exec/runner.ts

@@ -27,6 +27,9 @@ import {
 } from "../config/settings.js";
 import { codexProfileFromProviderName } from "../config/codex-providers.js";
 import { xaiProfileFromProviderName } from "../config/xai-providers.js";
+import { formatDirectorSystemPrompt } from "../agent/directors/identity.js";
+import { DIRECTOR_REGISTRY } from "../agent/directors/registry.js";
+import type { DirectorId } from "../agent/directors/types.js";
 import { createInferenceDependencies } from "../provider/inference-dependencies.js";
 import { getValidCodexToken } from "../auth/codex/session.js";
 import { getValidXaiToken } from "../auth/xai/session.js";
@@ -111,6 +114,40 @@ export function formatCaughtError(err: unknown): string {
   return err instanceof Error ? err.message : String(err);
 }
 
+/**
+ * Exec-primary director overlay. Omit / skywalker keep the product default
+ * (`loadSessionChatPrompt` + advertised session tools). Any other closed-fleet
+ * id uses the package prompt and allowlist. Worker effort/nudge are not applied.
+ */
+export type ExecDirectorOverlay = {
+  /** Package system prompt; omitted on the skywalker default path. */
+  systemPrompt?: string;
+  /** `pkg.tools.allow` (task stripped when `maySpawn` is false). */
+  advertisedAllow?: readonly string[];
+  mountTask: boolean;
+};
+
+export function resolveExecDirectorOverlay(
+  director: DirectorId | undefined,
+): ExecDirectorOverlay {
+  if (director === undefined || director === "skywalker") {
+    return { mountTask: true };
+  }
+  const pkg = DIRECTOR_REGISTRY[director];
+  const allow = pkg.tools?.allow;
+  const advertisedAllow =
+    allow !== undefined && allow.length > 0
+      ? pkg.spawn.maySpawn
+        ? [...allow]
+        : allow.filter((name) => name !== "task")
+      : undefined;
+  return {
+    systemPrompt: formatDirectorSystemPrompt(pkg),
+    ...(advertisedAllow !== undefined ? { advertisedAllow } : {}),
+    mountTask: pkg.spawn.maySpawn,
+  };
+}
+
 /** Content-less inbound used after compact so the reactor re-enters (matches TUI). */
 export function buildCompactionContinuationMessage(): InboundMessage {
   return {
@@ -348,6 +385,8 @@ export async function runExec(config: Config): Promise<ExecResult> {
 
     let currentAgent: Agent | null = null;
 
+    const overlay = resolveExecDirectorOverlay(config.director);
+
     const agentToolset = await createAgentToolset({
       cwd: config.cwd,
       permissionGate,
@@ -384,30 +423,39 @@ export async function runExec(config: Config): Promise<ExecResult> {
         );
         return result.kind === "option" && result.index === 0;
       },
-      subAgent: {
-        provider: () => liveSubAgentProvider.current,
-        sessions: subAgentSessions,
-        getWorkdirBase: () => sessionDir(config.cwd, sessionId),
-        onProgress: () => undefined,
-        ...(config.settings !== undefined ? { settings: () => config.settings! } : {}),
-        catalog: () => config.providers,
-        profiles: () => liveAgentProfiles,
-      },
+      ...(overlay.mountTask
+        ? {
+            subAgent: {
+              provider: () => liveSubAgentProvider.current,
+              sessions: subAgentSessions,
+              getWorkdirBase: () => sessionDir(config.cwd, sessionId),
+              onProgress: () => undefined,
+              ...(config.settings !== undefined ? { settings: () => config.settings! } : {}),
+              catalog: () => config.providers,
+              profiles: () => liveAgentProfiles,
+            },
+          }
+        : {}),
       ...(extraToolPlugins.length > 0 ? { extraToolPlugins } : {}),
     });
     toolset = agentToolset;
 
-    const { systemPrompt } = await loadSessionChatPrompt({
-      cwd: config.cwd,
-      skillDirs,
-      ...(config.systemPromptExtensions !== undefined
-        ? { systemPromptExtensions: config.systemPromptExtensions }
-        : {}),
-      sessionMode,
-      toolAvailability,
-    });
+    const systemPrompt =
+      overlay.systemPrompt ??
+      (
+        await loadSessionChatPrompt({
+          cwd: config.cwd,
+          skillDirs,
+          ...(config.systemPromptExtensions !== undefined
+            ? { systemPromptExtensions: config.systemPromptExtensions }
+            : {}),
+          sessionMode,
+          toolAvailability,
+        })
+      ).systemPrompt;
 
-    const advertisedBuiltInPrefix = advertisedToolNamesForSessionMode(sessionMode, toolAvailability);
+    const advertisedBuiltInPrefix =
+      overlay.advertisedAllow ?? advertisedToolNamesForSessionMode(sessionMode, toolAvailability);
     const activatedToolNames = createActivatedToolTracker();
     // Advertise then family-gate wire schemas (kimi gets a non-recursive present).
     const computeAdvertised = (all: readonly ToolDefinition[]): ToolDefinition[] =>

tests/unit/exec/runner.test.ts

@@ -1,6 +1,7 @@
 import { describe, expect, test } from "bun:test";
 import type { Config } from "../../../src/config/index.js";
-import { formatCaughtError, runExec } from "../../../src/exec/runner.js";
+import { formatCaughtError, resolveExecDirectorOverlay, runExec } from "../../../src/exec/runner.js";
+import { IMPLEMENT_TOOLS } from "../../../src/agent/directors/tool-sets.js";
 
 function bareConfig(task: string): Config {
   // Minimal unconfigured-shaped object is not enough — runExec only needs
@@ -48,3 +49,22 @@ describe("runExec", () => {
     }
   });
 });
+
+describe("resolveExecDirectorOverlay", () => {
+  test("implement exec primary does not mount task", () => {
+    const overlay = resolveExecDirectorOverlay("implement");
+    expect(overlay.mountTask).toBe(false);
+    expect(overlay.advertisedAllow).toBeDefined();
+    expect(overlay.advertisedAllow).not.toContain("task");
+    expect(overlay.advertisedAllow).toEqual([...IMPLEMENT_TOOLS]);
+    expect(overlay.systemPrompt).toContain("ImplementDirector");
+  });
+
+  test("skywalker default still can mount task", () => {
+    expect(resolveExecDirectorOverlay(undefined).mountTask).toBe(true);
+    expect(resolveExecDirectorOverlay(undefined).systemPrompt).toBeUndefined();
+    expect(resolveExecDirectorOverlay(undefined).advertisedAllow).toBeUndefined();
+    expect(resolveExecDirectorOverlay("skywalker").mountTask).toBe(true);
+    expect(resolveExecDirectorOverlay("skywalker").systemPrompt).toBeUndefined();
+  });
+});