Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,13 @@ parallel copies under `docs/` or `scripts/notes/`. At cut time: rename
write-free. Shell file-writes stay denied. Spawn is a judgment call,
not a tool ban.

- **Exec and capability evals can run as a chosen primary director.**
`corbits exec --director <id>` (and eval `--director`) overlays that
package's system prompt and tool allowlist on the product exec path.
Omit / skywalker keep the default Skywalker session. Directors that
cannot spawn (for example implement) do not mount `task`. This is an
exec/eval/CI override, not a TUI or single-agent mode.

## [0.2.99] - 2026-08-21

Skywalker is the primary orchestrator over a closed director fleet: product write tools stay off the primary, and you cannot spawn Skywalker as a task leaf. Workers are not done until they return the four-heading report. First-party action skills ship as slashes; eval runners require an explicit provider/model pair; the style skill no longer refuses non-git folders.
Expand Down
5 changes: 5 additions & 0 deletions evals/capability/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -132,6 +132,10 @@ bun run eval:capability -- --provider <name> --model <id> \
bun run eval:capability -- --provider <name> --model <id> --repeats 5 \
--out evals/capability/results/candidate.json \
--baseline evals/capability/results/baseline-0286.json

# Overlay a closed-fleet director on the product exec path (eval/CI override,
# not single-agent mode). Omit / skywalker keep the default Skywalker session.
bun run eval:capability -- --provider <name> --model <id> --director implement
```

## Confirmation gate for behavior changes
Expand Down Expand Up @@ -170,6 +174,7 @@ Flags:
| `--repeats <n>` | Runs per case×variant cell (default `1`; gate runs use `5`, baseline freezes `3`). Results record every repeat plus per-cell aggregates |
| `--concurrency <n>` | Independent case×variant×repeat cells in parallel (default `1`, env `CORBITS_EVAL_CONCURRENCY`). Each cell still uses its own temp workdir. Use `--concurrency 4` (or similar) to run a live matrix faster |
| `--dry-run` | Load cases × variants and print plan; no inference. Still requires `--provider`/`--model` or `--matrix` |
| `--director <id>` | Exec overlay: run the product `corbits exec` path as this closed-fleet director (default: skywalker). Eval/CI override, not single-agent mode. Directors that cannot spawn (for example `implement`) do not mount `task`. |

## Case format

Expand Down
17 changes: 17 additions & 0 deletions scripts/eval-capability.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -118,6 +118,22 @@ describe("parseArgs", () => {
/CORBITS_EVAL_CONCURRENCY must be a positive integer/,
);
});

test("--director implement is parsed", () => {
const opts = parseArgs(["--provider", "foo", "--model", "bar", "--director", "implement"]);
expect(opts.director).toBe("implement");
});

test("omitted --director stays undefined", () => {
const opts = parseArgs(["--provider", "foo", "--model", "bar"]);
expect(opts.director).toBeUndefined();
});

test("--director without a value throws", () => {
expect(() => parseArgs(["--provider", "foo", "--model", "bar", "--director"])).toThrow(
"--director requires a value",
);
});
});

describe("mapPool", () => {
Expand Down Expand Up @@ -150,6 +166,7 @@ describe("mapPool", () => {
test("rejects non-positive concurrency", async () => {
await expect(mapPool([1], 0, async (item) => item)).rejects.toThrow(/positive integer/);
});

});

describe("initEvalGitRepo", () => {
Expand Down
11 changes: 11 additions & 0 deletions scripts/eval-capability.ts
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,11 @@ type CliOptions = {
* differs from what was requested, instead of hard-failing.
*/
allowProviderFallback: boolean;
/**
* Exec overlay: run the product path as this closed-fleet director.
* Eval/CI override, not single-agent mode. Omitted = skywalker default.
*/
director?: string;
};

function printUsage(): void {
Expand All @@ -99,6 +104,8 @@ function printUsage(): void {
--dry-run List cases × variants only (still requires --provider/--model or --matrix)
--allow-provider-fallback Allow resolved provider/model to differ from
what was requested (default: hard-fail)
--director <id> Exec overlay: run as this director (default: skywalker).
Eval/CI override, not single-agent mode
-h, --help Show help
`);
}
Expand Down Expand Up @@ -230,6 +237,9 @@ export function parseArgs(argv: readonly string[]): CliOptions {
case "--allow-provider-fallback":
opts.allowProviderFallback = true;
break;
case "--director":
opts.director = next();
break;
default:
throw new Error(`Unknown argument: ${a}`);
}
Expand Down Expand Up @@ -608,6 +618,7 @@ async function runCase(
if (opts.configPath !== undefined) argv.push("--config", opts.configPath);
if (opts.skipPermissions) argv.push("--dangerously-skip-permissions");
argv.push("--force");
if (opts.director !== undefined) argv.push("--director", opts.director);

const maxTurns = opts.maxTurnsOverride ?? caseDef.maxTurns ?? null;
// maxTurns is a soft post-run budget (case fails if exceeded). It does not
Expand Down
50 changes: 50 additions & 0 deletions src/config.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import { tmpdir } from "node:os";
import { join } from "node:path";

import { buildBifrostSource, buildOpenAISource, buildXaiSource, buildProviderCatalog, catalogEntryAsProviderSettings, CliHelpError, CLI_HELP_TEXT, KEYLESS_API_KEY, loadConfig, providerCatalogToSettings, runtimeSettingsWithCatalog, SOURCE_MAX_TOKENS } from "./config/index.js";
import { DIRECTOR_IDS } from "./agent/directors/types.js";
import type { Config, UnconfiguredConfig } from "./config/index.js";
import { mergeProviderIntoSettings, type ResolvedProvider, type Settings } from "./config/settings.js";
import { OPENCODE_GO_BASE_URL } from "../packages/opencode-go/src/index.js";
Expand Down Expand Up @@ -183,6 +184,55 @@ describe("loadConfig", () => {
}
});

test("parses exec --director implement", async () => {
const cwd = await emptyCwd();
try {
const globalPath = await writeGlobalSettings(cwd);
const config = await loadConfig(["exec", "--cwd", cwd, "--director", "implement", "ship it"], {
globalSettingsPath: globalPath,
});
assertConfigured(config);
expect(config.command).toBe("exec");
expect(config.director).toBe("implement");
expect(config.task).toBe("ship it");
} finally {
await rm(cwd, { recursive: true, force: true });
}
});

test("omits director as undefined (skywalker default)", async () => {
const cwd = await emptyCwd();
try {
const globalPath = await writeGlobalSettings(cwd);
const config = await loadConfig(["exec", "--cwd", cwd, "ship it"], {
globalSettingsPath: globalPath,
});
assertConfigured(config);
expect(config.command).toBe("exec");
expect(config.director).toBeUndefined();
} finally {
await rm(cwd, { recursive: true, force: true });
}
});

test("unknown --director id errors listing DIRECTOR_IDS", async () => {
await expect(
loadConfig(["exec", "--director", "nope", "ship it"], { globalSettingsPath: NO_SETTINGS }),
).rejects.toThrow(new RegExp(`Unknown director "nope".*${DIRECTOR_IDS.join(", ")}`));
});

test("--director without a value errors", async () => {
await expect(loadConfig(["exec", "--director"], { globalSettingsPath: NO_SETTINGS })).rejects.toThrow(
"--director requires a value",
);
});

test("--director without exec/run is rejected", async () => {
await expect(
loadConfig(["--director", "implement", "ship it"], { globalSettingsPath: NO_SETTINGS }),
).rejects.toThrow("--director is only available in exec mode");
});

test("resume --pick opens the session picker without requiring prior sessions", async () => {
const cwd = await emptyCwd();
try {
Expand Down
24 changes: 24 additions & 0 deletions src/config/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@ import { generateSessionId, isSessionId, migrateLegacySessionIfNeeded } from "..
import { loadState } from "../session/state.js";


import { isDirectorId } from "../agent/directors/registry.js";
import { DIRECTOR_IDS, type DirectorId } from "../agent/directors/types.js";
import { validateEffort, type ReasoningEffort } from "../provider/reasoning-effort.js";
import { bootstrapPricingMetadata } from "../cost/pricing-metadata.js";
import { defaultPricingCachePath, type PricingFetcherOptions } from "../cost/pricing-fetcher.js";
Expand Down Expand Up @@ -278,6 +280,11 @@ export type Config = {
force: boolean;
dangerouslySkipPermissions: boolean;
auto: boolean;
/**
* Exec-only chosen primary director. Omitted = Skywalker (product default).
* `--director` is rejected in TUI mode.
*/
director?: DirectorId;
/**
* Entry mode. `"tui"` is the interactive Ink shell; `"exec"` is the non-TUI
* product agent path (`corbits exec "prompt"`). Same directors/tools/permissions.
Expand Down Expand Up @@ -343,6 +350,8 @@ export type UnconfiguredConfig = {
dangerouslySkipPermissions: boolean;
auto: boolean;
command: "tui" | "exec";
/** Exec-only chosen primary. Omitted on the unconfigured path too. */
director?: DirectorId;
// Path where the onboarding flow should write the new settings.
globalSettingsPath: string;
// The original error message, used for non-TUI (exec) error output.
Expand Down Expand Up @@ -377,6 +386,7 @@ Flags:
--profile <name> settings profile
--resume interactive session picker
--force override an existing run state
--director <id> exec-only: run as this director (default: skywalker)
--dangerously-skip-permissions
--auto / --no-auto auto mode on/off
--help, -h show this help
Expand Down Expand Up @@ -474,6 +484,7 @@ export async function loadConfig(
// to ask-on-every-write. There is currently no in-session key to toggle auto;
// Shift+Tab in the TUI cycles reasoning effort instead.
let auto = true;
let director: DirectorId | undefined;
let configPath: string | undefined;
let provider: string | undefined;
let model: string | undefined;
Expand Down Expand Up @@ -515,6 +526,17 @@ export async function loadConfig(
force = true;
continue;
}
if (arg === "--director") {
const value = requireValue("--director", args[++i]);
if (command !== "exec") {
throw new Error("--director is only available in exec mode");
}
if (!isDirectorId(value)) {
throw new Error(`Unknown director "${value}". Use one of: ${DIRECTOR_IDS.join(", ")}.`);
}
director = value;
continue;
}

if (arg === "--dangerously-skip-permissions") {
dangerouslySkipPermissions = true;
Expand Down Expand Up @@ -633,6 +655,7 @@ export async function loadConfig(
dangerouslySkipPermissions,
auto,
command,
...(director !== undefined ? { director } : {}),
globalSettingsPath: effectiveSettingsPath,
providerError: err instanceof Error ? err.message : String(err),
// Keep diagnostics even when provider setup fails early so junk local
Expand Down Expand Up @@ -684,6 +707,7 @@ export async function loadConfig(
dangerouslySkipPermissions,
auto,
command,
...(director !== undefined ? { director } : {}),
globalSettingsPath: effectiveSettingsPath,
sessionId,
noWorkflow,
Expand Down
86 changes: 67 additions & 19 deletions src/exec/runner.ts
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,9 @@ import {
} from "../config/settings.js";
import { codexProfileFromProviderName } from "../config/codex-providers.js";
import { xaiProfileFromProviderName } from "../config/xai-providers.js";
import { formatDirectorSystemPrompt } from "../agent/directors/identity.js";
import { DIRECTOR_REGISTRY } from "../agent/directors/registry.js";
import type { DirectorId } from "../agent/directors/types.js";
import { createInferenceDependencies } from "../provider/inference-dependencies.js";
import { getValidCodexToken } from "../auth/codex/session.js";
import { getValidXaiToken } from "../auth/xai/session.js";
Expand Down Expand Up @@ -111,6 +114,40 @@ export function formatCaughtError(err: unknown): string {
return err instanceof Error ? err.message : String(err);
}

/**
* Exec-primary director overlay. Omit / skywalker keep the product default
* (`loadSessionChatPrompt` + advertised session tools). Any other closed-fleet
* id uses the package prompt and allowlist. Worker effort/nudge are not applied.
*/
export type ExecDirectorOverlay = {
/** Package system prompt; omitted on the skywalker default path. */
systemPrompt?: string;
/** `pkg.tools.allow` (task stripped when `maySpawn` is false). */
advertisedAllow?: readonly string[];
mountTask: boolean;
};

export function resolveExecDirectorOverlay(
director: DirectorId | undefined,
): ExecDirectorOverlay {
if (director === undefined || director === "skywalker") {
return { mountTask: true };
}
const pkg = DIRECTOR_REGISTRY[director];
const allow = pkg.tools?.allow;
const advertisedAllow =
allow !== undefined && allow.length > 0
? pkg.spawn.maySpawn
? [...allow]
: allow.filter((name) => name !== "task")
: undefined;
return {
systemPrompt: formatDirectorSystemPrompt(pkg),
...(advertisedAllow !== undefined ? { advertisedAllow } : {}),
mountTask: pkg.spawn.maySpawn,
};
}

/** Content-less inbound used after compact so the reactor re-enters (matches TUI). */
export function buildCompactionContinuationMessage(): InboundMessage {
return {
Expand Down Expand Up @@ -348,6 +385,8 @@ export async function runExec(config: Config): Promise<ExecResult> {

let currentAgent: Agent | null = null;

const overlay = resolveExecDirectorOverlay(config.director);

const agentToolset = await createAgentToolset({
cwd: config.cwd,
permissionGate,
Expand Down Expand Up @@ -384,30 +423,39 @@ export async function runExec(config: Config): Promise<ExecResult> {
);
return result.kind === "option" && result.index === 0;
},
subAgent: {
provider: () => liveSubAgentProvider.current,
sessions: subAgentSessions,
getWorkdirBase: () => sessionDir(config.cwd, sessionId),
onProgress: () => undefined,
...(config.settings !== undefined ? { settings: () => config.settings! } : {}),
catalog: () => config.providers,
profiles: () => liveAgentProfiles,
},
...(overlay.mountTask
? {
subAgent: {
provider: () => liveSubAgentProvider.current,
sessions: subAgentSessions,
getWorkdirBase: () => sessionDir(config.cwd, sessionId),
onProgress: () => undefined,
...(config.settings !== undefined ? { settings: () => config.settings! } : {}),
catalog: () => config.providers,
profiles: () => liveAgentProfiles,
},
}
: {}),
...(extraToolPlugins.length > 0 ? { extraToolPlugins } : {}),
});
toolset = agentToolset;

const { systemPrompt } = await loadSessionChatPrompt({
cwd: config.cwd,
skillDirs,
...(config.systemPromptExtensions !== undefined
? { systemPromptExtensions: config.systemPromptExtensions }
: {}),
sessionMode,
toolAvailability,
});
const systemPrompt =
overlay.systemPrompt ??
(
await loadSessionChatPrompt({
cwd: config.cwd,
skillDirs,
...(config.systemPromptExtensions !== undefined
? { systemPromptExtensions: config.systemPromptExtensions }
: {}),
sessionMode,
toolAvailability,
})
).systemPrompt;

const advertisedBuiltInPrefix = advertisedToolNamesForSessionMode(sessionMode, toolAvailability);
const advertisedBuiltInPrefix =
overlay.advertisedAllow ?? advertisedToolNamesForSessionMode(sessionMode, toolAvailability);
const activatedToolNames = createActivatedToolTracker();
// Advertise then family-gate wire schemas (kimi gets a non-recursive present).
const computeAdvertised = (all: readonly ToolDefinition[]): ToolDefinition[] =>
Expand Down
22 changes: 21 additions & 1 deletion tests/unit/exec/runner.test.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import { describe, expect, test } from "bun:test";
import type { Config } from "../../../src/config/index.js";
import { formatCaughtError, runExec } from "../../../src/exec/runner.js";
import { formatCaughtError, resolveExecDirectorOverlay, runExec } from "../../../src/exec/runner.js";
import { IMPLEMENT_TOOLS } from "../../../src/agent/directors/tool-sets.js";

function bareConfig(task: string): Config {
// Minimal unconfigured-shaped object is not enough — runExec only needs
Expand Down Expand Up @@ -48,3 +49,22 @@ describe("runExec", () => {
}
});
});

describe("resolveExecDirectorOverlay", () => {
test("implement exec primary does not mount task", () => {
const overlay = resolveExecDirectorOverlay("implement");
expect(overlay.mountTask).toBe(false);
expect(overlay.advertisedAllow).toBeDefined();
expect(overlay.advertisedAllow).not.toContain("task");
expect(overlay.advertisedAllow).toEqual([...IMPLEMENT_TOOLS]);
expect(overlay.systemPrompt).toContain("ImplementDirector");
});

test("skywalker default still can mount task", () => {
expect(resolveExecDirectorOverlay(undefined).mountTask).toBe(true);
expect(resolveExecDirectorOverlay(undefined).systemPrompt).toBeUndefined();
expect(resolveExecDirectorOverlay(undefined).advertisedAllow).toBeUndefined();
expect(resolveExecDirectorOverlay("skywalker").mountTask).toBe(true);
expect(resolveExecDirectorOverlay("skywalker").systemPrompt).toBeUndefined();
});
});
Loading