Skip to content

Commit e40824c

Browse files
committed
Stop aborting long task and shell runs at eleven minutes
1 parent 9b78ffb commit e40824c

5 files changed

Lines changed: 163 additions & 58 deletions

File tree

CHANGELOG.md

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -17,13 +17,23 @@ parallel copies under `docs/` or `scripts/notes/`. At cut time: rename
1717

1818
- **Requested `run_shell` timeouts are no longer capped at 10 minutes.** The 15s
1919
default when timeout is omitted is unchanged. `shell.maxTimeoutMs` still
20-
clamps the command when set. The tool-execution watchdog follows a longer
21-
requested `run_shell` timeout instead of aborting at 11 minutes;
22-
`tools.timeoutMs` / `tools.maxTimeoutMs` still bound other tools only.
23-
Capability evals accept `--concurrency <n>` (env `CORBITS_EVAL_CONCURRENCY`,
20+
clamps the command when set.
21+
22+
- Capability evals accept `--concurrency <n>` (env `CORBITS_EVAL_CONCURRENCY`,
2423
default 1); overlapping `httpFixture` cells isolate `EVAL_HTTP_URL` so
2524
parallel web-bait runs do not share a process.env origin.
2625

26+
### TUI
27+
28+
- **Tool `run()` no longer has an implicit 11-minute wall-clock abort.** The
29+
outer watchdog arms only when Settings set `tools.timeoutMs` /
30+
`tools.maxTimeoutMs`, or when `run_shell` passes a positive `timeout`
31+
(requested plus slack, so this layer cannot beat shell-guard). Unset
32+
settings leave `task` and other tools unbounded; parent cancel, maxTurns,
33+
and eval `--agent-timeout-ms` still bound the run. `tools.maxTimeoutMs`
34+
still clamps non-shell tools when set and does not cap a longer requested
35+
`run_shell`.
36+
2737
## [0.2.99] - 2026-08-21
2838

2939
Skywalker is the primary orchestrator over a closed director fleet: product write tools stay off the primary, and you cannot spawn Skywalker as a task leaf. Workers are not done until they return the four-heading report. First-party action skills ship as slashes; eval runners require an explicit provider/model pair; the style skill no longer refuses non-git folders.

src/subagent/index.test.ts

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -792,6 +792,10 @@ describe("sub-agent stop helpers", () => {
792792
expect(resolveSubAgentDeadlineMs(45_000, 660_000)).toBe(45_000);
793793
});
794794

795+
test("resolveSubAgentDeadlineMs keeps an explicit deadline when the outer watchdog is omitted", () => {
796+
expect(resolveSubAgentDeadlineMs(18_000_000, undefined)).toBe(18_000_000);
797+
});
798+
795799
test("resolveSubAgentDeadlineMs skips arming when outer watchdog is at or below the margin", () => {
796800
expect(resolveSubAgentDeadlineMs(5_000, 5_000)).toBeUndefined();
797801
expect(resolveSubAgentDeadlineMs(5_000, SUBAGENT_DEADLINE_MARGIN_MS)).toBeUndefined();

src/subagent/stop-policy.ts

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,15 +32,20 @@ export const SUBAGENT_DEADLINE_MARGIN_MS = 30_000;
3232
* maxTurns + operator cancel are the primary bounds; callers pass deadlineMs
3333
* only when they want an extra wall-clock stop.
3434
*
35+
* When the outer watchdog is omitted (undefined), the requested deadline is
36+
* kept — an absent settings timeout must not clamp a 5-hour (or any) explicit
37+
* deadline down to a hidden default.
38+
*
3539
* Returns undefined (do not arm) when the outer watchdog is at or below the
3640
* salvage margin — an internal deadline would otherwise race or exceed outer
3741
* and leave no room to return a salvage report.
3842
*/
3943
export function resolveSubAgentDeadlineMs(
4044
requestedMs: number,
41-
outerWatchdogMs: number,
45+
outerWatchdogMs: number | undefined,
4246
): number | undefined {
4347
const requested = Math.max(1, Math.floor(requestedMs));
48+
if (outerWatchdogMs === undefined) return requested;
4449
if (outerWatchdogMs <= SUBAGENT_DEADLINE_MARGIN_MS) return undefined;
4550
// Ceiling must never exceed outer − margin (and stays ≥ 1 once outer > margin).
4651
const ceiling = Math.max(1, outerWatchdogMs - SUBAGENT_DEADLINE_MARGIN_MS);

src/tui/tool-execution-watchdog.test.ts

Lines changed: 78 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@ import { describe, expect, test } from "bun:test";
22
import type { AgentTool } from "@intx/agent";
33
import { createDynamicToolRunner } from "./dynamic-tool-runner.js";
44
import {
5-
DEFAULT_TOOL_EXECUTION_TIMEOUT_MS,
65
MAX_TOOL_EXECUTION_TIMEOUT_MS,
76
RUN_SHELL_WATCHDOG_SLACK_MS,
87
getToolApprovalBudget,
@@ -34,35 +33,59 @@ describe("tool execution watchdog", () => {
3433
expect(resolveToolExecutionTimeoutMs({ defaultMs: 9_999_999, maxMs: 100 })).toBe(100);
3534
});
3635

37-
test("run_shell requested timeout above 660s is not scheduled at the 11-minute default", () => {
38-
const requested = 5_400_000;
36+
test("task with no settings timeout is unbounded", () => {
37+
expect(
38+
resolveToolExecutionTimeoutMs(undefined, { id: "1", name: "task", arguments: {} }),
39+
).toBeUndefined();
40+
});
41+
42+
test("omitted config does not arm a default watchdog", () => {
43+
expect(resolveToolExecutionTimeoutMs(undefined)).toBeUndefined();
44+
expect(resolveToolExecutionTimeoutMs({})).toBeUndefined();
45+
expect(resolveToolExecutionTimeoutMs({ waitForApproval: true })).toBeUndefined();
46+
});
47+
48+
test("settings timeout without max clamps to MAX_TOOL_EXECUTION_TIMEOUT_MS", () => {
49+
expect(resolveToolExecutionTimeoutMs({ defaultMs: 9_999_999 })).toBe(
50+
MAX_TOOL_EXECUTION_TIMEOUT_MS,
51+
);
52+
});
53+
54+
test("run_shell requested 5-hour timeout is not clamped", () => {
55+
const requested = 18_000_000;
3956
const call = { id: "1", name: "run_shell", arguments: { timeout: requested } };
4057
const ms = resolveToolExecutionTimeoutMs(undefined, call);
4158
expect(ms).toBe(requested + RUN_SHELL_WATCHDOG_SLACK_MS);
42-
expect(ms).toBeGreaterThan(DEFAULT_TOOL_EXECUTION_TIMEOUT_MS);
4359
expect(ms).toBeGreaterThan(MAX_TOOL_EXECUTION_TIMEOUT_MS);
4460
});
4561

4662
test("tools.maxTimeoutMs does not cap a longer requested run_shell timeout", () => {
47-
const requested = 5_400_000;
63+
const requested = 18_000_000;
4864
const call = { id: "1", name: "run_shell", arguments: { timeout: requested } };
49-
const ms = resolveToolExecutionTimeoutMs(
50-
{ defaultMs: DEFAULT_TOOL_EXECUTION_TIMEOUT_MS, maxMs: 100_000 },
51-
call,
52-
);
65+
const ms = resolveToolExecutionTimeoutMs({ defaultMs: 660_000, maxMs: 100_000 }, call);
5366
expect(ms).toBe(requested + RUN_SHELL_WATCHDOG_SLACK_MS);
5467
});
5568

56-
test("omitted run_shell timeout keeps the default outer budget (shell-guard still 15s)", () => {
69+
test("omitted run_shell timeout is unbounded (shell-guard still 15s)", () => {
5770
expect(
5871
resolveToolExecutionTimeoutMs(undefined, { id: "1", name: "run_shell", arguments: {} }),
59-
).toBe(DEFAULT_TOOL_EXECUTION_TIMEOUT_MS);
72+
).toBeUndefined();
73+
expect(
74+
resolveToolExecutionTimeoutMs(undefined, {
75+
id: "1",
76+
name: "run_shell",
77+
arguments: { timeout: 0 },
78+
}),
79+
).toBeUndefined();
80+
});
81+
82+
test("omitted run_shell timeout still honors settings default", () => {
6083
expect(
6184
resolveToolExecutionTimeoutMs(
62-
{ maxMs: 100_000 },
63-
{ id: "1", name: "run_shell", arguments: { timeout: 0 } },
85+
{ defaultMs: 60_000, maxMs: 100_000 },
86+
{ id: "1", name: "run_shell", arguments: {} },
6487
),
65-
).toBe(DEFAULT_TOOL_EXECUTION_TIMEOUT_MS);
88+
).toBe(60_000);
6689
});
6790

6891
test("non-shell tools still honor tools.maxTimeoutMs", () => {
@@ -195,6 +218,47 @@ describe("tool execution watchdog", () => {
195218
expect(afterAbort.content).toBe("hang aborted");
196219
});
197220

221+
test("undefined timeout lets a 50ms tool complete", async () => {
222+
const result = await runWithToolExecutionWatchdog(
223+
{ id: "unbounded", name: "task", arguments: {} },
224+
new AbortController().signal,
225+
undefined,
226+
async () => {
227+
await new Promise((r) => setTimeout(r, 50));
228+
return { callId: "unbounded", content: "ok" };
229+
},
230+
{ salvageGraceMs: TEST_SALVAGE_GRACE_MS, waitForApproval: true },
231+
);
232+
expect(result.isError).not.toBe(true);
233+
expect(result.content).toBe("ok");
234+
});
235+
236+
test("undefined timeout still surfaces parent abort", async () => {
237+
const parent = new AbortController();
238+
const pending = runWithToolExecutionWatchdog(
239+
{ id: "unbounded-hang", name: "task", arguments: {} },
240+
parent.signal,
241+
undefined,
242+
async () => {
243+
await new Promise(() => {});
244+
return { callId: "unbounded-hang", content: "ok" };
245+
},
246+
{ salvageGraceMs: TEST_SALVAGE_GRACE_MS, waitForApproval: true },
247+
);
248+
parent.abort();
249+
const afterAbort = await Promise.race([
250+
pending,
251+
new Promise<never>((_, reject) =>
252+
setTimeout(
253+
() => reject(new Error("watchdog did not settle after parent abort + grace")),
254+
TEST_SALVAGE_GRACE_MS + 500,
255+
),
256+
),
257+
]);
258+
expect(afterAbort.isError).toBe(true);
259+
expect(afterAbort.content).toBe("task aborted");
260+
});
261+
198262
test("isUsableToolExecuteResult rejects errors and empty bodies", () => {
199263
expect(isUsableToolExecuteResult({ callId: "1", content: "ok" })).toBe(true);
200264
expect(isUsableToolExecuteResult({ callId: "1", content: " " })).toBe(false);

src/tui/tool-execution-watchdog.ts

Lines changed: 61 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -15,12 +15,8 @@ export type ToolWatchdogConfig = {
1515
waitForApproval?: boolean;
1616
};
1717

18-
// Outer budget for tools that have no per-call timeout. run_shell with a longer
19-
// requested timeout is resolved separately (see resolveToolExecutionTimeoutMs)
20-
// so this default — and MAX_TOOL_EXECUTION_TIMEOUT_MS / tools.maxTimeoutMs —
21-
// cannot abort it first. Omitting run_shell timeout still defaults to 15s
22-
// inside shell-guard.
23-
export const DEFAULT_TOOL_EXECUTION_TIMEOUT_MS = 660_000;
18+
// Cap applied when Settings set tools.timeoutMs without tools.maxTimeoutMs.
19+
// Not an implicit default — omitted settings leave the watchdog unarmed.
2420
export const MAX_TOOL_EXECUTION_TIMEOUT_MS = 1_800_000;
2521

2622
/**
@@ -46,16 +42,26 @@ export const MAX_TOOL_APPROVAL_PAUSE_MS = 1_800_000;
4642

4743
const BUDGET_EXPIRED = Symbol("tool-execution-budget-expired");
4844

45+
/**
46+
* Wall-clock budget for one tool `run()`, or undefined to leave the timer unarmed.
47+
* Parent cancel, maxTurns, and eval `--agent-timeout-ms` still bound the run.
48+
*
49+
* Arms only when Settings pass tools.timeoutMs / tools.maxTimeoutMs, or when
50+
* run_shell passes a positive arguments.timeout (requested + slack so this
51+
* layer cannot beat shell-guard). A requested run_shell timeout is not clamped
52+
* to MAX_TOOL_EXECUTION_TIMEOUT_MS or tools.maxTimeoutMs.
53+
*/
4954
export function resolveToolExecutionTimeoutMs(
5055
config?: ToolWatchdogConfig,
5156
call?: ToolCall,
52-
): number {
57+
): number | undefined {
5358
if (call?.name === "run_shell") {
54-
return resolveRunShellWatchdogTimeoutMs(config, call);
59+
const requested = requestedRunShellTimeoutMs(call);
60+
if (requested !== undefined) {
61+
return requested + RUN_SHELL_WATCHDOG_SLACK_MS;
62+
}
5563
}
56-
const max = config?.maxMs ?? MAX_TOOL_EXECUTION_TIMEOUT_MS;
57-
const raw = config?.defaultMs ?? DEFAULT_TOOL_EXECUTION_TIMEOUT_MS;
58-
return Math.min(max, Math.max(1, Math.floor(raw)));
64+
return resolveSettingsWatchdogTimeoutMs(config);
5965
}
6066

6167
function requestedRunShellTimeoutMs(call: ToolCall): number | undefined {
@@ -66,22 +72,15 @@ function requestedRunShellTimeoutMs(call: ToolCall): number | undefined {
6672
return Math.floor(timeout);
6773
}
6874

69-
/**
70-
* run_shell's watchdog floor is the requested command timeout (plus slack so
71-
* this outer timer cannot beat shell-guard). tools.maxTimeoutMs /
72-
* MAX_TOOL_EXECUTION_TIMEOUT_MS still bound other tools only — they must not
73-
* reimpose a cap when the operator/model passed a longer run_shell timeout.
74-
* Omitting timeout leaves the default outer budget (shell-guard still uses 15s).
75-
*/
76-
function resolveRunShellWatchdogTimeoutMs(
75+
function resolveSettingsWatchdogTimeoutMs(
7776
config: ToolWatchdogConfig | undefined,
78-
call: ToolCall,
79-
): number {
80-
const raw = config?.defaultMs ?? DEFAULT_TOOL_EXECUTION_TIMEOUT_MS;
81-
const floor = Math.max(1, Math.floor(raw));
82-
const requested = requestedRunShellTimeoutMs(call);
83-
if (requested === undefined) return floor;
84-
return Math.max(floor, requested + RUN_SHELL_WATCHDOG_SLACK_MS);
77+
): number | undefined {
78+
if (config === undefined || (config.defaultMs === undefined && config.maxMs === undefined)) {
79+
return undefined;
80+
}
81+
const max = config.maxMs ?? MAX_TOOL_EXECUTION_TIMEOUT_MS;
82+
const raw = config.defaultMs ?? max;
83+
return Math.min(max, Math.max(1, Math.floor(raw)));
8584
}
8685

8786
/** Default true: freeze tool budget while a permission prompt is open. */
@@ -122,6 +121,22 @@ export type PauseableTimeout = {
122121
resume: (token: PauseToken) => void;
123122
};
124123

124+
/** Chain parent cancel without arming a run-duration timer. */
125+
function withParentAbort(signal: AbortSignal): PauseableTimeout {
126+
const controller = new AbortController();
127+
const onParentAbort = () => controller.abort();
128+
signal.addEventListener("abort", onParentAbort, { once: true });
129+
if (signal.aborted) controller.abort();
130+
return {
131+
signal: controller.signal,
132+
dispose: () => {
133+
signal.removeEventListener("abort", onParentAbort);
134+
},
135+
pause: (): PauseToken => 0,
136+
resume: (_token: PauseToken) => {},
137+
};
138+
}
139+
125140
/**
126141
* Like withTimeout, but the remaining budget freezes while paused (e.g. while
127142
* a permission prompt is open). Pause/resume are refcounted so nested pauses
@@ -322,8 +337,10 @@ export type ToolExecutionWatchdogOptions = {
322337
};
323338

324339
/**
325-
* Runs `execute` under a wall-clock race against `parentSignal`, matching the
326-
* shell-guard search-tool pattern so non-abortable work still returns on time.
340+
* Runs `execute` under a race against `parentSignal` and, when `timeoutMs` is
341+
* set, a wall-clock budget. `undefined` timeout does not arm a timer — parent
342+
* cancel and the approval-budget ALS still apply. Permission pause ceiling
343+
* (`MAX_TOOL_APPROVAL_PAUSE_MS`) stays a stuck-prompt guard, not a run cap.
327344
*
328345
* When budget/parent abort wins the race, the signal is still aborted, but we
329346
* give the in-flight execute a short grace to return a usable non-error body
@@ -333,19 +350,22 @@ export type ToolExecutionWatchdogOptions = {
333350
export async function runWithToolExecutionWatchdog(
334351
call: ToolCall,
335352
parentSignal: AbortSignal,
336-
timeoutMs: number,
353+
timeoutMs: number | undefined,
337354
execute: (signal: AbortSignal) => Promise<ToolResult>,
338355
options: ToolExecutionWatchdogOptions,
339356
): Promise<ToolResult> {
340357
const salvageGraceMs = options.salvageGraceMs ?? TOOL_EXECUTION_SALVAGE_GRACE_MS;
341358
const waitForApproval = options.waitForApproval;
342-
const budget = waitForApproval
343-
? withPauseableTimeout(parentSignal, timeoutMs)
344-
: {
345-
...withTimeout(parentSignal, timeoutMs),
346-
pause: (): PauseToken => 0,
347-
resume: (_token: PauseToken) => {},
348-
};
359+
const budget: PauseableTimeout =
360+
timeoutMs === undefined
361+
? withParentAbort(parentSignal)
362+
: waitForApproval
363+
? withPauseableTimeout(parentSignal, timeoutMs)
364+
: {
365+
...withTimeout(parentSignal, timeoutMs),
366+
pause: (): PauseToken => 0,
367+
resume: (_token: PauseToken) => {},
368+
};
349369
// Nested runs (task tool → child tool call) shadow the parent store: the
350370
// gate captures the innermost budget, so pause/resume must chain outward or
351371
// the parent `task` budget keeps ticking under the permission modal.
@@ -375,9 +395,10 @@ export async function runWithToolExecutionWatchdog(
375395
if (salvaged !== undefined) return salvaged;
376396
// Avoid unhandled rejection if execute later fails after we move on.
377397
void executePromise.catch(() => {});
378-
const content = parentSignal.aborted
379-
? `${call.name} aborted`
380-
: formatToolExecutionTimeoutMessage(call.name, timeoutMs);
398+
const content =
399+
timeoutMs !== undefined && !parentSignal.aborted
400+
? formatToolExecutionTimeoutMessage(call.name, timeoutMs)
401+
: `${call.name} aborted`;
381402
return { callId: call.id, content, isError: true };
382403
}
383404

@@ -392,6 +413,7 @@ export async function runWithToolExecutionWatchdog(
392413
if (
393414
budget.signal.aborted &&
394415
!parentSignal.aborted &&
416+
timeoutMs !== undefined &&
395417
outcome.isError === true &&
396418
typeof outcome.content === "string" &&
397419
isAbortLikeToolError(outcome.content)

0 commit comments

Comments
 (0)