diff --git a/packages/agent-lifecycle/src/index.ts b/packages/agent-lifecycle/src/index.ts index 6255f60ac..d53bee5bc 100644 --- a/packages/agent-lifecycle/src/index.ts +++ b/packages/agent-lifecycle/src/index.ts @@ -25,11 +25,30 @@ type Logger = ReturnType; */ export const IDLE_HIBERNATE_UNDEPLOY_REASON = "idle-hibernate"; +/** + * `ensureAwake`'s default bound on one call to the injected `wake` port. + * A cold wake for a run whose deployed record was parked aside is a real + * deploy round-trip to the host, not a local check — long enough to allow + * for that, short enough that a wake the host never acks turns into a + * rejection (CL-6643) instead of a promise nothing ever observes. + */ +export const DEFAULT_WAKE_TIMEOUT_MS = 30_000; + export type CreateAgentLifecycleOptions = { /** How long an address may sit idle (no `recordActivity`) before the sweep sleeps it. Required — no default hidden in here. */ idleSleepMs: number; /** How often the sweep runs. Defaults to half the sleep threshold, floored at 5s. */ sweepIntervalMs?: number; + /** + * How long one `ensureAwake` call may wait on the injected `wake` port + * before treating it as failed. Defaults to `DEFAULT_WAKE_TIMEOUT_MS`. + * `wake` itself keeps running past the timeout — this only bounds how + * long a caller (and everyone coalesced behind it) waits before this + * package gives up and reports a rejection, so a wake the host never + * acks can never wedge an address's mail silently for the rest of the + * process's life (CL-6643). + */ + wakeTimeoutMs?: number; /** Whether the host currently has this address deployed/connected. */ isRoutable(address: string): boolean; /** Tears the address down on the host. Errors are the caller's to throw; the sweep catches and logs them per-address. */ @@ -76,6 +95,7 @@ export function createAgentLifecycle( const { idleSleepMs, isRoutable, undeploy, wake, log } = options; const sweepIntervalMs = options.sweepIntervalMs ?? Math.max(idleSleepMs / 2, 5_000); + const wakeTimeoutMs = options.wakeTimeoutMs ?? DEFAULT_WAKE_TIMEOUT_MS; const isBusy = options.isBusy ?? (() => false); const tracked = new Set(); @@ -148,13 +168,41 @@ export function createAgentLifecycle( }, sweepIntervalMs); if (typeof interval.unref === "function") interval.unref(); + /** + * Races `wake(address)` against `wakeTimeoutMs`. A timeout rejects + * with a distinct error so a caller (and everything coalesced behind + * it) always gets a settled outcome — never a hang — even though the + * underlying `wake` call keeps running unobserved past the deadline. + */ + function wakeWithTimeout(address: string): Promise { + return new Promise((resolve, reject) => { + const timer = setTimeout(() => { + reject( + new Error( + `wake for "${address}" did not settle within ${String(wakeTimeoutMs)}ms`, + ), + ); + }, wakeTimeoutMs); + wake(address).then( + (value) => { + clearTimeout(timer); + resolve(value); + }, + (cause: unknown) => { + clearTimeout(timer); + reject(cause); + }, + ); + }); + } + async function ensureAwake(address: string): Promise { if (isRoutable(address)) return; const pending = pendingWakes.get(address); if (pending !== undefined) return pending; - const waking = wake(address) + const waking = wakeWithTimeout(address) .then(() => { recordActivity(address); }) diff --git a/packages/agent-lifecycle/test/index.test.ts b/packages/agent-lifecycle/test/index.test.ts index 4afb4df2f..7ced65573 100644 --- a/packages/agent-lifecycle/test/index.test.ts +++ b/packages/agent-lifecycle/test/index.test.ts @@ -325,4 +325,50 @@ describe("createAgentLifecycle", () => { failure, ); }); + + // CL-6643: a run whose deployed record was parked aside (no sidecar + // record, cold wake required) can hang inside the injected `wake` port + // forever — the sidecar never acks a redeploy it has no state for. + // Without a bound, that first hung call never leaves `pendingWakes` + // (its `.finally` never runs because the promise it's attached to + // never settles), so every later `ensureAwake` for the same address + // coalesces onto that same dead promise and hangs too — silently, + // forever, for the rest of the process's life. This is the mechanism + // behind "message accepted, then silence: no fanout, no wake, no + // error" — the caller (`sendMail`) never sees a rejection to report as + // an undelivered notice, because nothing ever rejects. + test("a wake that never settles does not wedge every later call behind it forever", async () => { + const routable = new Set(); + let wakeCalls = 0; + + const lifecycle = createAgentLifecycle({ + idleSleepMs: 1_000, + wakeTimeoutMs: 20, + isRoutable: (address) => routable.has(address), + undeploy: async () => undefined, + wake: async () => { + wakeCalls += 1; + // Never resolves and never rejects — the hung sidecar RPC. + await new Promise(() => {}); + }, + log, + }); + stop = lifecycle.stop; + + // The first caller must not hang forever: the bounded wake times out + // and the caller sees a rejection it can turn into an undelivered + // notice, instead of an unsettled promise nothing ever observes. + await expect(lifecycle.ensureAwake("agent-1@t.test")).rejects.toThrow( + /wake/i, + ); + expect(wakeCalls).toBe(1); + + // A second, later call for the same address — the next message sent + // to the same workbench — must attempt its own wake rather than + // coalescing onto the first call's dead, already-timed-out promise. + await expect(lifecycle.ensureAwake("agent-1@t.test")).rejects.toThrow( + /wake/i, + ); + expect(wakeCalls).toBe(2); + }); });