diff --git a/packages/chat/src/platform-adapter.ts b/packages/chat/src/platform-adapter.ts index 5bdae3ea..6fc113a6 100644 --- a/packages/chat/src/platform-adapter.ts +++ b/packages/chat/src/platform-adapter.ts @@ -111,6 +111,13 @@ export type CreateHubChatPlatformDeps = { * backoff in milliseconds instead of the production ~8s budget. */ reclaimRetryDelaysMs?: readonly number[]; + /** + * Per-attempt wall-clock bound on `sendFoldedMail` inside + * `sendFoldedMailWithReclaimRetry` (CL-6644). Defaults to + * `DEFAULT_WAKE_TIMEOUT_MS`. Injectable so tests exercise the bound + * in milliseconds instead of the production ~30s budget. + */ + mailDeliveryTimeoutMs?: number; /** * The invite-launch model fallback (see `./inference-preferences.ts`'s * `createWorkbenchHostInferencePreferencesResolver`): a @@ -245,10 +252,42 @@ export function createHubChatPlatform( 250, 500, 1000, 2000, 4000, ]; + // CL-6644: the reclaim-retry loop's own delays only run between + // attempts that already failed LOUD with "agent is unreachable" -- + // they bound nothing about an attempt that instead stalls forever + // (a sidecar ack that never comes, a wedged promise anywhere in + // `sendFoldedMail`'s call chain) without ever throwing. Each + // attempt gets the same kind of wall-clock bound `wakeByAddressBounded` + // already puts on the wake itself, so that hang becomes a rejection + // `dispatchTurnBatch`'s catch can report and notify on, instead of a + // promise nothing ever settles. + const MAIL_DELIVERY_TIMEOUT_MS = + deps.mailDeliveryTimeoutMs ?? DEFAULT_WAKE_TIMEOUT_MS; + function sleep(ms: number): Promise { return new Promise((resolveSleep) => setTimeout(resolveSleep, ms)); } + function withTimeout( + promise: Promise, + ms: number, + message: string, + ): Promise { + return new Promise((resolve, reject) => { + const timer = setTimeout(() => reject(new Error(message)), ms); + promise.then( + (value) => { + clearTimeout(timer); + resolve(value); + }, + (cause: unknown) => { + clearTimeout(timer); + reject(cause); + }, + ); + }); + } + function isAgentUnreachable(err: unknown): boolean { return err instanceof Error && err.message.includes("agent is unreachable"); } @@ -565,25 +604,11 @@ export function createHubChatPlatform( * failing loud (CL-6644). */ function wakeByAddressBounded(address: string): Promise { - return new Promise((resolve, reject) => { - const timer = setTimeout(() => { - reject( - new Error( - `wake for "${address}" did not settle within ${String(DEFAULT_WAKE_TIMEOUT_MS)}ms`, - ), - ); - }, DEFAULT_WAKE_TIMEOUT_MS); - wakeByAddress(address).then( - (value) => { - clearTimeout(timer); - resolve(value); - }, - (cause: unknown) => { - clearTimeout(timer); - reject(cause); - }, - ); - }); + return withTimeout( + wakeByAddress(address), + DEFAULT_WAKE_TIMEOUT_MS, + `wake for "${address}" did not settle within ${String(DEFAULT_WAKE_TIMEOUT_MS)}ms`, + ); } /** @@ -682,7 +707,11 @@ export function createHubChatPlatform( let loggedRetryStart = false; for (let attempt = 0; ; attempt++) { try { - return await sendFoldedMail(foldedRunsDeps, params); + return await withTimeout( + sendFoldedMail(foldedRunsDeps, params), + MAIL_DELIVERY_TIMEOUT_MS, + `mail to ${params.agentAddress} did not settle within ${String(MAIL_DELIVERY_TIMEOUT_MS)}ms`, + ); } catch (err) { const delay = RECLAIM_RETRY_DELAYS_MS[attempt]; if (!isAgentUnreachable(err) || delay === undefined) { diff --git a/packages/chat/test/platform-adapter.test.ts b/packages/chat/test/platform-adapter.test.ts index c15e6993..c6529654 100644 --- a/packages/chat/test/platform-adapter.test.ts +++ b/packages/chat/test/platform-adapter.test.ts @@ -943,6 +943,70 @@ describe("createHubChatPlatform", () => { ); }); + test("sendMail rejects within the mail-delivery deadline instead of hanging forever when delivery never settles (CL-6644)", async () => { + resolveDefinitionSourcesResult = { + ok: true, + sources: [ + { + id: "off_1", + provider: "anthropic", + baseURL: "https://inference.invalid", + apiKey: "placeholder", + model: "claude-sonnet-5", + }, + ], + defaultSource: "off_1", + }; + + const db = createFakeDb({ + assetRow: { + tenantId: "ten_1", + creatorPrincipalId: "prin_creator", + name: "workbench-1", + displayName: null, + }, + definitionId: "wfd_workbench1", + workflowRunRow: { + id: "ins_workbench1", + address: "ins_workbench1@ten1.workbench.test", + principalId: "prin_run1", + }, + }); + db.inserted.push({ + table: agentSession, + values: { id: "ses_run1", principalId: "prin_run1" }, + }); + + const sessionService = createFakeSessionService(); + // Models the observed CL-6644 symptom: the post-deploy delivery + // step (`sessionService.sendUserMessage`, reached through + // `sendFoldedMail`) never resolves and never rejects -- a wedged + // ack, not a thrown "agent is unreachable" the reclaim-retry loop + // already knows how to handle. Before the fix, `sendMail`'s + // returned promise stayed pending forever with nothing logged. + sessionService.sendUserMessage = () => new Promise(() => {}); + const sidecarRouter = createFakeSidecarRouter(); + + const platform = createHubChatPlatform({ + toolGrantsForPins: () => [], + db: db as never, + sessionService, + assetService: createFakeAssetService(), + sidecarRouter, + eventCollectors: createFakeEventCollectors(), + mailDeliveryTimeoutMs: 20, + }); + + await expect( + platform.sendMail({ + tenantId: "ten_1", + workbenchId: "ins_workbench1", + principalId: "prin_sender", + content: { content: "hello workbench" }, + }), + ).rejects.toThrow(/did not settle within 20ms/); + }); + test("launchInvite mints from the target definition and ensureAwake deploys it", async () => { resolveDefinitionSourcesCalls.length = 0; resolveDefinitionSourcesResult = {