Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
69 changes: 49 additions & 20 deletions packages/chat/src/platform-adapter.ts
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,13 @@ export type CreateHubChatPlatformDeps = {
* backoff in milliseconds instead of the production ~8s budget.
*/
reclaimRetryDelaysMs?: readonly number[];
/**
* Per-attempt wall-clock bound on `sendFoldedMail` inside
* `sendFoldedMailWithReclaimRetry` (CL-6644). Defaults to
* `DEFAULT_WAKE_TIMEOUT_MS`. Injectable so tests exercise the bound
* in milliseconds instead of the production ~30s budget.
*/
mailDeliveryTimeoutMs?: number;
/**
* The invite-launch model fallback (see `./inference-preferences.ts`'s
* `createWorkbenchHostInferencePreferencesResolver`): a
Expand Down Expand Up @@ -245,10 +252,42 @@ export function createHubChatPlatform(
250, 500, 1000, 2000, 4000,
];

// CL-6644: the reclaim-retry loop's own delays only run between
// attempts that already failed LOUD with "agent is unreachable" --
// they bound nothing about an attempt that instead stalls forever
// (a sidecar ack that never comes, a wedged promise anywhere in
// `sendFoldedMail`'s call chain) without ever throwing. Each
// attempt gets the same kind of wall-clock bound `wakeByAddressBounded`
// already puts on the wake itself, so that hang becomes a rejection
// `dispatchTurnBatch`'s catch can report and notify on, instead of a
// promise nothing ever settles.
const MAIL_DELIVERY_TIMEOUT_MS =
deps.mailDeliveryTimeoutMs ?? DEFAULT_WAKE_TIMEOUT_MS;

function sleep(ms: number): Promise<void> {
return new Promise((resolveSleep) => setTimeout(resolveSleep, ms));
}

function withTimeout<T>(
promise: Promise<T>,
ms: number,
message: string,
): Promise<T> {
return new Promise((resolve, reject) => {
const timer = setTimeout(() => reject(new Error(message)), ms);
promise.then(
(value) => {
clearTimeout(timer);
resolve(value);
},
(cause: unknown) => {
clearTimeout(timer);
reject(cause);
},
);
});
}

function isAgentUnreachable(err: unknown): boolean {
return err instanceof Error && err.message.includes("agent is unreachable");
}
Expand Down Expand Up @@ -565,25 +604,11 @@ export function createHubChatPlatform(
* failing loud (CL-6644).
*/
function wakeByAddressBounded(address: string): Promise<void> {
return new Promise((resolve, reject) => {
const timer = setTimeout(() => {
reject(
new Error(
`wake for "${address}" did not settle within ${String(DEFAULT_WAKE_TIMEOUT_MS)}ms`,
),
);
}, DEFAULT_WAKE_TIMEOUT_MS);
wakeByAddress(address).then(
(value) => {
clearTimeout(timer);
resolve(value);
},
(cause: unknown) => {
clearTimeout(timer);
reject(cause);
},
);
});
return withTimeout(
wakeByAddress(address),
DEFAULT_WAKE_TIMEOUT_MS,
`wake for "${address}" did not settle within ${String(DEFAULT_WAKE_TIMEOUT_MS)}ms`,
);
}

/**
Expand Down Expand Up @@ -682,7 +707,11 @@ export function createHubChatPlatform(
let loggedRetryStart = false;
for (let attempt = 0; ; attempt++) {
try {
return await sendFoldedMail(foldedRunsDeps, params);
return await withTimeout(
sendFoldedMail(foldedRunsDeps, params),
MAIL_DELIVERY_TIMEOUT_MS,
`mail to ${params.agentAddress} did not settle within ${String(MAIL_DELIVERY_TIMEOUT_MS)}ms`,
);
} catch (err) {
const delay = RECLAIM_RETRY_DELAYS_MS[attempt];
if (!isAgentUnreachable(err) || delay === undefined) {
Expand Down
64 changes: 64 additions & 0 deletions packages/chat/test/platform-adapter.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -943,6 +943,70 @@ describe("createHubChatPlatform", () => {
);
});

test("sendMail rejects within the mail-delivery deadline instead of hanging forever when delivery never settles (CL-6644)", async () => {
resolveDefinitionSourcesResult = {
ok: true,
sources: [
{
id: "off_1",
provider: "anthropic",
baseURL: "https://inference.invalid",
apiKey: "placeholder",
model: "claude-sonnet-5",
},
],
defaultSource: "off_1",
};

const db = createFakeDb({
assetRow: {
tenantId: "ten_1",
creatorPrincipalId: "prin_creator",
name: "workbench-1",
displayName: null,
},
definitionId: "wfd_workbench1",
workflowRunRow: {
id: "ins_workbench1",
address: "ins_workbench1@ten1.workbench.test",
principalId: "prin_run1",
},
});
db.inserted.push({
table: agentSession,
values: { id: "ses_run1", principalId: "prin_run1" },
});

const sessionService = createFakeSessionService();
// Models the observed CL-6644 symptom: the post-deploy delivery
// step (`sessionService.sendUserMessage`, reached through
// `sendFoldedMail`) never resolves and never rejects -- a wedged
// ack, not a thrown "agent is unreachable" the reclaim-retry loop
// already knows how to handle. Before the fix, `sendMail`'s
// returned promise stayed pending forever with nothing logged.
sessionService.sendUserMessage = () => new Promise<never>(() => {});
const sidecarRouter = createFakeSidecarRouter();

const platform = createHubChatPlatform({
toolGrantsForPins: () => [],
db: db as never,
sessionService,
assetService: createFakeAssetService(),
sidecarRouter,
eventCollectors: createFakeEventCollectors(),
mailDeliveryTimeoutMs: 20,
});

await expect(
platform.sendMail({
tenantId: "ten_1",
workbenchId: "ins_workbench1",
principalId: "prin_sender",
content: { content: "hello workbench" },
}),
).rejects.toThrow(/did not settle within 20ms/);
});

test("launchInvite mints from the target definition and ensureAwake deploys it", async () => {
resolveDefinitionSourcesCalls.length = 0;
resolveDefinitionSourcesResult = {
Expand Down
Loading