From bb4f1a7688eac8419c3ff18be1ba8eefe964d5f7 Mon Sep 17 00:00:00 2001 From: Sawyer Cutler Date: Sat, 22 Aug 2026 23:22:43 -0700 Subject: [PATCH 1/2] Add tests for consumer inference error copy Failed turns must not persist or render HTTP status or raw provider JSON. Cover persist, preview, and the failed-turn strip. --- .../chat-ui/src/inference-failure.test.ts | 48 ++++++++++++++- packages/chat-ui/test/components.test.tsx | 16 +++++ .../chat-ui/test/failed-turn-strip.test.tsx | 60 +++++++++++++++++++ .../chat/src/consumer-inference-text.test.ts | 38 ++++++++++++ packages/chat/src/room-messages.test.ts | 50 ++++++++++++++++ 5 files changed, 211 insertions(+), 1 deletion(-) create mode 100644 packages/chat/src/consumer-inference-text.test.ts diff --git a/packages/chat-ui/src/inference-failure.test.ts b/packages/chat-ui/src/inference-failure.test.ts index 35c4dc260..80c9326d6 100644 --- a/packages/chat-ui/src/inference-failure.test.ts +++ b/packages/chat-ui/src/inference-failure.test.ts @@ -1,6 +1,42 @@ import { describe, expect, test } from "bun:test"; -import { isClassifiedInferenceFailureText } from "./inference-failure"; +import { + consumerFacingInferenceText, + isClassifiedInferenceFailureText, +} from "./inference-failure"; + +describe("consumerFacingInferenceText", () => { + test("keeps the vendor preamble and drops [HTTP …] plus the raw provider message", () => { + expect( + consumerFacingInferenceText( + "This agent could not complete your request due to a credential error [HTTP 401]: API key is invalid", + ), + ).toBe( + "This agent could not complete your request due to a credential error", + ); + expect( + consumerFacingInferenceText( + "This agent could not complete your request because the API quota has been exhausted [HTTP 429]: rate limited", + ), + ).toBe( + "This agent could not complete your request because the API quota has been exhausted", + ); + }); + + test("a forced [HTTP 401] dump is not consumer copy", () => { + const leaked = "[HTTP 401]: API key is invalid"; + const facing = consumerFacingInferenceText(leaked); + expect(facing).not.toContain("[HTTP"); + expect(facing).not.toContain("401"); + expect(facing).not.toContain("API key is invalid"); + }); + + test("leaves cause-aware undelivered-notice copy untouched", () => { + const notice = + "I can't reach a model right now — add or check your model key in Settings, then I'll pick this up."; + expect(consumerFacingInferenceText(notice)).toBe(notice); + }); +}); describe("isClassifiedInferenceFailureText", () => { test("matches a credential_failure reply, status code included", () => { @@ -11,6 +47,16 @@ describe("isClassifiedInferenceFailureText", () => { ).toBe(true); }); + test("matches a credential_failure reply after HTTP/raw is stripped", () => { + expect( + isClassifiedInferenceFailureText( + consumerFacingInferenceText( + "This agent could not complete your request due to a credential error [HTTP 401]: invalid api key", + ), + ), + ).toBe(true); + }); + test("matches a quota_exhausted reply", () => { expect( isClassifiedInferenceFailureText( diff --git a/packages/chat-ui/test/components.test.tsx b/packages/chat-ui/test/components.test.tsx index 48b124463..99bec22e9 100644 --- a/packages/chat-ui/test/components.test.tsx +++ b/packages/chat-ui/test/components.test.tsx @@ -498,6 +498,22 @@ describe("WorkbenchTimeline", () => { expect(markup).toContain("Fix this connection"); }); + test("never renders HTTP status or the raw provider message on a classified failure reply", () => { + const markup = renderToStaticMarkup( + {}} + />, + ); + expect(markup).not.toMatch(/\[HTTP/); + expect(markup).not.toContain("401"); + expect(markup).not.toContain("invalid api key"); + expect(markup).toContain( + "This agent could not complete your request due to a credential error", + ); + expect(markup).toContain("Fix this connection"); + }); + test("renders Fix this connection as a react-ui outline button, not a bare link", () => { const markup = renderToStaticMarkup( { ); }); + test("the expanded detail never shows HTTP status or a raw provider dump", async () => { + container = document.createElement("div"); + document.body.appendChild(container); + root = createRoot(container); + const items: MessageItem[] = [ + { + id: "msg_ok", + createdAt: "2026-01-01T00:00:00.000Z", + parts: [{ kind: "text", text: "hi @echo" }], + sender: { name: null, address: "prn_alice@agents.example" }, + }, + { + id: "msg_notice", + createdAt: "2026-01-01T00:00:05.000Z", + parts: [ + { + kind: "text", + text: "This agent could not complete your request due to a credential error [HTTP 401]: API key is invalid.", + turnFailed: true, + }, + ], + sender: { name: null, address: "ins_echo1@agents.example" }, + }, + ]; + await act(async () => { + root?.render( + , + ); + }); + + act(() => { + container + ?.querySelector(".chat-turn-failed-disclosure") + ?.dispatchEvent(new MouseEvent("click", { bubbles: true })); + }); + + const detail = container.querySelector(".chat-turn-failed-detail"); + expect(detail?.textContent).toBe( + "This agent could not complete your request due to a credential error", + ); + expect(detail?.textContent).not.toMatch(/\[HTTP/); + expect(detail?.textContent).not.toContain("401"); + expect(detail?.textContent).not.toContain("API key is invalid"); + }); + test("Retry auto-resends the recovered request text — no composer round trip", async () => { container = document.createElement("div"); document.body.appendChild(container); diff --git a/packages/chat/src/consumer-inference-text.test.ts b/packages/chat/src/consumer-inference-text.test.ts new file mode 100644 index 000000000..d604a0019 --- /dev/null +++ b/packages/chat/src/consumer-inference-text.test.ts @@ -0,0 +1,38 @@ +import { describe, expect, test } from "bun:test"; + +import { + CONSUMER_INFERENCE_FAILURE_NOTICE, + consumerFacingInferenceText, +} from "./consumer-inference-text"; + +describe("consumerFacingInferenceText", () => { + test("leaves ordinary replies unchanged", () => { + expect(consumerFacingInferenceText("Hello there.")).toBe("Hello there."); + }); + + test("keeps a classified preamble and drops a trailing HTTP dump", () => { + expect( + consumerFacingInferenceText( + "This agent could not complete your request due to a credential error [HTTP 401]: API key is invalid.", + ), + ).toBe( + "This agent could not complete your request due to a credential error", + ); + }); + + test("a forced HTTP dump is not consumer copy", () => { + const text = consumerFacingInferenceText("[HTTP 401]: API key is invalid."); + expect(text).toBe(CONSUMER_INFERENCE_FAILURE_NOTICE); + expect(text).not.toMatch(/\[HTTP/i); + expect(text).not.toMatch(/401/); + expect(text.toLowerCase()).not.toContain("api key is invalid"); + }); + + test("a JSON provider-error object is not consumer copy", () => { + const text = consumerFacingInferenceText( + '{"error":{"message":"Incorrect API key provided","type":"invalid_request_error","code":"invalid_api_key"}}', + ); + expect(text).toBe(CONSUMER_INFERENCE_FAILURE_NOTICE); + expect(text.toLowerCase()).not.toContain("invalid_api_key"); + }); +}); diff --git a/packages/chat/src/room-messages.test.ts b/packages/chat/src/room-messages.test.ts index 00ef3ae52..10c62adb4 100644 --- a/packages/chat/src/room-messages.test.ts +++ b/packages/chat/src/room-messages.test.ts @@ -58,6 +58,43 @@ describe("postRoomMessage", () => { ]); }); + test("persists a consumer sentence, not HTTP status or a raw provider dump", async () => { + const roomMessages = createInMemoryRoomMessageStore(); + const publisher = recordingPublisher(); + + const posted = await postRoomMessage( + { roomMessages, publish: publisher.publish }, + { + tenantId: TENANT, + workbenchId: WORKBENCH, + sender: { name: null, address: "run_myra@acme.example" }, + runId: "run_myra", + parts: [ + { + kind: "text", + text: "This agent could not complete your request due to a credential error [HTTP 401]: API key is invalid.", + }, + ], + }, + ); + + expect(posted.parts).toEqual([ + { + kind: "text", + text: "This agent could not complete your request due to a credential error", + }, + ]); + const listed = await roomMessages.listMessages({ + tenantId: TENANT, + workbenchId: WORKBENCH, + }); + expect(listed.items[0]?.parts).toEqual(posted.parts); + expect(JSON.stringify(publisher.published)).not.toMatch(/\[HTTP/); + expect(JSON.stringify(publisher.published)).not.toContain( + "API key is invalid", + ); + }); + test("an agent's message carries its run, a human's carries its principal", async () => { const roomMessages = createInMemoryRoomMessageStore(); const publisher = recordingPublisher(); @@ -202,4 +239,17 @@ describe("previewOf", () => { ]), ).toBe(""); }); + + test("does not preview HTTP status or raw provider dumps", () => { + expect( + previewOf([ + { + kind: "text", + text: "This agent could not complete your request due to a credential error [HTTP 401]: API key is invalid.", + }, + ]), + ).toBe( + "This agent could not complete your request due to a credential error", + ); + }); }); From dde8d8aad9359a3e76eec4173ba2339d81dcb2e0 Mon Sep 17 00:00:00 2001 From: Sawyer Cutler Date: Sat, 22 Aug 2026 23:22:46 -0700 Subject: [PATCH 2/2] Sanitize inference failures to consumer copy Persist and render failed turns as one consumer sentence, never an HTTP status or provider dump. --- packages/chat-ui/src/inference-failure.ts | 2 ++ packages/chat-ui/src/timeline.tsx | 15 +++++++--- packages/chat/package.json | 1 + packages/chat/src/consumer-inference-text.ts | 29 +++++++++++++++++++ packages/chat/src/index.ts | 4 +++ packages/chat/src/room-messages.ts | 30 ++++++++++++++------ 6 files changed, 69 insertions(+), 12 deletions(-) create mode 100644 packages/chat/src/consumer-inference-text.ts diff --git a/packages/chat-ui/src/inference-failure.ts b/packages/chat-ui/src/inference-failure.ts index 9037d65d7..0b3bb35a2 100644 --- a/packages/chat-ui/src/inference-failure.ts +++ b/packages/chat-ui/src/inference-failure.ts @@ -29,6 +29,8 @@ export const CLASSIFIED_INFERENCE_FAILURE_PREAMBLES: readonly string[] = [ "This agent could not complete your request because the API quota has been exhausted", ]; +export { consumerFacingInferenceText } from "@corbits/chat/consumer-inference-text"; + export function isClassifiedInferenceFailureText(text: string): boolean { return CLASSIFIED_INFERENCE_FAILURE_PREAMBLES.some((preamble) => text.startsWith(preamble), diff --git a/packages/chat-ui/src/timeline.tsx b/packages/chat-ui/src/timeline.tsx index 26160143f..cd99cfd01 100644 --- a/packages/chat-ui/src/timeline.tsx +++ b/packages/chat-ui/src/timeline.tsx @@ -57,7 +57,10 @@ import type { BlockResponseActions } from "./blocks/block-responses"; import type { ConnectGithubActions } from "./blocks/connect-github-actions"; import type { ConnectServiceActions } from "./blocks/connect-service-actions"; import { BlockPartView } from "./blocks/registry"; -import { isClassifiedInferenceFailureText } from "./inference-failure"; +import { + consumerFacingInferenceText, + isClassifiedInferenceFailureText, +} from "./inference-failure"; import { WorkbenchLoadingState } from "./loading-state"; import { Markdown } from "./markdown"; import type { ProfileSubject } from "./profile-subject"; @@ -455,6 +458,7 @@ function TextBubble({ pendingNonce?: string; pendingActions?: PendingActions; }) { + const consumerText = consumerFacingInferenceText(text); const display = senderDisplay(sender, participants, currentUser); const isOwn = currentUser !== undefined && @@ -545,10 +549,10 @@ function TextBubble({ )}
- +
{onFixConnection !== undefined && - isClassifiedInferenceFailureText(text) && ( + isClassifiedInferenceFailureText(consumerText) && (