From 4c733332741b2acdd50300ab1ddb24406a60aa59 Mon Sep 17 00:00:00 2001 From: Luke Parke <5702154+LukasParke@users.noreply.github.com> Date: Tue, 14 Jul 2026 14:58:40 -0500 Subject: [PATCH 1/4] fix(agent): tolerate empty final response after completed tool rounds Mini-class models intermittently return an empty final turn when the tool call was the answer; the SDK threw away successful runs. Now: retry the follow-up once; if still empty, resolve with empty text. strictFinalResponse: true restores the old throw. Runs with no completed tool rounds still throw. --- .changeset/fix-tool-terminal-empty-final.md | 5 + packages/agent/src/inner-loop/call-model.ts | 4 + packages/agent/src/lib/async-params.ts | 7 + packages/agent/src/lib/model-result.ts | 83 ++++++- .../unit/tool-terminal-empty-final.test.ts | 232 ++++++++++++++++++ 5 files changed, 328 insertions(+), 3 deletions(-) create mode 100644 .changeset/fix-tool-terminal-empty-final.md create mode 100644 packages/agent/tests/unit/tool-terminal-empty-final.test.ts diff --git a/.changeset/fix-tool-terminal-empty-final.md b/.changeset/fix-tool-terminal-empty-final.md new file mode 100644 index 0000000..a6fcfd5 --- /dev/null +++ b/.changeset/fix-tool-terminal-empty-final.md @@ -0,0 +1,5 @@ +--- +"@openrouter/agent": patch +--- + +Tolerate empty final `output` after completed tool rounds: retry the follow-up request once, then resolve successfully with empty text instead of throwing `Invalid final response: empty or invalid output`. Mini-class models intermittently treat a successful tool call as the terminal answer. Opt into the old throw with `strictFinalResponse: true`. Runs with no completed tool work still throw on empty output. diff --git a/packages/agent/src/inner-loop/call-model.ts b/packages/agent/src/inner-loop/call-model.ts index dc42df9..6169e02 100644 --- a/packages/agent/src/inner-loop/call-model.ts +++ b/packages/agent/src/inner-loop/call-model.ts @@ -101,6 +101,7 @@ export function callModel< onTurnStart, onTurnEnd, allowFinalResponse, + strictFinalResponse, ...apiRequest } = request; @@ -165,5 +166,8 @@ export function callModel< ...(allowFinalResponse !== undefined && { allowFinalResponse, }), + ...(strictFinalResponse !== undefined && { + strictFinalResponse, + }), } as GetResponseOptions); } diff --git a/packages/agent/src/lib/async-params.ts b/packages/agent/src/lib/async-params.ts index 18ba08d..6fbc46f 100644 --- a/packages/agent/src/lib/async-params.ts +++ b/packages/agent/src/lib/async-params.ts @@ -98,6 +98,12 @@ type BaseCallModelInput< * (HITL pause, approval pause, interruption, or natural completion). */ allowFinalResponse?: boolean | string; + /** + * When true, throw if the final response has an empty `output` array even + * after completed tool rounds (legacy behavior). Default false: empty + * finals after tool work are retried once, then accepted with empty text. + */ + strictFinalResponse?: boolean; }; /** @@ -199,6 +205,7 @@ export async function resolveAsyncFunctions { + if (!this.resolvedRequest) { + throw new Error('Request not initialized'); + } + + const newRequest: models.ResponsesRequest = { + ...this.resolvedRequest, + stream: true, + }; + + const newResult = await betaResponsesSend( + this.options.client, + { + responsesRequest: newRequest, + }, + this.options.options, + ); + + if (!newResult.ok) { + throw newResult.error; + } + + const value = newResult.value; + if (isEventStream(value)) { + const followUpStream = new ReusableReadableStream(value); + + if (this.turnBroadcaster) { + return this.pipeAndConsumeStream(followUpStream, turnNumber); + } + + return consumeStreamForCompletion(followUpStream); + } + if (this.isNonStreamingResponse(value)) { + return value; + } + throw new Error('Unexpected response type from API'); + } + /** * Resolve async functions in the request for a given turn context. * Extracts non-function fields and resolves any async parameter functions. @@ -1894,6 +1950,8 @@ export class ModelResult< ); if (!this.options.tools?.length || !hasToolCalls) { + // No tool work: keep hard throw on empty/invalid final output. + this.validateFinalResponse(currentResponse); this.finalResponse = currentResponse; await this.markStateComplete(); return; @@ -1908,6 +1966,7 @@ export class ModelResult< } if (!this.hasExecutableToolCalls(toolCalls)) { + this.validateFinalResponse(currentResponse); this.finalResponse = currentResponse; await this.markStateComplete(); return; @@ -2119,8 +2178,26 @@ export class ModelResult< await this.saveResponseToState(currentResponse); } - // Validate and finalize - this.validateFinalResponse(currentResponse); + // Validate and finalize. Mini-class models intermittently return an + // empty final turn after a successful tool round (the tool call was + // the answer). Retry once, then tolerate empty output so a completed + // run isn't reported as failure — unless `strictFinalResponse` is set. + const canTolerateEmptyFinal = + this.allToolExecutionRounds.length > 0 && this.options.strictFinalResponse !== true; + const isEmptyOutput = + Array.isArray(currentResponse.output) && currentResponse.output.length === 0; + + if (canTolerateEmptyFinal && isEmptyOutput) { + const turnNumber = this.allToolExecutionRounds.length + 1; + currentResponse = await this.retryCurrentRequest(turnNumber); + } + + const allowEmptyOutput = + canTolerateEmptyFinal && + Array.isArray(currentResponse.output) && + currentResponse.output.length === 0; + + this.validateFinalResponse(currentResponse, allowEmptyOutput); this.finalResponse = currentResponse; await this.markStateComplete(); })(); diff --git a/packages/agent/tests/unit/tool-terminal-empty-final.test.ts b/packages/agent/tests/unit/tool-terminal-empty-final.test.ts new file mode 100644 index 0000000..ce58c44 --- /dev/null +++ b/packages/agent/tests/unit/tool-terminal-empty-final.test.ts @@ -0,0 +1,232 @@ +import type { OpenRouterCore } from '@openrouter/sdk/core'; +import type * as models from '@openrouter/sdk/models'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { z } from 'zod/v4'; + +const mockBetaResponsesSend = vi.hoisted(() => vi.fn()); + +vi.mock('@openrouter/sdk/funcs/betaResponsesSend', () => ({ + betaResponsesSend: mockBetaResponsesSend, +})); + +import { callModel } from '../../src/inner-loop/call-model.js'; +import { ToolType } from '../../src/lib/tool-types.js'; + +function toolCallResponse(): models.OpenResponsesResult { + return { + id: 'resp_tool_call', + object: 'response', + createdAt: 0, + model: 'test-model', + status: 'completed', + completedAt: 0, + output: [ + { + type: 'function_call', + id: 'fc_1', + callId: 'call_abc', + name: 'post_comment', + arguments: '{"body":"looks good"}', + status: 'completed', + }, + ], + error: null, + incompleteDetails: null, + temperature: null, + topP: null, + presencePenalty: null, + frequencyPenalty: null, + metadata: null, + instructions: null, + tools: [], + toolChoice: 'auto', + parallelToolCalls: false, + } as models.OpenResponsesResult; +} + +function emptyOutputResponse(id = 'resp_empty'): models.OpenResponsesResult { + return { + id, + object: 'response', + createdAt: 0, + model: 'test-model', + status: 'completed', + completedAt: 0, + output: [], + error: null, + incompleteDetails: null, + temperature: null, + topP: null, + presencePenalty: null, + frequencyPenalty: null, + metadata: null, + instructions: null, + tools: [], + toolChoice: 'auto', + parallelToolCalls: false, + } as models.OpenResponsesResult; +} + +function textResponse(text: string, id = 'resp_text'): models.OpenResponsesResult { + return { + id, + object: 'response', + createdAt: 0, + model: 'test-model', + status: 'completed', + completedAt: 0, + output: [ + { + id: 'msg_text', + type: 'message', + role: 'assistant', + status: 'completed', + content: [ + { + type: 'output_text', + text, + annotations: [], + }, + ], + }, + ], + error: null, + incompleteDetails: null, + temperature: null, + topP: null, + presencePenalty: null, + frequencyPenalty: null, + metadata: null, + instructions: null, + tools: [], + toolChoice: 'auto', + parallelToolCalls: false, + } as models.OpenResponsesResult; +} + +const postCommentTool = { + type: ToolType.Function, + function: { + name: 'post_comment', + description: 'Post a comment.', + inputSchema: z.object({ + body: z.string(), + }), + outputSchema: z.object({ + ok: z.boolean(), + }), + execute: async (_params: { body: string }) => ({ + ok: true, + }), + }, +} as const; + +const client = {} as OpenRouterCore; + +describe('tool-terminal empty final response (PR-2)', () => { + beforeEach(() => { + mockBetaResponsesSend.mockReset(); + }); + + it('retries once then accepts empty final after a completed tool round', async () => { + mockBetaResponsesSend + .mockResolvedValueOnce({ + ok: true, + value: toolCallResponse(), + }) + .mockResolvedValueOnce({ + ok: true, + value: emptyOutputResponse('resp_empty_1'), + }) + .mockResolvedValueOnce({ + ok: true, + value: emptyOutputResponse('resp_empty_2'), + }); + + const result = callModel(client, { + model: 'test-model', + input: 'Review and post a comment.', + tools: [ + postCommentTool, + ] as const, + }); + + const text = await result.getText(); + expect(text).toBe(''); + // initial + follow-up + one empty-output retry + expect(mockBetaResponsesSend).toHaveBeenCalledTimes(3); + + const response = await result.getResponse(); + expect(response.id).toBe('resp_empty_2'); + expect(response.output).toEqual([]); + }); + + it('returns text when the empty-final retry succeeds', async () => { + mockBetaResponsesSend + .mockResolvedValueOnce({ + ok: true, + value: toolCallResponse(), + }) + .mockResolvedValueOnce({ + ok: true, + value: emptyOutputResponse('resp_empty'), + }) + .mockResolvedValueOnce({ + ok: true, + value: textResponse('Done posting.', 'resp_retry_text'), + }); + + const text = await callModel(client, { + model: 'test-model', + input: 'Review and post a comment.', + tools: [ + postCommentTool, + ] as const, + }).getText(); + + expect(text).toBe('Done posting.'); + expect(mockBetaResponsesSend).toHaveBeenCalledTimes(3); + }); + + it('throws on empty final after tool rounds when strictFinalResponse is true', async () => { + mockBetaResponsesSend + .mockResolvedValueOnce({ + ok: true, + value: toolCallResponse(), + }) + .mockResolvedValueOnce({ + ok: true, + value: emptyOutputResponse(), + }); + + await expect( + callModel(client, { + model: 'test-model', + input: 'Review and post a comment.', + tools: [ + postCommentTool, + ] as const, + strictFinalResponse: true, + }).getText(), + ).rejects.toThrow('Invalid final response: empty or invalid output'); + + // No retry when strict + expect(mockBetaResponsesSend).toHaveBeenCalledTimes(2); + }); + + it('still throws on empty output when no tool rounds completed', async () => { + mockBetaResponsesSend.mockResolvedValueOnce({ + ok: true, + value: emptyOutputResponse(), + }); + + await expect( + callModel(client, { + model: 'test-model', + input: 'Hello', + }).getText(), + ).rejects.toThrow('Invalid final response: empty or invalid output'); + + expect(mockBetaResponsesSend).toHaveBeenCalledTimes(1); + }); +}); From 5e4a3249b21db94bba4c2d056a8026bd70c30e4e Mon Sep 17 00:00:00 2001 From: Luke Parke <5702154+LukasParke@users.noreply.github.com> Date: Tue, 14 Jul 2026 15:11:12 -0500 Subject: [PATCH 2/4] =?UTF-8?q?fix:=20address=20review=20=E2=80=94=20persi?= =?UTF-8?q?st=20empty-final=20retry=20response=20to=20state,=20strip=20all?= =?UTF-8?q?=20client-only=20fields=20in=20non-async=20path?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - retryCurrentRequest result now goes through saveResponseToState like every other response in the loop (stateful conversations were silently losing the final turn on resume) - resolveRequestForContext now strips the same client-only field set as async-params.ts clientOnlyFields (strictFinalResponse, allowFinalResponse, sharedContextSchema, onTurnStart, onTurnEnd were leaking into API requests on the non-async path) - regression tests for both --- packages/agent/src/lib/model-result.ts | 15 +++- .../unit/tool-terminal-empty-final.test.ts | 76 +++++++++++++++++++ 2 files changed, 89 insertions(+), 2 deletions(-) diff --git a/packages/agent/src/lib/model-result.ts b/packages/agent/src/lib/model-result.ts index da6219a..5f61595 100644 --- a/packages/agent/src/lib/model-result.ts +++ b/packages/agent/src/lib/model-result.ts @@ -1362,8 +1362,11 @@ export class ModelResult< if (hasAsyncFunctions(this.options.request)) { return resolveAsyncFunctions(this.options.request, context); } - // Already resolved, extract non-function fields - // Filter out stopWhen and state-related fields that aren't part of the API request + // Already resolved, extract non-function fields. + // Strip ALL client-only fields — keep this list in sync with + // `clientOnlyFields` in async-params.ts, which handles the async path. + // (`sharedContextSchema` is absent here: call-model.ts destructures it + // before the request reaches ModelResult.) const { stopWhen: _, state: _s, @@ -1371,6 +1374,10 @@ export class ModelResult< approveToolCalls: _a, rejectToolCalls: _rj, context: _c, + onTurnStart: _ots, + onTurnEnd: _ote, + allowFinalResponse: _afr, + strictFinalResponse: _sfr, ...rest } = this.options.request; return rest as ResolvedCallModelInput; @@ -2190,6 +2197,10 @@ export class ModelResult< if (canTolerateEmptyFinal && isEmptyOutput) { const turnNumber = this.allToolExecutionRounds.length + 1; currentResponse = await this.retryCurrentRequest(turnNumber); + // Persist the retried response like every other response in the + // loop — otherwise stateful conversations silently lose the final + // turn's content on resume. + await this.saveResponseToState(currentResponse); } const allowEmptyOutput = diff --git a/packages/agent/tests/unit/tool-terminal-empty-final.test.ts b/packages/agent/tests/unit/tool-terminal-empty-final.test.ts index ce58c44..f7a5f7a 100644 --- a/packages/agent/tests/unit/tool-terminal-empty-final.test.ts +++ b/packages/agent/tests/unit/tool-terminal-empty-final.test.ts @@ -229,4 +229,80 @@ describe('tool-terminal empty final response (PR-2)', () => { expect(mockBetaResponsesSend).toHaveBeenCalledTimes(1); }); + + it('persists the successful empty-final retry response to conversation state', async () => { + mockBetaResponsesSend + .mockResolvedValueOnce({ + ok: true, + value: toolCallResponse(), + }) + .mockResolvedValueOnce({ + ok: true, + value: emptyOutputResponse('resp_empty'), + }) + .mockResolvedValueOnce({ + ok: true, + value: textResponse('Done posting.', 'resp_retry_text'), + }); + + let stored: unknown = null; + const result = callModel(client, { + model: 'test-model', + input: 'Review and post a comment.', + tools: [ + postCommentTool, + ] as const, + state: { + load: async () => stored as never, + save: async (s: unknown) => { + stored = s; + }, + }, + }); + + const text = await result.getText(); + expect(text).toBe('Done posting.'); + + // The retried final turn must be recorded in state — the assistant + // message from resp_retry_text must appear in messages. + const messages = ( + stored as { + messages: Array<{ + type?: string; + role?: string; + }>; + } + ).messages; + const assistantMessages = messages.filter( + (m) => m.type === 'message' && m.role === 'assistant', + ); + expect(assistantMessages.length).toBeGreaterThan(0); + expect(JSON.stringify(messages)).toContain('Done posting.'); + }); + + it('does not send strictFinalResponse (or other client-only fields) to the API', async () => { + mockBetaResponsesSend.mockResolvedValueOnce({ + ok: true, + value: textResponse('Hi.', 'resp_text'), + }); + + await callModel(client, { + model: 'test-model', + input: 'Hello', + strictFinalResponse: true, + allowFinalResponse: true, + }).getText(); + + const request = mockBetaResponsesSend.mock.calls[0]?.[1]?.responsesRequest as Record< + string, + unknown + >; + expect(request).toBeDefined(); + expect(request).not.toHaveProperty('strictFinalResponse'); + expect(request).not.toHaveProperty('allowFinalResponse'); + expect(request).not.toHaveProperty('stopWhen'); + expect(request).not.toHaveProperty('sharedContextSchema'); + expect(request).not.toHaveProperty('onTurnStart'); + expect(request).not.toHaveProperty('onTurnEnd'); + }); }); From 632011662a1b4c9c0d4a6523157b93af0b2ff35a Mon Sep 17 00:00:00 2001 From: Luke Parke <5702154+LukasParke@users.noreply.github.com> Date: Tue, 14 Jul 2026 16:02:14 -0500 Subject: [PATCH 3/4] fix(agent): retry the final no-tools request --- packages/agent/src/lib/model-result.ts | 1 + .../unit/tool-terminal-empty-final.test.ts | 45 +++++++++++++++++++ 2 files changed, 46 insertions(+) diff --git a/packages/agent/src/lib/model-result.ts b/packages/agent/src/lib/model-result.ts index 5f61595..4ce0b7f 100644 --- a/packages/agent/src/lib/model-result.ts +++ b/packages/agent/src/lib/model-result.ts @@ -1260,6 +1260,7 @@ export class ModelResult< input: newInput, stream: true, }; + this.resolvedRequest = finalRequest; const result = await betaResponsesSend( this.options.client, diff --git a/packages/agent/tests/unit/tool-terminal-empty-final.test.ts b/packages/agent/tests/unit/tool-terminal-empty-final.test.ts index f7a5f7a..e9aa550 100644 --- a/packages/agent/tests/unit/tool-terminal-empty-final.test.ts +++ b/packages/agent/tests/unit/tool-terminal-empty-final.test.ts @@ -10,6 +10,7 @@ vi.mock('@openrouter/sdk/funcs/betaResponsesSend', () => ({ })); import { callModel } from '../../src/inner-loop/call-model.js'; +import { stepCountIs } from '../../src/lib/stop-conditions.js'; import { ToolType } from '../../src/lib/tool-types.js'; function toolCallResponse(): models.OpenResponsesResult { @@ -188,6 +189,50 @@ describe('tool-terminal empty final response (PR-2)', () => { expect(mockBetaResponsesSend).toHaveBeenCalledTimes(3); }); + it('retries the same no-tools request when allowFinalResponse returns empty', async () => { + mockBetaResponsesSend + .mockResolvedValueOnce({ + ok: true, + value: toolCallResponse(), + }) + .mockResolvedValueOnce({ + ok: true, + value: emptyOutputResponse('resp_empty'), + }) + .mockResolvedValueOnce({ + ok: true, + value: textResponse('Done posting.', 'resp_retry_text'), + }); + + const text = await callModel(client, { + model: 'test-model', + input: 'Review and post a comment.', + tools: [ + postCommentTool, + ] as const, + stopWhen: stepCountIs(0), + allowFinalResponse: 'Summarize the result.', + }).getText(); + + expect(text).toBe('Done posting.'); + const finalRequest = mockBetaResponsesSend.mock.calls[1]?.[1]?.responsesRequest; + const retryRequest = mockBetaResponsesSend.mock.calls[2]?.[1]?.responsesRequest; + expect(retryRequest).toEqual(finalRequest); + expect(retryRequest).not.toHaveProperty('tools'); + expect(retryRequest.input).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + type: 'function_call_output', + callId: 'call_abc', + }), + expect.objectContaining({ + role: 'user', + content: 'Summarize the result.', + }), + ]), + ); + }); + it('throws on empty final after tool rounds when strictFinalResponse is true', async () => { mockBetaResponsesSend .mockResolvedValueOnce({ From e25af4a0643dd601a86c4467ecb2839f5fabd3b3 Mon Sep 17 00:00:00 2001 From: Luke Parke <5702154+LukasParke@users.noreply.github.com> Date: Wed, 15 Jul 2026 10:47:40 -0500 Subject: [PATCH 4/4] fix: strip tools from empty-final retry so it cannot emit an unexecuted tool call MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses Perry's review: on the natural-loop-completion path, resolvedRequest still carried tools/toolChoice/parallelToolCalls, so the empty-final retry could legitimately return a fresh function_call that passed validateFinalResponse but was never executed — silently dropped. The retry now strips those fields (mirroring makeFinalResponseRequest) to coerce a text turn, with a test pinning the behavior. --- packages/agent/src/lib/model-result.ts | 16 ++++++++- .../unit/tool-terminal-empty-final.test.ts | 36 +++++++++++++++++++ 2 files changed, 51 insertions(+), 1 deletion(-) diff --git a/packages/agent/src/lib/model-result.ts b/packages/agent/src/lib/model-result.ts index 4ce0b7f..c3c9175 100644 --- a/packages/agent/src/lib/model-result.ts +++ b/packages/agent/src/lib/model-result.ts @@ -1313,14 +1313,28 @@ export class ModelResult< /** * Re-send the current resolved request (same accumulated input) once. * Used when a follow-up after tool execution returned an empty `output`. + * + * `tools`, `toolChoice`, and `parallelToolCalls` are stripped (mirroring + * `makeFinalResponseRequest`) so the retry coerces a text turn: on the + * natural-loop-completion path the resolved request still carries tools, + * and a retry that emitted a fresh `function_call` would pass + * `validateFinalResponse` but never be executed — silently dropping a + * proposed tool call. */ private async retryCurrentRequest(turnNumber: number): Promise { if (!this.resolvedRequest) { throw new Error('Request not initialized'); } + const { + tools: _tools, + toolChoice: _toolChoice, + parallelToolCalls: _parallelToolCalls, + ...rest + } = this.resolvedRequest; + const newRequest: models.ResponsesRequest = { - ...this.resolvedRequest, + ...rest, stream: true, }; diff --git a/packages/agent/tests/unit/tool-terminal-empty-final.test.ts b/packages/agent/tests/unit/tool-terminal-empty-final.test.ts index e9aa550..c652cf2 100644 --- a/packages/agent/tests/unit/tool-terminal-empty-final.test.ts +++ b/packages/agent/tests/unit/tool-terminal-empty-final.test.ts @@ -189,6 +189,42 @@ describe('tool-terminal empty final response (PR-2)', () => { expect(mockBetaResponsesSend).toHaveBeenCalledTimes(3); }); + it('strips tools from the empty-final retry so it cannot emit an unexecuted tool call', async () => { + mockBetaResponsesSend + .mockResolvedValueOnce({ + ok: true, + value: toolCallResponse(), + }) + .mockResolvedValueOnce({ + ok: true, + value: emptyOutputResponse('resp_empty'), + }) + .mockResolvedValueOnce({ + ok: true, + value: textResponse('Done posting.', 'resp_retry_text'), + }); + + const text = await callModel(client, { + model: 'test-model', + input: 'Review and post a comment.', + tools: [ + postCommentTool, + ] as const, + }).getText(); + + expect(text).toBe('Done posting.'); + // The natural-loop follow-up request still carries tools… + const followupRequest = mockBetaResponsesSend.mock.calls[1]?.[1]?.responsesRequest; + expect(followupRequest).toHaveProperty('tools'); + // …but the retry must not, so it coerces a text turn instead of a + // fresh function_call that would be silently dropped. + const retryRequest = mockBetaResponsesSend.mock.calls[2]?.[1]?.responsesRequest; + expect(retryRequest).not.toHaveProperty('tools'); + expect(retryRequest).not.toHaveProperty('toolChoice'); + expect(retryRequest).not.toHaveProperty('parallelToolCalls'); + expect(retryRequest.input).toEqual(followupRequest.input); + }); + it('retries the same no-tools request when allowFinalResponse returns empty', async () => { mockBetaResponsesSend .mockResolvedValueOnce({