From d8e934ef79b7dff4ee5d509d64dafb25e7927ab9 Mon Sep 17 00:00:00 2001 From: Sawyer Cutler Date: Sat, 22 Aug 2026 19:06:39 -0700 Subject: [PATCH 1/2] Set prompt_cache_key on the Grok and OpenAI Responses adapters Codex's Responses adapter already keys prompt_cache_key by session id; grok-responses-adapter.ts and openai-responses-adapter.ts never set it, so xAI/OpenCode-Go Responses calls had no cache-routing signal beyond the implicit prefix hash. Threads the session id (parent session's sessionId, or a fresh id per subagent thread) through buildXaiSource/buildGoSource into each adapter's providerOptions, and sets prompt_cache_key in the request body when present. Per xAI's docs (docs.x.ai/developers/advanced-api-usage/prompt-caching), the Responses API honors prompt_cache_key the same way OpenAI does; verified this isn't a no-op before shipping it. --- src/config.test.ts | 10 +++- src/config/index.ts | 27 +++++++-- src/config/inference-sources.ts | 2 + src/exec/runner.ts | 1 + src/provider/grok-responses-adapter.ts | 7 ++- src/provider/openai-responses-adapter.ts | 12 ++++ src/tui/runner.ts | 1 + tests/unit/grok-responses-adapter.test.ts | 28 +++++++++ tests/unit/openai-responses-adapter.test.ts | 66 +++++++++++++++++++++ 9 files changed, 147 insertions(+), 7 deletions(-) create mode 100644 tests/unit/openai-responses-adapter.test.ts diff --git a/src/config.test.ts b/src/config.test.ts index 3b9d7b35b..47c82d662 100644 --- a/src/config.test.ts +++ b/src/config.test.ts @@ -892,7 +892,7 @@ describe("buildBifrostSource", () => { describe("buildXaiSource", () => { test("omits reasoning_effort when effort is absent", () => { - const source = buildXaiSource({ id: "xai/work", apiKey: "tok", model: "grok-4.6" }); + const source = buildXaiSource({ id: "xai/work", apiKey: "tok", model: "grok-4.6", sessionId: "sess-1" }); expect(source.provider).toBe("grok-responses"); expect(source.defaults?.providerOptions).not.toHaveProperty("reasoning_effort"); }); @@ -902,15 +902,21 @@ describe("buildXaiSource", () => { id: "xai/work", apiKey: "tok", model: "grok-4.6", + sessionId: "sess-1", reasoningEffort: "low", }); expect(source.defaults?.providerOptions).toMatchObject({ reasoning_effort: "low" }); }); test("does not invent high when effort is absent", () => { - const source = buildXaiSource({ id: "xai/work", apiKey: "tok", model: "grok-4.6" }); + const source = buildXaiSource({ id: "xai/work", apiKey: "tok", model: "grok-4.6", sessionId: "sess-1" }); expect(source.defaults?.providerOptions?.["reasoning_effort"]).toBeUndefined(); }); + + test("stashes the session id for the adapter's prompt_cache_key", () => { + const source = buildXaiSource({ id: "xai/work", apiKey: "tok", model: "grok-4.6", sessionId: "sess-1" }); + expect(source.defaults?.providerOptions).toMatchObject({ grokSessionId: "sess-1" }); + }); }); describe("buildProviderCatalog", () => { diff --git a/src/config/index.ts b/src/config/index.ts index eb98b4f26..28a94766d 100644 --- a/src/config/index.ts +++ b/src/config/index.ts @@ -32,9 +32,16 @@ import { CODEX_ACCOUNT_ID_OPTION, CODEX_SESSION_ID_OPTION, } from "../provider/codex-responses-adapter.js"; -import { GROK_RESPONSES_PROVIDER, GROK_USER_ID_OPTION } from "../provider/grok-responses-adapter.js"; +import { + GROK_RESPONSES_PROVIDER, + GROK_SESSION_ID_OPTION, + GROK_USER_ID_OPTION, +} from "../provider/grok-responses-adapter.js"; import { BIFROST_PROVIDER } from "../provider/bifrost-adapter.js"; -import { OPENAI_RESPONSES_PROVIDER } from "../provider/openai-responses-adapter.js"; +import { + OPENAI_RESPONSES_PROVIDER, + OPENAI_SESSION_ID_OPTION, +} from "../provider/openai-responses-adapter.js"; import { xaiUserIdFromAccessToken } from "../auth/xai/session.js"; import { OPENCODE_GO_BASE_URL, @@ -166,14 +173,19 @@ export function buildCodexSource(fields: { // "grok-responses" adapter (the grok-cli proxy speaks the Responses API, not // Chat Completions). The access token is the apiKey; the caller's user id is // decoded from it and lifted into the x-grok-user-id header by the adapter. +// The session id becomes the request's prompt_cache_key so every call in the +// thread routes to the same cache shard (store:false has no other signal). export function buildXaiSource(fields: { id: string; apiKey: string; model: string; + sessionId: string; reasoningEffort?: ReasoningEffort; }): InferenceSource { const userId = xaiUserIdFromAccessToken(fields.apiKey); - const providerOptions: Record = {}; + const providerOptions: Record = { + [GROK_SESSION_ID_OPTION]: fields.sessionId, + }; if (userId !== undefined) providerOptions[GROK_USER_ID_OPTION] = userId; if (fields.reasoningEffort !== undefined) providerOptions["reasoning_effort"] = fields.reasoningEffort; return { @@ -228,10 +240,12 @@ export function buildAnthropicSource(fields: { } // OpenCode Go: per-model protocol routing (chat completions / responses / messages). +// sessionId feeds the Responses-protocol prompt_cache_key (see buildXaiSource). export function buildGoSource(fields: { id: string; apiKey?: string; model: string; + sessionId?: string; reasoningEffort?: ReasoningEffort; }): InferenceSource { const endpoint = resolveGoEndpoint(fields.model); @@ -252,7 +266,12 @@ export function buildGoSource(fields: { baseURL: endpoint.baseURL, apiKey, model: fields.model, - defaults: { maxTokens: SOURCE_MAX_TOKENS }, + defaults: { + maxTokens: SOURCE_MAX_TOKENS, + ...(fields.sessionId !== undefined + ? { providerOptions: { [OPENAI_SESSION_ID_OPTION]: fields.sessionId } } + : {}), + }, }; } // chat-completions (default) diff --git a/src/config/inference-sources.ts b/src/config/inference-sources.ts index 390742042..761940fcb 100644 --- a/src/config/inference-sources.ts +++ b/src/config/inference-sources.ts @@ -98,6 +98,7 @@ export function buildInferenceSourceForRef( id: ref.provider, apiKey: entry.apiKey ?? "", model: ref.model, + sessionId: ctx.sessionId, ...(effort !== undefined ? { reasoningEffort: effort } : {}), }); } @@ -118,6 +119,7 @@ export function buildInferenceSourceForRef( ? { apiKey: providerSettings.apiKey } : {}), model: ref.model, + sessionId: ctx.sessionId, ...(effort !== undefined ? { reasoningEffort: effort } : {}), }); } diff --git a/src/exec/runner.ts b/src/exec/runner.ts index 692bd6b97..defe8d64e 100644 --- a/src/exec/runner.ts +++ b/src/exec/runner.ts @@ -562,6 +562,7 @@ export async function runExec(config: Config): Promise { id: config.providerName, apiKey: config.apiKey, model: config.model, + sessionId, ...(config.reasoningEffort !== undefined ? { reasoningEffort: config.reasoningEffort } : {}), diff --git a/src/provider/grok-responses-adapter.ts b/src/provider/grok-responses-adapter.ts index b3066d4a7..572f70f99 100644 --- a/src/provider/grok-responses-adapter.ts +++ b/src/provider/grok-responses-adapter.ts @@ -32,8 +32,9 @@ import { export const GROK_RESPONSES_PROVIDER = "grok-responses"; -// Key the source stashes in defaults.providerOptions for this adapter. +// Keys the source stashes in defaults.providerOptions for this adapter. export const GROK_USER_ID_OPTION = "grokUserId"; +export const GROK_SESSION_ID_OPTION = "grokSessionId"; type ResponsesInputContentPart = | { type: "input_text"; text: string } @@ -172,6 +173,10 @@ function buildRequest( body["tools"] = tools; body["tool_choice"] = "auto"; } + // With store:false this is the only cache-routing signal; keying it to the + // inference thread's session id keeps every request on the same cache shard. + const sessionId = optionString(options, GROK_SESSION_ID_OPTION); + if (sessionId !== undefined) body["prompt_cache_key"] = sessionId; const headers: Record = { "content-type": "application/json", diff --git a/src/provider/openai-responses-adapter.ts b/src/provider/openai-responses-adapter.ts index cb05e2c59..ffc31b72b 100644 --- a/src/provider/openai-responses-adapter.ts +++ b/src/provider/openai-responses-adapter.ts @@ -24,6 +24,9 @@ import { export const OPENAI_RESPONSES_PROVIDER = "openai-responses"; +// Key the source stashes in defaults.providerOptions for this adapter. +export const OPENAI_SESSION_ID_OPTION = "openaiSessionId"; + type ResponsesInputContentPart = | { type: "input_text"; text: string } | { type: "input_image"; image_url: string }; @@ -112,6 +115,11 @@ function toResponsesTools(options: InferenceOptions): unknown[] | undefined { })); } +function optionString(options: InferenceOptions, key: string): string | undefined { + const value = options.providerOptions?.[key]; + return typeof value === "string" && value.length > 0 ? value : undefined; +} + function dedupeToolOutputs(items: ResponsesInputItem[]): ResponsesInputItem[] { const seen = new Set(); const deduped: ResponsesInputItem[] = []; @@ -153,6 +161,10 @@ function buildRequest( } if (options.maxTokens !== undefined) body["max_output_tokens"] = options.maxTokens; if (options.temperature !== undefined) body["temperature"] = options.temperature; + // With store:false this is the only cache-routing signal; keying it to the + // inference thread's session id keeps every request on the same cache shard. + const sessionId = optionString(options, OPENAI_SESSION_ID_OPTION); + if (sessionId !== undefined) body["prompt_cache_key"] = sessionId; return { url: "/responses", diff --git a/src/tui/runner.ts b/src/tui/runner.ts index f994d39da..48295b47f 100644 --- a/src/tui/runner.ts +++ b/src/tui/runner.ts @@ -1387,6 +1387,7 @@ export async function runTUI(initialConfig: Config): Promise { id: config.providerName, apiKey: config.apiKey, model: config.model, + sessionId, ...(config.reasoningEffort !== undefined ? { reasoningEffort: config.reasoningEffort } : {}), }) : buildOpenAICompatibleInitialSource(); diff --git a/tests/unit/grok-responses-adapter.test.ts b/tests/unit/grok-responses-adapter.test.ts index 83ca29d74..a77093fff 100644 --- a/tests/unit/grok-responses-adapter.test.ts +++ b/tests/unit/grok-responses-adapter.test.ts @@ -1,6 +1,7 @@ import { test, expect, describe } from "bun:test"; import { createGrokResponsesAdapter, + GROK_SESSION_ID_OPTION, GROK_USER_ID_OPTION, } from "../../src/provider/grok-responses-adapter.js"; import { BEARER_CREDENTIAL_SENTINEL } from "@intx/inference"; @@ -68,6 +69,33 @@ describe("grok-responses buildRequest", () => { expect(body["tool_choice"]).toBe("auto"); }); + test("sets prompt_cache_key from the session id, stable across builds", () => { + const options: InferenceOptions = { + ...baseOptions, + providerOptions: { ...baseOptions.providerOptions, [GROK_SESSION_ID_OPTION]: "sess-1" }, + }; + const first = JSON.parse(adapter().buildRequest([userTurn("a")], "grok-4.5", options).body) as Record; + const second = JSON.parse(adapter().buildRequest([userTurn("b")], "grok-4.5", options).body) as Record; + expect(first["prompt_cache_key"]).toBe("sess-1"); + expect(second["prompt_cache_key"]).toBe("sess-1"); + }); + + test("distinct session ids yield distinct prompt_cache_keys", () => { + const bodyFor = (sessionId: string): Record => + JSON.parse( + adapter().buildRequest([userTurn("hi")], "grok-4.5", { + providerOptions: { [GROK_SESSION_ID_OPTION]: sessionId }, + }).body, + ) as Record; + expect(bodyFor("sess-1")["prompt_cache_key"]).toBe("sess-1"); + expect(bodyFor("sess-2")["prompt_cache_key"]).toBe("sess-2"); + }); + + test("omits prompt_cache_key when no session id is present", () => { + const body = JSON.parse(adapter().buildRequest([userTurn("hi")], "grok-4.5", baseOptions).body) as Record; + expect(body).not.toHaveProperty("prompt_cache_key"); + }); + test("drops duplicate tool results for a call id", () => { const turns: ConversationTurn[] = [ { role: "assistant", content: [{ type: "tool_call", id: "call-1", name: "read_file", arguments: { path: "a.ts" } }], timestamp: 0 }, diff --git a/tests/unit/openai-responses-adapter.test.ts b/tests/unit/openai-responses-adapter.test.ts new file mode 100644 index 000000000..978ba59b3 --- /dev/null +++ b/tests/unit/openai-responses-adapter.test.ts @@ -0,0 +1,66 @@ +import { test, expect, describe } from "bun:test"; +import { + createOpenAIResponsesAdapter, + OPENAI_SESSION_ID_OPTION, +} from "../../src/provider/openai-responses-adapter.js"; +import { BEARER_CREDENTIAL_SENTINEL } from "@intx/inference"; +import type { ConversationTurn, InferenceOptions, LastCycleSource } from "@intx/types/runtime"; + +const SOURCE: LastCycleSource = { + sourceId: "go/default", + provider: "openai-responses", + model: "gpt-5.6-luna", +}; + +function adapter() { + return createOpenAIResponsesAdapter(SOURCE); +} + +function userTurn(text: string): ConversationTurn { + return { role: "user", content: [{ type: "text", text }], timestamp: 0 }; +} + +describe("openai-responses buildRequest", () => { + test("targets the Responses path with store off and streaming on", () => { + const req = adapter().buildRequest([userTurn("hi")], "gpt-5.6-luna", {}); + expect(req.url).toBe("/responses"); + expect(req.headers["authorization"]).toBe(BEARER_CREDENTIAL_SENTINEL); + expect(req.headers["accept"]).toBe("text/event-stream"); + const body = JSON.parse(req.body) as Record; + expect(body["model"]).toBe("gpt-5.6-luna"); + expect(body["stream"]).toBe(true); + expect(body["store"]).toBe(false); + }); + + test("sets prompt_cache_key from the session id, stable across builds", () => { + const options: InferenceOptions = { + providerOptions: { [OPENAI_SESSION_ID_OPTION]: "sess-1" }, + }; + const first = JSON.parse( + adapter().buildRequest([userTurn("a")], "gpt-5.6-luna", options).body, + ) as Record; + const second = JSON.parse( + adapter().buildRequest([userTurn("b")], "gpt-5.6-luna", options).body, + ) as Record; + expect(first["prompt_cache_key"]).toBe("sess-1"); + expect(second["prompt_cache_key"]).toBe("sess-1"); + }); + + test("distinct session ids yield distinct prompt_cache_keys", () => { + const bodyFor = (sessionId: string): Record => + JSON.parse( + adapter().buildRequest([userTurn("hi")], "gpt-5.6-luna", { + providerOptions: { [OPENAI_SESSION_ID_OPTION]: sessionId }, + }).body, + ) as Record; + expect(bodyFor("sess-1")["prompt_cache_key"]).toBe("sess-1"); + expect(bodyFor("sess-2")["prompt_cache_key"]).toBe("sess-2"); + }); + + test("omits prompt_cache_key when no session id is present", () => { + const body = JSON.parse( + adapter().buildRequest([userTurn("hi")], "gpt-5.6-luna", {}).body, + ) as Record; + expect(body).not.toHaveProperty("prompt_cache_key"); + }); +}); From 728a1403a80b2a73e033c38fbde22352e29666c8 Mon Sep 17 00:00:00 2001 From: Sawyer Cutler Date: Sat, 22 Aug 2026 20:04:09 -0700 Subject: [PATCH 2/2] Apply prettier to files the main merge left unformatted Amp-Thread-ID: https://ampcode.com/threads/T-01a02c68-0d8d-777b-b717-81fb9a282023 Co-authored-by: Amp --- src/config.test.ts | 21 ++++++++++++++++++--- tests/unit/grok-responses-adapter.test.ts | 12 +++++++++--- 2 files changed, 27 insertions(+), 6 deletions(-) diff --git a/src/config.test.ts b/src/config.test.ts index 0bbdd28f4..799dabde0 100644 --- a/src/config.test.ts +++ b/src/config.test.ts @@ -927,7 +927,12 @@ describe("buildBifrostSource", () => { describe("buildXaiSource", () => { test("omits reasoning_effort when effort is absent", () => { - const source = buildXaiSource({ id: "xai/work", apiKey: "tok", model: "grok-4.6", sessionId: "sess-1" }); + const source = buildXaiSource({ + id: "xai/work", + apiKey: "tok", + model: "grok-4.6", + sessionId: "sess-1", + }); expect(source.provider).toBe("grok-responses"); expect(source.defaults?.providerOptions).not.toHaveProperty("reasoning_effort"); }); @@ -944,12 +949,22 @@ describe("buildXaiSource", () => { }); test("does not invent high when effort is absent", () => { - const source = buildXaiSource({ id: "xai/work", apiKey: "tok", model: "grok-4.6", sessionId: "sess-1" }); + const source = buildXaiSource({ + id: "xai/work", + apiKey: "tok", + model: "grok-4.6", + sessionId: "sess-1", + }); expect(source.defaults?.providerOptions?.["reasoning_effort"]).toBeUndefined(); }); test("stashes the session id for the adapter's prompt_cache_key", () => { - const source = buildXaiSource({ id: "xai/work", apiKey: "tok", model: "grok-4.6", sessionId: "sess-1" }); + const source = buildXaiSource({ + id: "xai/work", + apiKey: "tok", + model: "grok-4.6", + sessionId: "sess-1", + }); expect(source.defaults?.providerOptions).toMatchObject({ grokSessionId: "sess-1" }); }); }); diff --git a/tests/unit/grok-responses-adapter.test.ts b/tests/unit/grok-responses-adapter.test.ts index bf85f0e66..43b9cb3b9 100644 --- a/tests/unit/grok-responses-adapter.test.ts +++ b/tests/unit/grok-responses-adapter.test.ts @@ -105,8 +105,12 @@ describe("grok-responses buildRequest", () => { ...baseOptions, providerOptions: { ...baseOptions.providerOptions, [GROK_SESSION_ID_OPTION]: "sess-1" }, }; - const first = JSON.parse(adapter().buildRequest([userTurn("a")], "grok-4.5", options).body) as Record; - const second = JSON.parse(adapter().buildRequest([userTurn("b")], "grok-4.5", options).body) as Record; + const first = JSON.parse( + adapter().buildRequest([userTurn("a")], "grok-4.5", options).body, + ) as Record; + const second = JSON.parse( + adapter().buildRequest([userTurn("b")], "grok-4.5", options).body, + ) as Record; expect(first["prompt_cache_key"]).toBe("sess-1"); expect(second["prompt_cache_key"]).toBe("sess-1"); }); @@ -123,7 +127,9 @@ describe("grok-responses buildRequest", () => { }); test("omits prompt_cache_key when no session id is present", () => { - const body = JSON.parse(adapter().buildRequest([userTurn("hi")], "grok-4.5", baseOptions).body) as Record; + const body = JSON.parse( + adapter().buildRequest([userTurn("hi")], "grok-4.5", baseOptions).body, + ) as Record; expect(body).not.toHaveProperty("prompt_cache_key"); });