diff --git a/src/config.test.ts b/src/config.test.ts index ecd0bddc3..799dabde0 100644 --- a/src/config.test.ts +++ b/src/config.test.ts @@ -927,7 +927,12 @@ describe("buildBifrostSource", () => { describe("buildXaiSource", () => { test("omits reasoning_effort when effort is absent", () => { - const source = buildXaiSource({ id: "xai/work", apiKey: "tok", model: "grok-4.6" }); + const source = buildXaiSource({ + id: "xai/work", + apiKey: "tok", + model: "grok-4.6", + sessionId: "sess-1", + }); expect(source.provider).toBe("grok-responses"); expect(source.defaults?.providerOptions).not.toHaveProperty("reasoning_effort"); }); @@ -937,15 +942,31 @@ describe("buildXaiSource", () => { id: "xai/work", apiKey: "tok", model: "grok-4.6", + sessionId: "sess-1", reasoningEffort: "low", }); expect(source.defaults?.providerOptions).toMatchObject({ reasoning_effort: "low" }); }); test("does not invent high when effort is absent", () => { - const source = buildXaiSource({ id: "xai/work", apiKey: "tok", model: "grok-4.6" }); + const source = buildXaiSource({ + id: "xai/work", + apiKey: "tok", + model: "grok-4.6", + sessionId: "sess-1", + }); expect(source.defaults?.providerOptions?.["reasoning_effort"]).toBeUndefined(); }); + + test("stashes the session id for the adapter's prompt_cache_key", () => { + const source = buildXaiSource({ + id: "xai/work", + apiKey: "tok", + model: "grok-4.6", + sessionId: "sess-1", + }); + expect(source.defaults?.providerOptions).toMatchObject({ grokSessionId: "sess-1" }); + }); }); describe("buildProviderCatalog", () => { diff --git a/src/config/index.ts b/src/config/index.ts index 1230398dd..7bef84c2b 100644 --- a/src/config/index.ts +++ b/src/config/index.ts @@ -33,10 +33,14 @@ import { } from "../provider/codex-responses-adapter.js"; import { GROK_RESPONSES_PROVIDER, + GROK_SESSION_ID_OPTION, GROK_USER_ID_OPTION, } from "../provider/grok-responses-adapter.js"; import { BIFROST_PROVIDER } from "../provider/bifrost-adapter.js"; -import { OPENAI_RESPONSES_PROVIDER } from "../provider/openai-responses-adapter.js"; +import { + OPENAI_RESPONSES_PROVIDER, + OPENAI_SESSION_ID_OPTION, +} from "../provider/openai-responses-adapter.js"; import { xaiUserIdFromAccessToken } from "../auth/xai/session.js"; import { OPENCODE_GO_BASE_URL, @@ -169,14 +173,19 @@ export function buildCodexSource(fields: { // "grok-responses" adapter (the grok-cli proxy speaks the Responses API, not // Chat Completions). The access token is the apiKey; the caller's user id is // decoded from it and lifted into the x-grok-user-id header by the adapter. +// The session id becomes the request's prompt_cache_key so every call in the +// thread routes to the same cache shard (store:false has no other signal). export function buildXaiSource(fields: { id: string; apiKey: string; model: string; + sessionId: string; reasoningEffort?: ReasoningEffort; }): InferenceSource { const userId = xaiUserIdFromAccessToken(fields.apiKey); - const providerOptions: Record = {}; + const providerOptions: Record = { + [GROK_SESSION_ID_OPTION]: fields.sessionId, + }; if (userId !== undefined) providerOptions[GROK_USER_ID_OPTION] = userId; if (fields.reasoningEffort !== undefined) providerOptions["reasoning_effort"] = fields.reasoningEffort; @@ -234,10 +243,12 @@ export function buildAnthropicSource(fields: { } // OpenCode Go: per-model protocol routing (chat completions / responses / messages). +// sessionId feeds the Responses-protocol prompt_cache_key (see buildXaiSource). export function buildGoSource(fields: { id: string; apiKey?: string; model: string; + sessionId?: string; reasoningEffort?: ReasoningEffort; }): InferenceSource { const endpoint = resolveGoEndpoint(fields.model); @@ -258,7 +269,12 @@ export function buildGoSource(fields: { baseURL: endpoint.baseURL, apiKey, model: fields.model, - defaults: { maxTokens: SOURCE_MAX_TOKENS }, + defaults: { + maxTokens: SOURCE_MAX_TOKENS, + ...(fields.sessionId !== undefined + ? { providerOptions: { [OPENAI_SESSION_ID_OPTION]: fields.sessionId } } + : {}), + }, }; } // chat-completions (default) diff --git a/src/config/inference-sources.ts b/src/config/inference-sources.ts index ce4f4d57b..673ad6da7 100644 --- a/src/config/inference-sources.ts +++ b/src/config/inference-sources.ts @@ -98,6 +98,7 @@ export function buildInferenceSourceForRef( id: ref.provider, apiKey: entry.apiKey ?? "", model: ref.model, + sessionId: ctx.sessionId, ...(effort !== undefined ? { reasoningEffort: effort } : {}), }); } @@ -118,6 +119,7 @@ export function buildInferenceSourceForRef( ? { apiKey: providerSettings.apiKey } : {}), model: ref.model, + sessionId: ctx.sessionId, ...(effort !== undefined ? { reasoningEffort: effort } : {}), }); } diff --git a/src/exec/runner.ts b/src/exec/runner.ts index 4a2a6ce37..799da3f2e 100644 --- a/src/exec/runner.ts +++ b/src/exec/runner.ts @@ -556,6 +556,7 @@ export async function runExec(config: Config): Promise { id: config.providerName, apiKey: config.apiKey, model: config.model, + sessionId, ...(config.reasoningEffort !== undefined ? { reasoningEffort: config.reasoningEffort } : {}), diff --git a/src/provider/grok-responses-adapter.ts b/src/provider/grok-responses-adapter.ts index 3a8d7c47a..14f56dcdf 100644 --- a/src/provider/grok-responses-adapter.ts +++ b/src/provider/grok-responses-adapter.ts @@ -32,8 +32,9 @@ import { export const GROK_RESPONSES_PROVIDER = "grok-responses"; -// Key the source stashes in defaults.providerOptions for this adapter. +// Keys the source stashes in defaults.providerOptions for this adapter. export const GROK_USER_ID_OPTION = "grokUserId"; +export const GROK_SESSION_ID_OPTION = "grokSessionId"; type ResponsesInputContentPart = { type: "input_text"; text: string } | { type: "input_image"; image_url: string }; @@ -202,6 +203,10 @@ function buildRequest( body["tools"] = tools; body["tool_choice"] = "auto"; } + // With store:false this is the only cache-routing signal; keying it to the + // inference thread's session id keeps every request on the same cache shard. + const sessionId = optionString(options, GROK_SESSION_ID_OPTION); + if (sessionId !== undefined) body["prompt_cache_key"] = sessionId; const headers: Record = { "content-type": "application/json", diff --git a/src/provider/openai-responses-adapter.ts b/src/provider/openai-responses-adapter.ts index 46fc1bf57..b0092d927 100644 --- a/src/provider/openai-responses-adapter.ts +++ b/src/provider/openai-responses-adapter.ts @@ -24,6 +24,9 @@ import { export const OPENAI_RESPONSES_PROVIDER = "openai-responses"; +// Key the source stashes in defaults.providerOptions for this adapter. +export const OPENAI_SESSION_ID_OPTION = "openaiSessionId"; + type ResponsesInputContentPart = { type: "input_text"; text: string } | { type: "input_image"; image_url: string }; @@ -134,6 +137,11 @@ function toResponsesTools(options: InferenceOptions): unknown[] | undefined { })); } +function optionString(options: InferenceOptions, key: string): string | undefined { + const value = options.providerOptions?.[key]; + return typeof value === "string" && value.length > 0 ? value : undefined; +} + function dedupeToolOutputs(items: ResponsesInputItem[]): ResponsesInputItem[] { const seen = new Set(); const deduped: ResponsesInputItem[] = []; @@ -177,6 +185,10 @@ function buildRequest( } if (options.maxTokens !== undefined) body["max_output_tokens"] = options.maxTokens; if (options.temperature !== undefined) body["temperature"] = options.temperature; + // With store:false this is the only cache-routing signal; keying it to the + // inference thread's session id keeps every request on the same cache shard. + const sessionId = optionString(options, OPENAI_SESSION_ID_OPTION); + if (sessionId !== undefined) body["prompt_cache_key"] = sessionId; return { url: "/responses", diff --git a/src/tui/runner.ts b/src/tui/runner.ts index 2d1574054..65a94c78e 100644 --- a/src/tui/runner.ts +++ b/src/tui/runner.ts @@ -1446,6 +1446,7 @@ export async function runTUI(initialConfig: Config): Promise { id: config.providerName, apiKey: config.apiKey, model: config.model, + sessionId, ...(config.reasoningEffort !== undefined ? { reasoningEffort: config.reasoningEffort } : {}), diff --git a/tests/unit/grok-responses-adapter.test.ts b/tests/unit/grok-responses-adapter.test.ts index 949509f02..43b9cb3b9 100644 --- a/tests/unit/grok-responses-adapter.test.ts +++ b/tests/unit/grok-responses-adapter.test.ts @@ -1,6 +1,7 @@ import { test, expect, describe } from "bun:test"; import { createGrokResponsesAdapter, + GROK_SESSION_ID_OPTION, GROK_USER_ID_OPTION, } from "../../src/provider/grok-responses-adapter.js"; import { BEARER_CREDENTIAL_SENTINEL } from "@intx/inference"; @@ -99,6 +100,39 @@ describe("grok-responses buildRequest", () => { expect(body["tool_choice"]).toBe("auto"); }); + test("sets prompt_cache_key from the session id, stable across builds", () => { + const options: InferenceOptions = { + ...baseOptions, + providerOptions: { ...baseOptions.providerOptions, [GROK_SESSION_ID_OPTION]: "sess-1" }, + }; + const first = JSON.parse( + adapter().buildRequest([userTurn("a")], "grok-4.5", options).body, + ) as Record; + const second = JSON.parse( + adapter().buildRequest([userTurn("b")], "grok-4.5", options).body, + ) as Record; + expect(first["prompt_cache_key"]).toBe("sess-1"); + expect(second["prompt_cache_key"]).toBe("sess-1"); + }); + + test("distinct session ids yield distinct prompt_cache_keys", () => { + const bodyFor = (sessionId: string): Record => + JSON.parse( + adapter().buildRequest([userTurn("hi")], "grok-4.5", { + providerOptions: { [GROK_SESSION_ID_OPTION]: sessionId }, + }).body, + ) as Record; + expect(bodyFor("sess-1")["prompt_cache_key"]).toBe("sess-1"); + expect(bodyFor("sess-2")["prompt_cache_key"]).toBe("sess-2"); + }); + + test("omits prompt_cache_key when no session id is present", () => { + const body = JSON.parse( + adapter().buildRequest([userTurn("hi")], "grok-4.5", baseOptions).body, + ) as Record; + expect(body).not.toHaveProperty("prompt_cache_key"); + }); + test("drops duplicate tool results for a call id", () => { const turns: ConversationTurn[] = [ { diff --git a/tests/unit/openai-responses-adapter.test.ts b/tests/unit/openai-responses-adapter.test.ts new file mode 100644 index 000000000..978ba59b3 --- /dev/null +++ b/tests/unit/openai-responses-adapter.test.ts @@ -0,0 +1,66 @@ +import { test, expect, describe } from "bun:test"; +import { + createOpenAIResponsesAdapter, + OPENAI_SESSION_ID_OPTION, +} from "../../src/provider/openai-responses-adapter.js"; +import { BEARER_CREDENTIAL_SENTINEL } from "@intx/inference"; +import type { ConversationTurn, InferenceOptions, LastCycleSource } from "@intx/types/runtime"; + +const SOURCE: LastCycleSource = { + sourceId: "go/default", + provider: "openai-responses", + model: "gpt-5.6-luna", +}; + +function adapter() { + return createOpenAIResponsesAdapter(SOURCE); +} + +function userTurn(text: string): ConversationTurn { + return { role: "user", content: [{ type: "text", text }], timestamp: 0 }; +} + +describe("openai-responses buildRequest", () => { + test("targets the Responses path with store off and streaming on", () => { + const req = adapter().buildRequest([userTurn("hi")], "gpt-5.6-luna", {}); + expect(req.url).toBe("/responses"); + expect(req.headers["authorization"]).toBe(BEARER_CREDENTIAL_SENTINEL); + expect(req.headers["accept"]).toBe("text/event-stream"); + const body = JSON.parse(req.body) as Record; + expect(body["model"]).toBe("gpt-5.6-luna"); + expect(body["stream"]).toBe(true); + expect(body["store"]).toBe(false); + }); + + test("sets prompt_cache_key from the session id, stable across builds", () => { + const options: InferenceOptions = { + providerOptions: { [OPENAI_SESSION_ID_OPTION]: "sess-1" }, + }; + const first = JSON.parse( + adapter().buildRequest([userTurn("a")], "gpt-5.6-luna", options).body, + ) as Record; + const second = JSON.parse( + adapter().buildRequest([userTurn("b")], "gpt-5.6-luna", options).body, + ) as Record; + expect(first["prompt_cache_key"]).toBe("sess-1"); + expect(second["prompt_cache_key"]).toBe("sess-1"); + }); + + test("distinct session ids yield distinct prompt_cache_keys", () => { + const bodyFor = (sessionId: string): Record => + JSON.parse( + adapter().buildRequest([userTurn("hi")], "gpt-5.6-luna", { + providerOptions: { [OPENAI_SESSION_ID_OPTION]: sessionId }, + }).body, + ) as Record; + expect(bodyFor("sess-1")["prompt_cache_key"]).toBe("sess-1"); + expect(bodyFor("sess-2")["prompt_cache_key"]).toBe("sess-2"); + }); + + test("omits prompt_cache_key when no session id is present", () => { + const body = JSON.parse( + adapter().buildRequest([userTurn("hi")], "gpt-5.6-luna", {}).body, + ) as Record; + expect(body).not.toHaveProperty("prompt_cache_key"); + }); +});