Skip to content

Commit d8e934e

Browse files
committed
Set prompt_cache_key on the Grok and OpenAI Responses adapters
Codex's Responses adapter already keys prompt_cache_key by session id; grok-responses-adapter.ts and openai-responses-adapter.ts never set it, so xAI/OpenCode-Go Responses calls had no cache-routing signal beyond the implicit prefix hash. Threads the session id (parent session's sessionId, or a fresh id per subagent thread) through buildXaiSource/buildGoSource into each adapter's providerOptions, and sets prompt_cache_key in the request body when present. Per xAI's docs (docs.x.ai/developers/advanced-api-usage/prompt-caching), the Responses API honors prompt_cache_key the same way OpenAI does; verified this isn't a no-op before shipping it.
1 parent bebe563 commit d8e934e

9 files changed

Lines changed: 147 additions & 7 deletions

src/config.test.ts

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -892,7 +892,7 @@ describe("buildBifrostSource", () => {
892892

893893
describe("buildXaiSource", () => {
894894
test("omits reasoning_effort when effort is absent", () => {
895-
const source = buildXaiSource({ id: "xai/work", apiKey: "tok", model: "grok-4.6" });
895+
const source = buildXaiSource({ id: "xai/work", apiKey: "tok", model: "grok-4.6", sessionId: "sess-1" });
896896
expect(source.provider).toBe("grok-responses");
897897
expect(source.defaults?.providerOptions).not.toHaveProperty("reasoning_effort");
898898
});
@@ -902,15 +902,21 @@ describe("buildXaiSource", () => {
902902
id: "xai/work",
903903
apiKey: "tok",
904904
model: "grok-4.6",
905+
sessionId: "sess-1",
905906
reasoningEffort: "low",
906907
});
907908
expect(source.defaults?.providerOptions).toMatchObject({ reasoning_effort: "low" });
908909
});
909910

910911
test("does not invent high when effort is absent", () => {
911-
const source = buildXaiSource({ id: "xai/work", apiKey: "tok", model: "grok-4.6" });
912+
const source = buildXaiSource({ id: "xai/work", apiKey: "tok", model: "grok-4.6", sessionId: "sess-1" });
912913
expect(source.defaults?.providerOptions?.["reasoning_effort"]).toBeUndefined();
913914
});
915+
916+
test("stashes the session id for the adapter's prompt_cache_key", () => {
917+
const source = buildXaiSource({ id: "xai/work", apiKey: "tok", model: "grok-4.6", sessionId: "sess-1" });
918+
expect(source.defaults?.providerOptions).toMatchObject({ grokSessionId: "sess-1" });
919+
});
914920
});
915921

916922
describe("buildProviderCatalog", () => {

src/config/index.ts

Lines changed: 23 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -32,9 +32,16 @@ import {
3232
CODEX_ACCOUNT_ID_OPTION,
3333
CODEX_SESSION_ID_OPTION,
3434
} from "../provider/codex-responses-adapter.js";
35-
import { GROK_RESPONSES_PROVIDER, GROK_USER_ID_OPTION } from "../provider/grok-responses-adapter.js";
35+
import {
36+
GROK_RESPONSES_PROVIDER,
37+
GROK_SESSION_ID_OPTION,
38+
GROK_USER_ID_OPTION,
39+
} from "../provider/grok-responses-adapter.js";
3640
import { BIFROST_PROVIDER } from "../provider/bifrost-adapter.js";
37-
import { OPENAI_RESPONSES_PROVIDER } from "../provider/openai-responses-adapter.js";
41+
import {
42+
OPENAI_RESPONSES_PROVIDER,
43+
OPENAI_SESSION_ID_OPTION,
44+
} from "../provider/openai-responses-adapter.js";
3845
import { xaiUserIdFromAccessToken } from "../auth/xai/session.js";
3946
import {
4047
OPENCODE_GO_BASE_URL,
@@ -166,14 +173,19 @@ export function buildCodexSource(fields: {
166173
// "grok-responses" adapter (the grok-cli proxy speaks the Responses API, not
167174
// Chat Completions). The access token is the apiKey; the caller's user id is
168175
// decoded from it and lifted into the x-grok-user-id header by the adapter.
176+
// The session id becomes the request's prompt_cache_key so every call in the
177+
// thread routes to the same cache shard (store:false has no other signal).
169178
export function buildXaiSource(fields: {
170179
id: string;
171180
apiKey: string;
172181
model: string;
182+
sessionId: string;
173183
reasoningEffort?: ReasoningEffort;
174184
}): InferenceSource {
175185
const userId = xaiUserIdFromAccessToken(fields.apiKey);
176-
const providerOptions: Record<string, unknown> = {};
186+
const providerOptions: Record<string, unknown> = {
187+
[GROK_SESSION_ID_OPTION]: fields.sessionId,
188+
};
177189
if (userId !== undefined) providerOptions[GROK_USER_ID_OPTION] = userId;
178190
if (fields.reasoningEffort !== undefined) providerOptions["reasoning_effort"] = fields.reasoningEffort;
179191
return {
@@ -228,10 +240,12 @@ export function buildAnthropicSource(fields: {
228240
}
229241

230242
// OpenCode Go: per-model protocol routing (chat completions / responses / messages).
243+
// sessionId feeds the Responses-protocol prompt_cache_key (see buildXaiSource).
231244
export function buildGoSource(fields: {
232245
id: string;
233246
apiKey?: string;
234247
model: string;
248+
sessionId?: string;
235249
reasoningEffort?: ReasoningEffort;
236250
}): InferenceSource {
237251
const endpoint = resolveGoEndpoint(fields.model);
@@ -252,7 +266,12 @@ export function buildGoSource(fields: {
252266
baseURL: endpoint.baseURL,
253267
apiKey,
254268
model: fields.model,
255-
defaults: { maxTokens: SOURCE_MAX_TOKENS },
269+
defaults: {
270+
maxTokens: SOURCE_MAX_TOKENS,
271+
...(fields.sessionId !== undefined
272+
? { providerOptions: { [OPENAI_SESSION_ID_OPTION]: fields.sessionId } }
273+
: {}),
274+
},
256275
};
257276
}
258277
// chat-completions (default)

src/config/inference-sources.ts

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -98,6 +98,7 @@ export function buildInferenceSourceForRef(
9898
id: ref.provider,
9999
apiKey: entry.apiKey ?? "",
100100
model: ref.model,
101+
sessionId: ctx.sessionId,
101102
...(effort !== undefined ? { reasoningEffort: effort } : {}),
102103
});
103104
}
@@ -118,6 +119,7 @@ export function buildInferenceSourceForRef(
118119
? { apiKey: providerSettings.apiKey }
119120
: {}),
120121
model: ref.model,
122+
sessionId: ctx.sessionId,
121123
...(effort !== undefined ? { reasoningEffort: effort } : {}),
122124
});
123125
}

src/exec/runner.ts

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -562,6 +562,7 @@ export async function runExec(config: Config): Promise<ExecResult> {
562562
id: config.providerName,
563563
apiKey: config.apiKey,
564564
model: config.model,
565+
sessionId,
565566
...(config.reasoningEffort !== undefined
566567
? { reasoningEffort: config.reasoningEffort }
567568
: {}),

src/provider/grok-responses-adapter.ts

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,8 +32,9 @@ import {
3232

3333
export const GROK_RESPONSES_PROVIDER = "grok-responses";
3434

35-
// Key the source stashes in defaults.providerOptions for this adapter.
35+
// Keys the source stashes in defaults.providerOptions for this adapter.
3636
export const GROK_USER_ID_OPTION = "grokUserId";
37+
export const GROK_SESSION_ID_OPTION = "grokSessionId";
3738

3839
type ResponsesInputContentPart =
3940
| { type: "input_text"; text: string }
@@ -172,6 +173,10 @@ function buildRequest(
172173
body["tools"] = tools;
173174
body["tool_choice"] = "auto";
174175
}
176+
// With store:false this is the only cache-routing signal; keying it to the
177+
// inference thread's session id keeps every request on the same cache shard.
178+
const sessionId = optionString(options, GROK_SESSION_ID_OPTION);
179+
if (sessionId !== undefined) body["prompt_cache_key"] = sessionId;
175180

176181
const headers: Record<string, string> = {
177182
"content-type": "application/json",

src/provider/openai-responses-adapter.ts

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,9 @@ import {
2424

2525
export const OPENAI_RESPONSES_PROVIDER = "openai-responses";
2626

27+
// Key the source stashes in defaults.providerOptions for this adapter.
28+
export const OPENAI_SESSION_ID_OPTION = "openaiSessionId";
29+
2730
type ResponsesInputContentPart =
2831
| { type: "input_text"; text: string }
2932
| { type: "input_image"; image_url: string };
@@ -112,6 +115,11 @@ function toResponsesTools(options: InferenceOptions): unknown[] | undefined {
112115
}));
113116
}
114117

118+
function optionString(options: InferenceOptions, key: string): string | undefined {
119+
const value = options.providerOptions?.[key];
120+
return typeof value === "string" && value.length > 0 ? value : undefined;
121+
}
122+
115123
function dedupeToolOutputs(items: ResponsesInputItem[]): ResponsesInputItem[] {
116124
const seen = new Set<string>();
117125
const deduped: ResponsesInputItem[] = [];
@@ -153,6 +161,10 @@ function buildRequest(
153161
}
154162
if (options.maxTokens !== undefined) body["max_output_tokens"] = options.maxTokens;
155163
if (options.temperature !== undefined) body["temperature"] = options.temperature;
164+
// With store:false this is the only cache-routing signal; keying it to the
165+
// inference thread's session id keeps every request on the same cache shard.
166+
const sessionId = optionString(options, OPENAI_SESSION_ID_OPTION);
167+
if (sessionId !== undefined) body["prompt_cache_key"] = sessionId;
156168

157169
return {
158170
url: "/responses",

src/tui/runner.ts

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1387,6 +1387,7 @@ export async function runTUI(initialConfig: Config): Promise<number> {
13871387
id: config.providerName,
13881388
apiKey: config.apiKey,
13891389
model: config.model,
1390+
sessionId,
13901391
...(config.reasoningEffort !== undefined ? { reasoningEffort: config.reasoningEffort } : {}),
13911392
})
13921393
: buildOpenAICompatibleInitialSource();

tests/unit/grok-responses-adapter.test.ts

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import { test, expect, describe } from "bun:test";
22
import {
33
createGrokResponsesAdapter,
4+
GROK_SESSION_ID_OPTION,
45
GROK_USER_ID_OPTION,
56
} from "../../src/provider/grok-responses-adapter.js";
67
import { BEARER_CREDENTIAL_SENTINEL } from "@intx/inference";
@@ -68,6 +69,33 @@ describe("grok-responses buildRequest", () => {
6869
expect(body["tool_choice"]).toBe("auto");
6970
});
7071

72+
test("sets prompt_cache_key from the session id, stable across builds", () => {
73+
const options: InferenceOptions = {
74+
...baseOptions,
75+
providerOptions: { ...baseOptions.providerOptions, [GROK_SESSION_ID_OPTION]: "sess-1" },
76+
};
77+
const first = JSON.parse(adapter().buildRequest([userTurn("a")], "grok-4.5", options).body) as Record<string, unknown>;
78+
const second = JSON.parse(adapter().buildRequest([userTurn("b")], "grok-4.5", options).body) as Record<string, unknown>;
79+
expect(first["prompt_cache_key"]).toBe("sess-1");
80+
expect(second["prompt_cache_key"]).toBe("sess-1");
81+
});
82+
83+
test("distinct session ids yield distinct prompt_cache_keys", () => {
84+
const bodyFor = (sessionId: string): Record<string, unknown> =>
85+
JSON.parse(
86+
adapter().buildRequest([userTurn("hi")], "grok-4.5", {
87+
providerOptions: { [GROK_SESSION_ID_OPTION]: sessionId },
88+
}).body,
89+
) as Record<string, unknown>;
90+
expect(bodyFor("sess-1")["prompt_cache_key"]).toBe("sess-1");
91+
expect(bodyFor("sess-2")["prompt_cache_key"]).toBe("sess-2");
92+
});
93+
94+
test("omits prompt_cache_key when no session id is present", () => {
95+
const body = JSON.parse(adapter().buildRequest([userTurn("hi")], "grok-4.5", baseOptions).body) as Record<string, unknown>;
96+
expect(body).not.toHaveProperty("prompt_cache_key");
97+
});
98+
7199
test("drops duplicate tool results for a call id", () => {
72100
const turns: ConversationTurn[] = [
73101
{ role: "assistant", content: [{ type: "tool_call", id: "call-1", name: "read_file", arguments: { path: "a.ts" } }], timestamp: 0 },
Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
import { test, expect, describe } from "bun:test";
2+
import {
3+
createOpenAIResponsesAdapter,
4+
OPENAI_SESSION_ID_OPTION,
5+
} from "../../src/provider/openai-responses-adapter.js";
6+
import { BEARER_CREDENTIAL_SENTINEL } from "@intx/inference";
7+
import type { ConversationTurn, InferenceOptions, LastCycleSource } from "@intx/types/runtime";
8+
9+
const SOURCE: LastCycleSource = {
10+
sourceId: "go/default",
11+
provider: "openai-responses",
12+
model: "gpt-5.6-luna",
13+
};
14+
15+
function adapter() {
16+
return createOpenAIResponsesAdapter(SOURCE);
17+
}
18+
19+
function userTurn(text: string): ConversationTurn {
20+
return { role: "user", content: [{ type: "text", text }], timestamp: 0 };
21+
}
22+
23+
describe("openai-responses buildRequest", () => {
24+
test("targets the Responses path with store off and streaming on", () => {
25+
const req = adapter().buildRequest([userTurn("hi")], "gpt-5.6-luna", {});
26+
expect(req.url).toBe("/responses");
27+
expect(req.headers["authorization"]).toBe(BEARER_CREDENTIAL_SENTINEL);
28+
expect(req.headers["accept"]).toBe("text/event-stream");
29+
const body = JSON.parse(req.body) as Record<string, unknown>;
30+
expect(body["model"]).toBe("gpt-5.6-luna");
31+
expect(body["stream"]).toBe(true);
32+
expect(body["store"]).toBe(false);
33+
});
34+
35+
test("sets prompt_cache_key from the session id, stable across builds", () => {
36+
const options: InferenceOptions = {
37+
providerOptions: { [OPENAI_SESSION_ID_OPTION]: "sess-1" },
38+
};
39+
const first = JSON.parse(
40+
adapter().buildRequest([userTurn("a")], "gpt-5.6-luna", options).body,
41+
) as Record<string, unknown>;
42+
const second = JSON.parse(
43+
adapter().buildRequest([userTurn("b")], "gpt-5.6-luna", options).body,
44+
) as Record<string, unknown>;
45+
expect(first["prompt_cache_key"]).toBe("sess-1");
46+
expect(second["prompt_cache_key"]).toBe("sess-1");
47+
});
48+
49+
test("distinct session ids yield distinct prompt_cache_keys", () => {
50+
const bodyFor = (sessionId: string): Record<string, unknown> =>
51+
JSON.parse(
52+
adapter().buildRequest([userTurn("hi")], "gpt-5.6-luna", {
53+
providerOptions: { [OPENAI_SESSION_ID_OPTION]: sessionId },
54+
}).body,
55+
) as Record<string, unknown>;
56+
expect(bodyFor("sess-1")["prompt_cache_key"]).toBe("sess-1");
57+
expect(bodyFor("sess-2")["prompt_cache_key"]).toBe("sess-2");
58+
});
59+
60+
test("omits prompt_cache_key when no session id is present", () => {
61+
const body = JSON.parse(
62+
adapter().buildRequest([userTurn("hi")], "gpt-5.6-luna", {}).body,
63+
) as Record<string, unknown>;
64+
expect(body).not.toHaveProperty("prompt_cache_key");
65+
});
66+
});

0 commit comments

Comments
 (0)