From 708ab05cb0405f339bea48c5323643fa9c632f0a Mon Sep 17 00:00:00 2001 From: Ousama Ben Younes Date: Mon, 27 Jul 2026 19:19:48 +0000 Subject: [PATCH] feat(llm): cache recent Claude prompt context --- strix/core/inputs.py | 20 ++++++++++++++++---- tests/test_inputs.py | 3 +++ 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/strix/core/inputs.py b/strix/core/inputs.py index aef1fe137..30d503204 100644 --- a/strix/core/inputs.py +++ b/strix/core/inputs.py @@ -25,6 +25,11 @@ DEFAULT_MAX_TURNS = 500 +_CACHE_LOCATION_MESSAGE = "message" +_CACHE_LOCATION_TOOL_CONFIG = "tool_config" +_CACHE_ROLE_SYSTEM = "system" +_CACHE_INDEX_PREVIOUS_MESSAGE = -2 +_CACHE_INDEX_LAST_MESSAGE = -1 def _accepts_required_tool_choice(model_name: str | None) -> bool: @@ -163,7 +168,7 @@ def make_model_settings( def _prompt_cache_extra_args(model_name: str) -> dict[str, Any] | None: """LiteLLM ``cache_control_injection_points`` for Claude prompt caching. - System prompt + rolling last-message breakpoint everywhere; ``tool_config`` + System prompt + rolling recent-message breakpoints everywhere; ``tool_config`` only on Bedrock Converse (the only route whose LiteLLM transform consumes it — elsewhere it leaks onto the wire and native Anthropic 400s). Unmapped Bedrock models get no points at all: Bedrock rejects the passed-through @@ -174,10 +179,17 @@ def _prompt_cache_extra_args(model_name: str) -> dict[str, Any] | None: if is_bedrock_route(model_name) and not bedrock_route_supports_prompt_caching(model_name): return None - points: list[dict[str, Any]] = [{"location": "message", "role": "system"}] + points: list[dict[str, Any]] = [ + {"location": _CACHE_LOCATION_MESSAGE, "role": _CACHE_ROLE_SYSTEM} + ] if is_bedrock_route(model_name): - points.append({"location": "tool_config"}) - points.append({"location": "message", "index": -1}) + points.append({"location": _CACHE_LOCATION_TOOL_CONFIG}) + points.extend( + [ + {"location": _CACHE_LOCATION_MESSAGE, "index": _CACHE_INDEX_PREVIOUS_MESSAGE}, + {"location": _CACHE_LOCATION_MESSAGE, "index": _CACHE_INDEX_LAST_MESSAGE}, + ] + ) return {"cache_control_injection_points": points} diff --git a/tests/test_inputs.py b/tests/test_inputs.py index da914879a..5bb1643dd 100644 --- a/tests/test_inputs.py +++ b/tests/test_inputs.py @@ -65,6 +65,7 @@ def test_make_model_settings_enables_prompt_cache_for_bedrock_claude() -> None: assert _cache_points("bedrock/global.anthropic.claude-opus-4-8") == [ {"location": "message", "role": "system"}, {"location": "tool_config"}, + {"location": "message", "index": -2}, {"location": "message", "index": -1}, ] @@ -80,6 +81,7 @@ def test_make_model_settings_enables_prompt_cache_for_bedrock_claude() -> None: def test_make_model_settings_enables_prompt_cache_for_non_bedrock_claude(model_name: str) -> None: assert _cache_points(model_name) == [ {"location": "message", "role": "system"}, + {"location": "message", "index": -2}, {"location": "message", "index": -1}, ] @@ -125,6 +127,7 @@ def test_prompt_cache_kept_for_non_bedrock_claude_even_if_unmapped(monkeypatch: for model in ("anthropic/claude-brand-new-9", "openrouter/anthropic/claude-brand-new"): assert _cache_points(model) == [ {"location": "message", "role": "system"}, + {"location": "message", "index": -2}, {"location": "message", "index": -1}, ]