From 112287c1f884a467e3beb9dbfed26d2a81be5e4b Mon Sep 17 00:00:00 2001 From: kigland Date: Sun, 16 Aug 2026 10:35:33 +0800 Subject: [PATCH 1/2] fix Azure cache affinity detection --- hindsight-api-slim/hindsight_api/config.py | 2 +- hindsight-api-slim/hindsight_api/engine/cache_affinity.py | 4 ++-- hindsight-api-slim/tests/test_cache_affinity.py | 4 ++-- hindsight-docs/docs/developer/configuration.md | 2 +- hindsight-docs/docs/developer/models.mdx | 7 +++---- .../hindsight-docs/references/developer/configuration.md | 2 +- skills/hindsight-docs/references/developer/models.md | 7 +++---- 7 files changed, 13 insertions(+), 15 deletions(-) diff --git a/hindsight-api-slim/hindsight_api/config.py b/hindsight-api-slim/hindsight_api/config.py index 0dcdc2ba5..3b462c3e4 100644 --- a/hindsight-api-slim/hindsight_api/config.py +++ b/hindsight-api-slim/hindsight_api/config.py @@ -233,7 +233,7 @@ def normalize_config_dict(config: dict[str, Any]) -> dict[str, Any]: ) # "auto" is safe as a default because it is an allowlist, not a best-effort probe: # it emits a hint only for hosts documented to accept one (x.ai / grok.com get the -# header, native OpenAI / openai.com / Azure OpenAI get the field) and resolves to +# header, native OpenAI / openai.com get the field) and resolves to # "none" for every other backend, so vLLM, ollama, groq, openrouter and any custom # OpenAI-compatible endpoint keep receiving byte-identical requests. Measured on a # live xAI backend: 29% of a shared prefix cached without the header vs 99% with it, diff --git a/hindsight-api-slim/hindsight_api/engine/cache_affinity.py b/hindsight-api-slim/hindsight_api/engine/cache_affinity.py index 1fdddb02f..e7812baab 100644 --- a/hindsight-api-slim/hindsight_api/engine/cache_affinity.py +++ b/hindsight-api-slim/hindsight_api/engine/cache_affinity.py @@ -42,7 +42,7 @@ # Hosts (exact or parent domain) whose backends implement the xAI header. _XAI_DOMAINS = ("x.ai", "grok.com") # Hosts (exact or parent domain) that accept OpenAI's prompt_cache_key field. -_OPENAI_DOMAINS = ("openai.com", "openai.azure.com") +_OPENAI_DOMAINS = ("openai.com",) class CacheAffinityMode(StrEnum): @@ -87,7 +87,7 @@ def resolve_cache_affinity(mode: CacheAffinityMode, provider: str, base_url: str Non-``auto`` modes are returned unchanged. ``auto`` resolves to ``xai_conv_id`` for an x.ai / grok.com host, ``openai_prompt_cache_key`` for - native OpenAI (no base URL) or an openai.com / Azure OpenAI host, and + native OpenAI (no base URL) or an openai.com host, and ``none`` for everything else — an unknown backend gets no unfamiliar field. The xAI check is host-only and deliberately provider-independent: the diff --git a/hindsight-api-slim/tests/test_cache_affinity.py b/hindsight-api-slim/tests/test_cache_affinity.py index d27c84b5d..cab448f7a 100644 --- a/hindsight-api-slim/tests/test_cache_affinity.py +++ b/hindsight-api-slim/tests/test_cache_affinity.py @@ -211,7 +211,7 @@ def test_invalid_mode_raises(): ("openai", None, CacheAffinityMode.OPENAI_PROMPT_CACHE_KEY), ("openai", "", CacheAffinityMode.OPENAI_PROMPT_CACHE_KEY), ("openai", "https://api.openai.com/v1", CacheAffinityMode.OPENAI_PROMPT_CACHE_KEY), - ("openai", "https://my-res.openai.azure.com/openai/deployments/x", CacheAffinityMode.OPENAI_PROMPT_CACHE_KEY), + ("openai", "https://my-res.openai.azure.com/openai/deployments/x", CacheAffinityMode.NONE), ("fireworks", "https://api.fireworks.ai/inference/v1", CacheAffinityMode.NONE), ("fireworks", "", CacheAffinityMode.NONE), ("openai", "https://llm.internal.example/v1", CacheAffinityMode.NONE), @@ -470,7 +470,7 @@ def test_auto_default_sends_nothing_to_an_unrecognized_backend(provider, base_ur ("openai", "https://grok.com/v1", CacheAffinityMode.XAI_CONV_ID), ("openai", None, CacheAffinityMode.OPENAI_PROMPT_CACHE_KEY), ("openai", "https://api.openai.com/v1", CacheAffinityMode.OPENAI_PROMPT_CACHE_KEY), - ("openai", "https://myco.openai.azure.com/", CacheAffinityMode.OPENAI_PROMPT_CACHE_KEY), + ("openai", "https://myco.openai.azure.com/", CacheAffinityMode.NONE), ], ) def test_auto_default_sends_a_hint_only_to_documented_hosts(provider, base_url, expected): diff --git a/hindsight-docs/docs/developer/configuration.md b/hindsight-docs/docs/developer/configuration.md index 05adacecc..8c66ef47d 100644 --- a/hindsight-docs/docs/developer/configuration.md +++ b/hindsight-docs/docs/developer/configuration.md @@ -194,7 +194,7 @@ For non-English banks (especially CJK) and the language/extraction-language trad | `HINDSIGHT_API_LLM_GEMINI_SERVICE_TIER` | Gemini service tier: `flex` for 50% cost savings (best-effort inference) | Unset (default tier) | | `HINDSIGHT_API_LLM_EXTRA_BODY` | JSON dict of extra request-body params (e.g. `temperature`, `top_p`, `max_tokens`) merged into every LLM call. Applied across the OpenAI-compatible, Fireworks, Anthropic, Gemini/VertexAI and LiteLLM (incl. Bedrock/Router) providers. Each provider merges them in its own native parameter space, so use that provider's field names (e.g. `max_tokens` for OpenAI/Anthropic vs `max_output_tokens` for Gemini). The native Ollama structured-output path is a special case — see the note below. Also useful for custom model servers (e.g. vLLM `chat_template_kwargs`). | `null` | | `HINDSIGHT_API_LLM_DEFAULT_HEADERS` | JSON dict passed as `default_headers` to provider SDK clients. Used by operators routing through proxies / request-tracing middleware (e.g. Cloudflare AI Gateway, Helicone, corporate proxies). Wired into the Anthropic provider, the OpenAI-compatible providers, Fireworks, Nous, the OpenAI Responses API, and the LiteLLM-backed providers (`litellm`, `litellmrouter`, `bedrock`); other providers can opt in. | `null` | -| `HINDSIGHT_API_LLM_CACHE_AFFINITY` | Pin a conversation to one backend prompt cache on the OpenAI-compatible providers, Fireworks and Nous. Server-side prompt caches are per backend server, so the same conversation has to reach the same one for a cache hit. `xai_conv_id` sends xAI's documented `x-grok-conv-id` header; `openai_prompt_cache_key` sends OpenAI's `prompt_cache_key` field; `auto` picks one from the configured base URL host (x.ai / grok.com → `xai_conv_id`; native OpenAI or an openai.com / Azure OpenAI host → `openai_prompt_cache_key`; anything else → `none`); `none` sends nothing. The id is the operation's trace id, so every LLM call of one retain/reflect/consolidation run shares it. `auto` is an allowlist rather than a best-effort probe, so an unknown OpenAI-compatible backend (vLLM, ollama, groq, openrouter, a custom proxy) receives byte-identical requests to before; set `none` to disable entirely. Anthropic and Gemini/Vertex do their own caching and ignore this. | `auto` | +| `HINDSIGHT_API_LLM_CACHE_AFFINITY` | Pin a conversation to one backend prompt cache on the OpenAI-compatible providers, Fireworks and Nous. Server-side prompt caches are per backend server, so the same conversation has to reach the same one for a cache hit. `xai_conv_id` sends xAI's documented `x-grok-conv-id` header; `openai_prompt_cache_key` sends OpenAI's `prompt_cache_key` field; `auto` picks one from the configured base URL host (x.ai / grok.com → `xai_conv_id`; native OpenAI or an openai.com host → `openai_prompt_cache_key`; anything else, including Azure OpenAI, → `none`); `none` sends nothing. The id is the operation's trace id, so every LLM call of one retain/reflect/consolidation run shares it. `auto` is an allowlist rather than a best-effort probe, so an unknown OpenAI-compatible backend (vLLM, ollama, groq, openrouter, a custom proxy) receives byte-identical requests to before; set `none` to disable entirely. Anthropic and Gemini/Vertex do their own caching and ignore this. | `auto` | | `HINDSIGHT_API_LLM_STRICT_SCHEMA` | Grammar-enforce structured output via `json_schema` `strict: true` instead of the soft "schema-in-prompt + `json_object`" path. Typed Pydantic response models are serialized directly into the OpenAI strict subset: every object rejects additional properties, every declared property is required, and nullable fields remain nullable. Use it with weaker self-hosted models that return prose preambles, markdown ` ```json ` fences, or invalid JSON — which otherwise fail to parse and wedge retain/consolidation. Applies to OpenAI-compatible backends (OpenAI, LM Studio, Ollama, Volcano, llama.cpp, vLLM), Codex, and LiteLLM; Gemini already enforces its native `response_schema` regardless, and providers without a strict mode ignore it. LM Studio, Ollama, and Volcano are the most common cases: their soft path skips even `json_object` mode (they don't honor it reliably), so a small model's raw output goes unconstrained — enabling this flag is the fix for a `JSONDecodeError` during retain on those backends. | `false` | | `HINDSIGHT_API_LLM_STRICT_SCHEMA_RETAIN` | Override `HINDSIGHT_API_LLM_STRICT_SCHEMA` for retain (fact extraction) only. Applies to both the streaming and batch extraction paths. | Inherits global | | `HINDSIGHT_API_LLM_STRICT_SCHEMA_REFLECT` | Override `HINDSIGHT_API_LLM_STRICT_SCHEMA` for reflect's structured-output extraction only. | Inherits global | diff --git a/hindsight-docs/docs/developer/models.mdx b/hindsight-docs/docs/developer/models.mdx index 6f09112c5..97981f1d9 100644 --- a/hindsight-docs/docs/developer/models.mdx +++ b/hindsight-docs/docs/developer/models.mdx @@ -629,10 +629,9 @@ export HINDSIGHT_API_LLM_BASE_URL=https://.openai.azure.com/openai/dep directly first to isolate which layer is failing. - Gateways and proxies must preserve the same path shape (`/openai/v1` or `/openai/deployments/...?api-version=`). -- Azure accepts the `prompt_cache_key` field that - [`HINDSIGHT_API_LLM_CACHE_AFFINITY`](./configuration#llm-provider) sends under - `auto`, on every `api-version` from `2024-02-01` onward, so the default needs - no adjustment for Azure. +- Azure does not support OpenAI's `prompt_cache_key` field, so + [`HINDSIGHT_API_LLM_CACHE_AFFINITY`](./configuration#llm-provider) resolves + `auto` to `none` for Azure hosts. --- diff --git a/skills/hindsight-docs/references/developer/configuration.md b/skills/hindsight-docs/references/developer/configuration.md index 071fd7de5..acc9c5d4f 100644 --- a/skills/hindsight-docs/references/developer/configuration.md +++ b/skills/hindsight-docs/references/developer/configuration.md @@ -194,7 +194,7 @@ For non-English banks (especially CJK) and the language/extraction-language trad | `HINDSIGHT_API_LLM_GEMINI_SERVICE_TIER` | Gemini service tier: `flex` for 50% cost savings (best-effort inference) | Unset (default tier) | | `HINDSIGHT_API_LLM_EXTRA_BODY` | JSON dict of extra request-body params (e.g. `temperature`, `top_p`, `max_tokens`) merged into every LLM call. Applied across the OpenAI-compatible, Fireworks, Anthropic, Gemini/VertexAI and LiteLLM (incl. Bedrock/Router) providers. Each provider merges them in its own native parameter space, so use that provider's field names (e.g. `max_tokens` for OpenAI/Anthropic vs `max_output_tokens` for Gemini). The native Ollama structured-output path is a special case — see the note below. Also useful for custom model servers (e.g. vLLM `chat_template_kwargs`). | `null` | | `HINDSIGHT_API_LLM_DEFAULT_HEADERS` | JSON dict passed as `default_headers` to provider SDK clients. Used by operators routing through proxies / request-tracing middleware (e.g. Cloudflare AI Gateway, Helicone, corporate proxies). Wired into the Anthropic provider, the OpenAI-compatible providers, Fireworks, Nous, the OpenAI Responses API, and the LiteLLM-backed providers (`litellm`, `litellmrouter`, `bedrock`); other providers can opt in. | `null` | -| `HINDSIGHT_API_LLM_CACHE_AFFINITY` | Pin a conversation to one backend prompt cache on the OpenAI-compatible providers, Fireworks and Nous. Server-side prompt caches are per backend server, so the same conversation has to reach the same one for a cache hit. `xai_conv_id` sends xAI's documented `x-grok-conv-id` header; `openai_prompt_cache_key` sends OpenAI's `prompt_cache_key` field; `auto` picks one from the configured base URL host (x.ai / grok.com → `xai_conv_id`; native OpenAI or an openai.com / Azure OpenAI host → `openai_prompt_cache_key`; anything else → `none`); `none` sends nothing. The id is the operation's trace id, so every LLM call of one retain/reflect/consolidation run shares it. `auto` is an allowlist rather than a best-effort probe, so an unknown OpenAI-compatible backend (vLLM, ollama, groq, openrouter, a custom proxy) receives byte-identical requests to before; set `none` to disable entirely. Anthropic and Gemini/Vertex do their own caching and ignore this. | `auto` | +| `HINDSIGHT_API_LLM_CACHE_AFFINITY` | Pin a conversation to one backend prompt cache on the OpenAI-compatible providers, Fireworks and Nous. Server-side prompt caches are per backend server, so the same conversation has to reach the same one for a cache hit. `xai_conv_id` sends xAI's documented `x-grok-conv-id` header; `openai_prompt_cache_key` sends OpenAI's `prompt_cache_key` field; `auto` picks one from the configured base URL host (x.ai / grok.com → `xai_conv_id`; native OpenAI or an openai.com host → `openai_prompt_cache_key`; anything else, including Azure OpenAI, → `none`); `none` sends nothing. The id is the operation's trace id, so every LLM call of one retain/reflect/consolidation run shares it. `auto` is an allowlist rather than a best-effort probe, so an unknown OpenAI-compatible backend (vLLM, ollama, groq, openrouter, a custom proxy) receives byte-identical requests to before; set `none` to disable entirely. Anthropic and Gemini/Vertex do their own caching and ignore this. | `auto` | | `HINDSIGHT_API_LLM_STRICT_SCHEMA` | Grammar-enforce structured output via `json_schema` `strict: true` instead of the soft "schema-in-prompt + `json_object`" path. Typed Pydantic response models are serialized directly into the OpenAI strict subset: every object rejects additional properties, every declared property is required, and nullable fields remain nullable. Use it with weaker self-hosted models that return prose preambles, markdown ` ```json ` fences, or invalid JSON — which otherwise fail to parse and wedge retain/consolidation. Applies to OpenAI-compatible backends (OpenAI, LM Studio, Ollama, Volcano, llama.cpp, vLLM), Codex, and LiteLLM; Gemini already enforces its native `response_schema` regardless, and providers without a strict mode ignore it. LM Studio, Ollama, and Volcano are the most common cases: their soft path skips even `json_object` mode (they don't honor it reliably), so a small model's raw output goes unconstrained — enabling this flag is the fix for a `JSONDecodeError` during retain on those backends. | `false` | | `HINDSIGHT_API_LLM_STRICT_SCHEMA_RETAIN` | Override `HINDSIGHT_API_LLM_STRICT_SCHEMA` for retain (fact extraction) only. Applies to both the streaming and batch extraction paths. | Inherits global | | `HINDSIGHT_API_LLM_STRICT_SCHEMA_REFLECT` | Override `HINDSIGHT_API_LLM_STRICT_SCHEMA` for reflect's structured-output extraction only. | Inherits global | diff --git a/skills/hindsight-docs/references/developer/models.md b/skills/hindsight-docs/references/developer/models.md index cd9ce7f59..e0d203a67 100644 --- a/skills/hindsight-docs/references/developer/models.md +++ b/skills/hindsight-docs/references/developer/models.md @@ -688,10 +688,9 @@ export HINDSIGHT_API_LLM_BASE_URL=https://.openai.azure.com/openai/dep directly first to isolate which layer is failing. - Gateways and proxies must preserve the same path shape (`/openai/v1` or `/openai/deployments/...?api-version=`). -- Azure accepts the `prompt_cache_key` field that - [`HINDSIGHT_API_LLM_CACHE_AFFINITY`](./configuration#llm-provider) sends under - `auto`, on every `api-version` from `2024-02-01` onward, so the default needs - no adjustment for Azure. +- Azure does not support OpenAI's `prompt_cache_key` field, so + [`HINDSIGHT_API_LLM_CACHE_AFFINITY`](./configuration#llm-provider) resolves + `auto` to `none` for Azure hosts. --- From 069121555d826412bb0c3fd2652d86273ef45c59 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Mon, 17 Aug 2026 14:57:25 +0200 Subject: [PATCH 2/2] docs: state why Azure hosts opt out of prompt_cache_key Azure OpenAI does accept prompt_cache_key on GPT deployments; what rejects it is a non-OpenAI Foundry model (DeepSeek) served over the same *.openai.azure.com endpoint. The host can't distinguish the two, so auto stays off there - but operators on an Azure GPT deployment can opt back in with openai_prompt_cache_key. --- hindsight-api-slim/hindsight_api/engine/cache_affinity.py | 6 ++++++ hindsight-docs/docs/developer/models.mdx | 8 ++++++-- skills/hindsight-docs/references/developer/models.md | 8 ++++++-- 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/hindsight-api-slim/hindsight_api/engine/cache_affinity.py b/hindsight-api-slim/hindsight_api/engine/cache_affinity.py index e7812baab..beed125c1 100644 --- a/hindsight-api-slim/hindsight_api/engine/cache_affinity.py +++ b/hindsight-api-slim/hindsight_api/engine/cache_affinity.py @@ -42,6 +42,12 @@ # Hosts (exact or parent domain) whose backends implement the xAI header. _XAI_DOMAINS = ("x.ai", "grok.com") # Hosts (exact or parent domain) that accept OpenAI's prompt_cache_key field. +# Deliberately excludes openai.azure.com: Azure OpenAI itself accepts the field +# on GPT deployments, but the same *.openai.azure.com endpoint also fronts +# non-OpenAI Foundry models (DeepSeek, Llama, Mistral) that reject it with +# `unrecognized_request_argument` (#3518). The host says nothing about which +# model family the deployment serves, so `auto` stays off there and an Azure +# GPT operator opts in with openai_prompt_cache_key. _OPENAI_DOMAINS = ("openai.com",) diff --git a/hindsight-docs/docs/developer/models.mdx b/hindsight-docs/docs/developer/models.mdx index 97981f1d9..65b940cf3 100644 --- a/hindsight-docs/docs/developer/models.mdx +++ b/hindsight-docs/docs/developer/models.mdx @@ -629,9 +629,13 @@ export HINDSIGHT_API_LLM_BASE_URL=https://.openai.azure.com/openai/dep directly first to isolate which layer is failing. - Gateways and proxies must preserve the same path shape (`/openai/v1` or `/openai/deployments/...?api-version=`). -- Azure does not support OpenAI's `prompt_cache_key` field, so +- Azure OpenAI accepts `prompt_cache_key` on GPT deployments (it improves cache + routing on GPT-5.6 and later), but the same `*.openai.azure.com` endpoint also + serves non-OpenAI Foundry models (DeepSeek, Llama, Mistral) that reject it with + `unrecognized_request_argument`. The host alone can't tell the two apart, so [`HINDSIGHT_API_LLM_CACHE_AFFINITY`](./configuration#llm-provider) resolves - `auto` to `none` for Azure hosts. + `auto` to `none` for Azure hosts. If your deployment serves an OpenAI model, + set it explicitly to `openai_prompt_cache_key`. --- diff --git a/skills/hindsight-docs/references/developer/models.md b/skills/hindsight-docs/references/developer/models.md index e0d203a67..c0947cedb 100644 --- a/skills/hindsight-docs/references/developer/models.md +++ b/skills/hindsight-docs/references/developer/models.md @@ -688,9 +688,13 @@ export HINDSIGHT_API_LLM_BASE_URL=https://.openai.azure.com/openai/dep directly first to isolate which layer is failing. - Gateways and proxies must preserve the same path shape (`/openai/v1` or `/openai/deployments/...?api-version=`). -- Azure does not support OpenAI's `prompt_cache_key` field, so +- Azure OpenAI accepts `prompt_cache_key` on GPT deployments (it improves cache + routing on GPT-5.6 and later), but the same `*.openai.azure.com` endpoint also + serves non-OpenAI Foundry models (DeepSeek, Llama, Mistral) that reject it with + `unrecognized_request_argument`. The host alone can't tell the two apart, so [`HINDSIGHT_API_LLM_CACHE_AFFINITY`](./configuration#llm-provider) resolves - `auto` to `none` for Azure hosts. + `auto` to `none` for Azure hosts. If your deployment serves an OpenAI model, + set it explicitly to `openai_prompt_cache_key`. ---