From 7835dd17472a9021fab549f37d6134ab125c8a85 Mon Sep 17 00:00:00 2001 From: Chi Date: Sat, 5 Sep 2026 10:01:37 +0700 Subject: [PATCH 1/4] automatic endpoint type probing --- backend/inference/anthropic.py | 249 ++++++++++ backend/inference/client.py | 347 ++++++++----- backend/inference/endpoint_profiles.py | 252 +++++++++- backend/inference/errors.py | 17 + docs/architecture/endpoints.md | 90 ++++ mkdocs.yml | 1 + .../test_endpoint_transport_passes.py | 179 +++++++ tests/unit/test_endpoint_protocols.py | 463 ++++++++++++++++++ .../unit/test_forced_tool_choice_fallback.py | 18 +- tests/unit/test_model_discovery.py | 29 ++ 10 files changed, 1483 insertions(+), 162 deletions(-) create mode 100644 backend/inference/anthropic.py create mode 100644 docs/architecture/endpoints.md create mode 100644 tests/integration/test_endpoint_transport_passes.py create mode 100644 tests/unit/test_endpoint_protocols.py diff --git a/backend/inference/anthropic.py b/backend/inference/anthropic.py new file mode 100644 index 00000000..cacc42fc --- /dev/null +++ b/backend/inference/anthropic.py @@ -0,0 +1,249 @@ +"""Translate Orb's OpenAI-shaped chat contract to Anthropic Messages.""" + +from __future__ import annotations + +import base64 +import json +from collections.abc import Mapping, Sequence +from typing import Any + +# Anthropic rejects unknown top-level fields. These are the only user-provided +# extra_body keys accepted on a native Messages route; OpenAI-shaped escape +# hatches therefore cannot turn an otherwise-valid request into a hard 400. +EXTRA_BODY_ALLOWED: frozenset[str] = frozenset({"metadata", "service_tier"}) +DEFAULT_MAX_TOKENS = 4096 + +# Current families whose Messages endpoints reject the old sampling controls. +# Unknown proxy model names are tried once and learned from a provider rejection. +_NO_SAMPLING_MARKERS = ( + "opus-5", + "opus-4-8", + "opus-4.8", + "opus-4-7", + "opus-4.7", + "sonnet-5", + "fable-5", +) + +_SAMPLING_UNSUPPORTED: set[tuple[str, str]] = set() + + +def _text_parts(content: object) -> list[dict[str, Any]]: + if isinstance(content, str): + return [{"type": "text", "text": content}] if content else [] + if not isinstance(content, Sequence) or isinstance(content, (str, bytes)): + return [] + blocks: list[dict[str, Any]] = [] + for part in content: + if not isinstance(part, Mapping): + continue + kind = part.get("type") + if kind == "text" and isinstance(part.get("text"), str): + blocks.append({"type": "text", "text": part["text"]}) + elif kind == "image_url": + image = part.get("image_url") + url = image.get("url") if isinstance(image, Mapping) else image + if not isinstance(url, str): + continue + header, sep, data = url.partition(",") + if sep and header.startswith("data:") and ";base64" in header: + media_type = header[5:].split(";", 1)[0] + try: + base64.b64decode(data, validate=True) + except ValueError: + continue + blocks.append( + { + "type": "image", + "source": {"type": "base64", "media_type": media_type, "data": data}, + } + ) + return blocks + + +def _tool_use_blocks(tool_calls: object) -> list[dict[str, Any]]: + if not isinstance(tool_calls, Sequence) or isinstance(tool_calls, (str, bytes)): + return [] + out: list[dict[str, Any]] = [] + for call in tool_calls: + if not isinstance(call, Mapping): + continue + function = call.get("function") + if not isinstance(function, Mapping) or not isinstance(function.get("name"), str): + continue + raw = function.get("arguments", {}) + if isinstance(raw, str): + try: + decoded = json.loads(raw) + except ValueError: + decoded = {"_raw": raw} + else: + decoded = raw + if not isinstance(decoded, Mapping): + decoded = {"value": decoded} + out.append( + { + "type": "tool_use", + "id": str(call.get("id") or ""), + "name": function["name"], + "input": dict(decoded), + } + ) + return out + + +def translate_messages(messages: Sequence[Mapping[str, Any]]) -> tuple[str, list[dict[str, Any]]]: + """Hoist system text and translate/coalesce Anthropic user/assistant turns.""" + system_parts: list[str] = [] + translated: list[dict[str, Any]] = [] + + def append(role: str, blocks: list[dict[str, Any]]) -> None: + if not blocks: + return + if translated and translated[-1]["role"] == role: + translated[-1]["content"].extend(blocks) + else: + translated.append({"role": role, "content": blocks}) + + for message in messages: + role = message.get("role") + if role == "system": + for block in _text_parts(message.get("content")): + text = block.get("text") + if isinstance(text, str) and text: + system_parts.append(text) + continue + if role == "tool": + content = message.get("content", "") + tool_content: str | list[dict[str, Any]] + if isinstance(content, str): + tool_content = content + else: + tool_content = _text_parts(content) + append( + "user", + [ + { + "type": "tool_result", + "tool_use_id": str(message.get("tool_call_id") or ""), + "content": tool_content, + } + ], + ) + continue + if role not in {"user", "assistant"}: + continue + blocks = _text_parts(message.get("content")) + if role == "assistant": + blocks.extend(_tool_use_blocks(message.get("tool_calls"))) + append(role, blocks) + return "\n\n".join(system_parts), translated + + +def translate_tools(tools: object) -> list[dict[str, Any]]: + if not isinstance(tools, Sequence) or isinstance(tools, (str, bytes)): + return [] + out: list[dict[str, Any]] = [] + for tool in tools: + if not isinstance(tool, Mapping): + continue + function = tool.get("function") + if not isinstance(function, Mapping) or not isinstance(function.get("name"), str): + continue + translated: dict[str, Any] = { + "name": function["name"], + "input_schema": dict(function.get("parameters") or {"type": "object", "properties": {}}), + "strict": True, + } + if isinstance(function.get("description"), str): + translated["description"] = function["description"] + out.append(translated) + return out + + +def translate_tool_choice(choice: object) -> dict[str, Any] | None: + if choice is None: + return None + if choice == "auto": + return {"type": "auto"} + if choice == "none": + return {"type": "none"} + if choice == "required": + return {"type": "any"} + if isinstance(choice, Mapping): + function = choice.get("function") + name = function.get("name") if isinstance(function, Mapping) else None + if isinstance(name, str) and name: + return {"type": "tool", "name": name} + native_type = choice.get("type") + if native_type in {"auto", "none", "any"}: + return {"type": native_type} + return None + + +def _sampling_allowed(endpoint_url: str, model: str) -> bool: + low = model.lower().replace("_", "-") + return (endpoint_url, model) not in _SAMPLING_UNSUPPORTED and not any(marker in low for marker in _NO_SAMPLING_MARKERS) + + +def build_request_body( + openai_body: Mapping[str, Any], + endpoint_url: str, + model: str, + extra_body: Mapping[str, Any] | None = None, +) -> dict[str, Any]: + """Build a native Messages body from an allowlist.""" + system, messages = translate_messages(openai_body.get("messages") or []) + body: dict[str, Any] = { + "model": model, + "messages": messages, + "stream": True, + "max_tokens": openai_body.get("max_tokens") or DEFAULT_MAX_TOKENS, + } + if system: + body["system"] = system + tools = translate_tools(openai_body.get("tools")) + if tools: + body["tools"] = tools + choice = translate_tool_choice(openai_body.get("tool_choice")) + if choice is not None and tools: + body["tool_choice"] = choice + + reasoning = openai_body.get("reasoning") + thinking = openai_body.get("thinking") + reasoning_on = (isinstance(reasoning, Mapping) and reasoning.get("enabled") is True) or ( + isinstance(thinking, Mapping) and thinking.get("type") == "enabled" + ) + if reasoning_on: + body["thinking"] = {"type": "adaptive", "display": "summarized"} + effort = openai_body.get("reasoning_effort") + if effort in {"low", "medium", "high", "xhigh", "max"}: + body["output_config"] = {"effort": effort} + + if _sampling_allowed(endpoint_url, model): + for key in ("temperature", "top_p", "top_k"): + value = openai_body.get(key) + if value is not None: + body[key] = value + + if extra_body: + for key in EXTRA_BODY_ALLOWED: + if key in extra_body: + body[key] = extra_body[key] + return body + + +def recover_sampling_error(endpoint_url: str, model: str, body: dict[str, Any], status: int, text: str) -> str | None: + """Learn a sampling-field rejection and remove all three controls once.""" + if status != 400: + return None + low = text.lower() + present = [key for key in ("temperature", "top_p", "top_k") if key in body] + if not present or not any(key in low for key in present): + return None + if not any(marker in low for marker in ("unsupported", "not supported", "not allowed", "extra inputs")): + return None + _SAMPLING_UNSUPPORTED.add((endpoint_url, model)) + for key in present: + body.pop(key, None) + return f"Model {model} rejected Anthropic sampling fields {present}; retrying without them." diff --git a/backend/inference/client.py b/backend/inference/client.py index 75c87a05..60e62de7 100644 --- a/backend/inference/client.py +++ b/backend/inference/client.py @@ -9,8 +9,8 @@ import httpx -from . import endpoint_profiles, text_completion -from .errors import llm_call_error +from . import anthropic, endpoint_profiles, text_completion +from .errors import LLMCallError, llm_call_error, llm_stream_error from .gemma_tool_format import parse_gemma_tool_calls from .retry import RetryPolicy @@ -291,8 +291,23 @@ def _headers(self) -> dict: headers.update(self.extra_headers) return headers + def _headers_for(self, auth_family: endpoint_profiles.AuthFamily) -> dict: + """Return transport auth defaults with case-insensitive user overrides.""" + base: dict[str, str] = {} + if self.api_key: + if auth_family == "anthropic": + base["x-api-key"] = self.api_key + else: + base["Authorization"] = f"Bearer {self.api_key}" + if auth_family == "anthropic": + base["anthropic-version"] = "2023-06-01" + configured = {key.lower() for key in self.extra_headers} + headers = {key: value for key, value in base.items() if key.lower() not in configured} + headers.update(self.extra_headers) + return headers + def _url(self) -> str: - return f"{self.base_url}/chat/completions" + return endpoint_profiles.resolve_endpoint(self.base_url).url async def list_models(self) -> list[str]: """Return model ids advertised by an OpenAI-compatible ``GET /models``. @@ -302,9 +317,10 @@ async def list_models(self) -> list[str]: small non-streaming settings request and should fail back to Orb's editable model-name field promptly. """ - url = f"{self.base_url}/models" + route = endpoint_profiles.resolve_endpoint(self.base_url) + url = route.models_url async with httpx.AsyncClient(timeout=20.0, proxy=self.proxy, follow_redirects=True) as client: - response = await client.get(url, headers=self._headers()) + response = await client.get(url, headers=self._headers_for(route.auth_family)) response.raise_for_status() try: payload = response.json() @@ -319,7 +335,10 @@ async def list_models(self) -> list[str]: for item in data: model_id = item.get("id") if isinstance(item, dict) else None if isinstance(model_id, str) and model_id.strip(): - model_ids.add(model_id.strip()) + normalized = model_id.strip() + if "generativelanguage.googleapis.com" in route.url.lower() and normalized.startswith("models/"): + normalized = normalized.removeprefix("models/") + model_ids.add(normalized) return sorted(model_ids, key=str.casefold) def _server_root(self) -> str: @@ -420,7 +439,14 @@ async def _with_retry(self, open_stream: Callable[[], AsyncIterator[dict]]) -> A yield event return except httpx.HTTPError as exc: - if produced or self.is_aborted or attempt >= self.retry.count or not self.retry.should_retry(exc): + stream_event = isinstance(exc, LLMCallError) and exc.stream_event + if ( + produced + or stream_event + or self.is_aborted + or attempt >= self.retry.count + or not self.retry.should_retry(exc) + ): raise attempt += 1 detail = f"HTTP {exc.response.status_code}" if isinstance(exc, httpx.HTTPStatusError) else type(exc).__name__ @@ -562,19 +588,6 @@ def _plan() -> tuple[dict, str | None, bool]: apply_reasoning_effort(body, self.reasoning_effort, self.reasoning_effort_param, self.reasoning_effort_value) - # Same ordering as apply_reasoning_effort above, for the reason its - # docstring gives. Chat-only by design: the text transport builds its - # params from an allowlist. - if self.extra_body: - body.update(self.extra_body) - logger.info("LLM extra body fields: %s", sorted(self.extra_body)) - - # Provider-specific body translation (profiles + session-learned - # workarounds) lives entirely in endpoint_profiles; the client just - # applies whatever it returns. - for action in endpoint_profiles.prepare_request_body(self.base_url, model, body): - logger.info("LLM profile: %s", action) - logger.info( "LLM complete: model=%s, tools=%s, tool_choice=%s", model, @@ -584,6 +597,18 @@ def _plan() -> tuple[dict, str | None, bool]: logger.debug(messages) return body, forced_name, structured + def _outbound_body(body: dict, route: endpoint_profiles.EndpointRoute) -> dict: + """Copy the canonical OpenAI body into one route's wire dialect.""" + outbound = dict(body) + if route.protocol == "openai" and self.extra_body: + outbound.update(self.extra_body) + logger.info("LLM extra body fields: %s", sorted(self.extra_body)) + for action in endpoint_profiles.prepare_request_body(self.base_url, model, outbound): + logger.info("LLM profile: %s", action) + if route.protocol == "anthropic": + return anthropic.build_request_body(outbound, self.base_url, model, self.extra_body) + return outbound + content_parts: list[str] = [] reasoning_parts: list[str] = [] tool_calls_acc: dict[int, dict] = {} @@ -606,112 +631,184 @@ async def _issue(body: dict, forced_name: str | None) -> AsyncIterator[dict]: finish_reason = None usage = None - # At most one retry, solely to self-heal a provider quirk that - # endpoint_profiles.recover_from_error() recognises (e.g. an OpenRouter - # model rejecting tool_choice). The error lands before any SSE event, - # so the retry is clean. - for attempt in range(2): - # No read timeout on streaming calls: the server sends zero bytes - # while prefilling a large prompt (or queueing behind another - # request), and a long silence is normal there — a flat read - # timeout intermittently killed long turns. Abort/stop and the - # disconnect watcher remain the recovery paths. - async with httpx.AsyncClient(timeout=httpx.Timeout(self.timeout, read=None), proxy=self.proxy) as client: - async with client.stream("POST", self._url(), json=body, headers=self._headers()) as resp: - if resp.status_code >= 400: - # Concern 1: surface the error body. - err_text = await _read_error_body(resp, self._url()) - - # Concern 2: ask the provider layer whether this is a - # recognised quirk worth one retry. It mutates body in - # place and returns a log line, or None to propagate. - if attempt == 0: - fix = endpoint_profiles.recover_from_error( - self.base_url, model, body, resp.status_code, err_text + def tool_entry(index: int) -> dict: + if index not in tool_calls_acc: + tool_calls_acc[index] = { + "id": "", + "type": "function", + "function": {"name": "", "arguments": ""}, + } + return tool_calls_acc[index] + + async def consume_openai(resp) -> AsyncIterator[dict]: + nonlocal finish_reason, usage + async for payload in self._iter_sse_payloads(resp): + try: + chunk = json.loads(payload) + except json.JSONDecodeError: + continue + u = chunk.get("usage") + if isinstance(u, dict): + usage = u + choices = chunk.get("choices") or [] + if not choices: + continue + try: + choice = choices[0] + delta = choice.get("delta", {}) + rc = delta.get("reasoning_content") or delta.get("reasoning") + if rc: + reasoning_parts.append(rc) + yield {"type": "reasoning", "delta": rc} + content = delta.get("content") + if content: + content_parts.append(content) + if forced_name is None: + yield {"type": "content", "delta": content} + for rec in _parse_chat_logprobs(choice): + yield {"type": "token_probs", **rec} + for tc_delta in delta.get("tool_calls") or []: + entry = tool_entry(tc_delta.get("index", 0)) + if tc_delta.get("id"): + entry["id"] = tc_delta["id"] + fn = tc_delta.get("function", {}) + if fn.get("name"): + entry["function"]["name"] += fn["name"] + if fn.get("arguments"): + entry["function"]["arguments"] += fn["arguments"] + if choice.get("finish_reason"): + finish_reason = choice["finish_reason"] + except (KeyError, IndexError): + continue + + async def consume_anthropic(resp, url: str) -> AsyncIterator[dict]: + nonlocal finish_reason, usage + stopped = False + async for payload in self._iter_sse_payloads(resp): + try: + event = json.loads(payload) + except json.JSONDecodeError: + continue + event_type = event.get("type") + if event_type == "ping": + continue + if event_type == "error": + raise llm_stream_error(payload=event, url=url, model=model, api_key=self.api_key) + if event_type == "message_start": + initial = (event.get("message") or {}).get("usage") + if isinstance(initial, dict): + usage = dict(initial) + elif event_type == "content_block_start": + index = event.get("index", 0) + block = event.get("content_block") or {} + if block.get("type") == "tool_use": + entry = tool_entry(index) + entry["id"] = block.get("id", "") + entry["function"]["name"] = block.get("name", "") + if block.get("input"): + entry["function"]["arguments"] = json.dumps(block["input"], separators=(",", ":")) + elif block.get("type") == "text" and block.get("text"): + content_parts.append(block["text"]) + yield {"type": "content", "delta": block["text"]} + elif block.get("type") == "thinking" and block.get("thinking"): + reasoning_parts.append(block["thinking"]) + yield {"type": "reasoning", "delta": block["thinking"]} + elif event_type == "content_block_delta": + index = event.get("index", 0) + delta = event.get("delta") or {} + delta_type = delta.get("type") + if delta_type == "text_delta" and delta.get("text"): + content_parts.append(delta["text"]) + yield {"type": "content", "delta": delta["text"]} + elif delta_type == "thinking_delta" and delta.get("thinking"): + reasoning_parts.append(delta["thinking"]) + yield {"type": "reasoning", "delta": delta["thinking"]} + elif delta_type == "input_json_delta" and delta.get("partial_json"): + tool_entry(index)["function"]["arguments"] += delta["partial_json"] + elif event_type == "message_delta": + delta = event.get("delta") or {} + stop_reason = delta.get("stop_reason") + if stop_reason: + finish_reason = { + "end_turn": "stop", + "stop_sequence": "stop", + "tool_use": "tool_calls", + "max_tokens": "length", + }.get(stop_reason, stop_reason) + update = event.get("usage") + if isinstance(update, dict): + usage = {**(usage or {}), **update} + elif event_type == "message_stop": + stopped = True + break + if not stopped and not self.is_aborted: + raise llm_stream_error( + payload={"error": {"message": "Anthropic stream ended before message_stop"}}, + url=url, + model=model, + api_key=self.api_key, + ) + + routes = endpoint_profiles.endpoint_candidates(self.base_url, model) + route_index = 0 + while route_index < len(routes): + route = routes[route_index] + recovery_count = 0 + auth_family = route.auth_family + auth_retried = False + while True: + outbound = _outbound_body(body, route) + # No read timeout on streaming calls: a long prefill silence + # is normal; abort and disconnect close the stream instead. + async with httpx.AsyncClient(timeout=httpx.Timeout(self.timeout, read=None), proxy=self.proxy) as client: + async with client.stream( + "POST", route.url, json=outbound, headers=self._headers_for(auth_family) + ) as resp: + if resp.status_code >= 400: + err_text = await _read_error_body(resp, route.url) + if recovery_count < 2: + fix = endpoint_profiles.recover_from_error( + self.base_url, model, outbound, resp.status_code, err_text + ) + if fix is None and route.protocol == "anthropic": + fix = anthropic.recover_sampling_error( + self.base_url, model, outbound, resp.status_code, err_text + ) + if fix is not None: + recovery_count += 1 + logger.warning("LLM recovery: %s", fix) + continue + auths = endpoint_profiles.auth_families(route, self.base_url, model, resp.status_code) + if not auth_retried and len(auths) > 1: + auth_family = auths[1] + auth_retried = True + logger.warning("LLM auth recovery: retrying %s with %s auth", route.url, auth_family) + continue + if route_index + 1 < len(routes) and endpoint_profiles.should_probe_route( + resp.status_code, err_text + ): + logger.warning( + "LLM endpoint probe: %s rejected the route; trying %s", + route.url, + routes[route_index + 1].url, + ) + route_index += 1 + break + raise llm_call_error( + response=resp, + body=err_text, + url=route.url, + model=model, + api_key=self.api_key, ) - if fix is not None: - logger.warning("LLM recovery: %s", fix) - continue # leave async-with cleanly, then retry - - # Concern 3: keep the body. raise_for_status() would - # replace the provider's own sentence with httpx's canned - # status line, and it is the only part the user can act on. - raise llm_call_error( - response=resp, - body=err_text, - url=self._url(), - model=model, - api_key=self.api_key, - ) - async for payload in self._iter_sse_payloads(resp): - try: - chunk = json.loads(payload) - except json.JSONDecodeError: - continue - - # Usage may appear in a terminal chunk (choices=[]) or on the final content chunk; last-write-wins since totals are monotonic. - u = chunk.get("usage") - if isinstance(u, dict): - usage = u - - choices = chunk.get("choices") or [] - if not choices: - # Pure usage/metadata chunk — nothing else to do. - continue - - try: - choice = choices[0] - delta = choice.get("delta", {}) - - # Reasoning delta (field name varies by server) - rc = delta.get("reasoning_content") or delta.get("reasoning") - if rc: - reasoning_parts.append(rc) - yield {"type": "reasoning", "delta": rc} - - # Content delta. A structured forced call buffers - # instead of yielding: the content is the tool's - # arguments JSON, and chat mode never surfaces - # argument streams as content (they arrive as - # tool_calls deltas, which the pipeline hides). - c = delta.get("content") - if c: - content_parts.append(c) - if forced_name is None: - yield {"type": "content", "delta": c} - - # Per-token alternatives (Document mode steering) — - # present only when the caller passed logprobs and the - # provider honoured them; otherwise a no-op. - for rec in _parse_chat_logprobs(choice): - yield {"type": "token_probs", **rec} - - # Tool call argument deltas — accumulate by index - for tc_delta in delta.get("tool_calls") or []: - idx = tc_delta.get("index", 0) - if idx not in tool_calls_acc: - tool_calls_acc[idx] = { - "id": "", - "type": "function", - "function": {"name": "", "arguments": ""}, - } - entry = tool_calls_acc[idx] - if tc_delta.get("id"): - entry["id"] = tc_delta["id"] - fn = tc_delta.get("function", {}) - if fn.get("name"): - entry["function"]["name"] += fn["name"] - if fn.get("arguments"): - entry["function"]["arguments"] += fn["arguments"] - - if choice.get("finish_reason"): - finish_reason = choice["finish_reason"] - - except (KeyError, IndexError): - continue - # Streamed to completion (or aborted) without a retry-triggering - # error -- done, no second attempt. - break + if route.protocol == "anthropic": + async for event in consume_anthropic(resp, route.url): + yield event + else: + async for event in consume_openai(resp): + yield event + endpoint_profiles.note_successful_route(self.base_url, model, route) + return body, forced_name, structured = _plan() async for _ev in _issue(body, forced_name): diff --git a/backend/inference/endpoint_profiles.py b/backend/inference/endpoint_profiles.py index cf438a89..40ad8b9f 100644 --- a/backend/inference/endpoint_profiles.py +++ b/backend/inference/endpoint_profiles.py @@ -4,7 +4,8 @@ from collections.abc import Callable, Mapping from dataclasses import dataclass, field -from typing import Any +from typing import Any, Literal +from urllib.parse import urlsplit, urlunsplit # Body keys always sent; never subject to allowlist filtering. ALWAYS_ALLOWED: frozenset[str] = frozenset({"model", "messages", "stream", "tools", "tool_choice"}) @@ -50,6 +51,11 @@ class ModelProfile: # ``response_format: {"type": "json_schema", "strict": true}``. structured_tool_calls: bool = False + # If True, every value except ``"auto"`` is rewritten to ``"auto"``. + # Some routed providers reject not only forced choices but also ``"none"``; + # this is deliberately separate from allow_forced_tool_choice. + auto_tool_choice_only: bool = False + # Bespoke transforms applied after typed knobs, in order. Each callable # mutates body in place and may return a log line (or None for silent). custom: tuple[Transform, ...] = field(default_factory=tuple) @@ -72,6 +78,12 @@ def apply(self, body: dict) -> list[str]: body["tool_choice"] = "auto" actions.append(f"tool_choice {tc!r} -> 'auto'") + if self.auto_tool_choice_only: + tc = body.get("tool_choice") + if tc is not None and tc != "auto": + body["tool_choice"] = "auto" + actions.append(f"tool_choice {tc!r} -> 'auto' (auto-only endpoint)") + for fn in self.custom: log = fn(body) if log: @@ -149,16 +161,6 @@ def _deepseek_coerce_tool_choice_when_thinking(body: dict) -> str | None: allow_forced_tool_choice=False, ), }, - # No None-key default: unlisted OpenRouter models stay pass-through (most - # honor forcing). List only models known to reject a forced-function - # tool_choice; add a one-liner per newly-found one. llm_client self-heals - # the first hit of an unlisted model and logs a reminder to add it here. - "openrouter.ai": { - "minimax/minimax-m3": ModelProfile( - allow_extra=None, # OpenRouter is lenient; drop nothing - allow_forced_tool_choice=False, # forced -> "auto" - ), - }, # NanoGPT is a *proxy*: each model id it fronts sits behind a different # upstream engine with its own config, so no endpoint-wide statement about # decoding is true of every model. Its own tool-argument decoding is @@ -175,9 +177,185 @@ def _deepseek_coerce_tool_choice_when_thinking(body: dict) -> str | None: structured_tool_calls=True, ), }, + # Google's OpenAI compatibility API accepts ordinary OpenAI request fields + # (unknown additions are ignored) and honors strict json_schema output. + "generativelanguage.googleapis.com": { + None: ModelProfile( + allow_extra=None, + structured_tool_calls=True, + ), + }, } +Protocol = Literal["openai", "anthropic"] +AuthFamily = Literal["bearer", "anthropic"] + + +@dataclass(frozen=True) +class EndpointRoute: + """One concrete transport resource resolved from a configured endpoint.""" + + protocol: Protocol + url: str + models_url: str + auth_family: AuthFamily + authoritative: bool = False + + +# A successful route is remembered per configured URL and model because client +# objects are per-turn. Stored URLs are never rewritten. +_RESOLVED_ROUTES: dict[tuple[str, str], EndpointRoute] = {} + + +def _clean_url(url: str) -> str: + return url.strip().rstrip("/") + + +def _parsed_http_url(url: str): + parsed = urlsplit(_clean_url(url)) + if parsed.scheme.lower() not in {"http", "https"} or not parsed.netloc: + return None + return parsed + + +def _replace_path(parsed, path: str) -> str: + return urlunsplit((parsed.scheme, parsed.netloc, path, parsed.query, "")) + + +def _resource_route(protocol: Protocol, url: str, *, authoritative: bool) -> EndpointRoute: + parsed = _parsed_http_url(url) + clean = _clean_url(url) + path = parsed.path.rstrip("/") if parsed is not None else clean + resource = "/messages" if protocol == "anthropic" else "/chat/completions" + if path.endswith(resource): + models_path = f"{path[: -len(resource)]}/models" + else: + models_path = f"{path}/models" + models_url = _replace_path(parsed, models_path) if parsed is not None else f"{clean}/models" + return EndpointRoute( + protocol=protocol, + url=clean, + models_url=models_url, + auth_family="anthropic" if protocol == "anthropic" else "bearer", + authoritative=authoritative, + ) + + +def _base_route(protocol: Protocol, base_url: str, *, authoritative: bool = False) -> EndpointRoute: + clean = _clean_url(base_url) + suffix = "/messages" if protocol == "anthropic" else "/chat/completions" + parsed = _parsed_http_url(clean) + if parsed is None: + url = f"{clean}{suffix}" + else: + url = _replace_path(parsed, f"{parsed.path.rstrip('/')}{suffix}") + return _resource_route(protocol, url, authoritative=authoritative) + + +def _deterministic_route(endpoint_url: str) -> EndpointRoute: + """Resolve explicit resources and strong provider hints without probing.""" + clean = _clean_url(endpoint_url) + parsed = _parsed_http_url(clean) + low = clean.lower() + path = parsed.path.rstrip("/").lower() if parsed is not None else low + + # Full resource URLs are user intent and win over host heuristics. + if path.endswith("/chat/completions"): + return _resource_route("openai", clean, authoritative=True) + if path.endswith("/messages"): + return _resource_route("anthropic", clean, authoritative=True) + + host = (parsed.hostname or "").lower() if parsed is not None else "" + if host == "generativelanguage.googleapis.com" or host.endswith(".generativelanguage.googleapis.com"): + base = _replace_path(parsed, "/v1beta/openai") if parsed is not None else clean + return _base_route("openai", base, authoritative=True) + + segments = [segment for segment in path.split("/") if segment] + if host == "api.anthropic.com" or host.endswith(".api.anthropic.com"): + if not segments: + base = _replace_path(parsed, "/v1") if parsed is not None else f"{clean}/v1" + else: + base = clean + return _base_route("anthropic", base, authoritative=True) + + # Proxy prefixes such as /anthropic or /providers/anthropic/v1 are strong + # enough to select native Messages without replaying a prompt elsewhere. + if "anthropic" in segments: + base = f"{clean}/v1" if segments[-1] == "anthropic" else clean + return _base_route("anthropic", base, authoritative=True) + + return _base_route("openai", clean) + + +def resolve_endpoint(endpoint_url: str, model: str = "") -> EndpointRoute: + """Return the cached or deterministic route for one configured endpoint.""" + return _RESOLVED_ROUTES.get((endpoint_url, model)) or _deterministic_route(endpoint_url) + + +def endpoint_candidates(endpoint_url: str, model: str = "") -> list[EndpointRoute]: + """Return bounded same-host routes in request order. + + Explicit resources and provider hints are authoritative. Ambiguous URLs keep + Orb's historical ``{configured}/chat/completions`` request first, followed + by the conventional host-root OpenAI and Anthropic v1 resources. Candidates + are only attempted when :func:`should_probe_route` recognizes the response + body as a route mismatch. + """ + primary = resolve_endpoint(endpoint_url, model) + if primary.authoritative or (endpoint_url, model) in _RESOLVED_ROUTES: + return [primary] + parsed = _parsed_http_url(endpoint_url) + if parsed is None: + return [primary] + root = urlunsplit((parsed.scheme, parsed.netloc, "", "", "")) + candidates = [ + primary, + _base_route("openai", f"{root}/v1"), + _base_route("anthropic", f"{root}/v1"), + ] + out: list[EndpointRoute] = [] + seen: set[tuple[str, str]] = set() + for route in candidates: + key = (route.protocol, route.url) + if key not in seen: + seen.add(key) + out.append(route) + return out + + +def note_successful_route(endpoint_url: str, model: str, route: EndpointRoute) -> None: + """Cache a route after its stream completes successfully.""" + _RESOLVED_ROUTES[(endpoint_url, model)] = route + + +def should_probe_route(status: int, text: str) -> bool: + """Whether an error body specifically identifies an HTTP route mismatch.""" + del status # Deliberately a body fact; status-only routing is unsafe. + low = text.lower() + markers = ( + "cannot post /", + "route not found", + "unknown endpoint", + "unrecognized request url", + "unsupported endpoint", + "invalid url (post", + ) + return any(marker in low for marker in markers) + + +def auth_families(route: EndpointRoute, endpoint_url: str, model: str, status: int | None = None) -> tuple[AuthFamily, ...]: + """Return primary auth and, on supported 401/403 evidence, its peer.""" + primary = route.auth_family + if status not in {401, 403}: + return (primary,) + evidence = "claude" in model.lower() or "anthropic" in endpoint_url.lower() or route.protocol == "anthropic" + if not evidence: + return (primary,) + other: AuthFamily = "bearer" if primary == "anthropic" else "anthropic" + return (primary, other) + + # (endpoint_url, model) pairs observed to answer a forced tool_choice with a # different tool this session — either a profile coerced the choice to "auto" # or the provider ignored it silently (OpenRouter + a thinking-on model, @@ -266,22 +444,31 @@ def profile_for(endpoint_url: str, model: str = "") -> ModelProfile | None: # front instead of paying the round-trip + retry again. _TOOL_CHOICE_UNSUPPORTED: set[tuple[str, str]] = set() +# Pairs observed to accept only the literal ``"auto"`` value. Unlike +# _TOOL_CHOICE_UNSUPPORTED, these endpoints still need the field so forced and +# writer ``"none"`` requests are coerced rather than dropped. +_TOOL_CHOICE_AUTO_ONLY: set[tuple[str, str]] = set() + def _is_openrouter(endpoint_url: str) -> bool: return "openrouter.ai" in endpoint_url.lower() def _is_tool_choice_unsupported(status: int, text: str) -> bool: - """Return ``True`` for OpenRouter's ``tool_choice``-unsupported 404. + """Return ``True`` when the body says no ``tool_choice`` value is routed. Matches "No endpoints found that support the provided 'tool_choice' value." — meaning the routed provider rejects all ``tool_choice`` values. Kept narrow so genuine 404s (bad model id, etc.) don't match. """ - if status != 404: - return False low = text.lower() - return "tool_choice" in low and "no endpoints found" in low + return status in {400, 404} and "tool_choice" in low and "no endpoints found" in low + + +def _is_tool_choice_auto_only(status: int, text: str) -> bool: + """Return True when the body states that only ``auto`` is accepted.""" + low = text.lower() + return status in {400, 404} and "tool_choice" in low and "only" in low and "auto" in low and "support" in low def prepare_request_body(endpoint_url: str, model: str, body: dict) -> list[str]: @@ -301,6 +488,12 @@ def prepare_request_body(endpoint_url: str, model: str, body: dict) -> list[str] tc = body.pop("tool_choice") actions.append(f"tool_choice {tc!r} dropped (session-learned unsupported)") + if "tool_choice" in body and (endpoint_url, model) in _TOOL_CHOICE_AUTO_ONLY: + tc = body["tool_choice"] + if tc != "auto" and tc != {"type": "auto"}: + body["tool_choice"] = "auto" + actions.append(f"tool_choice {tc!r} -> 'auto' (session-learned auto-only)") + return actions @@ -311,16 +504,33 @@ def recover_from_error(endpoint_url: str, model: str, body: dict, status: int, t Currently handles one quirk: an OpenRouter model whose routed provider rejects ``tool_choice`` entirely. Recovery is to drop the param and retry; - the 404 lands before any SSE event so the retry is clean. Add such models - to ``PROFILES['openrouter.ai']`` for a zero-retry fix. + the 404 lands before any SSE event so the retry is clean. Model catalog ids + are deliberately never recorded here; learned capability facts expire with + the backend process. """ + tc = body.get("tool_choice") + low = text.lower() + native_forced_rejected = ( + status == 400 + and isinstance(tc, Mapping) + and tc.get("type") in {"any", "tool"} + and "tool_choice" in low + and any(marker in low for marker in ("not supported", "unsupported", "not allowed")) + ) + if native_forced_rejected: + _TOOL_CHOICE_AUTO_ONLY.add((endpoint_url, model)) + note_forced_tool_choice_ignored(endpoint_url, model) + body["tool_choice"] = {"type": "auto"} + return f"Model {model} rejected forced Anthropic tool choice; retrying with auto." + if "tool_choice" in body and _is_tool_choice_auto_only(status, text): + _TOOL_CHOICE_AUTO_ONLY.add((endpoint_url, model)) + tc = body["tool_choice"] + body["tool_choice"] = {"type": "auto"} if isinstance(tc, dict) and "function" not in tc else "auto" + return f"Model {model} accepts only tool_choice='auto'; retrying with auto." if not _is_openrouter(endpoint_url): return None if "tool_choice" in body and _is_tool_choice_unsupported(status, text): _TOOL_CHOICE_UNSUPPORTED.add((endpoint_url, model)) tc = body.pop("tool_choice") - return ( - f"Model {model} rejected tool_choice={tc!r}; retrying without it. " - f"Add it to endpoint_profiles.PROFILES['openrouter.ai'] for a zero-retry fix." - ) + return f"Model {model} rejected tool_choice={tc!r}; retrying without it." return None diff --git a/backend/inference/errors.py b/backend/inference/errors.py index c78455c0..53977257 100644 --- a/backend/inference/errors.py +++ b/backend/inference/errors.py @@ -39,6 +39,8 @@ class LLMCallError(httpx.HTTPStatusError): re-checking. """ + stream_event = False + def __init__( self, message: str, @@ -186,3 +188,18 @@ def llm_call_error( host=host, model=model, ) + + +def llm_stream_error(*, payload: object, url: str, model: str, api_key: str) -> LLMCallError: + """Build a normal sanitized provider error for an SSE error event. + + Anthropic can report a terminal provider error inside an HTTP-200 stream. + A synthetic 502 response represents that upstream failure without teaching + pipeline failure rendering a second exception shape. + """ + body = json.dumps(payload, ensure_ascii=False) if not isinstance(payload, str) else payload + request = httpx.Request("POST", url) + response = httpx.Response(502, request=request) + error = llm_call_error(response=response, body=body, url=url, model=model, api_key=api_key) + error.stream_event = True + return error diff --git a/docs/architecture/endpoints.md b/docs/architecture/endpoints.md new file mode 100644 index 00000000..c64c1a43 --- /dev/null +++ b/docs/architecture/endpoints.md @@ -0,0 +1,90 @@ +# LLM Endpoint Routing + +Orb keeps one OpenAI-shaped contract inside the pipeline: messages, tool calls, +stream events, terminal messages, and usage have the same shape regardless of +the provider. The inference layer resolves the configured URL and translates at +the network boundary. + +## Accepted endpoint forms + +A configured endpoint may be a versioned base or a full generation resource. + +| Configured form | Generation resource | +|---|---| +| `https://host/v1` | `https://host/v1/chat/completions` | +| `https://host/v1/chat/completions` | Used exactly as entered | +| `https://host/v1/messages` | Used exactly as entered with Anthropic Messages | +| `https://api.anthropic.com` | `https://api.anthropic.com/v1/messages` | +| `https://generativelanguage.googleapis.com` | `https://generativelanguage.googleapis.com/v1beta/openai/chat/completions` | + +An `anthropic` path segment is also a strong native-protocol hint. For example, +`https://gateway.example/providers/anthropic/v1` resolves to the sibling +`messages` resource. Full `chat/completions` and `messages` resource URLs are +authoritative even when their host would normally imply another protocol. + +Model discovery uses the sibling `models` resource and the matching auth family. +OpenAI and Gemini routes use Bearer authentication. Native Anthropic routes use +`x-api-key` and `anthropic-version`. Extra headers may replace those defaults +case-insensitively. + +## Automatic detection and probing + +Official hosts and path hints are deterministic and do not probe. An ambiguous +custom URL preserves Orb's historical OpenAI request first. Only when the +pre-stream response body specifically identifies a route mismatch does Orb try, +on the same host, these conventional resources: + +1. the configured base plus `chat/completions`; +2. host-root `/v1/chat/completions`; +3. host-root `/v1/messages`. + +The HTTP status alone never starts probing: a 400 or 404 can describe a bad +model, schema, or tool choice rather than a bad route. Known request recovery +runs first. No route is changed after the first streamed delta, and local +text-completion calls do not enter this chat probing path. + +Probing replays the complete POST, so a first request can upload the prompt up +to three times. Orb chooses that trade-off because native compatibility proxies +do not expose a reliable discovery contract. A successful protocol and path is +cached per configured URL and model for the life of the backend process; +configured settings are never rewritten. + +A 401 or 403 can trigger one alternate Anthropic/Bearer auth attempt only when +the model name, path, or resolved protocol supplies Claude/Anthropic evidence. +This retry is independently bounded and never changes hosts. + +## Provider request behavior + +Native Anthropic requests are built from an allowlist. System messages are +hoisted; text, base64 images, tool calls, and tool results are translated to +Messages content blocks; adjacent roles are coalesced. Tool definitions use +`input_schema` and `strict: true`. OpenAI `extra_body` fields are not passed +through; only Anthropic-native `metadata` and `service_tier` are accepted from +that escape hatch. A missing `max_tokens` defaults to 4096. + +Reasoning-on maps to adaptive thinking with summarized display, and supported +effort levels map to `output_config.effort`. Reasoning-off omits `thinking`. +Current Claude families that reject temperature, top-p, and top-k omit them; +unknown proxy model names try them once, learn from a specific rejection, and +omit them for later calls. `min_p`, repetition penalties, and logprobs are never +sent to Anthropic. Consequently, Document mode's per-token steering is not +available on native Anthropic endpoints. + +Some routed models accept only `tool_choice="auto"`. This is distinct from a +provider that rejects `tool_choice` entirely: Orb rewrites `none`, `required`, +and named choices to `auto`, including the Writer's normal `none`. If the body +reveals this restriction for an unlisted model, Orb learns it for the process. +Director and Editor already handle a model declining the intended forced call. + +Gemini uses Google's official OpenAI-compatible beta surface, Bearer auth, the +existing OpenAI stream parser, and strict structured output for forced calls. +Documented OpenAI fields, including `reasoning_effort`, remain intact. Native +Gemini features such as grounding and Files APIs are outside this version. + +## Stable external contracts + +Endpoint routing is internal and ephemeral. Public API and browser SSE shapes, +database settings, `LLMClient.complete()`, and `LLMClient.list_models()` do not +change. Anthropic stream events are translated to Orb's existing `content`, +`reasoning`, and terminal `done` events; provider error events inside an HTTP +200 stream use the same sanitized `LLMCallError` path as HTTP rejections. diff --git a/mkdocs.yml b/mkdocs.yml index 0ab15022..b210208d 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -67,6 +67,7 @@ nav: - Home: index.md - Getting Started: getting-started.md - Architecture: + - LLM Endpoint Routing: architecture/endpoints.md - KV Cache Reuse: architecture/kv-cache.md - SSE Turn Stream: architecture/sse-stream.md - Secondary Workflows: architecture/secondary-workflow.md diff --git a/tests/integration/test_endpoint_transport_passes.py b/tests/integration/test_endpoint_transport_passes.py new file mode 100644 index 00000000..a3dd2c65 --- /dev/null +++ b/tests/integration/test_endpoint_transport_passes.py @@ -0,0 +1,179 @@ +"""Pass-shaped calls through real LLMClient protocol adapters. + +The normal integration mock replaces ``LLMClient.complete`` wholesale. This +test stubs httpx one seam lower so Director/Writer/Editor request shapes and +stream translation run for OpenAI, Anthropic, and Gemini transports. +""" + +from __future__ import annotations + +import json +from unittest.mock import patch + +import pytest + +from backend.inference import client as llm_mod +from backend.inference import endpoint_profiles as ep +from backend.inference.client import LLMClient, parse_tool_calls + + +def _tool(name: str) -> dict: + return { + "type": "function", + "function": { + "name": name, + "description": name, + "parameters": { + "type": "object", + "properties": {"value": {"type": "string"}}, + "required": ["value"], + }, + }, + } + + +TOOLS = [_tool("direct_scene"), _tool("editor_apply_patch")] +DIRECTOR = {"type": "function", "function": {"name": "direct_scene"}} +EDITOR = {"type": "function", "function": {"name": "editor_apply_patch"}} + + +class _Response: + status_code = 200 + + def __init__(self, lines): + self.lines = lines + + async def __aenter__(self): + return self + + async def __aexit__(self, *_args): + return False + + async def aiter_lines(self): + for line in self.lines: + yield line + + +class _Transport: + def __init__(self, scripts): + self.scripts = list(scripts) + self.requests = [] + + async def __aenter__(self): + return self + + async def __aexit__(self, *_args): + return False + + def stream(self, method, url, json=None, headers=None): + self.requests.append({"url": url, "body": dict(json or {}), "headers": dict(headers or {})}) + return _Response(self.scripts.pop(0)) + + +def _data(payload: dict) -> str: + return f"data: {json.dumps(payload)}" + + +def _openai_tool(name: str) -> list[str]: + call = { + "index": 0, + "id": f"call-{name}", + "function": {"name": name, "arguments": '{"value":"ok"}'}, + } + return [_data({"choices": [{"delta": {"tool_calls": [call]}, "finish_reason": "tool_calls"}]}), "data: [DONE]"] + + +def _openai_text() -> list[str]: + return [_data({"choices": [{"delta": {"content": "draft"}, "finish_reason": "stop"}]}), "data: [DONE]"] + + +def _anthropic_tool(name: str) -> list[str]: + return [ + _data( + { + "type": "content_block_start", + "index": 0, + "content_block": {"type": "tool_use", "id": f"call-{name}", "name": name, "input": {}}, + } + ), + _data( + { + "type": "content_block_delta", + "index": 0, + "delta": {"type": "input_json_delta", "partial_json": '{"value":"ok"}'}, + } + ), + _data({"type": "message_delta", "delta": {"stop_reason": "tool_use"}}), + _data({"type": "message_stop"}), + ] + + +def _anthropic_text() -> list[str]: + return [ + _data({"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "draft"}}), + _data({"type": "message_delta", "delta": {"stop_reason": "end_turn"}}), + _data({"type": "message_stop"}), + ] + + +async def _complete(client: LLMClient, model: str, choice: dict | str) -> list[dict]: + return [ + event + async for event in client.complete( + [{"role": "system", "content": "system"}, {"role": "user", "content": "turn"}], + model, + tools=TOOLS, + tool_choice=choice, + max_tokens=100, + ) + ] + + +@pytest.mark.parametrize( + ("provider", "endpoint", "model"), + [ + ("openai", "https://openai.test/v1/chat/completions", "openai-model"), + ("anthropic", "https://api.anthropic.com/v1/messages", "claude-haiku-4-5"), + ("gemini", "https://generativelanguage.googleapis.com", "gemini-3-pro"), + ], +) +async def test_director_writer_editor_calls_cross_protocol_boundary(provider, endpoint, model): + ep._RESOLVED_ROUTES.clear() + if provider == "anthropic": + scripts = [_anthropic_tool("direct_scene"), _anthropic_text(), _anthropic_tool("editor_apply_patch")] + elif provider == "gemini": + # Gemini forced calls use strict response_format, whose JSON content is + # re-synthesized into Orb's ordinary tool-call message. + scripts = [ + [_data({"choices": [{"delta": {"content": '{"value":"ok"}'}, "finish_reason": "stop"}]}), "data: [DONE]"], + _openai_text(), + [_data({"choices": [{"delta": {"content": '{"value":"ok"}'}, "finish_reason": "stop"}]}), "data: [DONE]"], + ] + else: + scripts = [_openai_tool("direct_scene"), _openai_text(), _openai_tool("editor_apply_patch")] + + transport = _Transport(scripts) + client = LLMClient(endpoint, "secret") + with patch.object(llm_mod.httpx, "AsyncClient", lambda *args, **kwargs: transport): + director = await _complete(client, model, DIRECTOR) + writer = await _complete(client, model, "none") + editor = await _complete(client, model, EDITOR) + + assert parse_tool_calls(director[-1]["message"])[0]["name"] == "direct_scene" + assert writer[-1]["message"]["content"] == "draft" + assert parse_tool_calls(editor[-1]["message"])[0]["name"] == "editor_apply_patch" + assert len(transport.requests) == 3 + + bodies = [request["body"] for request in transport.requests] + if provider == "anthropic": + assert [body["tool_choice"] for body in bodies] == [ + {"type": "tool", "name": "direct_scene"}, + {"type": "none"}, + {"type": "tool", "name": "editor_apply_patch"}, + ] + assert all(body["tools"][0]["strict"] is True for body in bodies) + elif provider == "gemini": + assert "response_format" in bodies[0] and "response_format" in bodies[2] + assert "tools" not in bodies[1] and "tool_choice" not in bodies[1] + else: + assert [body["tool_choice"] for body in bodies] == [DIRECTOR, "none", EDITOR] diff --git a/tests/unit/test_endpoint_protocols.py b/tests/unit/test_endpoint_protocols.py new file mode 100644 index 00000000..fb1327b0 --- /dev/null +++ b/tests/unit/test_endpoint_protocols.py @@ -0,0 +1,463 @@ +"""Protocol resolution, Anthropic adaptation/streaming, and bounded probing.""" + +from __future__ import annotations + +import json +from unittest.mock import patch + +import pytest + +from backend.inference import anthropic +from backend.inference import client as llm_mod +from backend.inference import endpoint_profiles as ep +from backend.inference.client import LLMClient, parse_tool_calls, reasoning_cfg +from backend.inference.errors import LLMCallError + +TOOL = { + "type": "function", + "function": { + "name": "direct_scene", + "description": "Direct it", + "parameters": { + "type": "object", + "properties": {"mood": {"type": "string"}}, + "required": ["mood"], + }, + }, +} +FORCED = {"type": "function", "function": {"name": "direct_scene"}} + + +@pytest.fixture(autouse=True) +def _clear_learned_state(): + ep._RESOLVED_ROUTES.clear() + ep._TOOL_CHOICE_AUTO_ONLY.clear() + ep._TOOL_CHOICE_UNSUPPORTED.clear() + anthropic._SAMPLING_UNSUPPORTED.clear() + yield + ep._RESOLVED_ROUTES.clear() + ep._TOOL_CHOICE_AUTO_ONLY.clear() + ep._TOOL_CHOICE_UNSUPPORTED.clear() + anthropic._SAMPLING_UNSUPPORTED.clear() + + +@pytest.mark.parametrize( + ("configured", "protocol", "url", "models"), + [ + ( + "https://openai.test/v1/chat/completions", + "openai", + "https://openai.test/v1/chat/completions", + "https://openai.test/v1/models", + ), + ( + "https://proxy.test/prefix/v1/messages", + "anthropic", + "https://proxy.test/prefix/v1/messages", + "https://proxy.test/prefix/v1/models", + ), + ( + "https://api.anthropic.com", + "anthropic", + "https://api.anthropic.com/v1/messages", + "https://api.anthropic.com/v1/models", + ), + ( + "https://proxy.test/providers/anthropic/v1", + "anthropic", + "https://proxy.test/providers/anthropic/v1/messages", + "https://proxy.test/providers/anthropic/v1/models", + ), + ( + "https://generativelanguage.googleapis.com/v99/wrong", + "openai", + "https://generativelanguage.googleapis.com/v1beta/openai/chat/completions", + "https://generativelanguage.googleapis.com/v1beta/openai/models", + ), + ], +) +def test_deterministic_resolution(configured, protocol, url, models): + route = ep.resolve_endpoint(configured) + assert route.protocol == protocol + assert route.url == url + assert route.models_url == models + assert route.authoritative + + +def test_explicit_resource_is_authoritative_even_on_provider_host(): + route = ep.resolve_endpoint("https://api.anthropic.com/v1/chat/completions") + assert route.protocol == "openai" + assert route.url.endswith("/v1/chat/completions") + + +def test_ambiguous_candidates_preserve_old_request_then_same_host_v1(): + routes = ep.endpoint_candidates("https://custom.test/prefix", "m") + assert [(route.protocol, route.url) for route in routes] == [ + ("openai", "https://custom.test/prefix/chat/completions"), + ("openai", "https://custom.test/v1/chat/completions"), + ("anthropic", "https://custom.test/v1/messages"), + ] + + +def test_malformed_url_degrades_without_probe_candidates(): + routes = ep.endpoint_candidates("not a url", "m") + assert len(routes) == 1 + assert routes[0].protocol == "openai" + + +def test_route_probe_is_body_based(): + assert ep.should_probe_route(400, '{"error":"Unknown endpoint"}') + assert ep.should_probe_route(404, "Cannot POST /chat/completions") + assert not ep.should_probe_route(404, '{"error":"model not found"}') + assert not ep.should_probe_route(400, '{"error":"Unknown model"}') + + +def test_tool_schema_predicate_tracks_anthropic_and_gemini_wire_shapes(): + messages = [{"role": "user", "content": "hi"}] + assert LLMClient("https://api.anthropic.com").sends_tool_schemas(messages, "claude-opus-5") + assert not LLMClient("https://generativelanguage.googleapis.com").sends_tool_schemas(messages, "gemini-3-pro") + + +def test_translate_messages_system_images_tools_and_coalescing(): + messages = [ + {"role": "system", "content": "one"}, + {"role": "system", "content": [{"type": "text", "text": "two"}]}, + { + "role": "user", + "content": [ + {"type": "text", "text": "look"}, + {"type": "image_url", "image_url": {"url": "data:image/png;base64,eA=="}}, + ], + }, + { + "role": "assistant", + "content": "calling", + "tool_calls": [ + { + "id": "call-1", + "type": "function", + "function": {"name": "direct_scene", "arguments": '{"mood":"eerie"}'}, + } + ], + }, + {"role": "tool", "tool_call_id": "call-1", "content": "done"}, + {"role": "user", "content": "continue"}, + ] + system, out = anthropic.translate_messages(messages) + assert system == "one\n\ntwo" + assert out[0]["content"][1] == { + "type": "image", + "source": {"type": "base64", "media_type": "image/png", "data": "eA=="}, + } + assert out[1]["content"][1] == { + "type": "tool_use", + "id": "call-1", + "name": "direct_scene", + "input": {"mood": "eerie"}, + } + # tool result and following user content become one legal adjacent user turn. + assert out[2]["role"] == "user" + assert [block["type"] for block in out[2]["content"]] == ["tool_result", "text"] + + +def test_anthropic_body_allowlist_tools_choices_reasoning_and_sampling(): + canonical = { + "model": "claude-haiku-4-5", + "messages": [{"role": "user", "content": "hi"}], + "stream": True, + "tools": [TOOL], + "tool_choice": FORCED, + "temperature": 0.8, + "top_p": 0.95, + "top_k": 40, + "min_p": 0.1, + "repetition_penalty": 1.1, + "reasoning": {"enabled": True}, + "reasoning_effort": "xhigh", + "chat_template_kwargs": {"thinking": True}, + "stream_options": {"include_usage": True}, + "logprobs": True, + } + body = anthropic.build_request_body( + canonical, + "https://api.anthropic.com/v1/messages", + "claude-haiku-4-5", + {"metadata": {"user_id": "u"}, "response_format": {"type": "json"}, "seed": 7}, + ) + assert body["max_tokens"] == anthropic.DEFAULT_MAX_TOKENS + assert body["tools"][0]["strict"] is True + assert body["tools"][0]["input_schema"] == TOOL["function"]["parameters"] + assert body["tool_choice"] == {"type": "tool", "name": "direct_scene"} + assert body["thinking"] == {"type": "adaptive", "display": "summarized"} + assert body["output_config"] == {"effort": "xhigh"} + assert body["temperature"] == 0.8 and body["top_p"] == 0.95 and body["top_k"] == 40 + assert body["metadata"] == {"user_id": "u"} + for forbidden in ( + "reasoning", + "chat_template_kwargs", + "stream_options", + "logprobs", + "min_p", + "repetition_penalty", + "response_format", + "seed", + ): + assert forbidden not in body + + +def test_current_claude_sampling_is_withheld_and_reasoning_off_omits_thinking(): + body = anthropic.build_request_body( + { + "messages": [], + "temperature": 0.8, + "top_p": 0.95, + "top_k": 40, + "thinking": {"type": "disabled"}, + }, + "https://api.anthropic.com", + "claude-opus-5-20260801", + ) + assert "temperature" not in body and "top_p" not in body and "top_k" not in body + assert "thinking" not in body and "output_config" not in body + + +def test_tool_choice_mapping(): + assert anthropic.translate_tool_choice("none") == {"type": "none"} + assert anthropic.translate_tool_choice("auto") == {"type": "auto"} + assert anthropic.translate_tool_choice("required") == {"type": "any"} + assert anthropic.translate_tool_choice(FORCED) == {"type": "tool", "name": "direct_scene"} + + +class _Response: + def __init__(self, status=200, *, error="", lines=()): + self.status_code = status + self.error = error + self.lines = list(lines) + + async def __aenter__(self): + return self + + async def __aexit__(self, *_args): + return False + + async def aread(self): + return self.error.encode() + + async def aiter_lines(self): + for line in self.lines: + yield line + + +class _HTTP: + def __init__(self, responses): + self.responses = list(responses) + self.requests = [] + + async def __aenter__(self): + return self + + async def __aexit__(self, *_args): + return False + + def stream(self, method, url, json=None, headers=None): + self.requests.append({"method": method, "url": url, "body": dict(json or {}), "headers": dict(headers or {})}) + return self.responses.pop(0) + + +def _line(payload: dict) -> str: + return f"data: {json.dumps(payload)}" + + +ANTHROPIC_TOOL_STREAM = [ + _line({"type": "message_start", "message": {"usage": {"input_tokens": 10, "cache_read_input_tokens": 4}}}), + _line({"type": "ping"}), + _line({"type": "content_block_start", "index": 0, "content_block": {"type": "thinking", "thinking": "why"}}), + _line({"type": "content_block_delta", "index": 0, "delta": {"type": "thinking_delta", "thinking": " now"}}), + _line( + { + "type": "content_block_start", + "index": 1, + "content_block": {"type": "tool_use", "id": "tool-1", "name": "direct_scene", "input": {}}, + } + ), + _line({"type": "content_block_delta", "index": 1, "delta": {"type": "input_json_delta", "partial_json": '{"mood"'}}), + _line({"type": "content_block_delta", "index": 1, "delta": {"type": "input_json_delta", "partial_json": ':"eerie"}'}}), + _line({"type": "message_delta", "delta": {"stop_reason": "tool_use"}, "usage": {"output_tokens": 7}}), + _line({"type": "message_stop"}), + _line({"type": "content_block_delta", "index": 2, "delta": {"type": "text_delta", "text": "ignored"}}), +] + + +async def _run(client: LLMClient, fake: _HTTP, model="claude-haiku-4-5", **kwargs): + events = [] + with patch.object(llm_mod.httpx, "AsyncClient", lambda *args, **kw: fake): + async for event in client.complete([{"role": "user", "content": "hi"}], model, **kwargs): + events.append(event) + return events + + +async def test_anthropic_wire_headers_body_and_stream_translation(): + fake = _HTTP([_Response(lines=ANTHROPIC_TOOL_STREAM)]) + client = LLMClient( + "https://api.anthropic.com/v1/messages", + "sk-test", + reasoning_effort="high", + extra_headers="X-Custom: yes\nanthropic-version: 2026-01-01", + extra_body='{"seed": 9}', + ) + events = await _run( + client, + fake, + tools=[TOOL], + tool_choice=FORCED, + max_tokens=123, + temperature=0.4, + **reasoning_cfg(True), + ) + request = fake.requests[0] + assert request["url"] == "https://api.anthropic.com/v1/messages" + assert request["headers"] == {"x-api-key": "sk-test", "X-Custom": "yes", "anthropic-version": "2026-01-01"} + assert "Authorization" not in request["headers"] + assert request["body"]["max_tokens"] == 123 + assert request["body"]["thinking"] == {"type": "adaptive", "display": "summarized"} + assert request["body"]["tool_choice"] == {"type": "tool", "name": "direct_scene"} + assert "seed" not in request["body"] + assert [event["delta"] for event in events if event["type"] == "reasoning"] == ["why", " now"] + assert not [event for event in events if event["type"] == "content"] + done = events[-1] + assert done["usage"] == {"input_tokens": 10, "cache_read_input_tokens": 4, "output_tokens": 7} + assert done["message"]["finish_reason"] == "tool_calls" + assert parse_tool_calls(done["message"]) == [{"name": "direct_scene", "arguments": {"mood": "eerie"}}] + + +async def test_anthropic_message_stop_terminates_without_done_sentinel(): + lines = [ + _line({"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "hello"}}), + _line({"type": "message_delta", "delta": {"stop_reason": "end_turn"}, "usage": {"output_tokens": 1}}), + _line({"type": "message_stop"}), + ] + events = await _run(LLMClient("https://api.anthropic.com/v1/messages"), _HTTP([_Response(lines=lines)])) + assert events[0] == {"type": "content", "delta": "hello"} + assert events[-1]["message"] == {"content": "hello", "finish_reason": "stop"} + + +async def test_gemini_uses_normalized_openai_route_structured_output_and_effort(): + lines = [ + 'data: {"choices":[{"delta":{"content":"{\\"mood\\":\\"bright\\"}"},"finish_reason":"stop"}]}', + "data: [DONE]", + ] + fake = _HTTP([_Response(lines=lines)]) + client = LLMClient("https://generativelanguage.googleapis.com", "key", reasoning_effort="high") + events = await _run( + client, + fake, + model="gemini-3-pro", + tools=[TOOL], + tool_choice=FORCED, + **reasoning_cfg(True), + ) + request = fake.requests[0] + assert request["url"] == "https://generativelanguage.googleapis.com/v1beta/openai/chat/completions" + assert request["headers"] == {"Authorization": "Bearer key"} + assert "tools" not in request["body"] and "tool_choice" not in request["body"] + assert request["body"]["reasoning_effort"] == "high" + assert request["body"]["response_format"]["json_schema"]["strict"] is True + assert parse_tool_calls(events[-1]["message"]) == [{"name": "direct_scene", "arguments": {"mood": "bright"}}] + + +async def test_anthropic_midstream_error_uses_sanitized_llm_error(): + lines = [ + _line({"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "partial"}}), + _line({"type": "error", "error": {"type": "overloaded_error", "message": "busy sk-secret"}}), + ] + with pytest.raises(LLMCallError) as caught: + await _run(LLMClient("https://api.anthropic.com/v1/messages", "sk-secret"), _HTTP([_Response(lines=lines)])) + assert caught.value.response.status_code == 502 + assert caught.value.sentence == "busy [redacted]" + assert "sk-secret" not in caught.value.body + + +async def test_ambiguous_endpoint_probes_bounded_routes_and_caches_success(): + wrong = '{"error":"Cannot POST /prefix/chat/completions"}' + wrong_v1 = '{"error":"route not found"}' + success = [ + _line({"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "hi"}}), + _line({"type": "message_stop"}), + ] + fake = _HTTP([_Response(404, error=wrong), _Response(404, error=wrong_v1), _Response(lines=success)]) + client = LLMClient("https://custom.test/prefix") + events = await _run(client, fake, model="claude-proxy") + assert [request["url"] for request in fake.requests] == [ + "https://custom.test/prefix/chat/completions", + "https://custom.test/v1/chat/completions", + "https://custom.test/v1/messages", + ] + assert events[-1]["message"]["content"] == "hi" + assert ep.resolve_endpoint(client.base_url, "claude-proxy").protocol == "anthropic" + + again = _HTTP([_Response(lines=success)]) + await _run(client, again, model="claude-proxy") + assert [request["url"] for request in again.requests] == ["https://custom.test/v1/messages"] + + +async def test_auth_family_retry_is_evidence_gated_and_bounded(): + success = [_line({"type": "message_stop"})] + fake = _HTTP([_Response(401, error='{"error":"bad auth"}'), _Response(lines=success)]) + await _run(LLMClient("https://proxy.test/v1/messages", "key"), fake, model="claude-proxy") + assert fake.requests[0]["headers"]["x-api-key"] == "key" + assert fake.requests[1]["headers"]["Authorization"] == "Bearer key" + assert "x-api-key" not in fake.requests[1]["headers"] + + +async def test_openai_auth_rejection_without_claude_evidence_is_not_retried(): + fake = _HTTP([_Response(401, error='{"error":"bad auth"}')]) + with pytest.raises(LLMCallError): + await _run(LLMClient("https://custom.test/v1", "key"), fake, model="ordinary-model") + assert len(fake.requests) == 1 + + +async def test_sampling_rejection_is_learned_and_retried_once(): + rejection = '{"error":{"message":"temperature is not supported"}}' + fake = _HTTP([_Response(400, error=rejection), _Response(lines=[_line({"type": "message_stop"})])]) + client = LLMClient("https://proxy.test/v1/messages") + await _run(client, fake, model="hidden-model", temperature=0.8, top_p=0.9, top_k=20) + assert fake.requests[0]["body"]["temperature"] == 0.8 + assert not {"temperature", "top_p", "top_k"} & fake.requests[1]["body"].keys() + assert (client.base_url, "hidden-model") in anthropic._SAMPLING_UNSUPPORTED + + +async def test_anthropic_forced_choice_rejection_falls_back_to_auto(): + rejection = '{"error":{"message":"tool_choice type any is not supported for this model"}}' + fake = _HTTP( + [ + _Response(400, error=rejection), + _Response(lines=[_line({"type": "message_stop"})]), + ] + ) + client = LLMClient("https://proxy.test/v1/messages") + await _run(client, fake, model="claude-fable-5-1", tools=[TOOL], tool_choice="required") + assert fake.requests[0]["body"]["tool_choice"] == {"type": "any"} + assert fake.requests[1]["body"]["tool_choice"] == {"type": "auto"} + assert not ep.honors_forced_tool_choice(client.base_url, "claude-fable-5-1") + + +@pytest.mark.parametrize("choice", ["none", "required", FORCED]) +async def test_auto_only_tool_choice_recovery_and_process_memory(choice): + rejection = '{"error":{"message":"only \\"auto\\" is supported for tool_choice"}}' + openai_done = ['data: {"choices":[{"delta":{},"finish_reason":"stop"}]}', "data: [DONE]"] + fake = _HTTP([_Response(400, error=rejection), _Response(lines=openai_done)]) + client = LLMClient("https://openrouter.ai/api/v1") + await _run(client, fake, model="unlisted/model", tools=[TOOL], tool_choice=choice) + assert fake.requests[1]["body"]["tool_choice"] == "auto" + assert (client.base_url, "unlisted/model") in ep._TOOL_CHOICE_AUTO_ONLY + + again = _HTTP([_Response(lines=openai_done)]) + await _run(client, again, model="unlisted/model", tools=[TOOL], tool_choice=choice) + assert again.requests[0]["body"]["tool_choice"] == "auto" + + +async def test_unrelated_failure_does_not_probe(): + fake = _HTTP([_Response(404, error='{"error":"model not found"}')]) + with pytest.raises(LLMCallError): + await _run(LLMClient("https://custom.test/prefix"), fake, model="missing") + assert len(fake.requests) == 1 diff --git a/tests/unit/test_forced_tool_choice_fallback.py b/tests/unit/test_forced_tool_choice_fallback.py index 74ba97d2..258979f1 100644 --- a/tests/unit/test_forced_tool_choice_fallback.py +++ b/tests/unit/test_forced_tool_choice_fallback.py @@ -2,7 +2,7 @@ Covers: - ModelProfile.allow_extra=None disables drop-filtering entirely. - - The OpenRouter PROFILES entry coerces forced tool_choice proactively. + - OpenRouter catalog models are not encoded as permanent profiles. - LLMClient.complete()'s provider-gated, error-specific retry: drops tool_choice once for the matching OpenRouter 404 (regardless of its value), raises immediately for unrelated 404s, and never retries when no @@ -59,20 +59,6 @@ def test_allow_extra_frozenset_still_drops(): assert "weird" not in body -def test_openrouter_minimax_profile_coerces_forced_tool_choice(): - prof = profile_for("https://openrouter.ai/api/v1", "minimax/minimax-m3") - assert prof is not None - body = { - "model": "minimax/minimax-m3", - "messages": [], - "tool_choice": {"type": "function", "function": {"name": "direct_scene"}}, - "temperature": 0.7, - } - prof.apply(body) - assert body["tool_choice"] == "auto" - assert body["temperature"] == 0.7 # nothing dropped - - def test_openrouter_unlisted_model_is_passthrough(): assert profile_for("https://openrouter.ai/api/v1", "some/other-model") is None @@ -83,7 +69,7 @@ def test_openrouter_unlisted_model_is_passthrough(): def test_is_tool_choice_unsupported_signature(): txt = "No endpoints found that support the provided 'tool_choice' value." assert _is_tool_choice_unsupported(404, txt) - assert not _is_tool_choice_unsupported(400, txt) + assert _is_tool_choice_unsupported(400, txt) assert not _is_tool_choice_unsupported(404, "model not found") diff --git a/tests/unit/test_model_discovery.py b/tests/unit/test_model_discovery.py index 1c11e818..aa9b7684 100644 --- a/tests/unit/test_model_discovery.py +++ b/tests/unit/test_model_discovery.py @@ -61,3 +61,32 @@ async def test_list_models_rejects_non_openai_response(monkeypatch): with pytest.raises(ValueError, match="data list"): await LLMClient("https://models.test/v1").list_models() + + +@pytest.mark.asyncio +async def test_anthropic_models_use_sibling_resource_and_native_auth(monkeypatch): + _CatalogClient.payload = {"data": [{"id": "claude-opus-5"}]} + _CatalogClient.seen = {} + monkeypatch.setattr(client_module.httpx, "AsyncClient", _CatalogClient) + + models = await LLMClient("https://api.anthropic.com/v1/messages", "secret-key").list_models() + + assert models == ["claude-opus-5"] + assert _CatalogClient.seen["url"] == "https://api.anthropic.com/v1/models" + assert _CatalogClient.seen["headers"] == { + "x-api-key": "secret-key", + "anthropic-version": "2023-06-01", + } + + +@pytest.mark.asyncio +async def test_gemini_models_use_normalized_surface_and_strip_models_prefix(monkeypatch): + _CatalogClient.payload = {"data": [{"id": "models/gemini-3-pro"}, {"id": "gemini-3-flash"}]} + _CatalogClient.seen = {} + monkeypatch.setattr(client_module.httpx, "AsyncClient", _CatalogClient) + + models = await LLMClient("https://generativelanguage.googleapis.com", "gemini-key").list_models() + + assert models == ["gemini-3-flash", "gemini-3-pro"] + assert _CatalogClient.seen["url"] == "https://generativelanguage.googleapis.com/v1beta/openai/models" + assert _CatalogClient.seen["headers"] == {"Authorization": "Bearer gemini-key"} From 726fa6482e1b863b3f7492df14af4b485ba67703 Mon Sep 17 00:00:00 2001 From: Chi Date: Sat, 5 Sep 2026 10:57:56 +0700 Subject: [PATCH 2/4] handle edge cases --- backend/inference/anthropic.py | 34 +++++++++- backend/inference/client.py | 67 ++++++++----------- backend/inference/tool_registry.py | 31 +++++++++ tests/unit/test_endpoint_protocols.py | 94 ++++++++++++++++++++++++++- 4 files changed, 184 insertions(+), 42 deletions(-) diff --git a/backend/inference/anthropic.py b/backend/inference/anthropic.py index cacc42fc..beff524e 100644 --- a/backend/inference/anthropic.py +++ b/backend/inference/anthropic.py @@ -7,6 +7,8 @@ from collections.abc import Mapping, Sequence from typing import Any +from .tool_registry import strictify_schema + # Anthropic rejects unknown top-level fields. These are the only user-provided # extra_body keys accepted on a native Messages route; OpenAI-shaped escape # hatches therefore cannot turn an otherwise-valid request into a hard 400. @@ -27,6 +29,12 @@ _SAMPLING_UNSUPPORTED: set[tuple[str, str]] = set() +# Adaptive thinking and ``output_config.effort`` are 4.6-and-later fields. An +# older family behind a proxy (Haiku 4.5 and earlier want the retired +# ``budget_tokens`` shape) rejects them outright, so -- as with sampling -- +# they go out once and are learned from the provider's rejection. +_THINKING_UNSUPPORTED: set[tuple[str, str]] = set() + def _text_parts(content: object) -> list[dict[str, Any]]: if isinstance(content, str): @@ -152,7 +160,11 @@ def translate_tools(tools: object) -> list[dict[str, Any]]: continue translated: dict[str, Any] = { "name": function["name"], - "input_schema": dict(function.get("parameters") or {"type": "object", "properties": {}}), + # ``strict`` obliges the schema to close every object and mark every + # property required; Orb's own tools ship partial ``required`` lists, + # so shape them the way the OpenAI forced path already does rather + # than sending a schema the API will reject. + "input_schema": strictify_schema(dict(function.get("parameters") or {"type": "object", "properties": {}})), "strict": True, } if isinstance(function.get("description"), str): @@ -214,7 +226,7 @@ def build_request_body( reasoning_on = (isinstance(reasoning, Mapping) and reasoning.get("enabled") is True) or ( isinstance(thinking, Mapping) and thinking.get("type") == "enabled" ) - if reasoning_on: + if reasoning_on and (endpoint_url, model) not in _THINKING_UNSUPPORTED: body["thinking"] = {"type": "adaptive", "display": "summarized"} effort = openai_body.get("reasoning_effort") if effort in {"low", "medium", "high", "xhigh", "max"}: @@ -233,6 +245,24 @@ def build_request_body( return body +def recover_thinking_error(endpoint_url: str, model: str, body: dict[str, Any], status: int, text: str) -> str | None: + """Learn a rejection of the 4.6+ reasoning fields and drop them once.""" + if status != 400: + return None + low = text.lower() + present = [key for key in ("thinking", "output_config") if key in body] + if not present: + return None + if not any(key in low for key in (*present, "effort", "budget_tokens")): + return None + if not any(marker in low for marker in ("unsupported", "not supported", "not allowed", "extra inputs")): + return None + _THINKING_UNSUPPORTED.add((endpoint_url, model)) + for key in present: + body.pop(key, None) + return f"Model {model} rejected Anthropic reasoning fields {present}; retrying without them." + + def recover_sampling_error(endpoint_url: str, model: str, body: dict[str, Any], status: int, text: str) -> str | None: """Learn a sampling-field rejection and remove all three controls once.""" if status != 400: diff --git a/backend/inference/client.py b/backend/inference/client.py index 60e62de7..a2cbc436 100644 --- a/backend/inference/client.py +++ b/backend/inference/client.py @@ -13,6 +13,7 @@ from .errors import LLMCallError, llm_call_error, llm_stream_error from .gemma_tool_format import parse_gemma_tool_calls from .retry import RetryPolicy +from .tool_registry import strictify_schema logger = logging.getLogger(__name__) @@ -139,37 +140,6 @@ def parse_extra_body(text: str) -> dict: return parsed -def strictify_schema(schema: dict) -> dict: - """Copy *schema* into OpenAI strict-mode shape, recursively. - - Strict structured output requires every object to list all properties in - ``required`` and set ``additionalProperties: false``. Originally-optional - properties are made nullable so "may omit" survives as "may be null" -- - the passes' unpackers already discard empty/null argument values. - """ - node = dict(schema) - props = node.get("properties") - if isinstance(props, dict): - required = set(node.get("required") or []) - out_props: dict = {} - for key, prop in props.items(): - sub = strictify_schema(prop) if isinstance(prop, dict) else prop - if key not in required and isinstance(sub, dict) and "type" in sub: - t = sub["type"] - if isinstance(t, list): - t = t if "null" in t else [*t, "null"] - elif t != "null": - t = [t, "null"] - sub = {**sub, "type": t} - out_props[key] = sub - node["properties"] = out_props - node["required"] = list(props.keys()) - node["additionalProperties"] = False - if isinstance(node.get("items"), dict): - node["items"] = strictify_schema(node["items"]) - return node - - def _parse_chat_logprobs(choice: Mapping[str, Any]) -> list[dict]: """Normalize an OpenAI-compat ``choice.logprobs`` block to Orb's prob shape. @@ -306,9 +276,6 @@ def _headers_for(self, auth_family: endpoint_profiles.AuthFamily) -> dict: headers.update(self.extra_headers) return headers - def _url(self) -> str: - return endpoint_profiles.resolve_endpoint(self.base_url).url - async def list_models(self) -> list[str]: """Return model ids advertised by an OpenAI-compatible ``GET /models``. @@ -597,14 +564,25 @@ def _plan() -> tuple[dict, str | None, bool]: logger.debug(messages) return body, forced_name, structured + # The body is re-derived on every recovery, auth retry and route probe, so + # its INFO lines would otherwise repeat once per attempt. Keyed on the + # rendered line, not on "first attempt only": a recovery that learns a new + # quirk adds an action, and that line is exactly the one worth surfacing. + logged_lines: set[str] = set() + + def _log_once(line: str) -> None: + if line not in logged_lines: + logged_lines.add(line) + logger.info("%s", line) + def _outbound_body(body: dict, route: endpoint_profiles.EndpointRoute) -> dict: """Copy the canonical OpenAI body into one route's wire dialect.""" outbound = dict(body) if route.protocol == "openai" and self.extra_body: outbound.update(self.extra_body) - logger.info("LLM extra body fields: %s", sorted(self.extra_body)) + _log_once(f"LLM extra body fields: {sorted(self.extra_body)}") for action in endpoint_profiles.prepare_request_body(self.base_url, model, outbound): - logger.info("LLM profile: %s", action) + _log_once(f"LLM profile: {action}") if route.protocol == "anthropic": return anthropic.build_request_body(outbound, self.base_url, model, self.extra_body) return outbound @@ -709,7 +687,8 @@ async def consume_anthropic(resp, url: str) -> AsyncIterator[dict]: entry["function"]["arguments"] = json.dumps(block["input"], separators=(",", ":")) elif block.get("type") == "text" and block.get("text"): content_parts.append(block["text"]) - yield {"type": "content", "delta": block["text"]} + if forced_name is None: + yield {"type": "content", "delta": block["text"]} elif block.get("type") == "thinking" and block.get("thinking"): reasoning_parts.append(block["thinking"]) yield {"type": "reasoning", "delta": block["thinking"]} @@ -719,7 +698,11 @@ async def consume_anthropic(resp, url: str) -> AsyncIterator[dict]: delta_type = delta.get("type") if delta_type == "text_delta" and delta.get("text"): content_parts.append(delta["text"]) - yield {"type": "content", "delta": delta["text"]} + # Same gate as consume_openai: a forced pass buffers its + # text as the tool-arguments payload instead of streaming + # it, so the caller never sees a half-built JSON body. + if forced_name is None: + yield {"type": "content", "delta": delta["text"]} elif delta_type == "thinking_delta" and delta.get("thinking"): reasoning_parts.append(delta["thinking"]) yield {"type": "reasoning", "delta": delta["thinking"]} @@ -766,13 +749,19 @@ async def consume_anthropic(resp, url: str) -> AsyncIterator[dict]: ) as resp: if resp.status_code >= 400: err_text = await _read_error_body(resp, route.url) - if recovery_count < 2: + # One attempt per independent quirk class: the profile + # rules, Anthropic sampling controls, and the 4.6+ + # reasoning fields can each need their own rejection + # before a body this endpoint accepts is reached. + if recovery_count < 3: fix = endpoint_profiles.recover_from_error( self.base_url, model, outbound, resp.status_code, err_text ) if fix is None and route.protocol == "anthropic": fix = anthropic.recover_sampling_error( self.base_url, model, outbound, resp.status_code, err_text + ) or anthropic.recover_thinking_error( + self.base_url, model, outbound, resp.status_code, err_text ) if fix is not None: recovery_count += 1 diff --git a/backend/inference/tool_registry.py b/backend/inference/tool_registry.py index c2ce8e9b..aec3810f 100644 --- a/backend/inference/tool_registry.py +++ b/backend/inference/tool_registry.py @@ -445,3 +445,34 @@ def enabled_schemas( if enabled_tools is not None: eligible = [n for n in eligible if enabled_tools.get(n, False)] return [s for n in eligible if (s := overrides.get(n, TOOLS[n]["schema"])) is not None] + + +def strictify_schema(schema: dict) -> dict: + """Copy *schema* into OpenAI strict-mode shape, recursively. + + Strict structured output requires every object to list all properties in + ``required`` and set ``additionalProperties: false``. Originally-optional + properties are made nullable so "may omit" survives as "may be null" -- + the passes' unpackers already discard empty/null argument values. + """ + node = dict(schema) + props = node.get("properties") + if isinstance(props, dict): + required = set(node.get("required") or []) + out_props: dict = {} + for key, prop in props.items(): + sub = strictify_schema(prop) if isinstance(prop, dict) else prop + if key not in required and isinstance(sub, dict) and "type" in sub: + t = sub["type"] + if isinstance(t, list): + t = t if "null" in t else [*t, "null"] + elif t != "null": + t = [t, "null"] + sub = {**sub, "type": t} + out_props[key] = sub + node["properties"] = out_props + node["required"] = list(props.keys()) + node["additionalProperties"] = False + if isinstance(node.get("items"), dict): + node["items"] = strictify_schema(node["items"]) + return node diff --git a/tests/unit/test_endpoint_protocols.py b/tests/unit/test_endpoint_protocols.py index fb1327b0..b073077c 100644 --- a/tests/unit/test_endpoint_protocols.py +++ b/tests/unit/test_endpoint_protocols.py @@ -34,11 +34,13 @@ def _clear_learned_state(): ep._TOOL_CHOICE_AUTO_ONLY.clear() ep._TOOL_CHOICE_UNSUPPORTED.clear() anthropic._SAMPLING_UNSUPPORTED.clear() + anthropic._THINKING_UNSUPPORTED.clear() yield ep._RESOLVED_ROUTES.clear() ep._TOOL_CHOICE_AUTO_ONLY.clear() ep._TOOL_CHOICE_UNSUPPORTED.clear() anthropic._SAMPLING_UNSUPPORTED.clear() + anthropic._THINKING_UNSUPPORTED.clear() @pytest.mark.parametrize( @@ -186,7 +188,13 @@ def test_anthropic_body_allowlist_tools_choices_reasoning_and_sampling(): ) assert body["max_tokens"] == anthropic.DEFAULT_MAX_TOKENS assert body["tools"][0]["strict"] is True - assert body["tools"][0]["input_schema"] == TOOL["function"]["parameters"] + # ``strict`` obliges the schema to close and require everything. + assert body["tools"][0]["input_schema"] == { + "type": "object", + "properties": {"mood": {"type": "string"}}, + "required": ["mood"], + "additionalProperties": False, + } assert body["tool_choice"] == {"type": "tool", "name": "direct_scene"} assert body["thinking"] == {"type": "adaptive", "display": "summarized"} assert body["output_config"] == {"effort": "xhigh"} @@ -461,3 +469,87 @@ async def test_unrelated_failure_does_not_probe(): with pytest.raises(LLMCallError): await _run(LLMClient("https://custom.test/prefix"), fake, model="missing") assert len(fake.requests) == 1 + + +def test_v1_base_url_collapses_the_duplicate_openai_candidate(): + # The common local-server shape: the configured URL already ends in /v1, so + # the historical request and the host-root OpenAI guess are the same URL. + # Without the dedupe the probe loop would re-POST an identical request. + routes = ep.endpoint_candidates("http://localhost:1234/v1", "m") + assert [(route.protocol, route.url) for route in routes] == [ + ("openai", "http://localhost:1234/v1/chat/completions"), + ("anthropic", "http://localhost:1234/v1/messages"), + ] + + +def test_partial_required_tool_schema_is_closed_before_strict_goes_out(): + partial = { + "type": "function", + "function": { + "name": "direct_scene", + "parameters": { + "type": "object", + "properties": {"mood": {"type": "string"}, "moods": {"type": "array"}}, + "required": [], + }, + }, + } + schema = anthropic.translate_tools([partial])[0]["input_schema"] + assert schema["additionalProperties"] is False + assert sorted(schema["required"]) == ["mood", "moods"] + # Optional survives as nullable rather than as an omittable key. + assert schema["properties"]["mood"]["type"] == ["string", "null"] + + +def test_reasoning_fields_are_dropped_and_learned_on_rejection(): + url = "https://proxy.test/v1/messages" + body = anthropic.build_request_body( + {"messages": [], "reasoning": {"enabled": True}, "reasoning_effort": "high"}, + url, + "haiku-4-5-via-proxy", + ) + assert body["thinking"] == {"type": "adaptive", "display": "summarized"} + assert body["output_config"] == {"effort": "high"} + + rejection = '{"error":{"message":"thinking: Extra inputs are not permitted"}}' + fix = anthropic.recover_thinking_error(url, "haiku-4-5-via-proxy", body, 400, rejection) + assert fix is not None + assert "thinking" not in body and "output_config" not in body + + # Learned for the rest of the session, so the rebuilt body omits them too. + again = anthropic.build_request_body( + {"messages": [], "reasoning": {"enabled": True}, "reasoning_effort": "high"}, + url, + "haiku-4-5-via-proxy", + ) + assert "thinking" not in again and "output_config" not in again + + +def test_unrelated_400_does_not_strip_reasoning_fields(): + url = "https://proxy.test/v1/messages" + body = anthropic.build_request_body({"messages": [], "reasoning": {"enabled": True}}, url, "some-model") + assert anthropic.recover_thinking_error(url, "some-model", body, 400, '{"error":"model not found"}') is None + assert body["thinking"] == {"type": "adaptive", "display": "summarized"} + assert (url, "some-model") not in anthropic._THINKING_UNSUPPORTED + + +async def test_doc_mode_on_anthropic_buffers_text_instead_of_streaming_it(): + # tools_in_prompt=False makes _plan force a name, but the Anthropic allowlist + # drops response_format, so the endpoint answers with plain text. That text is + # the forced payload -- it must not reach the caller as content deltas. + lines = [ + _line({"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": '{"mood"'}}), + _line({"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": ':"eerie"}'}}), + _line({"type": "message_delta", "delta": {"stop_reason": "end_turn"}, "usage": {"output_tokens": 3}}), + _line({"type": "message_stop"}), + ] + fake = _HTTP([_Response(lines=lines)]) + events = await _run( + LLMClient("https://api.anthropic.com/v1/messages", "sk-test"), + fake, + tools=[TOOL], + tool_choice=FORCED, + tools_in_prompt=False, + ) + assert not [event for event in events if event["type"] == "content"] + assert parse_tool_calls(events[-1]["message"]) == [{"name": "direct_scene", "arguments": {"mood": "eerie"}}] From 72d5a0fc9c48783b8e037eacd59074377482c576 Mon Sep 17 00:00:00 2001 From: Chi Date: Sat, 5 Sep 2026 11:27:32 +0700 Subject: [PATCH 3/4] refine gemini dialect --- backend/inference/client.py | 40 +++- backend/inference/endpoint_profiles.py | 126 ++++++++++-- docs/architecture/endpoints.md | 39 +++- tests/unit/test_endpoint_protocols.py | 272 +++++++++++++++++++++++++ tests/unit/test_model_discovery.py | 24 +++ 5 files changed, 481 insertions(+), 20 deletions(-) diff --git a/backend/inference/client.py b/backend/inference/client.py index a2cbc436..7e3f9280 100644 --- a/backend/inference/client.py +++ b/backend/inference/client.py @@ -303,7 +303,9 @@ async def list_models(self) -> list[str]: model_id = item.get("id") if isinstance(item, dict) else None if isinstance(model_id, str) and model_id.strip(): normalized = model_id.strip() - if "generativelanguage.googleapis.com" in route.url.lower() and normalized.startswith("models/"): + # Google lists ids as ``models/gemini-...``; the generation + # resource takes either form, so strip it for the picker. + if normalized.startswith("models/") and endpoint_profiles.is_gemini_openai_surface(route.url): normalized = normalized.removeprefix("models/") model_ids.add(normalized) return sorted(model_ids, key=str.casefold) @@ -620,6 +622,31 @@ def tool_entry(index: int) -> dict: async def consume_openai(resp) -> AsyncIterator[dict]: nonlocal finish_reason, usage + # Slot last handed to an index-less delta; -1 before the first. + unindexed = -1 + + def slot_for(tc_delta: Mapping[str, Any]) -> int: + """Resolve one tool-call delta to an accumulator slot. + + Google's OpenAI-compatible surface omits ``index`` from + ``delta.tool_calls`` entirely, so keying on ``index`` with a + default of 0 merged every parallel call into one entry -- + names concatenated, all but the first argument payload lost. + Without an index, a delta that STARTS a call (it carries an + ``id`` or a function ``name``, which the OpenAI contract + sends only on a call's first chunk) opens the next free + slot; a bare argument continuation appends to the newest. + """ + nonlocal unindexed + index = tc_delta.get("index") + if isinstance(index, int) and not isinstance(index, bool): + return index + function = tc_delta.get("function") + starts = bool(tc_delta.get("id")) or bool(isinstance(function, Mapping) and function.get("name")) + if starts or unindexed < 0: + unindexed = max([*tool_calls_acc, unindexed], default=-1) + 1 + return unindexed + async for payload in self._iter_sse_payloads(resp): try: chunk = json.loads(payload) @@ -646,7 +673,7 @@ async def consume_openai(resp) -> AsyncIterator[dict]: for rec in _parse_chat_logprobs(choice): yield {"type": "token_probs", **rec} for tc_delta in delta.get("tool_calls") or []: - entry = tool_entry(tc_delta.get("index", 0)) + entry = tool_entry(slot_for(tc_delta)) if tc_delta.get("id"): entry["id"] = tc_delta["id"] fn = tc_delta.get("function", {}) @@ -750,9 +777,12 @@ async def consume_anthropic(resp, url: str) -> AsyncIterator[dict]: if resp.status_code >= 400: err_text = await _read_error_body(resp, route.url) # One attempt per independent quirk class: the profile - # rules, Anthropic sampling controls, and the 4.6+ - # reasoning fields can each need their own rejection - # before a body this endpoint accepts is reached. + # rules, a refused reasoning_effort level, Anthropic + # sampling controls, and the 4.6+ reasoning fields can + # each need their own rejection before a body this + # endpoint accepts is reached. No route reaches more + # than three of them -- reasoning_effort is an + # OpenAI-body field the Anthropic translation drops. if recovery_count < 3: fix = endpoint_profiles.recover_from_error( self.base_url, model, outbound, resp.status_code, err_text diff --git a/backend/inference/endpoint_profiles.py b/backend/inference/endpoint_profiles.py index 40ad8b9f..561ed890 100644 --- a/backend/inference/endpoint_profiles.py +++ b/backend/inference/endpoint_profiles.py @@ -177,17 +177,74 @@ def _deepseek_coerce_tool_choice_when_thinking(body: dict) -> str | None: structured_tool_calls=True, ), }, - # Google's OpenAI compatibility API accepts ordinary OpenAI request fields - # (unknown additions are ignored) and honors strict json_schema output. - "generativelanguage.googleapis.com": { - None: ModelProfile( - allow_extra=None, - structured_tool_calls=True, - ), - }, } +# Google's OpenAI compatibility surface. Matched by a predicate rather than a +# PROFILES substring because the same wire dialect is reached through more than +# one host: Google's own, and any proxy that mirrors the ``/v1beta/openai`` +# resource shape (Cloudflare AI Gateway, self-hosted key-pool proxies). Matching +# on the host string alone left every proxy user without the request policy, +# the reasoning translation and the catalogue normalization below. +# +# Deliberately NOT matched: a bare ``gemini`` or ``google`` path segment. Those +# appear on gateways whose route is an ordinary OpenAI one where native tool +# calls work, and ``structured_tool_calls`` would withhold ``tools`` from them +# for no reason. Vertex's ``endpoints/openapi`` surface is likewise excluded -- +# it is a different compatibility layer with its own tool behavior. +_GEMINI_HOST = "generativelanguage.googleapis.com" +_GEMINI_OPENAI_PATH = "/v1beta/openai" + + +def is_gemini_openai_surface(url: str) -> bool: + """Whether *url* addresses Google's OpenAI-compatible Gemini dialect.""" + parsed = _parsed_http_url(url) + if parsed is None: + return _GEMINI_OPENAI_PATH in _clean_url(url).lower() + host = (parsed.hostname or "").lower() + if host == _GEMINI_HOST or host.endswith(f".{_GEMINI_HOST}"): + return True + path = parsed.path.rstrip("/").lower() + return path == _GEMINI_OPENAI_PATH or f"{_GEMINI_OPENAI_PATH}/" in f"{path}/" + + +def _gemini_reasoning_off(body: dict) -> str | None: + """Ask Gemini to stop thinking when the call turned reasoning off. + + Orb's reasoning-off shape is three fields none of which Gemini's + compatibility layer reads (``reasoning``, ``chat_template_kwargs``, + ``thinking``); they are silently ignored, so the model kept thinking on its + default budget and the user paid for tokens they had disabled. The layer's + own control is ``reasoning_effort``, whose accepted set includes ``none``. + + Families that cannot disable thinking at all (2.5 Pro, the 3 series) answer + 400 to ``none``; :func:`recover_from_error` learns that from the rejection + and drops the field for the rest of the session, the same posture as every + other capability fact here. An explicit effort already in the body wins -- + that call asked for thinking. + """ + if "reasoning_effort" in body: + return None + reasoning = body.get("reasoning") + thinking = body.get("thinking") + disabled = (isinstance(reasoning, Mapping) and reasoning.get("enabled") is False) or ( + isinstance(thinking, Mapping) and thinking.get("type") == "disabled" + ) + if not disabled: + return None + body["reasoning_effort"] = "none" + return "reasoning_effort='none' (reasoning off)" + + +# Google's OpenAI compatibility API accepts ordinary OpenAI request fields +# (unknown additions are ignored) and honors strict json_schema output. +_GEMINI_PROFILE = ModelProfile( + allow_extra=None, + structured_tool_calls=True, + custom=(_gemini_reasoning_off,), +) + + Protocol = Literal["openai", "anthropic"] AuthFamily = Literal["bearer", "anthropic"] @@ -267,8 +324,8 @@ def _deterministic_route(endpoint_url: str) -> EndpointRoute: return _resource_route("anthropic", clean, authoritative=True) host = (parsed.hostname or "").lower() if parsed is not None else "" - if host == "generativelanguage.googleapis.com" or host.endswith(".generativelanguage.googleapis.com"): - base = _replace_path(parsed, "/v1beta/openai") if parsed is not None else clean + if host == _GEMINI_HOST or host.endswith(f".{_GEMINI_HOST}"): + base = _replace_path(parsed, _GEMINI_OPENAI_PATH) if parsed is not None else clean return _base_route("openai", base, authoritative=True) segments = [segment for segment in path.split("/") if segment] @@ -298,9 +355,15 @@ def endpoint_candidates(endpoint_url: str, model: str = "") -> list[EndpointRout Explicit resources and provider hints are authoritative. Ambiguous URLs keep Orb's historical ``{configured}/chat/completions`` request first, followed - by the conventional host-root OpenAI and Anthropic v1 resources. Candidates - are only attempted when :func:`should_probe_route` recognizes the response - body as a route mismatch. + by the conventional host-root OpenAI and Anthropic v1 resources, then + Google's ``/v1beta/openai`` compatibility resource. Candidates are only + attempted when :func:`should_probe_route` recognizes the response body as a + route mismatch. + + The Gemini candidate is last because it is the narrowest guess of the four + and costs a further prompt upload; it is still worth making, because a + Gemini-compat proxy configured at its bare root exposes that resource and + no other, and the two ``/v1`` guesses ahead of it cannot reach it. """ primary = resolve_endpoint(endpoint_url, model) if primary.authoritative or (endpoint_url, model) in _RESOLVED_ROUTES: @@ -313,6 +376,7 @@ def endpoint_candidates(endpoint_url: str, model: str = "") -> list[EndpointRout primary, _base_route("openai", f"{root}/v1"), _base_route("anthropic", f"{root}/v1"), + _base_route("openai", f"{root}{_GEMINI_OPENAI_PATH}"), ] out: list[EndpointRoute] = [] seen: set[tuple[str, str]] = set() @@ -418,9 +482,15 @@ def profile_for(endpoint_url: str, model: str = "") -> ModelProfile | None: A blank *model* falls through to the endpoint default. An unmatched URL returns ``None`` — the body is sent unchanged (local / unknown backends). + + Gemini is resolved by :func:`is_gemini_openai_surface` rather than by a + ``PROFILES`` substring so that a proxy mirroring Google's compatibility + resource is given the same request policy as Google's own host. """ if not endpoint_url: return None + if is_gemini_openai_surface(endpoint_url): + return _GEMINI_PROFILE haystack = endpoint_url.lower() for needle, models in PROFILES.items(): if needle in haystack: @@ -449,6 +519,14 @@ def profile_for(endpoint_url: str, model: str = "") -> ModelProfile | None: # writer ``"none"`` requests are coerced rather than dropped. _TOOL_CHOICE_AUTO_ONLY: set[tuple[str, str]] = set() +# Pairs whose reply rejected the ``reasoning_effort`` VALUE Orb sent. Orb offers +# a superset of levels (``xhigh`` is an Anthropic/OpenAI-ism; Gemini's set is +# high/low/medium/none) and a provider that validates the field answers 400 to +# anything outside its own. Learning the rejection rather than hard-coding each +# provider's accepted set is deliberate: those sets move under us, and a static +# list rots into wrongly clamping a level the provider has since added. +_REASONING_EFFORT_UNSUPPORTED: set[tuple[str, str]] = set() + def _is_openrouter(endpoint_url: str) -> bool: return "openrouter.ai" in endpoint_url.lower() @@ -471,6 +549,20 @@ def _is_tool_choice_auto_only(status: int, text: str) -> bool: return status in {400, 404} and "tool_choice" in low and "only" in low and "auto" in low and "support" in low +def _is_reasoning_effort_rejected(status: int, text: str) -> bool: + """Return True when the body names ``reasoning_effort`` as the bad field. + + Matches Google's "Invalid reasoning_effort: xhigh. Valid values are: high, + low, medium, none" and the equivalent from any provider that validates the + field. Kept to bodies that name the field so a generic 400 (bad model, + oversized prompt) never costs a reasoning setting the endpoint accepts. + """ + low = text.lower() + if status != 400 or "reasoning_effort" not in low: + return False + return any(marker in low for marker in ("invalid", "unsupported", "not supported", "not allowed", "valid values")) + + def prepare_request_body(endpoint_url: str, model: str, body: dict) -> list[str]: """Apply the matching profile and any session-learned workarounds to *body* in place. @@ -482,6 +574,10 @@ def prepare_request_body(endpoint_url: str, model: str, body: dict) -> list[str] if profile is not None: actions.extend(profile.apply(body)) + if "reasoning_effort" in body and (endpoint_url, model) in _REASONING_EFFORT_UNSUPPORTED: + effort = body.pop("reasoning_effort") + actions.append(f"reasoning_effort {effort!r} dropped (session-learned unsupported)") + # A model we already learned rejects tool_choice this session: drop it up # front so we skip the failing round-trip entirely. if "tool_choice" in body and (endpoint_url, model) in _TOOL_CHOICE_UNSUPPORTED: @@ -527,6 +623,10 @@ def recover_from_error(endpoint_url: str, model: str, body: dict, status: int, t tc = body["tool_choice"] body["tool_choice"] = {"type": "auto"} if isinstance(tc, dict) and "function" not in tc else "auto" return f"Model {model} accepts only tool_choice='auto'; retrying with auto." + if "reasoning_effort" in body and _is_reasoning_effort_rejected(status, text): + _REASONING_EFFORT_UNSUPPORTED.add((endpoint_url, model)) + effort = body.pop("reasoning_effort") + return f"Model {model} rejected reasoning_effort={effort!r}; retrying without it." if not _is_openrouter(endpoint_url): return None if "tool_choice" in body and _is_tool_choice_unsupported(status, text): diff --git a/docs/architecture/endpoints.md b/docs/architecture/endpoints.md index c64c1a43..a9d31612 100644 --- a/docs/architecture/endpoints.md +++ b/docs/architecture/endpoints.md @@ -22,6 +22,14 @@ An `anthropic` path segment is also a strong native-protocol hint. For example, `messages` resource. Full `chat/completions` and `messages` resource URLs are authoritative even when their host would normally imply another protocol. +Gemini is recognized by dialect, not by hostname. Google's host and any URL on +the `/v1beta/openai` resource path — a compatibility proxy, an AI gateway — +take the same request policy, reasoning translation, and catalogue +normalization. A bare `gemini` or `google` path segment deliberately does not +qualify: those appear on gateways whose route is an ordinary OpenAI one where +native tool calls work. Vertex AI's `endpoints/openapi` surface is a separate +compatibility layer and is treated as a plain OpenAI endpoint. + Model discovery uses the sibling `models` resource and the matching auth family. OpenAI and Gemini routes use Bearer authentication. Native Anthropic routes use `x-api-key` and `anthropic-version`. Extra headers may replace those defaults @@ -36,7 +44,11 @@ on the same host, these conventional resources: 1. the configured base plus `chat/completions`; 2. host-root `/v1/chat/completions`; -3. host-root `/v1/messages`. +3. host-root `/v1/messages`; +4. host-root `/v1beta/openai/chat/completions`. + +The last is Google's compatibility resource, which a Gemini-compat proxy +configured at its bare root exposes and the two `/v1` guesses cannot reach. The HTTP status alone never starts probing: a 400 or 404 can describe a bad model, schema, or tool choice rather than a bad route. Known request recovery @@ -44,11 +56,17 @@ runs first. No route is changed after the first streamed delta, and local text-completion calls do not enter this chat probing path. Probing replays the complete POST, so a first request can upload the prompt up -to three times. Orb chooses that trade-off because native compatibility proxies +to four times. Orb chooses that trade-off because native compatibility proxies do not expose a reliable discovery contract. A successful protocol and path is cached per configured URL and model for the life of the backend process; configured settings are never rewritten. +A rejected `reasoning_effort` value is recovered on any OpenAI-protocol route: +Orb offers a superset of levels (`xhigh` is not a Gemini value), so a body that +names the field as invalid drops it for one retry and for the rest of the +session. Providers' accepted sets move, so this is learned from the response +rather than held as a per-provider list. + A 401 or 403 can trigger one alternate Anthropic/Bearer auth attempt only when the model name, path, or resolved protocol supplies Claude/Anthropic evidence. This retry is independently bounded and never changes hosts. @@ -81,6 +99,23 @@ existing OpenAI stream parser, and strict structured output for forced calls. Documented OpenAI fields, including `reasoning_effort`, remain intact. Native Gemini features such as grounding and Files APIs are outside this version. +Structured output is what carries a forced call, so `tools` and `tool_choice` +are withheld from *every* Gemini request, not only the forced ones — the +argument-fidelity and prefix-stability reasons are the same ones set out for +any structured-output endpoint above. A pass that offers tools under +`tool_choice="auto"` therefore has none on the wire and answers as prose; the +Editor's unforced iteration is the one such pass, and it stops as it would for +any model that declined to call a tool. + +Reasoning-off is translated. Orb's `reasoning`, `chat_template_kwargs`, and +`thinking` fields mean nothing to the compatibility layer and are silently +ignored, so reasoning-off calls carry `reasoning_effort: "none"` — the control +the layer actually reads. Families that cannot disable thinking reject that +value; the rejection is learned per model like any other capability fact. + +`logprobs` is not supported on this surface, so Document mode's per-token +steering is unavailable on Gemini for the same reason it is on Anthropic. + ## Stable external contracts Endpoint routing is internal and ephemeral. Public API and browser SSE shapes, diff --git a/tests/unit/test_endpoint_protocols.py b/tests/unit/test_endpoint_protocols.py index b073077c..18dbc248 100644 --- a/tests/unit/test_endpoint_protocols.py +++ b/tests/unit/test_endpoint_protocols.py @@ -33,12 +33,14 @@ def _clear_learned_state(): ep._RESOLVED_ROUTES.clear() ep._TOOL_CHOICE_AUTO_ONLY.clear() ep._TOOL_CHOICE_UNSUPPORTED.clear() + ep._REASONING_EFFORT_UNSUPPORTED.clear() anthropic._SAMPLING_UNSUPPORTED.clear() anthropic._THINKING_UNSUPPORTED.clear() yield ep._RESOLVED_ROUTES.clear() ep._TOOL_CHOICE_AUTO_ONLY.clear() ep._TOOL_CHOICE_UNSUPPORTED.clear() + ep._REASONING_EFFORT_UNSUPPORTED.clear() anthropic._SAMPLING_UNSUPPORTED.clear() anthropic._THINKING_UNSUPPORTED.clear() @@ -98,6 +100,7 @@ def test_ambiguous_candidates_preserve_old_request_then_same_host_v1(): ("openai", "https://custom.test/prefix/chat/completions"), ("openai", "https://custom.test/v1/chat/completions"), ("anthropic", "https://custom.test/v1/messages"), + ("openai", "https://custom.test/v1beta/openai/chat/completions"), ] @@ -114,6 +117,83 @@ def test_route_probe_is_body_based(): assert not ep.should_probe_route(400, '{"error":"Unknown model"}') +@pytest.mark.parametrize( + ("url", "gemini"), + [ + ("https://generativelanguage.googleapis.com", True), + ("https://generativelanguage.googleapis.com/v1beta/openai/chat/completions", True), + # A proxy mirroring Google's compatibility resource is the same dialect. + ("https://gateway.ai.cloudflare.com/v1/a/g/google-ai-studio/v1beta/openai", True), + ("https://keys.example/v1beta/openai/chat/completions", True), + # Not Google's dialect: a bare vendor path segment could front an + # ordinary OpenAI route, and Vertex is a separate compatibility layer. + ("https://proxy.test/gemini/v1", False), + ("https://proxy.test/google/v1beta", False), + ("https://us-central1-aiplatform.googleapis.com/v1/projects/p/locations/l/endpoints/openapi", False), + ("https://openai.test/v1", False), + ], +) +def test_gemini_surface_predicate_covers_proxies_without_over_matching(url, gemini): + assert ep.is_gemini_openai_surface(url) is gemini + # The request policy follows the predicate, not the host string, so a proxy + # user gets the same structured-output posture as Google's own host. + assert ep.supports_structured_tool_calls(url, "gemini-3-pro") is gemini + + +def test_ambiguous_candidates_reach_the_gemini_compatibility_resource(): + # A Gemini-compat proxy configured at its bare root exposes /v1beta/openai + # and nothing the two /v1 guesses can reach. + routes = ep.endpoint_candidates("https://gemini-proxy.test", "gemini-3-pro") + assert ("openai", "https://gemini-proxy.test/v1beta/openai/chat/completions") in [ + (route.protocol, route.url) for route in routes + ] + + +def test_reasoning_off_asks_gemini_to_stop_thinking(): + # reasoning/chat_template_kwargs/thinking are all silently ignored by the + # compatibility layer; reasoning_effort is the control it actually reads. + body = {"model": "gemini-3-pro", **reasoning_cfg(False)} + ep.prepare_request_body("https://generativelanguage.googleapis.com", "gemini-3-pro", body) + assert body["reasoning_effort"] == "none" + + # An explicit effort means the call asked for thinking; leave it alone. + on = {"model": "gemini-3-pro", "reasoning_effort": "high", **reasoning_cfg(True)} + ep.prepare_request_body("https://generativelanguage.googleapis.com", "gemini-3-pro", on) + assert on["reasoning_effort"] == "high" + + # Non-Gemini endpoints keep Orb's historical reasoning-off shape untouched. + other = {"model": "m", **reasoning_cfg(False)} + ep.prepare_request_body("https://openai.test/v1", "m", other) + assert "reasoning_effort" not in other + + +def test_rejected_reasoning_effort_is_learned_and_dropped_for_the_session(): + url, model = "https://generativelanguage.googleapis.com", "gemini-3-pro" + body = {"model": model, "reasoning_effort": "xhigh"} + rejection = '{"error":{"code":400,"message":"Invalid reasoning_effort: xhigh. Valid values are: high, low, medium, none","status":"INVALID_ARGUMENT"}}' + + fix = ep.recover_from_error(url, model, body, 400, rejection) + assert fix is not None and "reasoning_effort" not in body + + # The recovery persists through the set, not the in-place pop: the client + # rebuilds the outbound body from scratch on every retry. + rebuilt = {"model": model, "reasoning_effort": "xhigh"} + ep.prepare_request_body(url, model, rebuilt) + assert "reasoning_effort" not in rebuilt + + # A learned rejection also wins over the profile's reasoning-off transform, + # which is how a model that cannot disable thinking at all settles down. + off = {"model": model, **reasoning_cfg(False)} + ep.prepare_request_body(url, model, off) + assert "reasoning_effort" not in off + + +def test_generic_400_does_not_cost_a_working_reasoning_effort(): + body = {"model": "m", "reasoning_effort": "high"} + assert ep.recover_from_error("https://openai.test/v1", "m", body, 400, '{"error":"model not found"}') is None + assert body["reasoning_effort"] == "high" + + def test_tool_schema_predicate_tracks_anthropic_and_gemini_wire_shapes(): messages = [{"role": "user", "content": "hi"}] assert LLMClient("https://api.anthropic.com").sends_tool_schemas(messages, "claude-opus-5") @@ -373,6 +453,152 @@ async def test_gemini_uses_normalized_openai_route_structured_output_and_effort( assert parse_tool_calls(events[-1]["message"]) == [{"name": "direct_scene", "arguments": {"mood": "bright"}}] +async def test_indexless_tool_call_deltas_stay_separate_calls(): + """Google's compatibility surface omits ``index`` from tool-call deltas. + + Reached on any Gemini-compat route that still sends ``tools`` -- a proxy the + profile does not claim, or a pair demoted by ``note_structured_output_ignored``. + Keying on a 0 default merged both calls into one entry whose name was the two + names concatenated and whose arguments were unparseable. + """ + lines = [ + _line( + { + "choices": [ + { + "delta": { + "tool_calls": [ + { + "id": "0", + "type": "function", + "function": {"name": "direct_scene", "arguments": '{"mood":"eerie"}'}, + } + ] + } + } + ] + } + ), + _line( + { + "choices": [ + { + "delta": { + "tool_calls": [ + { + "id": "1", + "type": "function", + "function": {"name": "editor_rewrite", "arguments": '{"text":"hi"}'}, + } + ] + } + } + ] + } + ), + _line({"choices": [{"delta": {}, "finish_reason": "tool_calls"}]}), + "data: [DONE]", + ] + events = await _run( + LLMClient("https://gemini-proxy.test/v1/chat/completions"), + _HTTP([_Response(lines=lines)]), + model="gemini-3-pro", + tools=[TOOL], + tool_choice="auto", + ) + assert parse_tool_calls(events[-1]["message"]) == [ + {"name": "direct_scene", "arguments": {"mood": "eerie"}}, + {"name": "editor_rewrite", "arguments": {"text": "hi"}}, + ] + + +async def test_indexless_argument_fragments_append_to_the_open_call(): + """Only a delta that STARTS a call opens a slot; continuations append. + + The OpenAI contract sends ``id``/``name`` on a call's first chunk alone, so + a bare ``arguments`` fragment must not be mistaken for a second call. + """ + lines = [ + _line( + { + "choices": [ + { + "delta": { + "tool_calls": [ + {"id": "0", "type": "function", "function": {"name": "direct_scene", "arguments": '{"mood"'}} + ] + } + } + ] + } + ), + _line({"choices": [{"delta": {"tool_calls": [{"function": {"arguments": ':"eerie"}'}}]}}]}), + _line({"choices": [{"delta": {}, "finish_reason": "tool_calls"}]}), + "data: [DONE]", + ] + events = await _run( + LLMClient("https://gemini-proxy.test/v1/chat/completions"), + _HTTP([_Response(lines=lines)]), + model="gemini-3-pro", + tools=[TOOL], + tool_choice="auto", + ) + assert parse_tool_calls(events[-1]["message"]) == [{"name": "direct_scene", "arguments": {"mood": "eerie"}}] + + +async def test_indexed_tool_call_deltas_are_unaffected(): + """The ordinary OpenAI shape must still be keyed by its own index.""" + lines = [ + _line( + { + "choices": [ + { + "delta": { + "tool_calls": [ + { + "index": 1, + "id": "b", + "type": "function", + "function": {"name": "editor_rewrite", "arguments": "{}"}, + } + ] + } + } + ] + } + ), + _line( + { + "choices": [ + { + "delta": { + "tool_calls": [ + { + "index": 0, + "id": "a", + "type": "function", + "function": {"name": "direct_scene", "arguments": "{}"}, + } + ] + } + } + ] + } + ), + _line({"choices": [{"delta": {}, "finish_reason": "tool_calls"}]}), + "data: [DONE]", + ] + events = await _run( + LLMClient("https://openai.test/v1/chat/completions"), + _HTTP([_Response(lines=lines)]), + model="openai-model", + tools=[TOOL], + tool_choice="auto", + ) + # Sorted by index, so the late index-0 chunk still leads. + assert [call["name"] for call in parse_tool_calls(events[-1]["message"])] == ["direct_scene", "editor_rewrite"] + + async def test_anthropic_midstream_error_uses_sanitized_llm_error(): lines = [ _line({"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "partial"}}), @@ -464,6 +690,51 @@ async def test_auto_only_tool_choice_recovery_and_process_memory(choice): assert again.requests[0]["body"]["tool_choice"] == "auto" +async def test_reasoning_effort_rejection_retries_and_is_remembered(): + """A level Orb offers but the endpoint refuses must self-heal, not fail the turn. + + Orb's picker spans none/minimal/low/medium/high/xhigh; Gemini's set is + high/low/medium/none. Learning the refusal from the body beats a hard-coded + per-provider list, which would have wrongly clamped ``minimal`` for the + months Google rejected a value its own table documented. + """ + rejection = ( + '{"error":{"code":400,"message":"Invalid reasoning_effort: xhigh. ' + 'Valid values are: high, low, medium, none","status":"INVALID_ARGUMENT"}}' + ) + openai_done = ['data: {"choices":[{"delta":{"content":"hi"},"finish_reason":"stop"}]}', "data: [DONE]"] + fake = _HTTP([_Response(400, error=rejection), _Response(lines=openai_done)]) + client = LLMClient("https://generativelanguage.googleapis.com", "key", reasoning_effort="xhigh") + + events = await _run(fake=fake, client=client, model="gemini-3-pro", **reasoning_cfg(True)) + + assert fake.requests[0]["body"]["reasoning_effort"] == "xhigh" + assert "reasoning_effort" not in fake.requests[1]["body"] + assert events[-1]["message"]["content"] == "hi" + + # Learned for the session: the next call skips the failing round-trip. + again = _HTTP([_Response(lines=openai_done)]) + await _run(client, again, model="gemini-3-pro", **reasoning_cfg(True)) + assert "reasoning_effort" not in again.requests[0]["body"] + + +async def test_gemini_model_that_cannot_disable_thinking_settles_after_one_rejection(): + """``reasoning_effort='none'`` is right for 2.5 Flash and refused by the 3 series. + + The profile asks anyway and lets the rejection teach it, the same posture as + the Anthropic sampling and thinking fields. + """ + rejection = '{"error":{"code":400,"message":"Invalid reasoning_effort: none. Valid values are: high, low, medium","status":"INVALID_ARGUMENT"}}' + openai_done = ['data: {"choices":[{"delta":{"content":"hi"},"finish_reason":"stop"}]}', "data: [DONE]"] + fake = _HTTP([_Response(400, error=rejection), _Response(lines=openai_done)]) + client = LLMClient("https://generativelanguage.googleapis.com", "key") + + await _run(client, fake, model="gemini-3-pro", **reasoning_cfg(False)) + + assert fake.requests[0]["body"]["reasoning_effort"] == "none" + assert "reasoning_effort" not in fake.requests[1]["body"] + + async def test_unrelated_failure_does_not_probe(): fake = _HTTP([_Response(404, error='{"error":"model not found"}')]) with pytest.raises(LLMCallError): @@ -479,6 +750,7 @@ def test_v1_base_url_collapses_the_duplicate_openai_candidate(): assert [(route.protocol, route.url) for route in routes] == [ ("openai", "http://localhost:1234/v1/chat/completions"), ("anthropic", "http://localhost:1234/v1/messages"), + ("openai", "http://localhost:1234/v1beta/openai/chat/completions"), ] diff --git a/tests/unit/test_model_discovery.py b/tests/unit/test_model_discovery.py index aa9b7684..cc8f716f 100644 --- a/tests/unit/test_model_discovery.py +++ b/tests/unit/test_model_discovery.py @@ -90,3 +90,27 @@ async def test_gemini_models_use_normalized_surface_and_strip_models_prefix(monk assert models == ["gemini-3-flash", "gemini-3-pro"] assert _CatalogClient.seen["url"] == "https://generativelanguage.googleapis.com/v1beta/openai/models" assert _CatalogClient.seen["headers"] == {"Authorization": "Bearer gemini-key"} + + +@pytest.mark.asyncio +async def test_gemini_proxy_catalogue_is_normalized_like_googles_own(monkeypatch): + # The prefix is a property of the dialect, not of Google's hostname: a proxy + # mirroring /v1beta/openai relays the same ``models/``-prefixed ids, and + # leaving them in put an unusable-looking id in the picker. + _CatalogClient.payload = {"data": [{"id": "models/gemini-3-pro"}]} + _CatalogClient.seen = {} + monkeypatch.setattr(client_module.httpx, "AsyncClient", _CatalogClient) + + models = await LLMClient("https://keys.example/v1beta/openai", "k").list_models() + + assert models == ["gemini-3-pro"] + assert _CatalogClient.seen["url"] == "https://keys.example/v1beta/openai/models" + + +@pytest.mark.asyncio +async def test_non_gemini_catalogue_keeps_a_models_prefixed_id(monkeypatch): + _CatalogClient.payload = {"data": [{"id": "models/local-thing"}]} + _CatalogClient.seen = {} + monkeypatch.setattr(client_module.httpx, "AsyncClient", _CatalogClient) + + assert await LLMClient("http://localhost:8080/v1", "").list_models() == ["models/local-thing"] From 7d9592460034db2f26d8af76d24ab5b1eb62e78a Mon Sep 17 00:00:00 2001 From: Chi Date: Sat, 5 Sep 2026 11:48:07 +0700 Subject: [PATCH 4/4] name-free automatic endpoint handling --- backend/inference/anthropic.py | 25 ++--- backend/inference/client.py | 79 +++++++++----- backend/inference/endpoint_profiles.py | 87 ++++++--------- docs/architecture/endpoints.md | 62 +++++------ .../test_endpoint_transport_passes.py | 2 +- tests/unit/test_endpoint_protocols.py | 103 +++++++++++++----- tests/unit/test_model_discovery.py | 58 +++++++++- 7 files changed, 251 insertions(+), 165 deletions(-) diff --git a/backend/inference/anthropic.py b/backend/inference/anthropic.py index beff524e..c2f54c5e 100644 --- a/backend/inference/anthropic.py +++ b/backend/inference/anthropic.py @@ -15,24 +15,14 @@ EXTRA_BODY_ALLOWED: frozenset[str] = frozenset({"metadata", "service_tier"}) DEFAULT_MAX_TOKENS = 4096 -# Current families whose Messages endpoints reject the old sampling controls. -# Unknown proxy model names are tried once and learned from a provider rejection. -_NO_SAMPLING_MARKERS = ( - "opus-5", - "opus-4-8", - "opus-4.8", - "opus-4-7", - "opus-4.7", - "sonnet-5", - "fable-5", -) - +# Sampling support is a capability of the concrete endpoint/model pair, not +# something that can be inferred from a provider-owned model id. Send the +# caller's controls optimistically and remember an explicit rejection. _SAMPLING_UNSUPPORTED: set[tuple[str, str]] = set() -# Adaptive thinking and ``output_config.effort`` are 4.6-and-later fields. An -# older family behind a proxy (Haiku 4.5 and earlier want the retired -# ``budget_tokens`` shape) rejects them outright, so -- as with sampling -- -# they go out once and are learned from the provider's rejection. +# Some Messages implementations accept an older ``budget_tokens`` shape and +# reject adaptive thinking or ``output_config.effort``. As with sampling, the +# modern fields go out once and an explicit rejection is remembered. _THINKING_UNSUPPORTED: set[tuple[str, str]] = set() @@ -194,8 +184,7 @@ def translate_tool_choice(choice: object) -> dict[str, Any] | None: def _sampling_allowed(endpoint_url: str, model: str) -> bool: - low = model.lower().replace("_", "-") - return (endpoint_url, model) not in _SAMPLING_UNSUPPORTED and not any(marker in low for marker in _NO_SAMPLING_MARKERS) + return (endpoint_url, model) not in _SAMPLING_UNSUPPORTED def build_request_body( diff --git a/backend/inference/client.py b/backend/inference/client.py index 7e3f9280..1b9d5ef1 100644 --- a/backend/inference/client.py +++ b/backend/inference/client.py @@ -277,38 +277,57 @@ def _headers_for(self, auth_family: endpoint_profiles.AuthFamily) -> dict: return headers async def list_models(self) -> list[str]: - """Return model ids advertised by an OpenAI-compatible ``GET /models``. + """Return model ids advertised by a compatible ``GET /models``. - Discovery uses the same bearer authentication and endpoint proxy as - generation, but a short finite timeout: unlike a completion, this is a - small non-streaming settings request and should fail back to Orb's - editable model-name field promptly. + Explicit resource shapes have one sibling catalogue. For an ambiguous + base, discovery walks the same bounded route/auth candidates as + generation and caches the first catalogue that satisfies the shared + ``data[].id`` contract. It uses a short finite timeout so failure still + falls back to Orb's editable model-name field promptly. """ - route = endpoint_profiles.resolve_endpoint(self.base_url) - url = route.models_url + routes = endpoint_profiles.endpoint_candidates(self.base_url) + seen: set[tuple[str, endpoint_profiles.AuthFamily]] = set() + contract_error: ValueError | None = None + last_response: httpx.Response | None = None async with httpx.AsyncClient(timeout=20.0, proxy=self.proxy, follow_redirects=True) as client: - response = await client.get(url, headers=self._headers_for(route.auth_family)) - response.raise_for_status() - try: - payload = response.json() - except ValueError as exc: - raise ValueError("Endpoint returned a non-JSON models response") from exc - - data = payload.get("data") if isinstance(payload, dict) else None - if not isinstance(data, list): - raise ValueError("Endpoint models response does not contain a data list") - - model_ids: set[str] = set() - for item in data: - model_id = item.get("id") if isinstance(item, dict) else None - if isinstance(model_id, str) and model_id.strip(): - normalized = model_id.strip() - # Google lists ids as ``models/gemini-...``; the generation - # resource takes either form, so strip it for the picker. - if normalized.startswith("models/") and endpoint_profiles.is_gemini_openai_surface(route.url): - normalized = normalized.removeprefix("models/") - model_ids.add(normalized) - return sorted(model_ids, key=str.casefold) + for route in routes: + candidate = (route.models_url, route.auth_family) + if candidate in seen: + continue + seen.add(candidate) + response = await client.get(route.models_url, headers=self._headers_for(route.auth_family)) + if response.status_code >= 400: + last_response = response + continue + try: + payload = response.json() + except ValueError: + contract_error = ValueError("Endpoint returned a non-JSON models response") + continue + + data = payload.get("data") if isinstance(payload, dict) else None + if not isinstance(data, list): + contract_error = ValueError("Endpoint models response does not contain a data list") + continue + + endpoint_profiles.note_successful_route(self.base_url, "", route) + model_ids: set[str] = set() + for item in data: + model_id = item.get("id") if isinstance(item, dict) else None + if isinstance(model_id, str) and model_id.strip(): + normalized = model_id.strip() + # This compatibility resource lists ids with a + # ``models/`` prefix that generation does not need. + if normalized.startswith("models/") and endpoint_profiles.is_gemini_openai_surface(route.url): + normalized = normalized.removeprefix("models/") + model_ids.add(normalized) + return sorted(model_ids, key=str.casefold) + + if contract_error is not None: + raise contract_error + if last_response is not None: + last_response.raise_for_status() + raise ValueError("Endpoint did not expose a models response") def _server_root(self) -> str: """Server root for llama.cpp native endpoints (/completion, /apply-template, @@ -797,7 +816,7 @@ async def consume_anthropic(resp, url: str) -> AsyncIterator[dict]: recovery_count += 1 logger.warning("LLM recovery: %s", fix) continue - auths = endpoint_profiles.auth_families(route, self.base_url, model, resp.status_code) + auths = endpoint_profiles.auth_families(route, resp.status_code, err_text) if not auth_retried and len(auths) > 1: auth_family = auths[1] auth_retried = True diff --git a/backend/inference/endpoint_profiles.py b/backend/inference/endpoint_profiles.py index 561ed890..b7cbe6c4 100644 --- a/backend/inference/endpoint_profiles.py +++ b/backend/inference/endpoint_profiles.py @@ -180,30 +180,17 @@ def _deepseek_coerce_tool_choice_when_thinking(body: dict) -> str | None: } -# Google's OpenAI compatibility surface. Matched by a predicate rather than a -# PROFILES substring because the same wire dialect is reached through more than -# one host: Google's own, and any proxy that mirrors the ``/v1beta/openai`` -# resource shape (Cloudflare AI Gateway, self-hosted key-pool proxies). Matching -# on the host string alone left every proxy user without the request policy, -# the reasoning translation and the catalogue normalization below. -# -# Deliberately NOT matched: a bare ``gemini`` or ``google`` path segment. Those -# appear on gateways whose route is an ordinary OpenAI one where native tool -# calls work, and ``structured_tool_calls`` would withhold ``tools`` from them -# for no reason. Vertex's ``endpoints/openapi`` surface is likewise excluded -- -# it is a different compatibility layer with its own tool behavior. -_GEMINI_HOST = "generativelanguage.googleapis.com" +# This OpenAI compatibility dialect is identified by its resource shape, not a +# provider hostname or a model id. Any proxy exposing the same path therefore +# receives the same request policy and catalogue normalization. _GEMINI_OPENAI_PATH = "/v1beta/openai" def is_gemini_openai_surface(url: str) -> bool: - """Whether *url* addresses Google's OpenAI-compatible Gemini dialect.""" + """Whether *url* has the ``/v1beta/openai`` compatibility shape.""" parsed = _parsed_http_url(url) if parsed is None: return _GEMINI_OPENAI_PATH in _clean_url(url).lower() - host = (parsed.hostname or "").lower() - if host == _GEMINI_HOST or host.endswith(f".{_GEMINI_HOST}"): - return True path = parsed.path.rstrip("/").lower() return path == _GEMINI_OPENAI_PATH or f"{_GEMINI_OPENAI_PATH}/" in f"{path}/" @@ -311,62 +298,49 @@ def _base_route(protocol: Protocol, base_url: str, *, authoritative: bool = Fals def _deterministic_route(endpoint_url: str) -> EndpointRoute: - """Resolve explicit resources and strong provider hints without probing.""" + """Resolve explicit resource shapes without provider/model-name guesses.""" clean = _clean_url(endpoint_url) parsed = _parsed_http_url(clean) low = clean.lower() path = parsed.path.rstrip("/").lower() if parsed is not None else low - # Full resource URLs are user intent and win over host heuristics. + # Full resource URLs are user intent. if path.endswith("/chat/completions"): return _resource_route("openai", clean, authoritative=True) if path.endswith("/messages"): return _resource_route("anthropic", clean, authoritative=True) - host = (parsed.hostname or "").lower() if parsed is not None else "" - if host == _GEMINI_HOST or host.endswith(f".{_GEMINI_HOST}"): - base = _replace_path(parsed, _GEMINI_OPENAI_PATH) if parsed is not None else clean - return _base_route("openai", base, authoritative=True) - - segments = [segment for segment in path.split("/") if segment] - if host == "api.anthropic.com" or host.endswith(".api.anthropic.com"): - if not segments: - base = _replace_path(parsed, "/v1") if parsed is not None else f"{clean}/v1" - else: - base = clean - return _base_route("anthropic", base, authoritative=True) - - # Proxy prefixes such as /anthropic or /providers/anthropic/v1 are strong - # enough to select native Messages without replaying a prompt elsewhere. - if "anthropic" in segments: - base = f"{clean}/v1" if segments[-1] == "anthropic" else clean - return _base_route("anthropic", base, authoritative=True) + if is_gemini_openai_surface(clean): + return _base_route("openai", clean, authoritative=True) return _base_route("openai", clean) def resolve_endpoint(endpoint_url: str, model: str = "") -> EndpointRoute: """Return the cached or deterministic route for one configured endpoint.""" - return _RESOLVED_ROUTES.get((endpoint_url, model)) or _deterministic_route(endpoint_url) + return ( + _RESOLVED_ROUTES.get((endpoint_url, model)) + or _RESOLVED_ROUTES.get((endpoint_url, "")) + or _deterministic_route(endpoint_url) + ) def endpoint_candidates(endpoint_url: str, model: str = "") -> list[EndpointRoute]: """Return bounded same-host routes in request order. - Explicit resources and provider hints are authoritative. Ambiguous URLs keep - Orb's historical ``{configured}/chat/completions`` request first, followed - by the conventional host-root OpenAI and Anthropic v1 resources, then - Google's ``/v1beta/openai`` compatibility resource. Candidates are only + Explicit resource shapes are authoritative. Ambiguous URLs keep Orb's + historical ``{configured}/chat/completions`` request first, followed by its + Messages sibling and the conventional host-root resources. Candidates are only attempted when :func:`should_probe_route` recognizes the response body as a route mismatch. - The Gemini candidate is last because it is the narrowest guess of the four + The ``/v1beta/openai`` candidate is last because it is the narrowest guess and costs a further prompt upload; it is still worth making, because a Gemini-compat proxy configured at its bare root exposes that resource and no other, and the two ``/v1`` guesses ahead of it cannot reach it. """ primary = resolve_endpoint(endpoint_url, model) - if primary.authoritative or (endpoint_url, model) in _RESOLVED_ROUTES: + if primary.authoritative or (endpoint_url, model) in _RESOLVED_ROUTES or (endpoint_url, "") in _RESOLVED_ROUTES: return [primary] parsed = _parsed_http_url(endpoint_url) if parsed is None: @@ -374,6 +348,7 @@ def endpoint_candidates(endpoint_url: str, model: str = "") -> list[EndpointRout root = urlunsplit((parsed.scheme, parsed.netloc, "", "", "")) candidates = [ primary, + _base_route("anthropic", endpoint_url), _base_route("openai", f"{root}/v1"), _base_route("anthropic", f"{root}/v1"), _base_route("openai", f"{root}{_GEMINI_OPENAI_PATH}"), @@ -395,7 +370,6 @@ def note_successful_route(endpoint_url: str, model: str, route: EndpointRoute) - def should_probe_route(status: int, text: str) -> bool: """Whether an error body specifically identifies an HTTP route mismatch.""" - del status # Deliberately a body fact; status-only routing is unsafe. low = text.lower() markers = ( "cannot post /", @@ -405,16 +379,20 @@ def should_probe_route(status: int, text: str) -> bool: "unsupported endpoint", "invalid url (post", ) - return any(marker in low for marker in markers) + native_not_found = status == 404 and "not_found_error" in low and "model" not in low + return any(marker in low for marker in markers) or native_not_found -def auth_families(route: EndpointRoute, endpoint_url: str, model: str, status: int | None = None) -> tuple[AuthFamily, ...]: - """Return primary auth and, on supported 401/403 evidence, its peer.""" +def auth_families(route: EndpointRoute, status: int | None = None, text: str = "") -> tuple[AuthFamily, ...]: + """Return primary auth and a bounded, evidence-based alternative.""" primary = route.auth_family if status not in {401, 403}: return (primary,) - evidence = "claude" in model.lower() or "anthropic" in endpoint_url.lower() or route.protocol == "anthropic" - if not evidence: + # A Messages route defines its auth default. Ambiguous routes may need the + # other family before the server will reveal that the resource is wrong; + # an explicit OpenAI route only retries when the response names the native + # API-key header. None of these signals depends on provider or model names. + if route.protocol != "anthropic" and route.authoritative and "x-api-key" not in text.lower(): return (primary,) other: AuthFamily = "bearer" if primary == "anthropic" else "anthropic" return (primary, other) @@ -483,13 +461,14 @@ def profile_for(endpoint_url: str, model: str = "") -> ModelProfile | None: A blank *model* falls through to the endpoint default. An unmatched URL returns ``None`` — the body is sent unchanged (local / unknown backends). - Gemini is resolved by :func:`is_gemini_openai_surface` rather than by a - ``PROFILES`` substring so that a proxy mirroring Google's compatibility - resource is given the same request policy as Google's own host. + The ``/v1beta/openai`` dialect is resolved by + :func:`is_gemini_openai_surface` rather than by a ``PROFILES`` substring, + so every proxy mirroring the resource gets the same request policy. """ if not endpoint_url: return None - if is_gemini_openai_surface(endpoint_url): + resolved_url = resolve_endpoint(endpoint_url, model).url + if is_gemini_openai_surface(endpoint_url) or is_gemini_openai_surface(resolved_url): return _GEMINI_PROFILE haystack = endpoint_url.lower() for needle, models in PROFILES.items(): diff --git a/docs/architecture/endpoints.md b/docs/architecture/endpoints.md index a9d31612..b778bd64 100644 --- a/docs/architecture/endpoints.md +++ b/docs/architecture/endpoints.md @@ -14,41 +14,38 @@ A configured endpoint may be a versioned base or a full generation resource. | `https://host/v1` | `https://host/v1/chat/completions` | | `https://host/v1/chat/completions` | Used exactly as entered | | `https://host/v1/messages` | Used exactly as entered with Anthropic Messages | -| `https://api.anthropic.com` | `https://api.anthropic.com/v1/messages` | -| `https://generativelanguage.googleapis.com` | `https://generativelanguage.googleapis.com/v1beta/openai/chat/completions` | - -An `anthropic` path segment is also a strong native-protocol hint. For example, -`https://gateway.example/providers/anthropic/v1` resolves to the sibling -`messages` resource. Full `chat/completions` and `messages` resource URLs are -authoritative even when their host would normally imply another protocol. - -Gemini is recognized by dialect, not by hostname. Google's host and any URL on -the `/v1beta/openai` resource path — a compatibility proxy, an AI gateway — -take the same request policy, reasoning translation, and catalogue -normalization. A bare `gemini` or `google` path segment deliberately does not -qualify: those appear on gateways whose route is an ordinary OpenAI one where -native tool calls work. Vertex AI's `endpoints/openapi` surface is a separate -compatibility layer and is treated as a plain OpenAI endpoint. +| `https://host/v1beta/openai` | `https://host/v1beta/openai/chat/completions` | + +Full `chat/completions` and `messages` resource URLs are authoritative. Bare and +versioned bases remain ambiguous: provider names in a hostname or path, and +family names in a model id, never select a protocol. + +The `/v1beta/openai` compatibility dialect is likewise recognized from its +resource path, not its hostname. Any proxy or gateway exposing that shape takes +the same request policy, reasoning translation, and catalogue normalization. Model discovery uses the sibling `models` resource and the matching auth family. +For an ambiguous base it walks the same route candidates until a catalogue +satisfies the shared `data[].id` contract, then caches that route for generation. OpenAI and Gemini routes use Bearer authentication. Native Anthropic routes use `x-api-key` and `anthropic-version`. Extra headers may replace those defaults case-insensitively. ## Automatic detection and probing -Official hosts and path hints are deterministic and do not probe. An ambiguous -custom URL preserves Orb's historical OpenAI request first. Only when the +Explicit resource shapes are deterministic and do not probe. An ambiguous URL +preserves Orb's historical OpenAI request first. Only when the pre-stream response body specifically identifies a route mismatch does Orb try, -on the same host, these conventional resources: +on the same host, these candidate resources: 1. the configured base plus `chat/completions`; -2. host-root `/v1/chat/completions`; -3. host-root `/v1/messages`; -4. host-root `/v1beta/openai/chat/completions`. +2. the configured base plus `messages`; +3. host-root `/v1/chat/completions`; +4. host-root `/v1/messages`; +5. host-root `/v1beta/openai/chat/completions`. -The last is Google's compatibility resource, which a Gemini-compat proxy -configured at its bare root exposes and the two `/v1` guesses cannot reach. +The last is the beta OpenAI compatibility resource, which some proxies expose +at their bare root and the two `/v1` guesses cannot reach. The HTTP status alone never starts probing: a 400 or 404 can describe a bad model, schema, or tool choice rather than a bad route. Known request recovery @@ -56,7 +53,7 @@ runs first. No route is changed after the first streamed delta, and local text-completion calls do not enter this chat probing path. Probing replays the complete POST, so a first request can upload the prompt up -to four times. Orb chooses that trade-off because native compatibility proxies +to five times. Orb chooses that trade-off because native compatibility proxies do not expose a reliable discovery contract. A successful protocol and path is cached per configured URL and model for the life of the backend process; configured settings are never rewritten. @@ -67,9 +64,10 @@ names the field as invalid drops it for one retry and for the rest of the session. Providers' accepted sets move, so this is learned from the response rather than held as a per-provider list. -A 401 or 403 can trigger one alternate Anthropic/Bearer auth attempt only when -the model name, path, or resolved protocol supplies Claude/Anthropic evidence. -This retry is independently bounded and never changes hosts. +A 401 or 403 can trigger one alternate native/Bearer auth attempt for a Messages +route or an ambiguous route. An explicit OpenAI resource only retries when the +response names the native `x-api-key` header. This retry is independently +bounded and never changes hosts; provider and model names are not evidence. ## Provider request behavior @@ -82,11 +80,11 @@ that escape hatch. A missing `max_tokens` defaults to 4096. Reasoning-on maps to adaptive thinking with summarized display, and supported effort levels map to `output_config.effort`. Reasoning-off omits `thinking`. -Current Claude families that reject temperature, top-p, and top-k omit them; -unknown proxy model names try them once, learn from a specific rejection, and -omit them for later calls. `min_p`, repetition penalties, and logprobs are never -sent to Anthropic. Consequently, Document mode's per-token steering is not -available on native Anthropic endpoints. +Sampling controls are sent optimistically. A specific rejection teaches Orb to +omit them for later calls to that endpoint/model pair; names never stand in for +capability evidence. `min_p`, repetition penalties, and logprobs are never sent +to Anthropic. Consequently, Document mode's per-token steering is not available +on native Anthropic endpoints. Some routed models accept only `tool_choice="auto"`. This is distinct from a provider that rejects `tool_choice` entirely: Orb rewrites `none`, `required`, diff --git a/tests/integration/test_endpoint_transport_passes.py b/tests/integration/test_endpoint_transport_passes.py index a3dd2c65..853f5934 100644 --- a/tests/integration/test_endpoint_transport_passes.py +++ b/tests/integration/test_endpoint_transport_passes.py @@ -134,7 +134,7 @@ async def _complete(client: LLMClient, model: str, choice: dict | str) -> list[d [ ("openai", "https://openai.test/v1/chat/completions", "openai-model"), ("anthropic", "https://api.anthropic.com/v1/messages", "claude-haiku-4-5"), - ("gemini", "https://generativelanguage.googleapis.com", "gemini-3-pro"), + ("gemini", "https://generativelanguage.googleapis.com/v1beta/openai", "gemini-3-pro"), ], ) async def test_director_writer_editor_calls_cross_protocol_boundary(provider, endpoint, model): diff --git a/tests/unit/test_endpoint_protocols.py b/tests/unit/test_endpoint_protocols.py index 18dbc248..bff82772 100644 --- a/tests/unit/test_endpoint_protocols.py +++ b/tests/unit/test_endpoint_protocols.py @@ -61,22 +61,16 @@ def _clear_learned_state(): "https://proxy.test/prefix/v1/models", ), ( - "https://api.anthropic.com", - "anthropic", - "https://api.anthropic.com/v1/messages", - "https://api.anthropic.com/v1/models", + "https://proxy.test/prefix/v1beta/openai", + "openai", + "https://proxy.test/prefix/v1beta/openai/chat/completions", + "https://proxy.test/prefix/v1beta/openai/models", ), ( - "https://proxy.test/providers/anthropic/v1", + "https://native.test/v1/messages", "anthropic", - "https://proxy.test/providers/anthropic/v1/messages", - "https://proxy.test/providers/anthropic/v1/models", - ), - ( - "https://generativelanguage.googleapis.com/v99/wrong", - "openai", - "https://generativelanguage.googleapis.com/v1beta/openai/chat/completions", - "https://generativelanguage.googleapis.com/v1beta/openai/models", + "https://native.test/v1/messages", + "https://native.test/v1/models", ), ], ) @@ -88,16 +82,31 @@ def test_deterministic_resolution(configured, protocol, url, models): assert route.authoritative -def test_explicit_resource_is_authoritative_even_on_provider_host(): +def test_explicit_resource_is_authoritative_regardless_of_host_name(): route = ep.resolve_endpoint("https://api.anthropic.com/v1/chat/completions") assert route.protocol == "openai" assert route.url.endswith("/v1/chat/completions") +@pytest.mark.parametrize( + "configured", + [ + "https://api.anthropic.com", + "https://proxy.test/providers/anthropic/v1", + "https://claude.example/v1", + ], +) +def test_provider_and_model_names_do_not_select_a_protocol(configured): + route = ep.resolve_endpoint(configured, "claude-opus-999") + assert route.protocol == "openai" + assert not route.authoritative + + def test_ambiguous_candidates_preserve_old_request_then_same_host_v1(): routes = ep.endpoint_candidates("https://custom.test/prefix", "m") assert [(route.protocol, route.url) for route in routes] == [ ("openai", "https://custom.test/prefix/chat/completions"), + ("anthropic", "https://custom.test/prefix/messages"), ("openai", "https://custom.test/v1/chat/completions"), ("anthropic", "https://custom.test/v1/messages"), ("openai", "https://custom.test/v1beta/openai/chat/completions"), @@ -113,14 +122,17 @@ def test_malformed_url_degrades_without_probe_candidates(): def test_route_probe_is_body_based(): assert ep.should_probe_route(400, '{"error":"Unknown endpoint"}') assert ep.should_probe_route(404, "Cannot POST /chat/completions") + assert ep.should_probe_route(404, '{"type":"error","error":{"type":"not_found_error"}}') assert not ep.should_probe_route(404, '{"error":"model not found"}') + assert not ep.should_probe_route(404, '{"error":{"type":"not_found_error","message":"model not found"}}') assert not ep.should_probe_route(400, '{"error":"Unknown model"}') @pytest.mark.parametrize( ("url", "gemini"), [ - ("https://generativelanguage.googleapis.com", True), + # A hostname alone is not protocol evidence. + ("https://generativelanguage.googleapis.com", False), ("https://generativelanguage.googleapis.com/v1beta/openai/chat/completions", True), # A proxy mirroring Google's compatibility resource is the same dialect. ("https://gateway.ai.cloudflare.com/v1/a/g/google-ai-studio/v1beta/openai", True), @@ -153,12 +165,12 @@ def test_reasoning_off_asks_gemini_to_stop_thinking(): # reasoning/chat_template_kwargs/thinking are all silently ignored by the # compatibility layer; reasoning_effort is the control it actually reads. body = {"model": "gemini-3-pro", **reasoning_cfg(False)} - ep.prepare_request_body("https://generativelanguage.googleapis.com", "gemini-3-pro", body) + ep.prepare_request_body("https://compat.test/v1beta/openai", "gemini-3-pro", body) assert body["reasoning_effort"] == "none" # An explicit effort means the call asked for thinking; leave it alone. on = {"model": "gemini-3-pro", "reasoning_effort": "high", **reasoning_cfg(True)} - ep.prepare_request_body("https://generativelanguage.googleapis.com", "gemini-3-pro", on) + ep.prepare_request_body("https://compat.test/v1beta/openai", "gemini-3-pro", on) assert on["reasoning_effort"] == "high" # Non-Gemini endpoints keep Orb's historical reasoning-off shape untouched. @@ -168,7 +180,7 @@ def test_reasoning_off_asks_gemini_to_stop_thinking(): def test_rejected_reasoning_effort_is_learned_and_dropped_for_the_session(): - url, model = "https://generativelanguage.googleapis.com", "gemini-3-pro" + url, model = "https://compat.test/v1beta/openai", "gemini-3-pro" body = {"model": model, "reasoning_effort": "xhigh"} rejection = '{"error":{"code":400,"message":"Invalid reasoning_effort: xhigh. Valid values are: high, low, medium, none","status":"INVALID_ARGUMENT"}}' @@ -196,8 +208,8 @@ def test_generic_400_does_not_cost_a_working_reasoning_effort(): def test_tool_schema_predicate_tracks_anthropic_and_gemini_wire_shapes(): messages = [{"role": "user", "content": "hi"}] - assert LLMClient("https://api.anthropic.com").sends_tool_schemas(messages, "claude-opus-5") - assert not LLMClient("https://generativelanguage.googleapis.com").sends_tool_schemas(messages, "gemini-3-pro") + assert LLMClient("https://native.test/v1/messages").sends_tool_schemas(messages, "opaque-model") + assert not LLMClient("https://compat.test/v1beta/openai").sends_tool_schemas(messages, "opaque-model") def test_translate_messages_system_images_tools_and_coalescing(): @@ -293,7 +305,7 @@ def test_anthropic_body_allowlist_tools_choices_reasoning_and_sampling(): assert forbidden not in body -def test_current_claude_sampling_is_withheld_and_reasoning_off_omits_thinking(): +def test_sampling_support_is_not_inferred_from_model_name(): body = anthropic.build_request_body( { "messages": [], @@ -305,7 +317,7 @@ def test_current_claude_sampling_is_withheld_and_reasoning_off_omits_thinking(): "https://api.anthropic.com", "claude-opus-5-20260801", ) - assert "temperature" not in body and "top_p" not in body and "top_k" not in body + assert body["temperature"] == 0.8 and body["top_p"] == 0.95 and body["top_k"] == 40 assert "thinking" not in body and "output_config" not in body @@ -435,7 +447,7 @@ async def test_gemini_uses_normalized_openai_route_structured_output_and_effort( "data: [DONE]", ] fake = _HTTP([_Response(lines=lines)]) - client = LLMClient("https://generativelanguage.googleapis.com", "key", reasoning_effort="high") + client = LLMClient("https://generativelanguage.googleapis.com/v1beta/openai", "key", reasoning_effort="high") events = await _run( client, fake, @@ -613,16 +625,24 @@ async def test_anthropic_midstream_error_uses_sanitized_llm_error(): async def test_ambiguous_endpoint_probes_bounded_routes_and_caches_success(): wrong = '{"error":"Cannot POST /prefix/chat/completions"}' - wrong_v1 = '{"error":"route not found"}' + wrong_route = '{"error":"route not found"}' success = [ _line({"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "hi"}}), _line({"type": "message_stop"}), ] - fake = _HTTP([_Response(404, error=wrong), _Response(404, error=wrong_v1), _Response(lines=success)]) + fake = _HTTP( + [ + _Response(404, error=wrong), + _Response(404, error=wrong_route), + _Response(404, error=wrong_route), + _Response(lines=success), + ] + ) client = LLMClient("https://custom.test/prefix") events = await _run(client, fake, model="claude-proxy") assert [request["url"] for request in fake.requests] == [ "https://custom.test/prefix/chat/completions", + "https://custom.test/prefix/messages", "https://custom.test/v1/chat/completions", "https://custom.test/v1/messages", ] @@ -643,10 +663,35 @@ async def test_auth_family_retry_is_evidence_gated_and_bounded(): assert "x-api-key" not in fake.requests[1]["headers"] -async def test_openai_auth_rejection_without_claude_evidence_is_not_retried(): +async def test_ambiguous_endpoint_detects_messages_dialect_without_name_hints(): + not_found = '{"type":"error","error":{"type":"not_found_error","message":"Not Found"}}' + success = [_line({"type": "message_stop"})] + fake = _HTTP( + [ + _Response(401, error='{"error":"authentication required"}'), + _Response(404, error=not_found), + _Response(lines=success), + ] + ) + client = LLMClient("https://opaque.test", "key") + + await _run(client, fake, model="model-7") + + assert [request["url"] for request in fake.requests] == [ + "https://opaque.test/chat/completions", + "https://opaque.test/chat/completions", + "https://opaque.test/messages", + ] + assert fake.requests[0]["headers"] == {"Authorization": "Bearer key"} + assert fake.requests[1]["headers"]["x-api-key"] == "key" + assert fake.requests[2]["headers"]["x-api-key"] == "key" + assert ep.resolve_endpoint(client.base_url, "model-7").protocol == "anthropic" + + +async def test_authoritative_openai_auth_rejection_without_header_evidence_is_not_retried(): fake = _HTTP([_Response(401, error='{"error":"bad auth"}')]) with pytest.raises(LLMCallError): - await _run(LLMClient("https://custom.test/v1", "key"), fake, model="ordinary-model") + await _run(LLMClient("https://custom.test/v1/chat/completions", "key"), fake, model="ordinary-model") assert len(fake.requests) == 1 @@ -704,7 +749,7 @@ async def test_reasoning_effort_rejection_retries_and_is_remembered(): ) openai_done = ['data: {"choices":[{"delta":{"content":"hi"},"finish_reason":"stop"}]}', "data: [DONE]"] fake = _HTTP([_Response(400, error=rejection), _Response(lines=openai_done)]) - client = LLMClient("https://generativelanguage.googleapis.com", "key", reasoning_effort="xhigh") + client = LLMClient("https://generativelanguage.googleapis.com/v1beta/openai", "key", reasoning_effort="xhigh") events = await _run(fake=fake, client=client, model="gemini-3-pro", **reasoning_cfg(True)) @@ -727,7 +772,7 @@ async def test_gemini_model_that_cannot_disable_thinking_settles_after_one_rejec rejection = '{"error":{"code":400,"message":"Invalid reasoning_effort: none. Valid values are: high, low, medium","status":"INVALID_ARGUMENT"}}' openai_done = ['data: {"choices":[{"delta":{"content":"hi"},"finish_reason":"stop"}]}', "data: [DONE]"] fake = _HTTP([_Response(400, error=rejection), _Response(lines=openai_done)]) - client = LLMClient("https://generativelanguage.googleapis.com", "key") + client = LLMClient("https://generativelanguage.googleapis.com/v1beta/openai", "key") await _run(client, fake, model="gemini-3-pro", **reasoning_cfg(False)) diff --git a/tests/unit/test_model_discovery.py b/tests/unit/test_model_discovery.py index cc8f716f..bda797ad 100644 --- a/tests/unit/test_model_discovery.py +++ b/tests/unit/test_model_discovery.py @@ -27,6 +27,26 @@ async def get(self, url, *, headers): return httpx.Response(200, json=self.payload, request=request) +class _ProbingCatalogClient: + responses: list[tuple[int, object]] = [] + requests: list[dict] = [] + + def __init__(self, **_kwargs): + pass + + async def __aenter__(self): + return self + + async def __aexit__(self, *_args): + return None + + async def get(self, url, *, headers): + self.requests.append({"url": url, "headers": headers}) + status, payload = self.responses.pop(0) + request = httpx.Request("GET", url) + return httpx.Response(status, json=payload, request=request) + + @pytest.mark.asyncio async def test_list_models_uses_openai_contract_auth_and_proxy(monkeypatch): _CatalogClient.payload = { @@ -85,7 +105,7 @@ async def test_gemini_models_use_normalized_surface_and_strip_models_prefix(monk _CatalogClient.seen = {} monkeypatch.setattr(client_module.httpx, "AsyncClient", _CatalogClient) - models = await LLMClient("https://generativelanguage.googleapis.com", "gemini-key").list_models() + models = await LLMClient("https://generativelanguage.googleapis.com/v1beta/openai", "gemini-key").list_models() assert models == ["gemini-3-flash", "gemini-3-pro"] assert _CatalogClient.seen["url"] == "https://generativelanguage.googleapis.com/v1beta/openai/models" @@ -114,3 +134,39 @@ async def test_non_gemini_catalogue_keeps_a_models_prefixed_id(monkeypatch): monkeypatch.setattr(client_module.httpx, "AsyncClient", _CatalogClient) assert await LLMClient("http://localhost:8080/v1", "").list_models() == ["models/local-thing"] + + +@pytest.mark.asyncio +async def test_ambiguous_catalogue_detection_uses_no_provider_or_model_names(monkeypatch): + from backend.inference import endpoint_profiles + + endpoint_profiles._RESOLVED_ROUTES.clear() + _ProbingCatalogClient.requests = [] + _ProbingCatalogClient.responses = [ + (404, {"error": "not found"}), + (404, {"error": "not found"}), + (401, {"error": "wrong auth"}), + (200, {"data": [{"id": "model-7"}]}), + ] + monkeypatch.setattr(client_module.httpx, "AsyncClient", _ProbingCatalogClient) + + client = LLMClient("https://opaque.test", "secret-key") + models = await client.list_models() + + assert models == ["model-7"] + assert _ProbingCatalogClient.requests == [ + {"url": "https://opaque.test/models", "headers": {"Authorization": "Bearer secret-key"}}, + { + "url": "https://opaque.test/models", + "headers": {"x-api-key": "secret-key", "anthropic-version": "2023-06-01"}, + }, + {"url": "https://opaque.test/v1/models", "headers": {"Authorization": "Bearer secret-key"}}, + { + "url": "https://opaque.test/v1/models", + "headers": {"x-api-key": "secret-key", "anthropic-version": "2023-06-01"}, + }, + ] + assert endpoint_profiles.resolve_endpoint(client.base_url, "model-7").protocol == "anthropic" + assert endpoint_profiles.endpoint_candidates(client.base_url, "model-7") == [ + endpoint_profiles.resolve_endpoint(client.base_url, "model-7") + ]