diff --git a/backend/inference/anthropic.py b/backend/inference/anthropic.py new file mode 100644 index 00000000..c2f54c5e --- /dev/null +++ b/backend/inference/anthropic.py @@ -0,0 +1,268 @@ +"""Translate Orb's OpenAI-shaped chat contract to Anthropic Messages.""" + +from __future__ import annotations + +import base64 +import json +from collections.abc import Mapping, Sequence +from typing import Any + +from .tool_registry import strictify_schema + +# Anthropic rejects unknown top-level fields. These are the only user-provided +# extra_body keys accepted on a native Messages route; OpenAI-shaped escape +# hatches therefore cannot turn an otherwise-valid request into a hard 400. +EXTRA_BODY_ALLOWED: frozenset[str] = frozenset({"metadata", "service_tier"}) +DEFAULT_MAX_TOKENS = 4096 + +# Sampling support is a capability of the concrete endpoint/model pair, not +# something that can be inferred from a provider-owned model id. Send the +# caller's controls optimistically and remember an explicit rejection. +_SAMPLING_UNSUPPORTED: set[tuple[str, str]] = set() + +# Some Messages implementations accept an older ``budget_tokens`` shape and +# reject adaptive thinking or ``output_config.effort``. As with sampling, the +# modern fields go out once and an explicit rejection is remembered. +_THINKING_UNSUPPORTED: set[tuple[str, str]] = set() + + +def _text_parts(content: object) -> list[dict[str, Any]]: + if isinstance(content, str): + return [{"type": "text", "text": content}] if content else [] + if not isinstance(content, Sequence) or isinstance(content, (str, bytes)): + return [] + blocks: list[dict[str, Any]] = [] + for part in content: + if not isinstance(part, Mapping): + continue + kind = part.get("type") + if kind == "text" and isinstance(part.get("text"), str): + blocks.append({"type": "text", "text": part["text"]}) + elif kind == "image_url": + image = part.get("image_url") + url = image.get("url") if isinstance(image, Mapping) else image + if not isinstance(url, str): + continue + header, sep, data = url.partition(",") + if sep and header.startswith("data:") and ";base64" in header: + media_type = header[5:].split(";", 1)[0] + try: + base64.b64decode(data, validate=True) + except ValueError: + continue + blocks.append( + { + "type": "image", + "source": {"type": "base64", "media_type": media_type, "data": data}, + } + ) + return blocks + + +def _tool_use_blocks(tool_calls: object) -> list[dict[str, Any]]: + if not isinstance(tool_calls, Sequence) or isinstance(tool_calls, (str, bytes)): + return [] + out: list[dict[str, Any]] = [] + for call in tool_calls: + if not isinstance(call, Mapping): + continue + function = call.get("function") + if not isinstance(function, Mapping) or not isinstance(function.get("name"), str): + continue + raw = function.get("arguments", {}) + if isinstance(raw, str): + try: + decoded = json.loads(raw) + except ValueError: + decoded = {"_raw": raw} + else: + decoded = raw + if not isinstance(decoded, Mapping): + decoded = {"value": decoded} + out.append( + { + "type": "tool_use", + "id": str(call.get("id") or ""), + "name": function["name"], + "input": dict(decoded), + } + ) + return out + + +def translate_messages(messages: Sequence[Mapping[str, Any]]) -> tuple[str, list[dict[str, Any]]]: + """Hoist system text and translate/coalesce Anthropic user/assistant turns.""" + system_parts: list[str] = [] + translated: list[dict[str, Any]] = [] + + def append(role: str, blocks: list[dict[str, Any]]) -> None: + if not blocks: + return + if translated and translated[-1]["role"] == role: + translated[-1]["content"].extend(blocks) + else: + translated.append({"role": role, "content": blocks}) + + for message in messages: + role = message.get("role") + if role == "system": + for block in _text_parts(message.get("content")): + text = block.get("text") + if isinstance(text, str) and text: + system_parts.append(text) + continue + if role == "tool": + content = message.get("content", "") + tool_content: str | list[dict[str, Any]] + if isinstance(content, str): + tool_content = content + else: + tool_content = _text_parts(content) + append( + "user", + [ + { + "type": "tool_result", + "tool_use_id": str(message.get("tool_call_id") or ""), + "content": tool_content, + } + ], + ) + continue + if role not in {"user", "assistant"}: + continue + blocks = _text_parts(message.get("content")) + if role == "assistant": + blocks.extend(_tool_use_blocks(message.get("tool_calls"))) + append(role, blocks) + return "\n\n".join(system_parts), translated + + +def translate_tools(tools: object) -> list[dict[str, Any]]: + if not isinstance(tools, Sequence) or isinstance(tools, (str, bytes)): + return [] + out: list[dict[str, Any]] = [] + for tool in tools: + if not isinstance(tool, Mapping): + continue + function = tool.get("function") + if not isinstance(function, Mapping) or not isinstance(function.get("name"), str): + continue + translated: dict[str, Any] = { + "name": function["name"], + # ``strict`` obliges the schema to close every object and mark every + # property required; Orb's own tools ship partial ``required`` lists, + # so shape them the way the OpenAI forced path already does rather + # than sending a schema the API will reject. + "input_schema": strictify_schema(dict(function.get("parameters") or {"type": "object", "properties": {}})), + "strict": True, + } + if isinstance(function.get("description"), str): + translated["description"] = function["description"] + out.append(translated) + return out + + +def translate_tool_choice(choice: object) -> dict[str, Any] | None: + if choice is None: + return None + if choice == "auto": + return {"type": "auto"} + if choice == "none": + return {"type": "none"} + if choice == "required": + return {"type": "any"} + if isinstance(choice, Mapping): + function = choice.get("function") + name = function.get("name") if isinstance(function, Mapping) else None + if isinstance(name, str) and name: + return {"type": "tool", "name": name} + native_type = choice.get("type") + if native_type in {"auto", "none", "any"}: + return {"type": native_type} + return None + + +def _sampling_allowed(endpoint_url: str, model: str) -> bool: + return (endpoint_url, model) not in _SAMPLING_UNSUPPORTED + + +def build_request_body( + openai_body: Mapping[str, Any], + endpoint_url: str, + model: str, + extra_body: Mapping[str, Any] | None = None, +) -> dict[str, Any]: + """Build a native Messages body from an allowlist.""" + system, messages = translate_messages(openai_body.get("messages") or []) + body: dict[str, Any] = { + "model": model, + "messages": messages, + "stream": True, + "max_tokens": openai_body.get("max_tokens") or DEFAULT_MAX_TOKENS, + } + if system: + body["system"] = system + tools = translate_tools(openai_body.get("tools")) + if tools: + body["tools"] = tools + choice = translate_tool_choice(openai_body.get("tool_choice")) + if choice is not None and tools: + body["tool_choice"] = choice + + reasoning = openai_body.get("reasoning") + thinking = openai_body.get("thinking") + reasoning_on = (isinstance(reasoning, Mapping) and reasoning.get("enabled") is True) or ( + isinstance(thinking, Mapping) and thinking.get("type") == "enabled" + ) + if reasoning_on and (endpoint_url, model) not in _THINKING_UNSUPPORTED: + body["thinking"] = {"type": "adaptive", "display": "summarized"} + effort = openai_body.get("reasoning_effort") + if effort in {"low", "medium", "high", "xhigh", "max"}: + body["output_config"] = {"effort": effort} + + if _sampling_allowed(endpoint_url, model): + for key in ("temperature", "top_p", "top_k"): + value = openai_body.get(key) + if value is not None: + body[key] = value + + if extra_body: + for key in EXTRA_BODY_ALLOWED: + if key in extra_body: + body[key] = extra_body[key] + return body + + +def recover_thinking_error(endpoint_url: str, model: str, body: dict[str, Any], status: int, text: str) -> str | None: + """Learn a rejection of the 4.6+ reasoning fields and drop them once.""" + if status != 400: + return None + low = text.lower() + present = [key for key in ("thinking", "output_config") if key in body] + if not present: + return None + if not any(key in low for key in (*present, "effort", "budget_tokens")): + return None + if not any(marker in low for marker in ("unsupported", "not supported", "not allowed", "extra inputs")): + return None + _THINKING_UNSUPPORTED.add((endpoint_url, model)) + for key in present: + body.pop(key, None) + return f"Model {model} rejected Anthropic reasoning fields {present}; retrying without them." + + +def recover_sampling_error(endpoint_url: str, model: str, body: dict[str, Any], status: int, text: str) -> str | None: + """Learn a sampling-field rejection and remove all three controls once.""" + if status != 400: + return None + low = text.lower() + present = [key for key in ("temperature", "top_p", "top_k") if key in body] + if not present or not any(key in low for key in present): + return None + if not any(marker in low for marker in ("unsupported", "not supported", "not allowed", "extra inputs")): + return None + _SAMPLING_UNSUPPORTED.add((endpoint_url, model)) + for key in present: + body.pop(key, None) + return f"Model {model} rejected Anthropic sampling fields {present}; retrying without them." diff --git a/backend/inference/client.py b/backend/inference/client.py index 75c87a05..1b9d5ef1 100644 --- a/backend/inference/client.py +++ b/backend/inference/client.py @@ -9,10 +9,11 @@ import httpx -from . import endpoint_profiles, text_completion -from .errors import llm_call_error +from . import anthropic, endpoint_profiles, text_completion +from .errors import LLMCallError, llm_call_error, llm_stream_error from .gemma_tool_format import parse_gemma_tool_calls from .retry import RetryPolicy +from .tool_registry import strictify_schema logger = logging.getLogger(__name__) @@ -139,37 +140,6 @@ def parse_extra_body(text: str) -> dict: return parsed -def strictify_schema(schema: dict) -> dict: - """Copy *schema* into OpenAI strict-mode shape, recursively. - - Strict structured output requires every object to list all properties in - ``required`` and set ``additionalProperties: false``. Originally-optional - properties are made nullable so "may omit" survives as "may be null" -- - the passes' unpackers already discard empty/null argument values. - """ - node = dict(schema) - props = node.get("properties") - if isinstance(props, dict): - required = set(node.get("required") or []) - out_props: dict = {} - for key, prop in props.items(): - sub = strictify_schema(prop) if isinstance(prop, dict) else prop - if key not in required and isinstance(sub, dict) and "type" in sub: - t = sub["type"] - if isinstance(t, list): - t = t if "null" in t else [*t, "null"] - elif t != "null": - t = [t, "null"] - sub = {**sub, "type": t} - out_props[key] = sub - node["properties"] = out_props - node["required"] = list(props.keys()) - node["additionalProperties"] = False - if isinstance(node.get("items"), dict): - node["items"] = strictify_schema(node["items"]) - return node - - def _parse_chat_logprobs(choice: Mapping[str, Any]) -> list[dict]: """Normalize an OpenAI-compat ``choice.logprobs`` block to Orb's prob shape. @@ -291,36 +261,73 @@ def _headers(self) -> dict: headers.update(self.extra_headers) return headers - def _url(self) -> str: - return f"{self.base_url}/chat/completions" + def _headers_for(self, auth_family: endpoint_profiles.AuthFamily) -> dict: + """Return transport auth defaults with case-insensitive user overrides.""" + base: dict[str, str] = {} + if self.api_key: + if auth_family == "anthropic": + base["x-api-key"] = self.api_key + else: + base["Authorization"] = f"Bearer {self.api_key}" + if auth_family == "anthropic": + base["anthropic-version"] = "2023-06-01" + configured = {key.lower() for key in self.extra_headers} + headers = {key: value for key, value in base.items() if key.lower() not in configured} + headers.update(self.extra_headers) + return headers async def list_models(self) -> list[str]: - """Return model ids advertised by an OpenAI-compatible ``GET /models``. + """Return model ids advertised by a compatible ``GET /models``. - Discovery uses the same bearer authentication and endpoint proxy as - generation, but a short finite timeout: unlike a completion, this is a - small non-streaming settings request and should fail back to Orb's - editable model-name field promptly. + Explicit resource shapes have one sibling catalogue. For an ambiguous + base, discovery walks the same bounded route/auth candidates as + generation and caches the first catalogue that satisfies the shared + ``data[].id`` contract. It uses a short finite timeout so failure still + falls back to Orb's editable model-name field promptly. """ - url = f"{self.base_url}/models" + routes = endpoint_profiles.endpoint_candidates(self.base_url) + seen: set[tuple[str, endpoint_profiles.AuthFamily]] = set() + contract_error: ValueError | None = None + last_response: httpx.Response | None = None async with httpx.AsyncClient(timeout=20.0, proxy=self.proxy, follow_redirects=True) as client: - response = await client.get(url, headers=self._headers()) - response.raise_for_status() - try: - payload = response.json() - except ValueError as exc: - raise ValueError("Endpoint returned a non-JSON models response") from exc + for route in routes: + candidate = (route.models_url, route.auth_family) + if candidate in seen: + continue + seen.add(candidate) + response = await client.get(route.models_url, headers=self._headers_for(route.auth_family)) + if response.status_code >= 400: + last_response = response + continue + try: + payload = response.json() + except ValueError: + contract_error = ValueError("Endpoint returned a non-JSON models response") + continue - data = payload.get("data") if isinstance(payload, dict) else None - if not isinstance(data, list): - raise ValueError("Endpoint models response does not contain a data list") + data = payload.get("data") if isinstance(payload, dict) else None + if not isinstance(data, list): + contract_error = ValueError("Endpoint models response does not contain a data list") + continue - model_ids: set[str] = set() - for item in data: - model_id = item.get("id") if isinstance(item, dict) else None - if isinstance(model_id, str) and model_id.strip(): - model_ids.add(model_id.strip()) - return sorted(model_ids, key=str.casefold) + endpoint_profiles.note_successful_route(self.base_url, "", route) + model_ids: set[str] = set() + for item in data: + model_id = item.get("id") if isinstance(item, dict) else None + if isinstance(model_id, str) and model_id.strip(): + normalized = model_id.strip() + # This compatibility resource lists ids with a + # ``models/`` prefix that generation does not need. + if normalized.startswith("models/") and endpoint_profiles.is_gemini_openai_surface(route.url): + normalized = normalized.removeprefix("models/") + model_ids.add(normalized) + return sorted(model_ids, key=str.casefold) + + if contract_error is not None: + raise contract_error + if last_response is not None: + last_response.raise_for_status() + raise ValueError("Endpoint did not expose a models response") def _server_root(self) -> str: """Server root for llama.cpp native endpoints (/completion, /apply-template, @@ -420,7 +427,14 @@ async def _with_retry(self, open_stream: Callable[[], AsyncIterator[dict]]) -> A yield event return except httpx.HTTPError as exc: - if produced or self.is_aborted or attempt >= self.retry.count or not self.retry.should_retry(exc): + stream_event = isinstance(exc, LLMCallError) and exc.stream_event + if ( + produced + or stream_event + or self.is_aborted + or attempt >= self.retry.count + or not self.retry.should_retry(exc) + ): raise attempt += 1 detail = f"HTTP {exc.response.status_code}" if isinstance(exc, httpx.HTTPStatusError) else type(exc).__name__ @@ -562,19 +576,6 @@ def _plan() -> tuple[dict, str | None, bool]: apply_reasoning_effort(body, self.reasoning_effort, self.reasoning_effort_param, self.reasoning_effort_value) - # Same ordering as apply_reasoning_effort above, for the reason its - # docstring gives. Chat-only by design: the text transport builds its - # params from an allowlist. - if self.extra_body: - body.update(self.extra_body) - logger.info("LLM extra body fields: %s", sorted(self.extra_body)) - - # Provider-specific body translation (profiles + session-learned - # workarounds) lives entirely in endpoint_profiles; the client just - # applies whatever it returns. - for action in endpoint_profiles.prepare_request_body(self.base_url, model, body): - logger.info("LLM profile: %s", action) - logger.info( "LLM complete: model=%s, tools=%s, tool_choice=%s", model, @@ -584,6 +585,29 @@ def _plan() -> tuple[dict, str | None, bool]: logger.debug(messages) return body, forced_name, structured + # The body is re-derived on every recovery, auth retry and route probe, so + # its INFO lines would otherwise repeat once per attempt. Keyed on the + # rendered line, not on "first attempt only": a recovery that learns a new + # quirk adds an action, and that line is exactly the one worth surfacing. + logged_lines: set[str] = set() + + def _log_once(line: str) -> None: + if line not in logged_lines: + logged_lines.add(line) + logger.info("%s", line) + + def _outbound_body(body: dict, route: endpoint_profiles.EndpointRoute) -> dict: + """Copy the canonical OpenAI body into one route's wire dialect.""" + outbound = dict(body) + if route.protocol == "openai" and self.extra_body: + outbound.update(self.extra_body) + _log_once(f"LLM extra body fields: {sorted(self.extra_body)}") + for action in endpoint_profiles.prepare_request_body(self.base_url, model, outbound): + _log_once(f"LLM profile: {action}") + if route.protocol == "anthropic": + return anthropic.build_request_body(outbound, self.base_url, model, self.extra_body) + return outbound + content_parts: list[str] = [] reasoning_parts: list[str] = [] tool_calls_acc: dict[int, dict] = {} @@ -606,112 +630,223 @@ async def _issue(body: dict, forced_name: str | None) -> AsyncIterator[dict]: finish_reason = None usage = None - # At most one retry, solely to self-heal a provider quirk that - # endpoint_profiles.recover_from_error() recognises (e.g. an OpenRouter - # model rejecting tool_choice). The error lands before any SSE event, - # so the retry is clean. - for attempt in range(2): - # No read timeout on streaming calls: the server sends zero bytes - # while prefilling a large prompt (or queueing behind another - # request), and a long silence is normal there — a flat read - # timeout intermittently killed long turns. Abort/stop and the - # disconnect watcher remain the recovery paths. - async with httpx.AsyncClient(timeout=httpx.Timeout(self.timeout, read=None), proxy=self.proxy) as client: - async with client.stream("POST", self._url(), json=body, headers=self._headers()) as resp: - if resp.status_code >= 400: - # Concern 1: surface the error body. - err_text = await _read_error_body(resp, self._url()) - - # Concern 2: ask the provider layer whether this is a - # recognised quirk worth one retry. It mutates body in - # place and returns a log line, or None to propagate. - if attempt == 0: - fix = endpoint_profiles.recover_from_error( - self.base_url, model, body, resp.status_code, err_text + def tool_entry(index: int) -> dict: + if index not in tool_calls_acc: + tool_calls_acc[index] = { + "id": "", + "type": "function", + "function": {"name": "", "arguments": ""}, + } + return tool_calls_acc[index] + + async def consume_openai(resp) -> AsyncIterator[dict]: + nonlocal finish_reason, usage + # Slot last handed to an index-less delta; -1 before the first. + unindexed = -1 + + def slot_for(tc_delta: Mapping[str, Any]) -> int: + """Resolve one tool-call delta to an accumulator slot. + + Google's OpenAI-compatible surface omits ``index`` from + ``delta.tool_calls`` entirely, so keying on ``index`` with a + default of 0 merged every parallel call into one entry -- + names concatenated, all but the first argument payload lost. + Without an index, a delta that STARTS a call (it carries an + ``id`` or a function ``name``, which the OpenAI contract + sends only on a call's first chunk) opens the next free + slot; a bare argument continuation appends to the newest. + """ + nonlocal unindexed + index = tc_delta.get("index") + if isinstance(index, int) and not isinstance(index, bool): + return index + function = tc_delta.get("function") + starts = bool(tc_delta.get("id")) or bool(isinstance(function, Mapping) and function.get("name")) + if starts or unindexed < 0: + unindexed = max([*tool_calls_acc, unindexed], default=-1) + 1 + return unindexed + + async for payload in self._iter_sse_payloads(resp): + try: + chunk = json.loads(payload) + except json.JSONDecodeError: + continue + u = chunk.get("usage") + if isinstance(u, dict): + usage = u + choices = chunk.get("choices") or [] + if not choices: + continue + try: + choice = choices[0] + delta = choice.get("delta", {}) + rc = delta.get("reasoning_content") or delta.get("reasoning") + if rc: + reasoning_parts.append(rc) + yield {"type": "reasoning", "delta": rc} + content = delta.get("content") + if content: + content_parts.append(content) + if forced_name is None: + yield {"type": "content", "delta": content} + for rec in _parse_chat_logprobs(choice): + yield {"type": "token_probs", **rec} + for tc_delta in delta.get("tool_calls") or []: + entry = tool_entry(slot_for(tc_delta)) + if tc_delta.get("id"): + entry["id"] = tc_delta["id"] + fn = tc_delta.get("function", {}) + if fn.get("name"): + entry["function"]["name"] += fn["name"] + if fn.get("arguments"): + entry["function"]["arguments"] += fn["arguments"] + if choice.get("finish_reason"): + finish_reason = choice["finish_reason"] + except (KeyError, IndexError): + continue + + async def consume_anthropic(resp, url: str) -> AsyncIterator[dict]: + nonlocal finish_reason, usage + stopped = False + async for payload in self._iter_sse_payloads(resp): + try: + event = json.loads(payload) + except json.JSONDecodeError: + continue + event_type = event.get("type") + if event_type == "ping": + continue + if event_type == "error": + raise llm_stream_error(payload=event, url=url, model=model, api_key=self.api_key) + if event_type == "message_start": + initial = (event.get("message") or {}).get("usage") + if isinstance(initial, dict): + usage = dict(initial) + elif event_type == "content_block_start": + index = event.get("index", 0) + block = event.get("content_block") or {} + if block.get("type") == "tool_use": + entry = tool_entry(index) + entry["id"] = block.get("id", "") + entry["function"]["name"] = block.get("name", "") + if block.get("input"): + entry["function"]["arguments"] = json.dumps(block["input"], separators=(",", ":")) + elif block.get("type") == "text" and block.get("text"): + content_parts.append(block["text"]) + if forced_name is None: + yield {"type": "content", "delta": block["text"]} + elif block.get("type") == "thinking" and block.get("thinking"): + reasoning_parts.append(block["thinking"]) + yield {"type": "reasoning", "delta": block["thinking"]} + elif event_type == "content_block_delta": + index = event.get("index", 0) + delta = event.get("delta") or {} + delta_type = delta.get("type") + if delta_type == "text_delta" and delta.get("text"): + content_parts.append(delta["text"]) + # Same gate as consume_openai: a forced pass buffers its + # text as the tool-arguments payload instead of streaming + # it, so the caller never sees a half-built JSON body. + if forced_name is None: + yield {"type": "content", "delta": delta["text"]} + elif delta_type == "thinking_delta" and delta.get("thinking"): + reasoning_parts.append(delta["thinking"]) + yield {"type": "reasoning", "delta": delta["thinking"]} + elif delta_type == "input_json_delta" and delta.get("partial_json"): + tool_entry(index)["function"]["arguments"] += delta["partial_json"] + elif event_type == "message_delta": + delta = event.get("delta") or {} + stop_reason = delta.get("stop_reason") + if stop_reason: + finish_reason = { + "end_turn": "stop", + "stop_sequence": "stop", + "tool_use": "tool_calls", + "max_tokens": "length", + }.get(stop_reason, stop_reason) + update = event.get("usage") + if isinstance(update, dict): + usage = {**(usage or {}), **update} + elif event_type == "message_stop": + stopped = True + break + if not stopped and not self.is_aborted: + raise llm_stream_error( + payload={"error": {"message": "Anthropic stream ended before message_stop"}}, + url=url, + model=model, + api_key=self.api_key, + ) + + routes = endpoint_profiles.endpoint_candidates(self.base_url, model) + route_index = 0 + while route_index < len(routes): + route = routes[route_index] + recovery_count = 0 + auth_family = route.auth_family + auth_retried = False + while True: + outbound = _outbound_body(body, route) + # No read timeout on streaming calls: a long prefill silence + # is normal; abort and disconnect close the stream instead. + async with httpx.AsyncClient(timeout=httpx.Timeout(self.timeout, read=None), proxy=self.proxy) as client: + async with client.stream( + "POST", route.url, json=outbound, headers=self._headers_for(auth_family) + ) as resp: + if resp.status_code >= 400: + err_text = await _read_error_body(resp, route.url) + # One attempt per independent quirk class: the profile + # rules, a refused reasoning_effort level, Anthropic + # sampling controls, and the 4.6+ reasoning fields can + # each need their own rejection before a body this + # endpoint accepts is reached. No route reaches more + # than three of them -- reasoning_effort is an + # OpenAI-body field the Anthropic translation drops. + if recovery_count < 3: + fix = endpoint_profiles.recover_from_error( + self.base_url, model, outbound, resp.status_code, err_text + ) + if fix is None and route.protocol == "anthropic": + fix = anthropic.recover_sampling_error( + self.base_url, model, outbound, resp.status_code, err_text + ) or anthropic.recover_thinking_error( + self.base_url, model, outbound, resp.status_code, err_text + ) + if fix is not None: + recovery_count += 1 + logger.warning("LLM recovery: %s", fix) + continue + auths = endpoint_profiles.auth_families(route, resp.status_code, err_text) + if not auth_retried and len(auths) > 1: + auth_family = auths[1] + auth_retried = True + logger.warning("LLM auth recovery: retrying %s with %s auth", route.url, auth_family) + continue + if route_index + 1 < len(routes) and endpoint_profiles.should_probe_route( + resp.status_code, err_text + ): + logger.warning( + "LLM endpoint probe: %s rejected the route; trying %s", + route.url, + routes[route_index + 1].url, + ) + route_index += 1 + break + raise llm_call_error( + response=resp, + body=err_text, + url=route.url, + model=model, + api_key=self.api_key, ) - if fix is not None: - logger.warning("LLM recovery: %s", fix) - continue # leave async-with cleanly, then retry - - # Concern 3: keep the body. raise_for_status() would - # replace the provider's own sentence with httpx's canned - # status line, and it is the only part the user can act on. - raise llm_call_error( - response=resp, - body=err_text, - url=self._url(), - model=model, - api_key=self.api_key, - ) - async for payload in self._iter_sse_payloads(resp): - try: - chunk = json.loads(payload) - except json.JSONDecodeError: - continue - - # Usage may appear in a terminal chunk (choices=[]) or on the final content chunk; last-write-wins since totals are monotonic. - u = chunk.get("usage") - if isinstance(u, dict): - usage = u - - choices = chunk.get("choices") or [] - if not choices: - # Pure usage/metadata chunk — nothing else to do. - continue - - try: - choice = choices[0] - delta = choice.get("delta", {}) - - # Reasoning delta (field name varies by server) - rc = delta.get("reasoning_content") or delta.get("reasoning") - if rc: - reasoning_parts.append(rc) - yield {"type": "reasoning", "delta": rc} - - # Content delta. A structured forced call buffers - # instead of yielding: the content is the tool's - # arguments JSON, and chat mode never surfaces - # argument streams as content (they arrive as - # tool_calls deltas, which the pipeline hides). - c = delta.get("content") - if c: - content_parts.append(c) - if forced_name is None: - yield {"type": "content", "delta": c} - - # Per-token alternatives (Document mode steering) — - # present only when the caller passed logprobs and the - # provider honoured them; otherwise a no-op. - for rec in _parse_chat_logprobs(choice): - yield {"type": "token_probs", **rec} - - # Tool call argument deltas — accumulate by index - for tc_delta in delta.get("tool_calls") or []: - idx = tc_delta.get("index", 0) - if idx not in tool_calls_acc: - tool_calls_acc[idx] = { - "id": "", - "type": "function", - "function": {"name": "", "arguments": ""}, - } - entry = tool_calls_acc[idx] - if tc_delta.get("id"): - entry["id"] = tc_delta["id"] - fn = tc_delta.get("function", {}) - if fn.get("name"): - entry["function"]["name"] += fn["name"] - if fn.get("arguments"): - entry["function"]["arguments"] += fn["arguments"] - - if choice.get("finish_reason"): - finish_reason = choice["finish_reason"] - - except (KeyError, IndexError): - continue - # Streamed to completion (or aborted) without a retry-triggering - # error -- done, no second attempt. - break + if route.protocol == "anthropic": + async for event in consume_anthropic(resp, route.url): + yield event + else: + async for event in consume_openai(resp): + yield event + endpoint_profiles.note_successful_route(self.base_url, model, route) + return body, forced_name, structured = _plan() async for _ev in _issue(body, forced_name): diff --git a/backend/inference/endpoint_profiles.py b/backend/inference/endpoint_profiles.py index cf438a89..b7cbe6c4 100644 --- a/backend/inference/endpoint_profiles.py +++ b/backend/inference/endpoint_profiles.py @@ -4,7 +4,8 @@ from collections.abc import Callable, Mapping from dataclasses import dataclass, field -from typing import Any +from typing import Any, Literal +from urllib.parse import urlsplit, urlunsplit # Body keys always sent; never subject to allowlist filtering. ALWAYS_ALLOWED: frozenset[str] = frozenset({"model", "messages", "stream", "tools", "tool_choice"}) @@ -50,6 +51,11 @@ class ModelProfile: # ``response_format: {"type": "json_schema", "strict": true}``. structured_tool_calls: bool = False + # If True, every value except ``"auto"`` is rewritten to ``"auto"``. + # Some routed providers reject not only forced choices but also ``"none"``; + # this is deliberately separate from allow_forced_tool_choice. + auto_tool_choice_only: bool = False + # Bespoke transforms applied after typed knobs, in order. Each callable # mutates body in place and may return a log line (or None for silent). custom: tuple[Transform, ...] = field(default_factory=tuple) @@ -72,6 +78,12 @@ def apply(self, body: dict) -> list[str]: body["tool_choice"] = "auto" actions.append(f"tool_choice {tc!r} -> 'auto'") + if self.auto_tool_choice_only: + tc = body.get("tool_choice") + if tc is not None and tc != "auto": + body["tool_choice"] = "auto" + actions.append(f"tool_choice {tc!r} -> 'auto' (auto-only endpoint)") + for fn in self.custom: log = fn(body) if log: @@ -149,16 +161,6 @@ def _deepseek_coerce_tool_choice_when_thinking(body: dict) -> str | None: allow_forced_tool_choice=False, ), }, - # No None-key default: unlisted OpenRouter models stay pass-through (most - # honor forcing). List only models known to reject a forced-function - # tool_choice; add a one-liner per newly-found one. llm_client self-heals - # the first hit of an unlisted model and logs a reminder to add it here. - "openrouter.ai": { - "minimax/minimax-m3": ModelProfile( - allow_extra=None, # OpenRouter is lenient; drop nothing - allow_forced_tool_choice=False, # forced -> "auto" - ), - }, # NanoGPT is a *proxy*: each model id it fronts sits behind a different # upstream engine with its own config, so no endpoint-wide statement about # decoding is true of every model. Its own tool-argument decoding is @@ -178,6 +180,224 @@ def _deepseek_coerce_tool_choice_when_thinking(body: dict) -> str | None: } +# This OpenAI compatibility dialect is identified by its resource shape, not a +# provider hostname or a model id. Any proxy exposing the same path therefore +# receives the same request policy and catalogue normalization. +_GEMINI_OPENAI_PATH = "/v1beta/openai" + + +def is_gemini_openai_surface(url: str) -> bool: + """Whether *url* has the ``/v1beta/openai`` compatibility shape.""" + parsed = _parsed_http_url(url) + if parsed is None: + return _GEMINI_OPENAI_PATH in _clean_url(url).lower() + path = parsed.path.rstrip("/").lower() + return path == _GEMINI_OPENAI_PATH or f"{_GEMINI_OPENAI_PATH}/" in f"{path}/" + + +def _gemini_reasoning_off(body: dict) -> str | None: + """Ask Gemini to stop thinking when the call turned reasoning off. + + Orb's reasoning-off shape is three fields none of which Gemini's + compatibility layer reads (``reasoning``, ``chat_template_kwargs``, + ``thinking``); they are silently ignored, so the model kept thinking on its + default budget and the user paid for tokens they had disabled. The layer's + own control is ``reasoning_effort``, whose accepted set includes ``none``. + + Families that cannot disable thinking at all (2.5 Pro, the 3 series) answer + 400 to ``none``; :func:`recover_from_error` learns that from the rejection + and drops the field for the rest of the session, the same posture as every + other capability fact here. An explicit effort already in the body wins -- + that call asked for thinking. + """ + if "reasoning_effort" in body: + return None + reasoning = body.get("reasoning") + thinking = body.get("thinking") + disabled = (isinstance(reasoning, Mapping) and reasoning.get("enabled") is False) or ( + isinstance(thinking, Mapping) and thinking.get("type") == "disabled" + ) + if not disabled: + return None + body["reasoning_effort"] = "none" + return "reasoning_effort='none' (reasoning off)" + + +# Google's OpenAI compatibility API accepts ordinary OpenAI request fields +# (unknown additions are ignored) and honors strict json_schema output. +_GEMINI_PROFILE = ModelProfile( + allow_extra=None, + structured_tool_calls=True, + custom=(_gemini_reasoning_off,), +) + + +Protocol = Literal["openai", "anthropic"] +AuthFamily = Literal["bearer", "anthropic"] + + +@dataclass(frozen=True) +class EndpointRoute: + """One concrete transport resource resolved from a configured endpoint.""" + + protocol: Protocol + url: str + models_url: str + auth_family: AuthFamily + authoritative: bool = False + + +# A successful route is remembered per configured URL and model because client +# objects are per-turn. Stored URLs are never rewritten. +_RESOLVED_ROUTES: dict[tuple[str, str], EndpointRoute] = {} + + +def _clean_url(url: str) -> str: + return url.strip().rstrip("/") + + +def _parsed_http_url(url: str): + parsed = urlsplit(_clean_url(url)) + if parsed.scheme.lower() not in {"http", "https"} or not parsed.netloc: + return None + return parsed + + +def _replace_path(parsed, path: str) -> str: + return urlunsplit((parsed.scheme, parsed.netloc, path, parsed.query, "")) + + +def _resource_route(protocol: Protocol, url: str, *, authoritative: bool) -> EndpointRoute: + parsed = _parsed_http_url(url) + clean = _clean_url(url) + path = parsed.path.rstrip("/") if parsed is not None else clean + resource = "/messages" if protocol == "anthropic" else "/chat/completions" + if path.endswith(resource): + models_path = f"{path[: -len(resource)]}/models" + else: + models_path = f"{path}/models" + models_url = _replace_path(parsed, models_path) if parsed is not None else f"{clean}/models" + return EndpointRoute( + protocol=protocol, + url=clean, + models_url=models_url, + auth_family="anthropic" if protocol == "anthropic" else "bearer", + authoritative=authoritative, + ) + + +def _base_route(protocol: Protocol, base_url: str, *, authoritative: bool = False) -> EndpointRoute: + clean = _clean_url(base_url) + suffix = "/messages" if protocol == "anthropic" else "/chat/completions" + parsed = _parsed_http_url(clean) + if parsed is None: + url = f"{clean}{suffix}" + else: + url = _replace_path(parsed, f"{parsed.path.rstrip('/')}{suffix}") + return _resource_route(protocol, url, authoritative=authoritative) + + +def _deterministic_route(endpoint_url: str) -> EndpointRoute: + """Resolve explicit resource shapes without provider/model-name guesses.""" + clean = _clean_url(endpoint_url) + parsed = _parsed_http_url(clean) + low = clean.lower() + path = parsed.path.rstrip("/").lower() if parsed is not None else low + + # Full resource URLs are user intent. + if path.endswith("/chat/completions"): + return _resource_route("openai", clean, authoritative=True) + if path.endswith("/messages"): + return _resource_route("anthropic", clean, authoritative=True) + + if is_gemini_openai_surface(clean): + return _base_route("openai", clean, authoritative=True) + + return _base_route("openai", clean) + + +def resolve_endpoint(endpoint_url: str, model: str = "") -> EndpointRoute: + """Return the cached or deterministic route for one configured endpoint.""" + return ( + _RESOLVED_ROUTES.get((endpoint_url, model)) + or _RESOLVED_ROUTES.get((endpoint_url, "")) + or _deterministic_route(endpoint_url) + ) + + +def endpoint_candidates(endpoint_url: str, model: str = "") -> list[EndpointRoute]: + """Return bounded same-host routes in request order. + + Explicit resource shapes are authoritative. Ambiguous URLs keep Orb's + historical ``{configured}/chat/completions`` request first, followed by its + Messages sibling and the conventional host-root resources. Candidates are only + attempted when :func:`should_probe_route` recognizes the response body as a + route mismatch. + + The ``/v1beta/openai`` candidate is last because it is the narrowest guess + and costs a further prompt upload; it is still worth making, because a + Gemini-compat proxy configured at its bare root exposes that resource and + no other, and the two ``/v1`` guesses ahead of it cannot reach it. + """ + primary = resolve_endpoint(endpoint_url, model) + if primary.authoritative or (endpoint_url, model) in _RESOLVED_ROUTES or (endpoint_url, "") in _RESOLVED_ROUTES: + return [primary] + parsed = _parsed_http_url(endpoint_url) + if parsed is None: + return [primary] + root = urlunsplit((parsed.scheme, parsed.netloc, "", "", "")) + candidates = [ + primary, + _base_route("anthropic", endpoint_url), + _base_route("openai", f"{root}/v1"), + _base_route("anthropic", f"{root}/v1"), + _base_route("openai", f"{root}{_GEMINI_OPENAI_PATH}"), + ] + out: list[EndpointRoute] = [] + seen: set[tuple[str, str]] = set() + for route in candidates: + key = (route.protocol, route.url) + if key not in seen: + seen.add(key) + out.append(route) + return out + + +def note_successful_route(endpoint_url: str, model: str, route: EndpointRoute) -> None: + """Cache a route after its stream completes successfully.""" + _RESOLVED_ROUTES[(endpoint_url, model)] = route + + +def should_probe_route(status: int, text: str) -> bool: + """Whether an error body specifically identifies an HTTP route mismatch.""" + low = text.lower() + markers = ( + "cannot post /", + "route not found", + "unknown endpoint", + "unrecognized request url", + "unsupported endpoint", + "invalid url (post", + ) + native_not_found = status == 404 and "not_found_error" in low and "model" not in low + return any(marker in low for marker in markers) or native_not_found + + +def auth_families(route: EndpointRoute, status: int | None = None, text: str = "") -> tuple[AuthFamily, ...]: + """Return primary auth and a bounded, evidence-based alternative.""" + primary = route.auth_family + if status not in {401, 403}: + return (primary,) + # A Messages route defines its auth default. Ambiguous routes may need the + # other family before the server will reveal that the resource is wrong; + # an explicit OpenAI route only retries when the response names the native + # API-key header. None of these signals depends on provider or model names. + if route.protocol != "anthropic" and route.authoritative and "x-api-key" not in text.lower(): + return (primary,) + other: AuthFamily = "bearer" if primary == "anthropic" else "anthropic" + return (primary, other) + + # (endpoint_url, model) pairs observed to answer a forced tool_choice with a # different tool this session — either a profile coerced the choice to "auto" # or the provider ignored it silently (OpenRouter + a thinking-on model, @@ -240,9 +460,16 @@ def profile_for(endpoint_url: str, model: str = "") -> ModelProfile | None: A blank *model* falls through to the endpoint default. An unmatched URL returns ``None`` — the body is sent unchanged (local / unknown backends). + + The ``/v1beta/openai`` dialect is resolved by + :func:`is_gemini_openai_surface` rather than by a ``PROFILES`` substring, + so every proxy mirroring the resource gets the same request policy. """ if not endpoint_url: return None + resolved_url = resolve_endpoint(endpoint_url, model).url + if is_gemini_openai_surface(endpoint_url) or is_gemini_openai_surface(resolved_url): + return _GEMINI_PROFILE haystack = endpoint_url.lower() for needle, models in PROFILES.items(): if needle in haystack: @@ -266,22 +493,53 @@ def profile_for(endpoint_url: str, model: str = "") -> ModelProfile | None: # front instead of paying the round-trip + retry again. _TOOL_CHOICE_UNSUPPORTED: set[tuple[str, str]] = set() +# Pairs observed to accept only the literal ``"auto"`` value. Unlike +# _TOOL_CHOICE_UNSUPPORTED, these endpoints still need the field so forced and +# writer ``"none"`` requests are coerced rather than dropped. +_TOOL_CHOICE_AUTO_ONLY: set[tuple[str, str]] = set() + +# Pairs whose reply rejected the ``reasoning_effort`` VALUE Orb sent. Orb offers +# a superset of levels (``xhigh`` is an Anthropic/OpenAI-ism; Gemini's set is +# high/low/medium/none) and a provider that validates the field answers 400 to +# anything outside its own. Learning the rejection rather than hard-coding each +# provider's accepted set is deliberate: those sets move under us, and a static +# list rots into wrongly clamping a level the provider has since added. +_REASONING_EFFORT_UNSUPPORTED: set[tuple[str, str]] = set() + def _is_openrouter(endpoint_url: str) -> bool: return "openrouter.ai" in endpoint_url.lower() def _is_tool_choice_unsupported(status: int, text: str) -> bool: - """Return ``True`` for OpenRouter's ``tool_choice``-unsupported 404. + """Return ``True`` when the body says no ``tool_choice`` value is routed. Matches "No endpoints found that support the provided 'tool_choice' value." — meaning the routed provider rejects all ``tool_choice`` values. Kept narrow so genuine 404s (bad model id, etc.) don't match. """ - if status != 404: - return False low = text.lower() - return "tool_choice" in low and "no endpoints found" in low + return status in {400, 404} and "tool_choice" in low and "no endpoints found" in low + + +def _is_tool_choice_auto_only(status: int, text: str) -> bool: + """Return True when the body states that only ``auto`` is accepted.""" + low = text.lower() + return status in {400, 404} and "tool_choice" in low and "only" in low and "auto" in low and "support" in low + + +def _is_reasoning_effort_rejected(status: int, text: str) -> bool: + """Return True when the body names ``reasoning_effort`` as the bad field. + + Matches Google's "Invalid reasoning_effort: xhigh. Valid values are: high, + low, medium, none" and the equivalent from any provider that validates the + field. Kept to bodies that name the field so a generic 400 (bad model, + oversized prompt) never costs a reasoning setting the endpoint accepts. + """ + low = text.lower() + if status != 400 or "reasoning_effort" not in low: + return False + return any(marker in low for marker in ("invalid", "unsupported", "not supported", "not allowed", "valid values")) def prepare_request_body(endpoint_url: str, model: str, body: dict) -> list[str]: @@ -295,12 +553,22 @@ def prepare_request_body(endpoint_url: str, model: str, body: dict) -> list[str] if profile is not None: actions.extend(profile.apply(body)) + if "reasoning_effort" in body and (endpoint_url, model) in _REASONING_EFFORT_UNSUPPORTED: + effort = body.pop("reasoning_effort") + actions.append(f"reasoning_effort {effort!r} dropped (session-learned unsupported)") + # A model we already learned rejects tool_choice this session: drop it up # front so we skip the failing round-trip entirely. if "tool_choice" in body and (endpoint_url, model) in _TOOL_CHOICE_UNSUPPORTED: tc = body.pop("tool_choice") actions.append(f"tool_choice {tc!r} dropped (session-learned unsupported)") + if "tool_choice" in body and (endpoint_url, model) in _TOOL_CHOICE_AUTO_ONLY: + tc = body["tool_choice"] + if tc != "auto" and tc != {"type": "auto"}: + body["tool_choice"] = "auto" + actions.append(f"tool_choice {tc!r} -> 'auto' (session-learned auto-only)") + return actions @@ -311,16 +579,37 @@ def recover_from_error(endpoint_url: str, model: str, body: dict, status: int, t Currently handles one quirk: an OpenRouter model whose routed provider rejects ``tool_choice`` entirely. Recovery is to drop the param and retry; - the 404 lands before any SSE event so the retry is clean. Add such models - to ``PROFILES['openrouter.ai']`` for a zero-retry fix. + the 404 lands before any SSE event so the retry is clean. Model catalog ids + are deliberately never recorded here; learned capability facts expire with + the backend process. """ + tc = body.get("tool_choice") + low = text.lower() + native_forced_rejected = ( + status == 400 + and isinstance(tc, Mapping) + and tc.get("type") in {"any", "tool"} + and "tool_choice" in low + and any(marker in low for marker in ("not supported", "unsupported", "not allowed")) + ) + if native_forced_rejected: + _TOOL_CHOICE_AUTO_ONLY.add((endpoint_url, model)) + note_forced_tool_choice_ignored(endpoint_url, model) + body["tool_choice"] = {"type": "auto"} + return f"Model {model} rejected forced Anthropic tool choice; retrying with auto." + if "tool_choice" in body and _is_tool_choice_auto_only(status, text): + _TOOL_CHOICE_AUTO_ONLY.add((endpoint_url, model)) + tc = body["tool_choice"] + body["tool_choice"] = {"type": "auto"} if isinstance(tc, dict) and "function" not in tc else "auto" + return f"Model {model} accepts only tool_choice='auto'; retrying with auto." + if "reasoning_effort" in body and _is_reasoning_effort_rejected(status, text): + _REASONING_EFFORT_UNSUPPORTED.add((endpoint_url, model)) + effort = body.pop("reasoning_effort") + return f"Model {model} rejected reasoning_effort={effort!r}; retrying without it." if not _is_openrouter(endpoint_url): return None if "tool_choice" in body and _is_tool_choice_unsupported(status, text): _TOOL_CHOICE_UNSUPPORTED.add((endpoint_url, model)) tc = body.pop("tool_choice") - return ( - f"Model {model} rejected tool_choice={tc!r}; retrying without it. " - f"Add it to endpoint_profiles.PROFILES['openrouter.ai'] for a zero-retry fix." - ) + return f"Model {model} rejected tool_choice={tc!r}; retrying without it." return None diff --git a/backend/inference/errors.py b/backend/inference/errors.py index c78455c0..53977257 100644 --- a/backend/inference/errors.py +++ b/backend/inference/errors.py @@ -39,6 +39,8 @@ class LLMCallError(httpx.HTTPStatusError): re-checking. """ + stream_event = False + def __init__( self, message: str, @@ -186,3 +188,18 @@ def llm_call_error( host=host, model=model, ) + + +def llm_stream_error(*, payload: object, url: str, model: str, api_key: str) -> LLMCallError: + """Build a normal sanitized provider error for an SSE error event. + + Anthropic can report a terminal provider error inside an HTTP-200 stream. + A synthetic 502 response represents that upstream failure without teaching + pipeline failure rendering a second exception shape. + """ + body = json.dumps(payload, ensure_ascii=False) if not isinstance(payload, str) else payload + request = httpx.Request("POST", url) + response = httpx.Response(502, request=request) + error = llm_call_error(response=response, body=body, url=url, model=model, api_key=api_key) + error.stream_event = True + return error diff --git a/backend/inference/tool_registry.py b/backend/inference/tool_registry.py index c2ce8e9b..aec3810f 100644 --- a/backend/inference/tool_registry.py +++ b/backend/inference/tool_registry.py @@ -445,3 +445,34 @@ def enabled_schemas( if enabled_tools is not None: eligible = [n for n in eligible if enabled_tools.get(n, False)] return [s for n in eligible if (s := overrides.get(n, TOOLS[n]["schema"])) is not None] + + +def strictify_schema(schema: dict) -> dict: + """Copy *schema* into OpenAI strict-mode shape, recursively. + + Strict structured output requires every object to list all properties in + ``required`` and set ``additionalProperties: false``. Originally-optional + properties are made nullable so "may omit" survives as "may be null" -- + the passes' unpackers already discard empty/null argument values. + """ + node = dict(schema) + props = node.get("properties") + if isinstance(props, dict): + required = set(node.get("required") or []) + out_props: dict = {} + for key, prop in props.items(): + sub = strictify_schema(prop) if isinstance(prop, dict) else prop + if key not in required and isinstance(sub, dict) and "type" in sub: + t = sub["type"] + if isinstance(t, list): + t = t if "null" in t else [*t, "null"] + elif t != "null": + t = [t, "null"] + sub = {**sub, "type": t} + out_props[key] = sub + node["properties"] = out_props + node["required"] = list(props.keys()) + node["additionalProperties"] = False + if isinstance(node.get("items"), dict): + node["items"] = strictify_schema(node["items"]) + return node diff --git a/docs/architecture/endpoints.md b/docs/architecture/endpoints.md new file mode 100644 index 00000000..b778bd64 --- /dev/null +++ b/docs/architecture/endpoints.md @@ -0,0 +1,123 @@ +# LLM Endpoint Routing + +Orb keeps one OpenAI-shaped contract inside the pipeline: messages, tool calls, +stream events, terminal messages, and usage have the same shape regardless of +the provider. The inference layer resolves the configured URL and translates at +the network boundary. + +## Accepted endpoint forms + +A configured endpoint may be a versioned base or a full generation resource. + +| Configured form | Generation resource | +|---|---| +| `https://host/v1` | `https://host/v1/chat/completions` | +| `https://host/v1/chat/completions` | Used exactly as entered | +| `https://host/v1/messages` | Used exactly as entered with Anthropic Messages | +| `https://host/v1beta/openai` | `https://host/v1beta/openai/chat/completions` | + +Full `chat/completions` and `messages` resource URLs are authoritative. Bare and +versioned bases remain ambiguous: provider names in a hostname or path, and +family names in a model id, never select a protocol. + +The `/v1beta/openai` compatibility dialect is likewise recognized from its +resource path, not its hostname. Any proxy or gateway exposing that shape takes +the same request policy, reasoning translation, and catalogue normalization. + +Model discovery uses the sibling `models` resource and the matching auth family. +For an ambiguous base it walks the same route candidates until a catalogue +satisfies the shared `data[].id` contract, then caches that route for generation. +OpenAI and Gemini routes use Bearer authentication. Native Anthropic routes use +`x-api-key` and `anthropic-version`. Extra headers may replace those defaults +case-insensitively. + +## Automatic detection and probing + +Explicit resource shapes are deterministic and do not probe. An ambiguous URL +preserves Orb's historical OpenAI request first. Only when the +pre-stream response body specifically identifies a route mismatch does Orb try, +on the same host, these candidate resources: + +1. the configured base plus `chat/completions`; +2. the configured base plus `messages`; +3. host-root `/v1/chat/completions`; +4. host-root `/v1/messages`; +5. host-root `/v1beta/openai/chat/completions`. + +The last is the beta OpenAI compatibility resource, which some proxies expose +at their bare root and the two `/v1` guesses cannot reach. + +The HTTP status alone never starts probing: a 400 or 404 can describe a bad +model, schema, or tool choice rather than a bad route. Known request recovery +runs first. No route is changed after the first streamed delta, and local +text-completion calls do not enter this chat probing path. + +Probing replays the complete POST, so a first request can upload the prompt up +to five times. Orb chooses that trade-off because native compatibility proxies +do not expose a reliable discovery contract. A successful protocol and path is +cached per configured URL and model for the life of the backend process; +configured settings are never rewritten. + +A rejected `reasoning_effort` value is recovered on any OpenAI-protocol route: +Orb offers a superset of levels (`xhigh` is not a Gemini value), so a body that +names the field as invalid drops it for one retry and for the rest of the +session. Providers' accepted sets move, so this is learned from the response +rather than held as a per-provider list. + +A 401 or 403 can trigger one alternate native/Bearer auth attempt for a Messages +route or an ambiguous route. An explicit OpenAI resource only retries when the +response names the native `x-api-key` header. This retry is independently +bounded and never changes hosts; provider and model names are not evidence. + +## Provider request behavior + +Native Anthropic requests are built from an allowlist. System messages are +hoisted; text, base64 images, tool calls, and tool results are translated to +Messages content blocks; adjacent roles are coalesced. Tool definitions use +`input_schema` and `strict: true`. OpenAI `extra_body` fields are not passed +through; only Anthropic-native `metadata` and `service_tier` are accepted from +that escape hatch. A missing `max_tokens` defaults to 4096. + +Reasoning-on maps to adaptive thinking with summarized display, and supported +effort levels map to `output_config.effort`. Reasoning-off omits `thinking`. +Sampling controls are sent optimistically. A specific rejection teaches Orb to +omit them for later calls to that endpoint/model pair; names never stand in for +capability evidence. `min_p`, repetition penalties, and logprobs are never sent +to Anthropic. Consequently, Document mode's per-token steering is not available +on native Anthropic endpoints. + +Some routed models accept only `tool_choice="auto"`. This is distinct from a +provider that rejects `tool_choice` entirely: Orb rewrites `none`, `required`, +and named choices to `auto`, including the Writer's normal `none`. If the body +reveals this restriction for an unlisted model, Orb learns it for the process. +Director and Editor already handle a model declining the intended forced call. + +Gemini uses Google's official OpenAI-compatible beta surface, Bearer auth, the +existing OpenAI stream parser, and strict structured output for forced calls. +Documented OpenAI fields, including `reasoning_effort`, remain intact. Native +Gemini features such as grounding and Files APIs are outside this version. + +Structured output is what carries a forced call, so `tools` and `tool_choice` +are withheld from *every* Gemini request, not only the forced ones — the +argument-fidelity and prefix-stability reasons are the same ones set out for +any structured-output endpoint above. A pass that offers tools under +`tool_choice="auto"` therefore has none on the wire and answers as prose; the +Editor's unforced iteration is the one such pass, and it stops as it would for +any model that declined to call a tool. + +Reasoning-off is translated. Orb's `reasoning`, `chat_template_kwargs`, and +`thinking` fields mean nothing to the compatibility layer and are silently +ignored, so reasoning-off calls carry `reasoning_effort: "none"` — the control +the layer actually reads. Families that cannot disable thinking reject that +value; the rejection is learned per model like any other capability fact. + +`logprobs` is not supported on this surface, so Document mode's per-token +steering is unavailable on Gemini for the same reason it is on Anthropic. + +## Stable external contracts + +Endpoint routing is internal and ephemeral. Public API and browser SSE shapes, +database settings, `LLMClient.complete()`, and `LLMClient.list_models()` do not +change. Anthropic stream events are translated to Orb's existing `content`, +`reasoning`, and terminal `done` events; provider error events inside an HTTP +200 stream use the same sanitized `LLMCallError` path as HTTP rejections. diff --git a/mkdocs.yml b/mkdocs.yml index 0ab15022..b210208d 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -67,6 +67,7 @@ nav: - Home: index.md - Getting Started: getting-started.md - Architecture: + - LLM Endpoint Routing: architecture/endpoints.md - KV Cache Reuse: architecture/kv-cache.md - SSE Turn Stream: architecture/sse-stream.md - Secondary Workflows: architecture/secondary-workflow.md diff --git a/tests/integration/test_endpoint_transport_passes.py b/tests/integration/test_endpoint_transport_passes.py new file mode 100644 index 00000000..853f5934 --- /dev/null +++ b/tests/integration/test_endpoint_transport_passes.py @@ -0,0 +1,179 @@ +"""Pass-shaped calls through real LLMClient protocol adapters. + +The normal integration mock replaces ``LLMClient.complete`` wholesale. This +test stubs httpx one seam lower so Director/Writer/Editor request shapes and +stream translation run for OpenAI, Anthropic, and Gemini transports. +""" + +from __future__ import annotations + +import json +from unittest.mock import patch + +import pytest + +from backend.inference import client as llm_mod +from backend.inference import endpoint_profiles as ep +from backend.inference.client import LLMClient, parse_tool_calls + + +def _tool(name: str) -> dict: + return { + "type": "function", + "function": { + "name": name, + "description": name, + "parameters": { + "type": "object", + "properties": {"value": {"type": "string"}}, + "required": ["value"], + }, + }, + } + + +TOOLS = [_tool("direct_scene"), _tool("editor_apply_patch")] +DIRECTOR = {"type": "function", "function": {"name": "direct_scene"}} +EDITOR = {"type": "function", "function": {"name": "editor_apply_patch"}} + + +class _Response: + status_code = 200 + + def __init__(self, lines): + self.lines = lines + + async def __aenter__(self): + return self + + async def __aexit__(self, *_args): + return False + + async def aiter_lines(self): + for line in self.lines: + yield line + + +class _Transport: + def __init__(self, scripts): + self.scripts = list(scripts) + self.requests = [] + + async def __aenter__(self): + return self + + async def __aexit__(self, *_args): + return False + + def stream(self, method, url, json=None, headers=None): + self.requests.append({"url": url, "body": dict(json or {}), "headers": dict(headers or {})}) + return _Response(self.scripts.pop(0)) + + +def _data(payload: dict) -> str: + return f"data: {json.dumps(payload)}" + + +def _openai_tool(name: str) -> list[str]: + call = { + "index": 0, + "id": f"call-{name}", + "function": {"name": name, "arguments": '{"value":"ok"}'}, + } + return [_data({"choices": [{"delta": {"tool_calls": [call]}, "finish_reason": "tool_calls"}]}), "data: [DONE]"] + + +def _openai_text() -> list[str]: + return [_data({"choices": [{"delta": {"content": "draft"}, "finish_reason": "stop"}]}), "data: [DONE]"] + + +def _anthropic_tool(name: str) -> list[str]: + return [ + _data( + { + "type": "content_block_start", + "index": 0, + "content_block": {"type": "tool_use", "id": f"call-{name}", "name": name, "input": {}}, + } + ), + _data( + { + "type": "content_block_delta", + "index": 0, + "delta": {"type": "input_json_delta", "partial_json": '{"value":"ok"}'}, + } + ), + _data({"type": "message_delta", "delta": {"stop_reason": "tool_use"}}), + _data({"type": "message_stop"}), + ] + + +def _anthropic_text() -> list[str]: + return [ + _data({"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "draft"}}), + _data({"type": "message_delta", "delta": {"stop_reason": "end_turn"}}), + _data({"type": "message_stop"}), + ] + + +async def _complete(client: LLMClient, model: str, choice: dict | str) -> list[dict]: + return [ + event + async for event in client.complete( + [{"role": "system", "content": "system"}, {"role": "user", "content": "turn"}], + model, + tools=TOOLS, + tool_choice=choice, + max_tokens=100, + ) + ] + + +@pytest.mark.parametrize( + ("provider", "endpoint", "model"), + [ + ("openai", "https://openai.test/v1/chat/completions", "openai-model"), + ("anthropic", "https://api.anthropic.com/v1/messages", "claude-haiku-4-5"), + ("gemini", "https://generativelanguage.googleapis.com/v1beta/openai", "gemini-3-pro"), + ], +) +async def test_director_writer_editor_calls_cross_protocol_boundary(provider, endpoint, model): + ep._RESOLVED_ROUTES.clear() + if provider == "anthropic": + scripts = [_anthropic_tool("direct_scene"), _anthropic_text(), _anthropic_tool("editor_apply_patch")] + elif provider == "gemini": + # Gemini forced calls use strict response_format, whose JSON content is + # re-synthesized into Orb's ordinary tool-call message. + scripts = [ + [_data({"choices": [{"delta": {"content": '{"value":"ok"}'}, "finish_reason": "stop"}]}), "data: [DONE]"], + _openai_text(), + [_data({"choices": [{"delta": {"content": '{"value":"ok"}'}, "finish_reason": "stop"}]}), "data: [DONE]"], + ] + else: + scripts = [_openai_tool("direct_scene"), _openai_text(), _openai_tool("editor_apply_patch")] + + transport = _Transport(scripts) + client = LLMClient(endpoint, "secret") + with patch.object(llm_mod.httpx, "AsyncClient", lambda *args, **kwargs: transport): + director = await _complete(client, model, DIRECTOR) + writer = await _complete(client, model, "none") + editor = await _complete(client, model, EDITOR) + + assert parse_tool_calls(director[-1]["message"])[0]["name"] == "direct_scene" + assert writer[-1]["message"]["content"] == "draft" + assert parse_tool_calls(editor[-1]["message"])[0]["name"] == "editor_apply_patch" + assert len(transport.requests) == 3 + + bodies = [request["body"] for request in transport.requests] + if provider == "anthropic": + assert [body["tool_choice"] for body in bodies] == [ + {"type": "tool", "name": "direct_scene"}, + {"type": "none"}, + {"type": "tool", "name": "editor_apply_patch"}, + ] + assert all(body["tools"][0]["strict"] is True for body in bodies) + elif provider == "gemini": + assert "response_format" in bodies[0] and "response_format" in bodies[2] + assert "tools" not in bodies[1] and "tool_choice" not in bodies[1] + else: + assert [body["tool_choice"] for body in bodies] == [DIRECTOR, "none", EDITOR] diff --git a/tests/unit/test_endpoint_protocols.py b/tests/unit/test_endpoint_protocols.py new file mode 100644 index 00000000..bff82772 --- /dev/null +++ b/tests/unit/test_endpoint_protocols.py @@ -0,0 +1,872 @@ +"""Protocol resolution, Anthropic adaptation/streaming, and bounded probing.""" + +from __future__ import annotations + +import json +from unittest.mock import patch + +import pytest + +from backend.inference import anthropic +from backend.inference import client as llm_mod +from backend.inference import endpoint_profiles as ep +from backend.inference.client import LLMClient, parse_tool_calls, reasoning_cfg +from backend.inference.errors import LLMCallError + +TOOL = { + "type": "function", + "function": { + "name": "direct_scene", + "description": "Direct it", + "parameters": { + "type": "object", + "properties": {"mood": {"type": "string"}}, + "required": ["mood"], + }, + }, +} +FORCED = {"type": "function", "function": {"name": "direct_scene"}} + + +@pytest.fixture(autouse=True) +def _clear_learned_state(): + ep._RESOLVED_ROUTES.clear() + ep._TOOL_CHOICE_AUTO_ONLY.clear() + ep._TOOL_CHOICE_UNSUPPORTED.clear() + ep._REASONING_EFFORT_UNSUPPORTED.clear() + anthropic._SAMPLING_UNSUPPORTED.clear() + anthropic._THINKING_UNSUPPORTED.clear() + yield + ep._RESOLVED_ROUTES.clear() + ep._TOOL_CHOICE_AUTO_ONLY.clear() + ep._TOOL_CHOICE_UNSUPPORTED.clear() + ep._REASONING_EFFORT_UNSUPPORTED.clear() + anthropic._SAMPLING_UNSUPPORTED.clear() + anthropic._THINKING_UNSUPPORTED.clear() + + +@pytest.mark.parametrize( + ("configured", "protocol", "url", "models"), + [ + ( + "https://openai.test/v1/chat/completions", + "openai", + "https://openai.test/v1/chat/completions", + "https://openai.test/v1/models", + ), + ( + "https://proxy.test/prefix/v1/messages", + "anthropic", + "https://proxy.test/prefix/v1/messages", + "https://proxy.test/prefix/v1/models", + ), + ( + "https://proxy.test/prefix/v1beta/openai", + "openai", + "https://proxy.test/prefix/v1beta/openai/chat/completions", + "https://proxy.test/prefix/v1beta/openai/models", + ), + ( + "https://native.test/v1/messages", + "anthropic", + "https://native.test/v1/messages", + "https://native.test/v1/models", + ), + ], +) +def test_deterministic_resolution(configured, protocol, url, models): + route = ep.resolve_endpoint(configured) + assert route.protocol == protocol + assert route.url == url + assert route.models_url == models + assert route.authoritative + + +def test_explicit_resource_is_authoritative_regardless_of_host_name(): + route = ep.resolve_endpoint("https://api.anthropic.com/v1/chat/completions") + assert route.protocol == "openai" + assert route.url.endswith("/v1/chat/completions") + + +@pytest.mark.parametrize( + "configured", + [ + "https://api.anthropic.com", + "https://proxy.test/providers/anthropic/v1", + "https://claude.example/v1", + ], +) +def test_provider_and_model_names_do_not_select_a_protocol(configured): + route = ep.resolve_endpoint(configured, "claude-opus-999") + assert route.protocol == "openai" + assert not route.authoritative + + +def test_ambiguous_candidates_preserve_old_request_then_same_host_v1(): + routes = ep.endpoint_candidates("https://custom.test/prefix", "m") + assert [(route.protocol, route.url) for route in routes] == [ + ("openai", "https://custom.test/prefix/chat/completions"), + ("anthropic", "https://custom.test/prefix/messages"), + ("openai", "https://custom.test/v1/chat/completions"), + ("anthropic", "https://custom.test/v1/messages"), + ("openai", "https://custom.test/v1beta/openai/chat/completions"), + ] + + +def test_malformed_url_degrades_without_probe_candidates(): + routes = ep.endpoint_candidates("not a url", "m") + assert len(routes) == 1 + assert routes[0].protocol == "openai" + + +def test_route_probe_is_body_based(): + assert ep.should_probe_route(400, '{"error":"Unknown endpoint"}') + assert ep.should_probe_route(404, "Cannot POST /chat/completions") + assert ep.should_probe_route(404, '{"type":"error","error":{"type":"not_found_error"}}') + assert not ep.should_probe_route(404, '{"error":"model not found"}') + assert not ep.should_probe_route(404, '{"error":{"type":"not_found_error","message":"model not found"}}') + assert not ep.should_probe_route(400, '{"error":"Unknown model"}') + + +@pytest.mark.parametrize( + ("url", "gemini"), + [ + # A hostname alone is not protocol evidence. + ("https://generativelanguage.googleapis.com", False), + ("https://generativelanguage.googleapis.com/v1beta/openai/chat/completions", True), + # A proxy mirroring Google's compatibility resource is the same dialect. + ("https://gateway.ai.cloudflare.com/v1/a/g/google-ai-studio/v1beta/openai", True), + ("https://keys.example/v1beta/openai/chat/completions", True), + # Not Google's dialect: a bare vendor path segment could front an + # ordinary OpenAI route, and Vertex is a separate compatibility layer. + ("https://proxy.test/gemini/v1", False), + ("https://proxy.test/google/v1beta", False), + ("https://us-central1-aiplatform.googleapis.com/v1/projects/p/locations/l/endpoints/openapi", False), + ("https://openai.test/v1", False), + ], +) +def test_gemini_surface_predicate_covers_proxies_without_over_matching(url, gemini): + assert ep.is_gemini_openai_surface(url) is gemini + # The request policy follows the predicate, not the host string, so a proxy + # user gets the same structured-output posture as Google's own host. + assert ep.supports_structured_tool_calls(url, "gemini-3-pro") is gemini + + +def test_ambiguous_candidates_reach_the_gemini_compatibility_resource(): + # A Gemini-compat proxy configured at its bare root exposes /v1beta/openai + # and nothing the two /v1 guesses can reach. + routes = ep.endpoint_candidates("https://gemini-proxy.test", "gemini-3-pro") + assert ("openai", "https://gemini-proxy.test/v1beta/openai/chat/completions") in [ + (route.protocol, route.url) for route in routes + ] + + +def test_reasoning_off_asks_gemini_to_stop_thinking(): + # reasoning/chat_template_kwargs/thinking are all silently ignored by the + # compatibility layer; reasoning_effort is the control it actually reads. + body = {"model": "gemini-3-pro", **reasoning_cfg(False)} + ep.prepare_request_body("https://compat.test/v1beta/openai", "gemini-3-pro", body) + assert body["reasoning_effort"] == "none" + + # An explicit effort means the call asked for thinking; leave it alone. + on = {"model": "gemini-3-pro", "reasoning_effort": "high", **reasoning_cfg(True)} + ep.prepare_request_body("https://compat.test/v1beta/openai", "gemini-3-pro", on) + assert on["reasoning_effort"] == "high" + + # Non-Gemini endpoints keep Orb's historical reasoning-off shape untouched. + other = {"model": "m", **reasoning_cfg(False)} + ep.prepare_request_body("https://openai.test/v1", "m", other) + assert "reasoning_effort" not in other + + +def test_rejected_reasoning_effort_is_learned_and_dropped_for_the_session(): + url, model = "https://compat.test/v1beta/openai", "gemini-3-pro" + body = {"model": model, "reasoning_effort": "xhigh"} + rejection = '{"error":{"code":400,"message":"Invalid reasoning_effort: xhigh. Valid values are: high, low, medium, none","status":"INVALID_ARGUMENT"}}' + + fix = ep.recover_from_error(url, model, body, 400, rejection) + assert fix is not None and "reasoning_effort" not in body + + # The recovery persists through the set, not the in-place pop: the client + # rebuilds the outbound body from scratch on every retry. + rebuilt = {"model": model, "reasoning_effort": "xhigh"} + ep.prepare_request_body(url, model, rebuilt) + assert "reasoning_effort" not in rebuilt + + # A learned rejection also wins over the profile's reasoning-off transform, + # which is how a model that cannot disable thinking at all settles down. + off = {"model": model, **reasoning_cfg(False)} + ep.prepare_request_body(url, model, off) + assert "reasoning_effort" not in off + + +def test_generic_400_does_not_cost_a_working_reasoning_effort(): + body = {"model": "m", "reasoning_effort": "high"} + assert ep.recover_from_error("https://openai.test/v1", "m", body, 400, '{"error":"model not found"}') is None + assert body["reasoning_effort"] == "high" + + +def test_tool_schema_predicate_tracks_anthropic_and_gemini_wire_shapes(): + messages = [{"role": "user", "content": "hi"}] + assert LLMClient("https://native.test/v1/messages").sends_tool_schemas(messages, "opaque-model") + assert not LLMClient("https://compat.test/v1beta/openai").sends_tool_schemas(messages, "opaque-model") + + +def test_translate_messages_system_images_tools_and_coalescing(): + messages = [ + {"role": "system", "content": "one"}, + {"role": "system", "content": [{"type": "text", "text": "two"}]}, + { + "role": "user", + "content": [ + {"type": "text", "text": "look"}, + {"type": "image_url", "image_url": {"url": "data:image/png;base64,eA=="}}, + ], + }, + { + "role": "assistant", + "content": "calling", + "tool_calls": [ + { + "id": "call-1", + "type": "function", + "function": {"name": "direct_scene", "arguments": '{"mood":"eerie"}'}, + } + ], + }, + {"role": "tool", "tool_call_id": "call-1", "content": "done"}, + {"role": "user", "content": "continue"}, + ] + system, out = anthropic.translate_messages(messages) + assert system == "one\n\ntwo" + assert out[0]["content"][1] == { + "type": "image", + "source": {"type": "base64", "media_type": "image/png", "data": "eA=="}, + } + assert out[1]["content"][1] == { + "type": "tool_use", + "id": "call-1", + "name": "direct_scene", + "input": {"mood": "eerie"}, + } + # tool result and following user content become one legal adjacent user turn. + assert out[2]["role"] == "user" + assert [block["type"] for block in out[2]["content"]] == ["tool_result", "text"] + + +def test_anthropic_body_allowlist_tools_choices_reasoning_and_sampling(): + canonical = { + "model": "claude-haiku-4-5", + "messages": [{"role": "user", "content": "hi"}], + "stream": True, + "tools": [TOOL], + "tool_choice": FORCED, + "temperature": 0.8, + "top_p": 0.95, + "top_k": 40, + "min_p": 0.1, + "repetition_penalty": 1.1, + "reasoning": {"enabled": True}, + "reasoning_effort": "xhigh", + "chat_template_kwargs": {"thinking": True}, + "stream_options": {"include_usage": True}, + "logprobs": True, + } + body = anthropic.build_request_body( + canonical, + "https://api.anthropic.com/v1/messages", + "claude-haiku-4-5", + {"metadata": {"user_id": "u"}, "response_format": {"type": "json"}, "seed": 7}, + ) + assert body["max_tokens"] == anthropic.DEFAULT_MAX_TOKENS + assert body["tools"][0]["strict"] is True + # ``strict`` obliges the schema to close and require everything. + assert body["tools"][0]["input_schema"] == { + "type": "object", + "properties": {"mood": {"type": "string"}}, + "required": ["mood"], + "additionalProperties": False, + } + assert body["tool_choice"] == {"type": "tool", "name": "direct_scene"} + assert body["thinking"] == {"type": "adaptive", "display": "summarized"} + assert body["output_config"] == {"effort": "xhigh"} + assert body["temperature"] == 0.8 and body["top_p"] == 0.95 and body["top_k"] == 40 + assert body["metadata"] == {"user_id": "u"} + for forbidden in ( + "reasoning", + "chat_template_kwargs", + "stream_options", + "logprobs", + "min_p", + "repetition_penalty", + "response_format", + "seed", + ): + assert forbidden not in body + + +def test_sampling_support_is_not_inferred_from_model_name(): + body = anthropic.build_request_body( + { + "messages": [], + "temperature": 0.8, + "top_p": 0.95, + "top_k": 40, + "thinking": {"type": "disabled"}, + }, + "https://api.anthropic.com", + "claude-opus-5-20260801", + ) + assert body["temperature"] == 0.8 and body["top_p"] == 0.95 and body["top_k"] == 40 + assert "thinking" not in body and "output_config" not in body + + +def test_tool_choice_mapping(): + assert anthropic.translate_tool_choice("none") == {"type": "none"} + assert anthropic.translate_tool_choice("auto") == {"type": "auto"} + assert anthropic.translate_tool_choice("required") == {"type": "any"} + assert anthropic.translate_tool_choice(FORCED) == {"type": "tool", "name": "direct_scene"} + + +class _Response: + def __init__(self, status=200, *, error="", lines=()): + self.status_code = status + self.error = error + self.lines = list(lines) + + async def __aenter__(self): + return self + + async def __aexit__(self, *_args): + return False + + async def aread(self): + return self.error.encode() + + async def aiter_lines(self): + for line in self.lines: + yield line + + +class _HTTP: + def __init__(self, responses): + self.responses = list(responses) + self.requests = [] + + async def __aenter__(self): + return self + + async def __aexit__(self, *_args): + return False + + def stream(self, method, url, json=None, headers=None): + self.requests.append({"method": method, "url": url, "body": dict(json or {}), "headers": dict(headers or {})}) + return self.responses.pop(0) + + +def _line(payload: dict) -> str: + return f"data: {json.dumps(payload)}" + + +ANTHROPIC_TOOL_STREAM = [ + _line({"type": "message_start", "message": {"usage": {"input_tokens": 10, "cache_read_input_tokens": 4}}}), + _line({"type": "ping"}), + _line({"type": "content_block_start", "index": 0, "content_block": {"type": "thinking", "thinking": "why"}}), + _line({"type": "content_block_delta", "index": 0, "delta": {"type": "thinking_delta", "thinking": " now"}}), + _line( + { + "type": "content_block_start", + "index": 1, + "content_block": {"type": "tool_use", "id": "tool-1", "name": "direct_scene", "input": {}}, + } + ), + _line({"type": "content_block_delta", "index": 1, "delta": {"type": "input_json_delta", "partial_json": '{"mood"'}}), + _line({"type": "content_block_delta", "index": 1, "delta": {"type": "input_json_delta", "partial_json": ':"eerie"}'}}), + _line({"type": "message_delta", "delta": {"stop_reason": "tool_use"}, "usage": {"output_tokens": 7}}), + _line({"type": "message_stop"}), + _line({"type": "content_block_delta", "index": 2, "delta": {"type": "text_delta", "text": "ignored"}}), +] + + +async def _run(client: LLMClient, fake: _HTTP, model="claude-haiku-4-5", **kwargs): + events = [] + with patch.object(llm_mod.httpx, "AsyncClient", lambda *args, **kw: fake): + async for event in client.complete([{"role": "user", "content": "hi"}], model, **kwargs): + events.append(event) + return events + + +async def test_anthropic_wire_headers_body_and_stream_translation(): + fake = _HTTP([_Response(lines=ANTHROPIC_TOOL_STREAM)]) + client = LLMClient( + "https://api.anthropic.com/v1/messages", + "sk-test", + reasoning_effort="high", + extra_headers="X-Custom: yes\nanthropic-version: 2026-01-01", + extra_body='{"seed": 9}', + ) + events = await _run( + client, + fake, + tools=[TOOL], + tool_choice=FORCED, + max_tokens=123, + temperature=0.4, + **reasoning_cfg(True), + ) + request = fake.requests[0] + assert request["url"] == "https://api.anthropic.com/v1/messages" + assert request["headers"] == {"x-api-key": "sk-test", "X-Custom": "yes", "anthropic-version": "2026-01-01"} + assert "Authorization" not in request["headers"] + assert request["body"]["max_tokens"] == 123 + assert request["body"]["thinking"] == {"type": "adaptive", "display": "summarized"} + assert request["body"]["tool_choice"] == {"type": "tool", "name": "direct_scene"} + assert "seed" not in request["body"] + assert [event["delta"] for event in events if event["type"] == "reasoning"] == ["why", " now"] + assert not [event for event in events if event["type"] == "content"] + done = events[-1] + assert done["usage"] == {"input_tokens": 10, "cache_read_input_tokens": 4, "output_tokens": 7} + assert done["message"]["finish_reason"] == "tool_calls" + assert parse_tool_calls(done["message"]) == [{"name": "direct_scene", "arguments": {"mood": "eerie"}}] + + +async def test_anthropic_message_stop_terminates_without_done_sentinel(): + lines = [ + _line({"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "hello"}}), + _line({"type": "message_delta", "delta": {"stop_reason": "end_turn"}, "usage": {"output_tokens": 1}}), + _line({"type": "message_stop"}), + ] + events = await _run(LLMClient("https://api.anthropic.com/v1/messages"), _HTTP([_Response(lines=lines)])) + assert events[0] == {"type": "content", "delta": "hello"} + assert events[-1]["message"] == {"content": "hello", "finish_reason": "stop"} + + +async def test_gemini_uses_normalized_openai_route_structured_output_and_effort(): + lines = [ + 'data: {"choices":[{"delta":{"content":"{\\"mood\\":\\"bright\\"}"},"finish_reason":"stop"}]}', + "data: [DONE]", + ] + fake = _HTTP([_Response(lines=lines)]) + client = LLMClient("https://generativelanguage.googleapis.com/v1beta/openai", "key", reasoning_effort="high") + events = await _run( + client, + fake, + model="gemini-3-pro", + tools=[TOOL], + tool_choice=FORCED, + **reasoning_cfg(True), + ) + request = fake.requests[0] + assert request["url"] == "https://generativelanguage.googleapis.com/v1beta/openai/chat/completions" + assert request["headers"] == {"Authorization": "Bearer key"} + assert "tools" not in request["body"] and "tool_choice" not in request["body"] + assert request["body"]["reasoning_effort"] == "high" + assert request["body"]["response_format"]["json_schema"]["strict"] is True + assert parse_tool_calls(events[-1]["message"]) == [{"name": "direct_scene", "arguments": {"mood": "bright"}}] + + +async def test_indexless_tool_call_deltas_stay_separate_calls(): + """Google's compatibility surface omits ``index`` from tool-call deltas. + + Reached on any Gemini-compat route that still sends ``tools`` -- a proxy the + profile does not claim, or a pair demoted by ``note_structured_output_ignored``. + Keying on a 0 default merged both calls into one entry whose name was the two + names concatenated and whose arguments were unparseable. + """ + lines = [ + _line( + { + "choices": [ + { + "delta": { + "tool_calls": [ + { + "id": "0", + "type": "function", + "function": {"name": "direct_scene", "arguments": '{"mood":"eerie"}'}, + } + ] + } + } + ] + } + ), + _line( + { + "choices": [ + { + "delta": { + "tool_calls": [ + { + "id": "1", + "type": "function", + "function": {"name": "editor_rewrite", "arguments": '{"text":"hi"}'}, + } + ] + } + } + ] + } + ), + _line({"choices": [{"delta": {}, "finish_reason": "tool_calls"}]}), + "data: [DONE]", + ] + events = await _run( + LLMClient("https://gemini-proxy.test/v1/chat/completions"), + _HTTP([_Response(lines=lines)]), + model="gemini-3-pro", + tools=[TOOL], + tool_choice="auto", + ) + assert parse_tool_calls(events[-1]["message"]) == [ + {"name": "direct_scene", "arguments": {"mood": "eerie"}}, + {"name": "editor_rewrite", "arguments": {"text": "hi"}}, + ] + + +async def test_indexless_argument_fragments_append_to_the_open_call(): + """Only a delta that STARTS a call opens a slot; continuations append. + + The OpenAI contract sends ``id``/``name`` on a call's first chunk alone, so + a bare ``arguments`` fragment must not be mistaken for a second call. + """ + lines = [ + _line( + { + "choices": [ + { + "delta": { + "tool_calls": [ + {"id": "0", "type": "function", "function": {"name": "direct_scene", "arguments": '{"mood"'}} + ] + } + } + ] + } + ), + _line({"choices": [{"delta": {"tool_calls": [{"function": {"arguments": ':"eerie"}'}}]}}]}), + _line({"choices": [{"delta": {}, "finish_reason": "tool_calls"}]}), + "data: [DONE]", + ] + events = await _run( + LLMClient("https://gemini-proxy.test/v1/chat/completions"), + _HTTP([_Response(lines=lines)]), + model="gemini-3-pro", + tools=[TOOL], + tool_choice="auto", + ) + assert parse_tool_calls(events[-1]["message"]) == [{"name": "direct_scene", "arguments": {"mood": "eerie"}}] + + +async def test_indexed_tool_call_deltas_are_unaffected(): + """The ordinary OpenAI shape must still be keyed by its own index.""" + lines = [ + _line( + { + "choices": [ + { + "delta": { + "tool_calls": [ + { + "index": 1, + "id": "b", + "type": "function", + "function": {"name": "editor_rewrite", "arguments": "{}"}, + } + ] + } + } + ] + } + ), + _line( + { + "choices": [ + { + "delta": { + "tool_calls": [ + { + "index": 0, + "id": "a", + "type": "function", + "function": {"name": "direct_scene", "arguments": "{}"}, + } + ] + } + } + ] + } + ), + _line({"choices": [{"delta": {}, "finish_reason": "tool_calls"}]}), + "data: [DONE]", + ] + events = await _run( + LLMClient("https://openai.test/v1/chat/completions"), + _HTTP([_Response(lines=lines)]), + model="openai-model", + tools=[TOOL], + tool_choice="auto", + ) + # Sorted by index, so the late index-0 chunk still leads. + assert [call["name"] for call in parse_tool_calls(events[-1]["message"])] == ["direct_scene", "editor_rewrite"] + + +async def test_anthropic_midstream_error_uses_sanitized_llm_error(): + lines = [ + _line({"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "partial"}}), + _line({"type": "error", "error": {"type": "overloaded_error", "message": "busy sk-secret"}}), + ] + with pytest.raises(LLMCallError) as caught: + await _run(LLMClient("https://api.anthropic.com/v1/messages", "sk-secret"), _HTTP([_Response(lines=lines)])) + assert caught.value.response.status_code == 502 + assert caught.value.sentence == "busy [redacted]" + assert "sk-secret" not in caught.value.body + + +async def test_ambiguous_endpoint_probes_bounded_routes_and_caches_success(): + wrong = '{"error":"Cannot POST /prefix/chat/completions"}' + wrong_route = '{"error":"route not found"}' + success = [ + _line({"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "hi"}}), + _line({"type": "message_stop"}), + ] + fake = _HTTP( + [ + _Response(404, error=wrong), + _Response(404, error=wrong_route), + _Response(404, error=wrong_route), + _Response(lines=success), + ] + ) + client = LLMClient("https://custom.test/prefix") + events = await _run(client, fake, model="claude-proxy") + assert [request["url"] for request in fake.requests] == [ + "https://custom.test/prefix/chat/completions", + "https://custom.test/prefix/messages", + "https://custom.test/v1/chat/completions", + "https://custom.test/v1/messages", + ] + assert events[-1]["message"]["content"] == "hi" + assert ep.resolve_endpoint(client.base_url, "claude-proxy").protocol == "anthropic" + + again = _HTTP([_Response(lines=success)]) + await _run(client, again, model="claude-proxy") + assert [request["url"] for request in again.requests] == ["https://custom.test/v1/messages"] + + +async def test_auth_family_retry_is_evidence_gated_and_bounded(): + success = [_line({"type": "message_stop"})] + fake = _HTTP([_Response(401, error='{"error":"bad auth"}'), _Response(lines=success)]) + await _run(LLMClient("https://proxy.test/v1/messages", "key"), fake, model="claude-proxy") + assert fake.requests[0]["headers"]["x-api-key"] == "key" + assert fake.requests[1]["headers"]["Authorization"] == "Bearer key" + assert "x-api-key" not in fake.requests[1]["headers"] + + +async def test_ambiguous_endpoint_detects_messages_dialect_without_name_hints(): + not_found = '{"type":"error","error":{"type":"not_found_error","message":"Not Found"}}' + success = [_line({"type": "message_stop"})] + fake = _HTTP( + [ + _Response(401, error='{"error":"authentication required"}'), + _Response(404, error=not_found), + _Response(lines=success), + ] + ) + client = LLMClient("https://opaque.test", "key") + + await _run(client, fake, model="model-7") + + assert [request["url"] for request in fake.requests] == [ + "https://opaque.test/chat/completions", + "https://opaque.test/chat/completions", + "https://opaque.test/messages", + ] + assert fake.requests[0]["headers"] == {"Authorization": "Bearer key"} + assert fake.requests[1]["headers"]["x-api-key"] == "key" + assert fake.requests[2]["headers"]["x-api-key"] == "key" + assert ep.resolve_endpoint(client.base_url, "model-7").protocol == "anthropic" + + +async def test_authoritative_openai_auth_rejection_without_header_evidence_is_not_retried(): + fake = _HTTP([_Response(401, error='{"error":"bad auth"}')]) + with pytest.raises(LLMCallError): + await _run(LLMClient("https://custom.test/v1/chat/completions", "key"), fake, model="ordinary-model") + assert len(fake.requests) == 1 + + +async def test_sampling_rejection_is_learned_and_retried_once(): + rejection = '{"error":{"message":"temperature is not supported"}}' + fake = _HTTP([_Response(400, error=rejection), _Response(lines=[_line({"type": "message_stop"})])]) + client = LLMClient("https://proxy.test/v1/messages") + await _run(client, fake, model="hidden-model", temperature=0.8, top_p=0.9, top_k=20) + assert fake.requests[0]["body"]["temperature"] == 0.8 + assert not {"temperature", "top_p", "top_k"} & fake.requests[1]["body"].keys() + assert (client.base_url, "hidden-model") in anthropic._SAMPLING_UNSUPPORTED + + +async def test_anthropic_forced_choice_rejection_falls_back_to_auto(): + rejection = '{"error":{"message":"tool_choice type any is not supported for this model"}}' + fake = _HTTP( + [ + _Response(400, error=rejection), + _Response(lines=[_line({"type": "message_stop"})]), + ] + ) + client = LLMClient("https://proxy.test/v1/messages") + await _run(client, fake, model="claude-fable-5-1", tools=[TOOL], tool_choice="required") + assert fake.requests[0]["body"]["tool_choice"] == {"type": "any"} + assert fake.requests[1]["body"]["tool_choice"] == {"type": "auto"} + assert not ep.honors_forced_tool_choice(client.base_url, "claude-fable-5-1") + + +@pytest.mark.parametrize("choice", ["none", "required", FORCED]) +async def test_auto_only_tool_choice_recovery_and_process_memory(choice): + rejection = '{"error":{"message":"only \\"auto\\" is supported for tool_choice"}}' + openai_done = ['data: {"choices":[{"delta":{},"finish_reason":"stop"}]}', "data: [DONE]"] + fake = _HTTP([_Response(400, error=rejection), _Response(lines=openai_done)]) + client = LLMClient("https://openrouter.ai/api/v1") + await _run(client, fake, model="unlisted/model", tools=[TOOL], tool_choice=choice) + assert fake.requests[1]["body"]["tool_choice"] == "auto" + assert (client.base_url, "unlisted/model") in ep._TOOL_CHOICE_AUTO_ONLY + + again = _HTTP([_Response(lines=openai_done)]) + await _run(client, again, model="unlisted/model", tools=[TOOL], tool_choice=choice) + assert again.requests[0]["body"]["tool_choice"] == "auto" + + +async def test_reasoning_effort_rejection_retries_and_is_remembered(): + """A level Orb offers but the endpoint refuses must self-heal, not fail the turn. + + Orb's picker spans none/minimal/low/medium/high/xhigh; Gemini's set is + high/low/medium/none. Learning the refusal from the body beats a hard-coded + per-provider list, which would have wrongly clamped ``minimal`` for the + months Google rejected a value its own table documented. + """ + rejection = ( + '{"error":{"code":400,"message":"Invalid reasoning_effort: xhigh. ' + 'Valid values are: high, low, medium, none","status":"INVALID_ARGUMENT"}}' + ) + openai_done = ['data: {"choices":[{"delta":{"content":"hi"},"finish_reason":"stop"}]}', "data: [DONE]"] + fake = _HTTP([_Response(400, error=rejection), _Response(lines=openai_done)]) + client = LLMClient("https://generativelanguage.googleapis.com/v1beta/openai", "key", reasoning_effort="xhigh") + + events = await _run(fake=fake, client=client, model="gemini-3-pro", **reasoning_cfg(True)) + + assert fake.requests[0]["body"]["reasoning_effort"] == "xhigh" + assert "reasoning_effort" not in fake.requests[1]["body"] + assert events[-1]["message"]["content"] == "hi" + + # Learned for the session: the next call skips the failing round-trip. + again = _HTTP([_Response(lines=openai_done)]) + await _run(client, again, model="gemini-3-pro", **reasoning_cfg(True)) + assert "reasoning_effort" not in again.requests[0]["body"] + + +async def test_gemini_model_that_cannot_disable_thinking_settles_after_one_rejection(): + """``reasoning_effort='none'`` is right for 2.5 Flash and refused by the 3 series. + + The profile asks anyway and lets the rejection teach it, the same posture as + the Anthropic sampling and thinking fields. + """ + rejection = '{"error":{"code":400,"message":"Invalid reasoning_effort: none. Valid values are: high, low, medium","status":"INVALID_ARGUMENT"}}' + openai_done = ['data: {"choices":[{"delta":{"content":"hi"},"finish_reason":"stop"}]}', "data: [DONE]"] + fake = _HTTP([_Response(400, error=rejection), _Response(lines=openai_done)]) + client = LLMClient("https://generativelanguage.googleapis.com/v1beta/openai", "key") + + await _run(client, fake, model="gemini-3-pro", **reasoning_cfg(False)) + + assert fake.requests[0]["body"]["reasoning_effort"] == "none" + assert "reasoning_effort" not in fake.requests[1]["body"] + + +async def test_unrelated_failure_does_not_probe(): + fake = _HTTP([_Response(404, error='{"error":"model not found"}')]) + with pytest.raises(LLMCallError): + await _run(LLMClient("https://custom.test/prefix"), fake, model="missing") + assert len(fake.requests) == 1 + + +def test_v1_base_url_collapses_the_duplicate_openai_candidate(): + # The common local-server shape: the configured URL already ends in /v1, so + # the historical request and the host-root OpenAI guess are the same URL. + # Without the dedupe the probe loop would re-POST an identical request. + routes = ep.endpoint_candidates("http://localhost:1234/v1", "m") + assert [(route.protocol, route.url) for route in routes] == [ + ("openai", "http://localhost:1234/v1/chat/completions"), + ("anthropic", "http://localhost:1234/v1/messages"), + ("openai", "http://localhost:1234/v1beta/openai/chat/completions"), + ] + + +def test_partial_required_tool_schema_is_closed_before_strict_goes_out(): + partial = { + "type": "function", + "function": { + "name": "direct_scene", + "parameters": { + "type": "object", + "properties": {"mood": {"type": "string"}, "moods": {"type": "array"}}, + "required": [], + }, + }, + } + schema = anthropic.translate_tools([partial])[0]["input_schema"] + assert schema["additionalProperties"] is False + assert sorted(schema["required"]) == ["mood", "moods"] + # Optional survives as nullable rather than as an omittable key. + assert schema["properties"]["mood"]["type"] == ["string", "null"] + + +def test_reasoning_fields_are_dropped_and_learned_on_rejection(): + url = "https://proxy.test/v1/messages" + body = anthropic.build_request_body( + {"messages": [], "reasoning": {"enabled": True}, "reasoning_effort": "high"}, + url, + "haiku-4-5-via-proxy", + ) + assert body["thinking"] == {"type": "adaptive", "display": "summarized"} + assert body["output_config"] == {"effort": "high"} + + rejection = '{"error":{"message":"thinking: Extra inputs are not permitted"}}' + fix = anthropic.recover_thinking_error(url, "haiku-4-5-via-proxy", body, 400, rejection) + assert fix is not None + assert "thinking" not in body and "output_config" not in body + + # Learned for the rest of the session, so the rebuilt body omits them too. + again = anthropic.build_request_body( + {"messages": [], "reasoning": {"enabled": True}, "reasoning_effort": "high"}, + url, + "haiku-4-5-via-proxy", + ) + assert "thinking" not in again and "output_config" not in again + + +def test_unrelated_400_does_not_strip_reasoning_fields(): + url = "https://proxy.test/v1/messages" + body = anthropic.build_request_body({"messages": [], "reasoning": {"enabled": True}}, url, "some-model") + assert anthropic.recover_thinking_error(url, "some-model", body, 400, '{"error":"model not found"}') is None + assert body["thinking"] == {"type": "adaptive", "display": "summarized"} + assert (url, "some-model") not in anthropic._THINKING_UNSUPPORTED + + +async def test_doc_mode_on_anthropic_buffers_text_instead_of_streaming_it(): + # tools_in_prompt=False makes _plan force a name, but the Anthropic allowlist + # drops response_format, so the endpoint answers with plain text. That text is + # the forced payload -- it must not reach the caller as content deltas. + lines = [ + _line({"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": '{"mood"'}}), + _line({"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": ':"eerie"}'}}), + _line({"type": "message_delta", "delta": {"stop_reason": "end_turn"}, "usage": {"output_tokens": 3}}), + _line({"type": "message_stop"}), + ] + fake = _HTTP([_Response(lines=lines)]) + events = await _run( + LLMClient("https://api.anthropic.com/v1/messages", "sk-test"), + fake, + tools=[TOOL], + tool_choice=FORCED, + tools_in_prompt=False, + ) + assert not [event for event in events if event["type"] == "content"] + assert parse_tool_calls(events[-1]["message"]) == [{"name": "direct_scene", "arguments": {"mood": "eerie"}}] diff --git a/tests/unit/test_forced_tool_choice_fallback.py b/tests/unit/test_forced_tool_choice_fallback.py index 74ba97d2..258979f1 100644 --- a/tests/unit/test_forced_tool_choice_fallback.py +++ b/tests/unit/test_forced_tool_choice_fallback.py @@ -2,7 +2,7 @@ Covers: - ModelProfile.allow_extra=None disables drop-filtering entirely. - - The OpenRouter PROFILES entry coerces forced tool_choice proactively. + - OpenRouter catalog models are not encoded as permanent profiles. - LLMClient.complete()'s provider-gated, error-specific retry: drops tool_choice once for the matching OpenRouter 404 (regardless of its value), raises immediately for unrelated 404s, and never retries when no @@ -59,20 +59,6 @@ def test_allow_extra_frozenset_still_drops(): assert "weird" not in body -def test_openrouter_minimax_profile_coerces_forced_tool_choice(): - prof = profile_for("https://openrouter.ai/api/v1", "minimax/minimax-m3") - assert prof is not None - body = { - "model": "minimax/minimax-m3", - "messages": [], - "tool_choice": {"type": "function", "function": {"name": "direct_scene"}}, - "temperature": 0.7, - } - prof.apply(body) - assert body["tool_choice"] == "auto" - assert body["temperature"] == 0.7 # nothing dropped - - def test_openrouter_unlisted_model_is_passthrough(): assert profile_for("https://openrouter.ai/api/v1", "some/other-model") is None @@ -83,7 +69,7 @@ def test_openrouter_unlisted_model_is_passthrough(): def test_is_tool_choice_unsupported_signature(): txt = "No endpoints found that support the provided 'tool_choice' value." assert _is_tool_choice_unsupported(404, txt) - assert not _is_tool_choice_unsupported(400, txt) + assert _is_tool_choice_unsupported(400, txt) assert not _is_tool_choice_unsupported(404, "model not found") diff --git a/tests/unit/test_model_discovery.py b/tests/unit/test_model_discovery.py index 1c11e818..bda797ad 100644 --- a/tests/unit/test_model_discovery.py +++ b/tests/unit/test_model_discovery.py @@ -27,6 +27,26 @@ async def get(self, url, *, headers): return httpx.Response(200, json=self.payload, request=request) +class _ProbingCatalogClient: + responses: list[tuple[int, object]] = [] + requests: list[dict] = [] + + def __init__(self, **_kwargs): + pass + + async def __aenter__(self): + return self + + async def __aexit__(self, *_args): + return None + + async def get(self, url, *, headers): + self.requests.append({"url": url, "headers": headers}) + status, payload = self.responses.pop(0) + request = httpx.Request("GET", url) + return httpx.Response(status, json=payload, request=request) + + @pytest.mark.asyncio async def test_list_models_uses_openai_contract_auth_and_proxy(monkeypatch): _CatalogClient.payload = { @@ -61,3 +81,92 @@ async def test_list_models_rejects_non_openai_response(monkeypatch): with pytest.raises(ValueError, match="data list"): await LLMClient("https://models.test/v1").list_models() + + +@pytest.mark.asyncio +async def test_anthropic_models_use_sibling_resource_and_native_auth(monkeypatch): + _CatalogClient.payload = {"data": [{"id": "claude-opus-5"}]} + _CatalogClient.seen = {} + monkeypatch.setattr(client_module.httpx, "AsyncClient", _CatalogClient) + + models = await LLMClient("https://api.anthropic.com/v1/messages", "secret-key").list_models() + + assert models == ["claude-opus-5"] + assert _CatalogClient.seen["url"] == "https://api.anthropic.com/v1/models" + assert _CatalogClient.seen["headers"] == { + "x-api-key": "secret-key", + "anthropic-version": "2023-06-01", + } + + +@pytest.mark.asyncio +async def test_gemini_models_use_normalized_surface_and_strip_models_prefix(monkeypatch): + _CatalogClient.payload = {"data": [{"id": "models/gemini-3-pro"}, {"id": "gemini-3-flash"}]} + _CatalogClient.seen = {} + monkeypatch.setattr(client_module.httpx, "AsyncClient", _CatalogClient) + + models = await LLMClient("https://generativelanguage.googleapis.com/v1beta/openai", "gemini-key").list_models() + + assert models == ["gemini-3-flash", "gemini-3-pro"] + assert _CatalogClient.seen["url"] == "https://generativelanguage.googleapis.com/v1beta/openai/models" + assert _CatalogClient.seen["headers"] == {"Authorization": "Bearer gemini-key"} + + +@pytest.mark.asyncio +async def test_gemini_proxy_catalogue_is_normalized_like_googles_own(monkeypatch): + # The prefix is a property of the dialect, not of Google's hostname: a proxy + # mirroring /v1beta/openai relays the same ``models/``-prefixed ids, and + # leaving them in put an unusable-looking id in the picker. + _CatalogClient.payload = {"data": [{"id": "models/gemini-3-pro"}]} + _CatalogClient.seen = {} + monkeypatch.setattr(client_module.httpx, "AsyncClient", _CatalogClient) + + models = await LLMClient("https://keys.example/v1beta/openai", "k").list_models() + + assert models == ["gemini-3-pro"] + assert _CatalogClient.seen["url"] == "https://keys.example/v1beta/openai/models" + + +@pytest.mark.asyncio +async def test_non_gemini_catalogue_keeps_a_models_prefixed_id(monkeypatch): + _CatalogClient.payload = {"data": [{"id": "models/local-thing"}]} + _CatalogClient.seen = {} + monkeypatch.setattr(client_module.httpx, "AsyncClient", _CatalogClient) + + assert await LLMClient("http://localhost:8080/v1", "").list_models() == ["models/local-thing"] + + +@pytest.mark.asyncio +async def test_ambiguous_catalogue_detection_uses_no_provider_or_model_names(monkeypatch): + from backend.inference import endpoint_profiles + + endpoint_profiles._RESOLVED_ROUTES.clear() + _ProbingCatalogClient.requests = [] + _ProbingCatalogClient.responses = [ + (404, {"error": "not found"}), + (404, {"error": "not found"}), + (401, {"error": "wrong auth"}), + (200, {"data": [{"id": "model-7"}]}), + ] + monkeypatch.setattr(client_module.httpx, "AsyncClient", _ProbingCatalogClient) + + client = LLMClient("https://opaque.test", "secret-key") + models = await client.list_models() + + assert models == ["model-7"] + assert _ProbingCatalogClient.requests == [ + {"url": "https://opaque.test/models", "headers": {"Authorization": "Bearer secret-key"}}, + { + "url": "https://opaque.test/models", + "headers": {"x-api-key": "secret-key", "anthropic-version": "2023-06-01"}, + }, + {"url": "https://opaque.test/v1/models", "headers": {"Authorization": "Bearer secret-key"}}, + { + "url": "https://opaque.test/v1/models", + "headers": {"x-api-key": "secret-key", "anthropic-version": "2023-06-01"}, + }, + ] + assert endpoint_profiles.resolve_endpoint(client.base_url, "model-7").protocol == "anthropic" + assert endpoint_profiles.endpoint_candidates(client.base_url, "model-7") == [ + endpoint_profiles.resolve_endpoint(client.base_url, "model-7") + ]