diff --git a/CHANGELOG.md b/CHANGELOG.md index efc2c02..09edb4c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,57 @@ All notable changes to blockrun-llm will be documented in this file. +## 1.12.0 — 2026-08-19 + +### Added +- **Smart routing on the Solana clients.** `SolanaLLMClient` and + `AsyncSolanaLLMClient` had no routing at all — `smart_chat`, `route` and the + routing profiles were Base-only, so a Solana user got no model selection while + the TypeScript SDK offered it on both chains. All four Python clients (Base and + Solana, sync and async) now expose the same surface: `route()`, `smart_chat()` + and `smart_chat_completion()`. Both chains run the same Router Core engine + against the same catalog, so an identical request picks an identical model; + only the x402 payment floor in the cost metadata differs ($0.002 Base, + $0.001 Solana). Pinned by `tests/unit/test_routing_parity.py`. + +- **`smart_chat_completion(messages, ...)`** on every client — routing for a full + message list rather than a single prompt. `tools`, `tool_choice` and + `response_format` are inputs to the *decision*, not just the request: a turn + that must call a tool routes to a tool-capable model, a JSON schema forces the + structured-output tier, and image parts route to a vision model. Capacity is + checked against the whole transcript, because an agent conversation can be + 100x its final turn and a context overflow is a non-transient error the + fallback chain cannot rescue. + +- **`blockrun/auto`, `blockrun/eco` and `blockrun/premium` virtual model ids.** + Passing one to `chat()` or `chat_completion()` routes the turn instead of + calling a model of that name, ranked fallback chain included — TypeScript SDK + parity, and it lets OpenAI-compatible code opt into routing by changing one + string. + +- **`fallback_models` on the Solana `chat()` / `chat_completion()`.** The + parameter existed only on the Solana streaming path, so a routed Solana call + had a recovery chain it could not walk. The chain now steps to the next ranked + model on a timeout, network error or 5xx, using the same + `_should_fallback_solana` classifier as the stream path — a settled payment is + never retried, so a second model cannot sign a second transfer for one call. + +### Fixed +- **A 429 now walks the fallback chain instead of failing the call.** Both + clients treated only 5xx as retriable, so a saturated upstream ended the + request even with capable models left in the chain. Found live: a rate-limited + free model answered 429 and the three remaining free models were never tried. + The TypeScript adapter has always counted 429 as transient — the next model in + the chain is a different upstream. Settled payments and permanent payment + failures are still refused before the status check, so no call can pay twice. + +### Changed +- The `/v1/models` → pricing-map conversion moved to + `router_adapter.build_model_pricing()`, shared by all four clients instead of + being written out per client. Rows the catalog marks `available: false` are + skipped everywhere now (previously only the Base sync client did this, as of + 1.11.0). + ## 1.11.0 — 2026-08-15 ### Added diff --git a/README.md b/README.md index 96ec70a..4fe95de 100644 --- a/README.md +++ b/README.md @@ -142,6 +142,27 @@ result = client.smart_chat("Prove the Riemann hypothesis step by step") print(result.model) # 'deepseek/deepseek-v4-pro' ``` +Routing works the same on every client — `LLMClient`, `AsyncLLMClient`, +`SolanaLLMClient` and `AsyncSolanaLLMClient` all expose `route()`, +`smart_chat()` and `smart_chat_completion()`. Both chains run the same engine +against the same catalog, so an identical request picks an identical model; only +the x402 minimum in the cost estimate differs. + +```python +# Route a full message list — tools and response_format shape the decision, +# not just the request +result = client.smart_chat_completion( + [{"role": "user", "content": "Cancel order B-42"}], + tools=[{"type": "function", "function": {"name": "cancel_order", "parameters": {}}}], + tool_choice="required", +) +print(result.model) # a tool-capable model +print(result.routing.task_type) # 'tool_agent' + +# Or opt in from OpenAI-compatible code by changing one string +response = client.chat_completion("blockrun/auto", messages) +``` + Want to see the decision without paying for a call? `client.route(...)` runs the same routing locally and returns the decision only: diff --git a/VERSION b/VERSION index 1cac385..0eed1a2 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.11.0 +1.12.0 diff --git a/blockrun_llm/__init__.py b/blockrun_llm/__init__.py index a7b88c1..b424cbf 100644 --- a/blockrun_llm/__init__.py +++ b/blockrun_llm/__init__.py @@ -145,6 +145,7 @@ SearchParameters, # Standalone search SearchResult, + SmartChatCompletionResponse, SmartChatResponse, SpeechAudio, # Speech (TTS / sound effects) types @@ -186,7 +187,7 @@ create_wallet as generate_wallet, # User-friendly alias ) -__version__ = "1.11.0" +__version__ = "1.12.0" __all__ = [ "NETWORK_ALIASES", "SUPPORTED_NETWORKS", @@ -248,6 +249,7 @@ "SearchParameters", # Standalone search "SearchResult", + "SmartChatCompletionResponse", "SmartChatResponse", "SolanaLLMClient", "SpeechAudio", diff --git a/blockrun_llm/client.py b/blockrun_llm/client.py index f6ba258..d6b117d 100644 --- a/blockrun_llm/client.py +++ b/blockrun_llm/client.py @@ -50,7 +50,13 @@ from dotenv import load_dotenv from eth_account import Account -from .router_adapter import BASE_MINIMUM_PAYMENT_USD, route_with_catalog +from .router_adapter import ( + BASE_MINIMUM_PAYMENT_USD, + build_model_pricing, + route_with_catalog, + routing_profile_for_model, + routing_text, +) from .tx_log import ( TransactionLogger, _resolve_log_dir, @@ -68,6 +74,7 @@ RoutingDecision, RoutingProfile, SearchResult, + SmartChatCompletionResponse, SmartChatResponse, chunk_meta, chunk_usage_dict, @@ -213,7 +220,13 @@ def _should_fallback(exc: Exception) -> bool: return True if isinstance(exc, httpx.NetworkError): return True - return bool(isinstance(exc, APIError) and exc.status_code in (502, 503, 504, 522, 524)) + # 429 is retriable here for the same reason the TypeScript adapter treats it + # as transient: it means THIS upstream is saturated, and the next model in + # the chain is a different upstream. Observed live on the free tier — a + # rate-limited free model returned 429 and the three remaining free models + # in the ranked chain were never tried. Permanent payment failures and + # settled calls are refused above, before this line. + return bool(isinstance(exc, APIError) and exc.status_code in (429, 502, 503, 504, 522, 524)) # The gateway states the output-token ceiling it actually quoted in the 402's @@ -472,39 +485,17 @@ def _capture_settlement(self, response: httpx.Response) -> dict[str, Any] | None def _get_model_pricing(self) -> dict[str, dict[str, float]]: """ - Get model pricing for smart routing. + Get model pricing for smart routing (cached for the client's lifetime). Returns: Dict mapping model_id -> {"input_price": x, "output_price": y, "flat_price": z}. ``flat_price`` is 0 for per-token billing and non-zero (USD per call) for flat-billed models. - - The /v1/models response uses the nested ``pricing.input``/``pricing.output`` - shape today; older snapshots used top-level ``inputPrice``/``outputPrice``. - Both are accepted so the SDK keeps working through backend transitions. """ if self._model_pricing_cache is not None: return self._model_pricing_cache - models = self.list_models() - pricing: dict[str, dict[str, float]] = {} - for model in models: - model_id = model.get("id", "") - # A model the catalog marks unavailable must not win routing — every - # smart call to it would fail with a non-transient error. - if model.get("available") is False: - continue - block = model.get("pricing") or {} - input_price = block.get("input", model.get("inputPrice", model.get("input_price", 0))) - output_price = block.get( - "output", model.get("outputPrice", model.get("output_price", 0)) - ) - flat_price = block.get("flat", model.get("flatPrice", 0)) - pricing[model_id] = { - "input_price": float(input_price or 0), - "output_price": float(output_price or 0), - "flat_price": float(flat_price or 0), - } + pricing = build_model_pricing(self.list_models()) self._model_pricing_cache = pricing return pricing @@ -613,6 +604,81 @@ def smart_chat( routing=RoutingDecision(**decision), ) + def smart_chat_completion( + self, + messages: list[dict[str, Any]], + *, + max_tokens: int | None = None, + temperature: float | None = None, + top_p: float | None = None, + search: bool | None = None, + search_parameters: dict[str, Any] | None = None, + tools: list[dict[str, Any]] | None = None, + tool_choice: Any | None = None, + response_format: dict[str, Any] | None = None, + stop: str | list[str] | None = None, + fallback_models: list[str] | None = None, + routing_profile: RoutingProfile = "auto", + **extra: Any, + ) -> SmartChatCompletionResponse: + """ + Smart routing for a full message list (OpenAI-compatible). + + The routing counterpart of ``chat_completion``: tools, tool_choice and + response_format are part of the routing decision, not just the request. + A turn that must call a tool routes to a tool-capable model, a JSON + schema forces a structured-output-capable tier, and image parts route to + a vision model. + + Capacity is checked against the WHOLE transcript, not the last message — + an agent conversation can be 100x its final turn, and a context overflow + is a non-transient error the fallback chain cannot rescue. + + Example: + result = client.smart_chat_completion( + [{"role": "user", "content": "Cancel order B-42"}], + tools=[{"type": "function", "function": {"name": "cancel_order", ...}}], + tool_choice="required", + ) + print(result.model) # a tool-capable model + print(result.routing.task_type) # 'tool_agent' + """ + view = routing_text(messages) + decision = route_with_catalog( + view["prompt"], + view["system_prompt"], + max_tokens or self.DEFAULT_MAX_TOKENS, + self._get_model_pricing(), + routing_profile=routing_profile, + requires_structured_output=response_format is not None, + tools=tools, + tool_choice=tool_choice, + conversation_chars=view["conversation_chars"], + has_vision=view["has_vision"], + minimum_payment_usd=BASE_MINIMUM_PAYMENT_USD, + ) + response = self.chat_completion( + decision["model"], + messages, + max_tokens=max_tokens, + temperature=temperature, + top_p=top_p, + search=search, + search_parameters=search_parameters, + tools=tools, + tool_choice=tool_choice, + response_format=response_format, + stop=stop, + # An explicit caller-supplied chain wins over the routed one. + fallback_models=fallback_models or decision.get("fallbacks") or None, + **extra, + ) + return SmartChatCompletionResponse( + response=response, + model=decision["model"], + routing=RoutingDecision(**decision), + ) + def get_spending(self) -> dict[str, Any]: """ Get current session spending. @@ -780,7 +846,31 @@ def chat_completion( if result.choices[0].message.tool_calls: for tc in result.choices[0].message.tool_calls: print(f"Call: {tc.function.name}({tc.function.arguments})") + + # Virtual routing ids pick the model for you + result = client.chat_completion("blockrun/auto", messages) """ + # `blockrun/auto` | `blockrun/eco` | `blockrun/premium` are not models — + # they select a routing profile. Hand the turn to the routed path, which + # also supplies the ranked fallback chain. + virtual_profile = routing_profile_for_model(model) + if virtual_profile is not None: + return self.smart_chat_completion( + messages, + max_tokens=max_tokens, + temperature=temperature, + top_p=top_p, + search=search, + search_parameters=search_parameters, + tools=tools, + tool_choice=tool_choice, + response_format=response_format, + stop=stop, + fallback_models=fallback_models, + routing_profile=virtual_profile, # type: ignore[arg-type] + **extra, + ).response + # Validate inputs validate_model(model) validate_max_tokens(max_tokens) @@ -2546,6 +2636,8 @@ def __init__( self._tx_logger: TransactionLogger | None = ( TransactionLogger(log_dir) if log_dir is not None else None ) + # Model pricing cache for smart routing + self._model_pricing_cache: dict[str, dict[str, float]] | None = None self._last_settlement: dict[str, Any] | None = None def _capture_settlement(self, response: httpx.Response) -> dict[str, Any] | None: @@ -2555,6 +2647,125 @@ def _capture_settlement(self, response: httpx.Response) -> dict[str, Any] | None self._last_settlement = settlement return settlement + async def _get_model_pricing(self) -> dict[str, dict[str, float]]: + """Model pricing for smart routing (cached for the client's lifetime).""" + if self._model_pricing_cache is not None: + return self._model_pricing_cache + pricing = build_model_pricing(await self.list_models()) + self._model_pricing_cache = pricing + return pricing + + async def route( + self, + prompt: str, + *, + system: str | None = None, + max_tokens: int | None = None, + routing_profile: RoutingProfile = "auto", + requires_structured_output: bool = False, + ) -> RoutingDecision: + """Inspect a routing decision without making or paying for a call.""" + decision = route_with_catalog( + prompt, + system, + max_tokens or self.DEFAULT_MAX_TOKENS, + await self._get_model_pricing(), + routing_profile=routing_profile, + requires_structured_output=requires_structured_output, + minimum_payment_usd=BASE_MINIMUM_PAYMENT_USD, + ) + return RoutingDecision(**decision) + + async def smart_chat( + self, + prompt: str, + *, + system: str | None = None, + max_tokens: int | None = None, + temperature: float | None = None, + routing_profile: RoutingProfile = "auto", + ) -> SmartChatResponse: + """Async smart chat with automatic model routing. + + Same Router Core portfolio strategy as the sync client — see + :meth:`LLMClient.smart_chat`. + """ + decision = route_with_catalog( + prompt, + system, + max_tokens or self.DEFAULT_MAX_TOKENS, + await self._get_model_pricing(), + routing_profile=routing_profile, + minimum_payment_usd=BASE_MINIMUM_PAYMENT_USD, + ) + response = await self.chat( + decision["model"], + prompt, + system=system, + max_tokens=max_tokens, + temperature=temperature, + fallback_models=decision.get("fallbacks") or None, + ) + return SmartChatResponse( + response=response, + model=decision["model"], + routing=RoutingDecision(**decision), + ) + + async def smart_chat_completion( + self, + messages: list[dict[str, Any]], + *, + max_tokens: int | None = None, + temperature: float | None = None, + top_p: float | None = None, + search: bool | None = None, + search_parameters: dict[str, Any] | None = None, + tools: list[dict[str, Any]] | None = None, + tool_choice: Any | None = None, + response_format: dict[str, Any] | None = None, + stop: str | list[str] | None = None, + fallback_models: list[str] | None = None, + routing_profile: RoutingProfile = "auto", + **extra: Any, + ) -> SmartChatCompletionResponse: + """Async smart routing for a full message list — see + :meth:`LLMClient.smart_chat_completion`.""" + view = routing_text(messages) + decision = route_with_catalog( + view["prompt"], + view["system_prompt"], + max_tokens or self.DEFAULT_MAX_TOKENS, + await self._get_model_pricing(), + routing_profile=routing_profile, + requires_structured_output=response_format is not None, + tools=tools, + tool_choice=tool_choice, + conversation_chars=view["conversation_chars"], + has_vision=view["has_vision"], + minimum_payment_usd=BASE_MINIMUM_PAYMENT_USD, + ) + response = await self.chat_completion( + decision["model"], + messages, + max_tokens=max_tokens, + temperature=temperature, + top_p=top_p, + search=search, + search_parameters=search_parameters, + tools=tools, + tool_choice=tool_choice, + response_format=response_format, + stop=stop, + fallback_models=fallback_models or decision.get("fallbacks") or None, + **extra, + ) + return SmartChatCompletionResponse( + response=response, + model=decision["model"], + routing=RoutingDecision(**decision), + ) + async def chat( self, model: str, @@ -2610,7 +2821,32 @@ async def chat_completion( fallback_models: list[str] | None = None, **extra: Any, ) -> ChatResponse: - """Async full chat completion interface with optional xAI Live Search and tool calling.""" + """Async full chat completion interface with optional xAI Live Search and tool calling. + + ``blockrun/auto`` | ``blockrun/eco`` | ``blockrun/premium`` are routing + profiles rather than models: passing one routes the turn and returns the + routed response, ranked fallback chain included. + """ + virtual_profile = routing_profile_for_model(model) + if virtual_profile is not None: + return ( + await self.smart_chat_completion( + messages, + max_tokens=max_tokens, + temperature=temperature, + top_p=top_p, + search=search, + search_parameters=search_parameters, + tools=tools, + tool_choice=tool_choice, + response_format=response_format, + stop=stop, + fallback_models=fallback_models, + routing_profile=virtual_profile, # type: ignore[arg-type] + **extra, + ) + ).response + # Validate inputs validate_model(model) validate_max_tokens(max_tokens) diff --git a/blockrun_llm/router_adapter.py b/blockrun_llm/router_adapter.py index 67405a1..cc3b073 100644 --- a/blockrun_llm/router_adapter.py +++ b/blockrun_llm/router_adapter.py @@ -107,6 +107,37 @@ class ResolvedRoutingDecision(RoutingDecision, total=False): } +def build_model_pricing(models: list[dict[str, Any]]) -> dict[str, ModelPricing]: + """Build the router's pricing map from a ``/v1/models`` payload. + + Shared by every client (Base and Solana, sync and async) so the four copies + cannot drift. Rows the catalog marks unavailable are skipped: a model that + cannot serve a request must not win routing, since every call to it would + fail with a non-transient error. + + The catalog uses the nested ``pricing.input`` / ``pricing.output`` shape; + older snapshots used top-level ``inputPrice`` / ``outputPrice``. Both are + accepted so the SDK keeps working through backend transitions. + """ + pricing: dict[str, ModelPricing] = {} + for model in models: + if model.get("available") is False: + continue + model_id = model.get("id", "") + if not model_id: + continue + block = model.get("pricing") or {} + input_price = block.get("input", model.get("inputPrice", model.get("input_price", 0))) + output_price = block.get("output", model.get("outputPrice", model.get("output_price", 0))) + flat_price = block.get("flat", model.get("flatPrice", model.get("flat_price", 0))) + pricing[model_id] = { + "input_price": float(input_price or 0), + "output_price": float(output_price or 0), + "flat_price": float(flat_price or 0), + } + return pricing + + def routing_profile_for_model(model: str) -> str | None: """Map a ``blockrun/auto``-style virtual model id to a routing profile.""" return AUTO_ROUTING_PROFILES.get(model.lower()) diff --git a/blockrun_llm/solana_client.py b/blockrun_llm/solana_client.py index cdf750f..93c86d4 100644 --- a/blockrun_llm/solana_client.py +++ b/blockrun_llm/solana_client.py @@ -36,6 +36,13 @@ from .client import _SETTLED_ATTR, _enforce_spend_limits, _mark_settled from .price import Category, Market, Resolution, Session from .realface import _GROUP_ID_RE +from .router_adapter import ( + SOLANA_MINIMUM_PAYMENT_USD, + build_model_pricing, + route_with_catalog, + routing_profile_for_model, + routing_text, +) from .solana_wallet import get_solana_public_key from .tx_log import ( TransactionLogger, @@ -60,8 +67,12 @@ RealFaceList, RealFaceStatus, RetiredEndpointError, + RoutingDecision, + RoutingProfile, RpcResponse, SearchResult, + SmartChatCompletionResponse, + SmartChatResponse, SpeechResponse, SymbolListResponse, VideoResponse, @@ -371,7 +382,13 @@ def _should_fallback_solana(exc: Exception) -> bool: return True if isinstance(exc, httpx.NetworkError): return True - return bool(isinstance(exc, APIError) and exc.status_code in (502, 503, 504, 522, 524)) + # 429 is retriable here for the same reason the TypeScript adapter treats it + # as transient: it means THIS upstream is saturated, and the next model in + # the chain is a different upstream. Observed live on the free tier — a + # rate-limited free model returned 429 and the three remaining free models + # in the ranked chain were never tried. Permanent payment failures and + # settled calls are refused above, before this line. + return bool(isinstance(exc, APIError) and exc.status_code in (429, 502, 503, 504, 522, 524)) # Characters safe to interpolate into a single URL path segment. network / @@ -517,6 +534,8 @@ def __init__( self._private_key = key validate_api_url(api_url) self._api_url = api_url.rstrip("/") + # Model pricing cache for smart routing + self._model_pricing_cache: dict[str, dict[str, float]] | None = None # Resolve effective RPC URL + headers (explicit args > env vars > default). resolved_url, resolved_headers = _resolve_rpc_config(rpc_url, rpc_headers) @@ -666,6 +685,142 @@ def _log_transaction( except Exception: pass + def _get_model_pricing(self) -> dict[str, dict[str, float]]: + """Model pricing for smart routing (cached for the client's lifetime).""" + if self._model_pricing_cache is not None: + return self._model_pricing_cache + pricing = build_model_pricing(self.list_models()) + self._model_pricing_cache = pricing + return pricing + + def route( + self, + prompt: str, + *, + system: str | None = None, + max_tokens: int | None = None, + routing_profile: RoutingProfile = "auto", + requires_structured_output: bool = False, + ) -> RoutingDecision: + """Inspect a Solana routing decision without making or paying for a call. + + Identical routing to the Base client — same Router Core engine, same + catalog — with the Solana x402 minimum applied to the cost estimate. + """ + decision = route_with_catalog( + prompt, + system, + max_tokens or DEFAULT_MAX_TOKENS, + self._get_model_pricing(), + routing_profile=routing_profile, + requires_structured_output=requires_structured_output, + minimum_payment_usd=SOLANA_MINIMUM_PAYMENT_USD, + ) + return RoutingDecision(**decision) + + def smart_chat( + self, + prompt: str, + *, + system: str | None = None, + max_tokens: int | None = None, + temperature: float | None = None, + routing_profile: RoutingProfile = "auto", + timeout: float | None = None, + ) -> SmartChatResponse: + """Smart chat with automatic model routing, paid on Solana. + + Uses BlockRun's Router Core portfolio strategy — the same engine the + Base client, the TypeScript SDK and the gateway run. Routing is local + (<1ms, no extra model call); only the payment leg differs by chain. + + Example: + result = client.smart_chat("What is 2+2?") + print(result.model) # 'google/gemini-2.5-flash' + print(result.routing.method) # 'portfolio' + """ + decision = route_with_catalog( + prompt, + system, + max_tokens or DEFAULT_MAX_TOKENS, + self._get_model_pricing(), + routing_profile=routing_profile, + minimum_payment_usd=SOLANA_MINIMUM_PAYMENT_USD, + ) + response = self.chat( + decision["model"], + prompt, + system=system, + max_tokens=max_tokens or DEFAULT_MAX_TOKENS, + temperature=temperature, + timeout=timeout, + fallback_models=decision.get("fallbacks") or None, + ) + return SmartChatResponse( + response=response, + model=decision["model"], + routing=RoutingDecision(**decision), + ) + + def smart_chat_completion( + self, + messages: list[dict[str, Any]], + *, + max_tokens: int | None = None, + temperature: float | None = None, + top_p: float | None = None, + search: bool = False, + search_parameters: dict[str, Any] | None = None, + tools: list[dict[str, Any]] | None = None, + tool_choice: Any | None = None, + timeout: float | None = None, + response_format: dict[str, Any] | None = None, + stop: str | list[str] | None = None, + fallback_models: list[str] | None = None, + routing_profile: RoutingProfile = "auto", + ) -> SmartChatCompletionResponse: + """Smart routing for a full message list, paid on Solana. + + Tools, tool_choice and response_format are part of the routing + decision, and capacity is checked against the whole transcript — see + :meth:`blockrun_llm.LLMClient.smart_chat_completion`. + """ + view = routing_text(messages) + decision = route_with_catalog( + view["prompt"], + view["system_prompt"], + max_tokens or DEFAULT_MAX_TOKENS, + self._get_model_pricing(), + routing_profile=routing_profile, + requires_structured_output=response_format is not None, + tools=tools, + tool_choice=tool_choice, + conversation_chars=view["conversation_chars"], + has_vision=view["has_vision"], + minimum_payment_usd=SOLANA_MINIMUM_PAYMENT_USD, + ) + response = self.chat_completion( + decision["model"], + messages, + max_tokens=max_tokens or DEFAULT_MAX_TOKENS, + temperature=temperature, + top_p=top_p, + search=search, + search_parameters=search_parameters, + tools=tools, + tool_choice=tool_choice, + timeout=timeout, + response_format=response_format, + stop=stop, + # An explicit caller-supplied chain wins over the routed one. + fallback_models=fallback_models or decision.get("fallbacks") or None, + ) + return SmartChatCompletionResponse( + response=response, + model=decision["model"], + routing=RoutingDecision(**decision), + ) + def chat( self, model: str, @@ -677,6 +832,7 @@ def chat( timeout: float | None = None, response_format: dict[str, Any] | None = None, stop: str | list[str] | None = None, + fallback_models: list[str] | None = None, ) -> str: """Simple 1-line chat.""" messages: list[dict[str, str]] = [] @@ -692,6 +848,7 @@ def chat( timeout=timeout, response_format=response_format, stop=stop, + fallback_models=fallback_models, ) return result.choices[0].message.content or "" @@ -709,6 +866,7 @@ def chat_completion( timeout: float | None = None, response_format: dict[str, Any] | None = None, stop: str | list[str] | None = None, + fallback_models: list[str] | None = None, ) -> ChatResponse: """Full chat completion (OpenAI-compatible). @@ -721,6 +879,26 @@ def chat_completion( client's chat baseline, ``DEFAULT_CHAT_TIMEOUT``). Raise it for large ``max_tokens`` runs against slow models. """ + # `blockrun/auto` | `blockrun/eco` | `blockrun/premium` are routing + # profiles rather than models — hand the turn to the routed path. + virtual_profile = routing_profile_for_model(model) + if virtual_profile is not None: + return self.smart_chat_completion( + messages, + max_tokens=max_tokens, + temperature=temperature, + top_p=top_p, + search=search, + search_parameters=search_parameters, + tools=tools, + tool_choice=tool_choice, + timeout=timeout, + response_format=response_format, + stop=stop, + fallback_models=fallback_models, + routing_profile=virtual_profile, # type: ignore[arg-type] + ).response + validate_max_tokens(max_tokens) body: dict[str, Any] = {"model": model, "messages": messages, "max_tokens": max_tokens} if temperature is not None: @@ -739,7 +917,28 @@ def chat_completion( body["response_format"] = response_format if stop is not None: body["stop"] = stop - return self._request_with_payment("/v1/chat/completions", body, timeout=timeout) + + # Walk [model, *fallback_models] on retriable errors (timeouts, 5xx, + # network) exactly as the streaming path and the Base client do. A + # settled payment is never retried — _should_fallback_solana refuses + # anything tagged as settled, so the next model cannot sign a second + # transfer for the same call. + attempts = [model, *(fallback_models or [])] + last_exc: Exception | None = None + for i, attempt_model in enumerate(attempts): + body["model"] = attempt_model + try: + return self._request_with_payment("/v1/chat/completions", body, timeout=timeout) + except Exception as exc: + if not _should_fallback_solana(exc) or i + 1 >= len(attempts): + raise + last_exc = exc + sys.stderr.write( + f"[blockrun_llm] solana {attempt_model} -> {attempts[i + 1]} " + f"({type(exc).__name__}: {str(exc)[:80]})\n" + ) + assert last_exc is not None + raise last_exc def close(self) -> None: """Close the HTTP client.""" @@ -2907,6 +3106,8 @@ def __init__( self._private_key = key validate_api_url(api_url) self._api_url = api_url.rstrip("/") + # Model pricing cache for smart routing + self._model_pricing_cache: dict[str, dict[str, float]] | None = None resolved_url, resolved_headers = _resolve_rpc_config(rpc_url, rpc_headers) self._rpc_url = resolved_url @@ -3052,6 +3253,122 @@ def _billing_meta(self) -> dict[str, str | None]: # Non-streaming chat # ------------------------------------------------------------------ + async def _get_model_pricing(self) -> dict[str, dict[str, float]]: + """Model pricing for smart routing (cached for the client's lifetime).""" + if self._model_pricing_cache is not None: + return self._model_pricing_cache + pricing = build_model_pricing(await self.list_models()) + self._model_pricing_cache = pricing + return pricing + + async def route( + self, + prompt: str, + *, + system: str | None = None, + max_tokens: int | None = None, + routing_profile: RoutingProfile = "auto", + requires_structured_output: bool = False, + ) -> RoutingDecision: + """Inspect a Solana routing decision without making or paying for a call.""" + decision = route_with_catalog( + prompt, + system, + max_tokens or DEFAULT_MAX_TOKENS, + await self._get_model_pricing(), + routing_profile=routing_profile, + requires_structured_output=requires_structured_output, + minimum_payment_usd=SOLANA_MINIMUM_PAYMENT_USD, + ) + return RoutingDecision(**decision) + + async def smart_chat( + self, + prompt: str, + *, + system: str | None = None, + max_tokens: int | None = None, + temperature: float | None = None, + routing_profile: RoutingProfile = "auto", + timeout: float | None = None, + ) -> SmartChatResponse: + """Async smart chat with automatic model routing, paid on Solana.""" + decision = route_with_catalog( + prompt, + system, + max_tokens or DEFAULT_MAX_TOKENS, + await self._get_model_pricing(), + routing_profile=routing_profile, + minimum_payment_usd=SOLANA_MINIMUM_PAYMENT_USD, + ) + response = await self.chat( + decision["model"], + prompt, + system=system, + max_tokens=max_tokens or DEFAULT_MAX_TOKENS, + temperature=temperature, + timeout=timeout, + fallback_models=decision.get("fallbacks") or None, + ) + return SmartChatResponse( + response=response, + model=decision["model"], + routing=RoutingDecision(**decision), + ) + + async def smart_chat_completion( + self, + messages: list[dict[str, Any]], + *, + max_tokens: int | None = None, + temperature: float | None = None, + top_p: float | None = None, + search: bool = False, + search_parameters: dict[str, Any] | None = None, + tools: list[dict[str, Any]] | None = None, + tool_choice: Any | None = None, + timeout: float | None = None, + response_format: dict[str, Any] | None = None, + stop: str | list[str] | None = None, + fallback_models: list[str] | None = None, + routing_profile: RoutingProfile = "auto", + ) -> SmartChatCompletionResponse: + """Async smart routing for a full message list, paid on Solana.""" + view = routing_text(messages) + decision = route_with_catalog( + view["prompt"], + view["system_prompt"], + max_tokens or DEFAULT_MAX_TOKENS, + await self._get_model_pricing(), + routing_profile=routing_profile, + requires_structured_output=response_format is not None, + tools=tools, + tool_choice=tool_choice, + conversation_chars=view["conversation_chars"], + has_vision=view["has_vision"], + minimum_payment_usd=SOLANA_MINIMUM_PAYMENT_USD, + ) + response = await self.chat_completion( + decision["model"], + messages, + max_tokens=max_tokens or DEFAULT_MAX_TOKENS, + temperature=temperature, + top_p=top_p, + search=search, + search_parameters=search_parameters, + tools=tools, + tool_choice=tool_choice, + timeout=timeout, + response_format=response_format, + stop=stop, + fallback_models=fallback_models or decision.get("fallbacks") or None, + ) + return SmartChatCompletionResponse( + response=response, + model=decision["model"], + routing=RoutingDecision(**decision), + ) + async def chat( self, model: str, @@ -3063,6 +3380,7 @@ async def chat( timeout: float | None = None, response_format: dict[str, Any] | None = None, stop: str | list[str] | None = None, + fallback_models: list[str] | None = None, ) -> str: messages: list[dict[str, str]] = [] if system: @@ -3077,6 +3395,7 @@ async def chat( timeout=timeout, response_format=response_format, stop=stop, + fallback_models=fallback_models, ) return result.choices[0].message.content or "" @@ -3094,7 +3413,30 @@ async def chat_completion( timeout: float | None = None, response_format: dict[str, Any] | None = None, stop: str | list[str] | None = None, + fallback_models: list[str] | None = None, ) -> ChatResponse: + # `blockrun/auto` | `blockrun/eco` | `blockrun/premium` select a routing + # profile rather than a model. + virtual_profile = routing_profile_for_model(model) + if virtual_profile is not None: + return ( + await self.smart_chat_completion( + messages, + max_tokens=max_tokens, + temperature=temperature, + top_p=top_p, + search=search, + search_parameters=search_parameters, + tools=tools, + tool_choice=tool_choice, + timeout=timeout, + response_format=response_format, + stop=stop, + fallback_models=fallback_models, + routing_profile=virtual_profile, # type: ignore[arg-type] + ) + ).response + validate_max_tokens(max_tokens) body: dict[str, Any] = {"model": model, "messages": messages, "max_tokens": max_tokens} if temperature is not None: @@ -3113,7 +3455,27 @@ async def chat_completion( body["response_format"] = response_format if stop is not None: body["stop"] = stop - return await self._request_with_payment("/v1/chat/completions", body, timeout=timeout) + + # Same recovery walk as the sync client: transient upstream failures + # step to the next ranked model, a settled payment never retries. + attempts = [model, *(fallback_models or [])] + last_exc: Exception | None = None + for i, attempt_model in enumerate(attempts): + body["model"] = attempt_model + try: + return await self._request_with_payment( + "/v1/chat/completions", body, timeout=timeout + ) + except Exception as exc: + if not _should_fallback_solana(exc) or i + 1 >= len(attempts): + raise + last_exc = exc + sys.stderr.write( + f"[blockrun_llm] solana {attempt_model} -> {attempts[i + 1]} " + f"({type(exc).__name__}: {str(exc)[:80]})\n" + ) + assert last_exc is not None + raise last_exc async def list_models(self) -> list[dict[str, Any]]: resp = await self._client.get(f"{self._api_url}/v1/models") diff --git a/blockrun_llm/types.py b/blockrun_llm/types.py index f6cd42e..50a7e04 100644 --- a/blockrun_llm/types.py +++ b/blockrun_llm/types.py @@ -713,6 +713,25 @@ class RoutingDecision(BaseModel): agentic_score: Optional[float] = None +class SmartChatCompletionResponse(BaseModel): + """ + Response from smart_chat_completion — the routed full completion. + + ``response`` is the ordinary ChatResponse (choices, usage, citations), so + tool calls and structured output work exactly as with chat_completion. + + Example: + result = client.smart_chat_completion([{"role": "user", "content": "hi"}]) + print(result.model) # the model routing picked + print(result.response.choices[0].message.content) + print(result.routing.task_type) # 'chat' + """ + + response: ChatResponse + model: str + routing: RoutingDecision + + class SmartChatResponse(BaseModel): """ Response from smart_chat with routing information. diff --git a/pyproject.toml b/pyproject.toml index 7748f0a..33b606a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "blockrun-llm" -version = "1.11.0" +version = "1.12.0" description = "BlockRun SDK - Pay-per-request AI (LLM, Image, Video, Music, Voice) via x402 on Base and Solana" readme = "README.md" license = "MIT" diff --git a/tests/unit/test_routing_parity.py b/tests/unit/test_routing_parity.py new file mode 100644 index 0000000..a272d5e --- /dev/null +++ b/tests/unit/test_routing_parity.py @@ -0,0 +1,198 @@ +""" +Routing surface parity across the four clients. + +Base and Solana, sync and async, must expose the same routing: route(), +smart_chat(), smart_chat_completion(), the blockrun/* virtual model ids, and a +ranked fallback chain on the ordinary chat paths. Before 1.12.0 the Solana +clients had none of it and the Base clients had no message-list routing, so a +Solana user got no routing at all and an agent transcript could not be routed. +""" + +from __future__ import annotations + +import inspect +from unittest.mock import patch + +import pytest + +from blockrun_llm import AsyncLLMClient, AsyncSolanaLLMClient, LLMClient, SolanaLLMClient +from blockrun_llm.router_adapter import build_model_pricing, routing_profile_for_model + +CLIENTS = [LLMClient, AsyncLLMClient, SolanaLLMClient, AsyncSolanaLLMClient] + +CATALOG = [ + {"id": "google/gemini-2.5-flash", "pricing": {"input": 0.15, "output": 0.6}}, + {"id": "google/gemini-3.5-flash", "pricing": {"input": 0.5, "output": 3}}, + {"id": "google/gemini-3-flash-preview", "pricing": {"input": 0.5, "output": 3}}, + {"id": "google/gemini-3.1-flash-lite", "pricing": {"input": 0.25, "output": 1.5}}, + {"id": "google/gemini-3.1-pro", "pricing": {"input": 1.25, "output": 10}}, + {"id": "anthropic/claude-opus-4.7", "pricing": {"input": 5, "output": 25}}, + {"id": "anthropic/claude-sonnet-5", "pricing": {"input": 3, "output": 15}}, + {"id": "openai/gpt-5-mini", "pricing": {"input": 0.25, "output": 2}}, + {"id": "openai/gpt-5.3-codex", "pricing": {"input": 1.75, "output": 14}}, + {"id": "deepseek/deepseek-v4-pro", "pricing": {"input": 0.435, "output": 0.87}}, + {"id": "moonshot/kimi-k2.7", "pricing": {"input": 0.95, "output": 4}}, + {"id": "nvidia/step-3.7-flash", "pricing": {"input": 0, "output": 0}}, + {"id": "nvidia/mistral-nemotron", "pricing": {"input": 0, "output": 0}}, + {"id": "nvidia/nemotron-nano-9b-v2", "pricing": {"input": 0, "output": 0}}, + {"id": "nvidia/nemotron-3-nano-omni-30b-a3b-reasoning", "pricing": {"input": 0, "output": 0}}, + # Unavailable rows must never win routing. + {"id": "dead/model", "pricing": {"input": 0.01, "output": 0.01}, "available": False}, +] + + +class TestSurfaceParity: + @pytest.mark.parametrize("client", CLIENTS, ids=lambda c: c.__name__) + @pytest.mark.parametrize("method", ["route", "smart_chat", "smart_chat_completion"]) + def test_every_client_exposes_the_routing_surface(self, client, method): + assert hasattr(client, method), f"{client.__name__} is missing {method}()" + + @pytest.mark.parametrize("client", CLIENTS, ids=lambda c: c.__name__) + def test_the_ordinary_chat_paths_accept_a_fallback_chain(self, client): + # Routing hands back a ranked chain; it is useless if chat() cannot walk it. + for method in ("chat", "chat_completion"): + params = inspect.signature(getattr(client, method)).parameters + assert "fallback_models" in params, f"{client.__name__}.{method}" + + @pytest.mark.parametrize("client", CLIENTS, ids=lambda c: c.__name__) + def test_routing_profile_is_selectable_everywhere(self, client): + for method in ("route", "smart_chat", "smart_chat_completion"): + params = inspect.signature(getattr(client, method)).parameters + assert "routing_profile" in params, f"{client.__name__}.{method}" + + +class TestVirtualModelIds: + @pytest.mark.parametrize( + ("model", "expected"), + [ + ("blockrun/auto", "auto"), + ("blockrun/eco", "eco"), + ("blockrun/premium", "premium"), + ("BLOCKRUN/AUTO", "auto"), + ("google/gemini-2.5-flash", None), + ("blockrun/nonsense", None), + ], + ) + def test_only_the_three_profiles_are_virtual(self, model, expected): + assert routing_profile_for_model(model) == expected + + def test_chat_completion_routes_a_virtual_id_instead_of_calling_it(self): + client = LLMClient(private_key="0x" + "11" * 32) + with ( + patch.object(LLMClient, "list_models", return_value=CATALOG), + patch.object(LLMClient, "chat_completion", wraps=client.chat_completion) as spy, + patch.object(LLMClient, "_request_with_payment") as request, + ): + request.return_value = None + try: + client.chat_completion("blockrun/auto", [{"role": "user", "content": "hi"}]) + except Exception: # the transport is stubbed; routing is the subject + pass + # Re-entered through the routed path with a concrete model. + routed = [call.args[0] for call in spy.call_args_list if call.args] + assert "blockrun/auto" in routed + assert any(m != "blockrun/auto" for m in routed), "never resolved to a real model" + + +class TestPricingMap: + def test_skips_rows_the_catalog_marks_unavailable(self): + pricing = build_model_pricing(CATALOG) + + assert "dead/model" not in pricing + assert pricing["google/gemini-2.5-flash"] == { + "input_price": 0.15, + "output_price": 0.6, + "flat_price": 0.0, + } + + def test_accepts_the_legacy_top_level_price_shape(self): + pricing = build_model_pricing([{"id": "a/b", "inputPrice": 1, "outputPrice": 2}]) + + assert pricing["a/b"]["input_price"] == 1 + assert pricing["a/b"]["output_price"] == 2 + + +class TestDecisionsMatchAcrossChains: + """Base and Solana share one engine: same catalog in, same model out. + + Only the cost floor differs — Base signs a $0.002 minimum, Solana $0.001. + """ + + @pytest.mark.parametrize( + "prompt", + [ + "Summarize this changelog entry in one line", + "Prove that the square root of 2 is irrational, step by step", + "Refactor this TypeScript function to use async/await", + ], + ) + def test_same_model_on_both_chains(self, prompt): + base = LLMClient(private_key="0x" + "11" * 32) + solana = SolanaLLMClient.__new__(SolanaLLMClient) + solana._model_pricing_cache = build_model_pricing(CATALOG) + + with patch.object(LLMClient, "list_models", return_value=CATALOG): + base_decision = base.route(prompt) + solana_decision = SolanaLLMClient.route(solana, prompt) + + assert base_decision.model == solana_decision.model + assert base_decision.tier == solana_decision.tier + assert base_decision.task_type == solana_decision.task_type + assert base_decision.candidates == solana_decision.candidates + # Chain-specific payment floor, same routing. + assert base_decision.cost_estimate >= solana_decision.cost_estimate + + def test_free_profile_is_free_on_solana_too(self): + solana = SolanaLLMClient.__new__(SolanaLLMClient) + solana._model_pricing_cache = build_model_pricing(CATALOG) + + decision = SolanaLLMClient.route(solana, "What is 2+2?", routing_profile="free") + + assert decision.cost_estimate == 0 + for model in [decision.model, *decision.fallbacks]: + assert solana._model_pricing_cache[model]["input_price"] == 0 + assert solana._model_pricing_cache[model]["output_price"] == 0 + + +class TestRetriableStatuses: + """A saturated upstream must hand the turn to the next ranked model. + + Observed live: a rate-limited free model answered 429 and the three + remaining free models in the chain were never tried, because 429 was not in + the retriable set. The TypeScript adapter has always treated it as + transient — same upstream saturated, next model is a different upstream. + """ + + @pytest.mark.parametrize("status", [429, 502, 503, 504, 522, 524]) + def test_saturation_and_availability_errors_walk_the_chain(self, status): + from blockrun_llm.client import _should_fallback + from blockrun_llm.solana_client import _should_fallback_solana + from blockrun_llm.types import APIError + + exc = APIError(f"API error: {status}", status_code=status) + + assert _should_fallback(exc), f"Base refuses to fall back on {status}" + assert _should_fallback_solana(exc), f"Solana refuses to fall back on {status}" + + @pytest.mark.parametrize("status", [400, 401, 403, 404, 422]) + def test_client_errors_do_not_walk_the_chain(self, status): + from blockrun_llm.client import _should_fallback + from blockrun_llm.solana_client import _should_fallback_solana + from blockrun_llm.types import APIError + + exc = APIError(f"API error: {status}", status_code=status) + + assert not _should_fallback(exc) + assert not _should_fallback_solana(exc) + + def test_a_settled_payment_is_never_retried(self): + # The next model would sign a second transfer for one call. + from blockrun_llm.client import _mark_settled, _should_fallback + from blockrun_llm.solana_client import _should_fallback_solana + from blockrun_llm.types import APIError + + exc = APIError("API error: 503", status_code=503) + _mark_settled(exc) + + assert not _should_fallback(exc) + assert not _should_fallback_solana(exc)