From dd16691f8158fde3f8a2235d6f377fa679ea93c9 Mon Sep 17 00:00:00 2001 From: XiaoHuo888 Date: Fri, 14 Aug 2026 18:28:08 +0800 Subject: [PATCH] feat: add OrcaRouter as a named provider Add PREFERRED_PROVIDER=orcarouter, mapping haiku/sonnet to anthropic/claude-haiku-4.5 and anthropic/claude-sonnet-4.6 by default and routing them through OrcaRouter's OpenAI-compatible endpoint (ORCAROUTER_API_KEY + ORCAROUTER_BASE_URL, default https://api.orcarouter.ai/v1). Skips the 16384 max_tokens cap for OrcaRouter models so Claude output limits are not truncated. Documents the provider in the README and .env.example. Co-Authored-By: Claude Signed-off-by: XiaoHuo888 --- .env.example | 10 +++++++++- README.md | 29 ++++++++++++++++++++++------- server.py | 50 ++++++++++++++++++++++++++++++++++++-------------- 3 files changed, 67 insertions(+), 22 deletions(-) diff --git a/.env.example b/.env.example index f61e441b..cfdf98e3 100644 --- a/.env.example +++ b/.env.example @@ -2,18 +2,21 @@ ANTHROPIC_API_KEY="your-anthropic-api-key" # Needed if proxying *to* Anthropic OPENAI_API_KEY="sk-..." GEMINI_API_KEY="your-google-ai-studio-key" +ORCAROUTER_API_KEY="sk-orca-..." # Needed if PREFERRED_PROVIDER=orcarouter # Optional: Provider Preference and Model Mapping -# Controls which provider (google, openai, or anthropic) is preferred for mapping haiku/sonnet. +# Controls which provider (google, openai, orcarouter, or anthropic) is preferred for mapping haiku/sonnet. # Defaults to openai if not set. # Set to "anthropic" for "just an Anthropic proxy" mode (no remapping) PREFERRED_PROVIDER="openai" OPENAI_BASE_URL="https://api.openai.com/v1" +ORCAROUTER_BASE_URL="https://api.orcarouter.ai/v1" # Optional, it's the default # Optional: Specify the exact models to map haiku/sonnet to. # If PREFERRED_PROVIDER=google, these MUST be valid Gemini model names known to the server. # Defaults to gemini-2.5-pro and gemini-2.5-flash if PREFERRED_PROVIDER=google. # Defaults to gpt-4.1 and gpt-4.1-mini if PREFERRED_PROVIDER=openai. +# Defaults to anthropic/claude-sonnet-4.6 and anthropic/claude-haiku-4.5 if PREFERRED_PROVIDER=orcarouter. # These are IGNORED when PREFERRED_PROVIDER=anthropic (models are not remapped). # BIG_MODEL="gpt-4.1" # SMALL_MODEL="gpt-4.1-mini" @@ -29,6 +32,11 @@ OPENAI_BASE_URL="https://api.openai.com/v1" # BIG_MODEL="gemini-2.5-pro" # SMALL_MODEL="gemini-2.5-flash" +# Example OrcaRouter mapping: +# PREFERRED_PROVIDER="orcarouter" +# BIG_MODEL="anthropic/claude-sonnet-4.6" # any model id from OrcaRouter's catalog, e.g. orcarouter/auto +# SMALL_MODEL="anthropic/claude-haiku-4.5" + # Example "just an Anthropic proxy" mode: # PREFERRED_PROVIDER="anthropic" # (BIG_MODEL and SMALL_MODEL are ignored in this mode) diff --git a/README.md b/README.md index d9ae04c8..ab054c08 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,8 @@ -# Anthropic API Proxy for Gemini & OpenAI Models 🔄 +# Anthropic API Proxy for Gemini, OpenAI & OrcaRouter Models 🔄 -**Use Anthropic clients (like Claude Code) with Gemini, OpenAI, or direct Anthropic backends.** 🤝 +**Use Anthropic clients (like Claude Code) with Gemini, OpenAI, OrcaRouter, or direct Anthropic backends.** 🤝 -A proxy server that lets you use Anthropic clients with Gemini, OpenAI, or Anthropic models themselves (a transparent proxy of sorts), all via LiteLLM. 🌉 +A proxy server that lets you use Anthropic clients with Gemini, OpenAI, OrcaRouter, or Anthropic models themselves (a transparent proxy of sorts), all via LiteLLM. 🌉 ![Anthropic API Proxy](pic.png) @@ -13,6 +13,7 @@ A proxy server that lets you use Anthropic clients with Gemini, OpenAI, or Anthr - OpenAI API key 🔑 - Google AI Studio (Gemini) API key (if using Google provider) 🔑 +- OrcaRouter API key (if using OrcaRouter provider) 🔑 - Google Cloud Project with Vertex AI API enabled (if using Application Default Credentials for Gemini) ☁️ - [uv](https://github.com/astral-sh/uv) installed. @@ -42,16 +43,19 @@ A proxy server that lets you use Anthropic clients with Gemini, OpenAI, or Anthr * `ANTHROPIC_API_KEY`: (Optional) Needed only if proxying *to* Anthropic models. * `OPENAI_API_KEY`: Your OpenAI API key (Required if using the default OpenAI preference or as fallback). * `GEMINI_API_KEY`: Your Google AI Studio (Gemini) API key (Required if `PREFERRED_PROVIDER=google` and `USE_VERTEX_AUTH=true`). + * `ORCAROUTER_API_KEY`: Your [OrcaRouter](https://www.orcarouter.ai) API key (Required if `PREFERRED_PROVIDER=orcarouter`). + * `ORCAROUTER_BASE_URL` (Optional): OrcaRouter's OpenAI-compatible endpoint. Defaults to `https://api.orcarouter.ai/v1`. * `USE_VERTEX_AUTH` (Optional): Set to `true` to use Application Default Credentials (ADC) will be used (no static API key required). Note: when USE_VERTEX_AUTH=true, you must configure `VERTEX_PROJECT` and `VERTEX_LOCATION`. * `VERTEX_PROJECT` (Optional): Your Google Cloud Project ID (Required if `PREFERRED_PROVIDER=google` and `USE_VERTEX_AUTH=true`). * `VERTEX_LOCATION` (Optional): The Google Cloud region for Vertex AI (e.g., `us-central1`) (Required if `PREFERRED_PROVIDER=google` and `USE_VERTEX_AUTH=true`). - * `PREFERRED_PROVIDER` (Optional): Set to `openai` (default), `google`, or `anthropic`. This determines the primary backend for mapping `haiku`/`sonnet`. - * `BIG_MODEL` (Optional): The model to map `sonnet` requests to. Defaults to `gpt-4.1` (if `PREFERRED_PROVIDER=openai`) or `gemini-2.5-pro-preview-03-25`. Ignored when `PREFERRED_PROVIDER=anthropic`. - * `SMALL_MODEL` (Optional): The model to map `haiku` requests to. Defaults to `gpt-4.1-mini` (if `PREFERRED_PROVIDER=openai`) or `gemini-2.0-flash`. Ignored when `PREFERRED_PROVIDER=anthropic`. + * `PREFERRED_PROVIDER` (Optional): Set to `openai` (default), `google`, `orcarouter`, or `anthropic`. This determines the primary backend for mapping `haiku`/`sonnet`. + * `BIG_MODEL` (Optional): The model to map `sonnet` requests to. Defaults to `gpt-4.1` (if `PREFERRED_PROVIDER=openai`), `gemini-2.5-pro-preview-03-25` (if `PREFERRED_PROVIDER=google`), or `anthropic/claude-sonnet-4.6` (if `PREFERRED_PROVIDER=orcarouter`). Ignored when `PREFERRED_PROVIDER=anthropic`. + * `SMALL_MODEL` (Optional): The model to map `haiku` requests to. Defaults to `gpt-4.1-mini` (if `PREFERRED_PROVIDER=openai`), `gemini-2.0-flash` (if `PREFERRED_PROVIDER=google`), or `anthropic/claude-haiku-4.5` (if `PREFERRED_PROVIDER=orcarouter`). Ignored when `PREFERRED_PROVIDER=anthropic`. **Mapping Logic:** - If `PREFERRED_PROVIDER=openai` (default), `haiku`/`sonnet` map to `SMALL_MODEL`/`BIG_MODEL` prefixed with `openai/`. - If `PREFERRED_PROVIDER=google`, `haiku`/`sonnet` map to `SMALL_MODEL`/`BIG_MODEL` prefixed with `gemini/` *if* those models are in the server's known `GEMINI_MODELS` list (otherwise falls back to OpenAI mapping). + - If `PREFERRED_PROVIDER=orcarouter`, `haiku`/`sonnet` map to `SMALL_MODEL`/`BIG_MODEL` prefixed with `openai/` (OrcaRouter speaks the OpenAI wire format) and are routed through [OrcaRouter](https://www.orcarouter.ai) using `ORCAROUTER_API_KEY` + `ORCAROUTER_BASE_URL`. - If `PREFERRED_PROVIDER=anthropic`, `haiku`/`sonnet` requests are passed directly to Anthropic with the `anthropic/` prefix without remapping to different models. 4. **Run the server**: @@ -194,13 +198,24 @@ BIG_MODEL="gpt-4o" # Example specific model SMALL_MODEL="gpt-4o-mini" # Example specific model ``` +**Example 5: Prefer [OrcaRouter](https://www.orcarouter.ai)** +```dotenv +ORCAROUTER_API_KEY="sk-orca-..." +PREFERRED_PROVIDER="orcarouter" +# ORCAROUTER_BASE_URL="https://api.orcarouter.ai/v1" # Optional, it's the default +# BIG_MODEL="anthropic/claude-sonnet-4.6" # Optional, it's the default for OrcaRouter pref +# SMALL_MODEL="anthropic/claude-haiku-4.5" # Optional, it's the default for OrcaRouter pref +``` + +*Use case: [OrcaRouter](https://www.orcarouter.ai) is an OpenAI-compatible LLM router with one API key for 150+ models (Anthropic, OpenAI, Google, DeepSeek, and more). Set `BIG_MODEL`/`SMALL_MODEL` to any model id from its catalog — e.g. `anthropic/claude-sonnet-4.6`, `openai/gpt-4o`, `google/gemini-2.5-flash`, or the smart `orcarouter/auto` router. It also runs gateway-level, zero-trust security for AI agents on the same endpoint — screening every prompt/response and governing every tool call on a default-deny basis, with no application code changes.* + ## How It Works 🧩 This proxy works by: 1. **Receiving requests** in Anthropic's API format 📥 2. **Translating** the requests to OpenAI format via LiteLLM 🔄 -3. **Sending** the translated request to OpenAI 📤 +3. **Sending** the translated request to the configured backend (OpenAI, Gemini, OrcaRouter, or Anthropic) 📤 4. **Converting** the response back to Anthropic format 🔄 5. **Returning** the formatted response to the client ✅ diff --git a/server.py b/server.py index 15dacf35..cf3d5af5 100644 --- a/server.py +++ b/server.py @@ -100,13 +100,23 @@ def format(self, record): # Get OpenAI base URL from environment (if set) OPENAI_BASE_URL = os.environ.get("OPENAI_BASE_URL") +# Get OrcaRouter API key and base URL from environment (if set) +ORCAROUTER_API_KEY = os.environ.get("ORCAROUTER_API_KEY") +ORCAROUTER_BASE_URL = os.environ.get( + "ORCAROUTER_BASE_URL", "https://api.orcarouter.ai/v1" +) + # Get preferred provider (default to openai) PREFERRED_PROVIDER = os.environ.get("PREFERRED_PROVIDER", "openai").lower() # Get model mapping configuration from environment # Default to latest OpenAI models if not set -BIG_MODEL = os.environ.get("BIG_MODEL", "gpt-4.1") -SMALL_MODEL = os.environ.get("SMALL_MODEL", "gpt-4.1-mini") +if PREFERRED_PROVIDER == "orcarouter": + BIG_MODEL = os.environ.get("BIG_MODEL", "anthropic/claude-sonnet-4.6") + SMALL_MODEL = os.environ.get("SMALL_MODEL", "anthropic/claude-haiku-4.5") +else: + BIG_MODEL = os.environ.get("BIG_MODEL", "gpt-4.1") + SMALL_MODEL = os.environ.get("SMALL_MODEL", "gpt-4.1-mini") # List of OpenAI models OPENAI_MODELS = [ @@ -631,11 +641,13 @@ def convert_anthropic_to_litellm(anthropic_request: MessagesRequest) -> Dict[str messages.append({"role": msg.role, "content": processed_content}) - # Cap max_tokens for OpenAI models to their limit of 16384 + # Cap max_tokens for OpenAI models to their limit of 16384. + # Skip for OrcaRouter: it routes to Claude models with larger output limits. max_tokens = anthropic_request.max_tokens - if anthropic_request.model.startswith( - "openai/" - ) or anthropic_request.model.startswith("gemini/"): + if ( + anthropic_request.model.startswith("openai/") + or anthropic_request.model.startswith("gemini/") + ) and PREFERRED_PROVIDER != "orcarouter": max_tokens = min(max_tokens, 16384) logger.debug( f"Capping max_tokens to 16384 for OpenAI/Gemini model (original value: {anthropic_request.max_tokens})" @@ -1242,15 +1254,22 @@ async def create_message(request: MessagesRequest, raw_request: Request): # Determine which API key to use based on the model if request.model.startswith("openai/"): - litellm_request["api_key"] = OPENAI_API_KEY - # Use custom OpenAI base URL if configured - if OPENAI_BASE_URL: - litellm_request["api_base"] = OPENAI_BASE_URL + if PREFERRED_PROVIDER == "orcarouter": + litellm_request["api_key"] = ORCAROUTER_API_KEY + litellm_request["api_base"] = ORCAROUTER_BASE_URL logger.debug( - f"Using OpenAI API key and custom base URL {OPENAI_BASE_URL} for model: {request.model}" + f"Using OrcaRouter API key and base URL {ORCAROUTER_BASE_URL} for model: {request.model}" ) else: - logger.debug(f"Using OpenAI API key for model: {request.model}") + litellm_request["api_key"] = OPENAI_API_KEY + # Use custom OpenAI base URL if configured + if OPENAI_BASE_URL: + litellm_request["api_base"] = OPENAI_BASE_URL + logger.debug( + f"Using OpenAI API key and custom base URL {OPENAI_BASE_URL} for model: {request.model}" + ) + else: + logger.debug(f"Using OpenAI API key for model: {request.model}") elif request.model.startswith("gemini/"): if USE_VERTEX_AUTH: litellm_request["vertex_project"] = VERTEX_PROJECT @@ -1620,8 +1639,11 @@ async def count_tokens(request: TokenCountRequest, raw_request: Request): } # Add custom base URL for OpenAI models if configured - if request.model.startswith("openai/") and OPENAI_BASE_URL: - token_counter_args["api_base"] = OPENAI_BASE_URL + if request.model.startswith("openai/"): + if PREFERRED_PROVIDER == "orcarouter": + token_counter_args["api_base"] = ORCAROUTER_BASE_URL + elif OPENAI_BASE_URL: + token_counter_args["api_base"] = OPENAI_BASE_URL # Count tokens token_count = token_counter(**token_counter_args)