diff --git a/CHANGELOG.md b/CHANGELOG.md index 025506956..995648431 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,11 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ### Added +- **Per-target `reasoning_effort`** — a target can force the reasoning effort + of every request it serves, replacing the caller's value (`reasoning.effort` + on the Responses wire, `reasoning_effort` on Chat Completions), so a strong + tier can run at `max` behind a client that sends `high`. `extra_body` only + fills absent keys and could not do this. Rejected on Anthropic clients. - **Raw Responses stream trace** — an opt-in trace of every upstream Responses event as received, under `RUST_LOG=switchyard_translation::responses::raw=trace`, for diagnosing provider-specific event shapes. (#646) diff --git a/crates/libsy-llm-client/src/backend.rs b/crates/libsy-llm-client/src/backend.rs index a526712fc..71f5d281d 100644 --- a/crates/libsy-llm-client/src/backend.rs +++ b/crates/libsy-llm-client/src/backend.rs @@ -54,6 +54,10 @@ pub struct HttpBackendConfig { pub extra_headers: BTreeMap, /// Default top-level request fields, applied only when the request omits the key. pub extra_body: BTreeMap, + /// Reasoning effort forced on every request to this backend, replacing whatever the caller + /// sent. Responses carries it as `reasoning.effort`, Chat Completions as `reasoning_effort`; + /// Anthropic has no equivalent and rejects the setting at configuration time. + pub reasoning_effort: Option, /// Additional attempts after the initial upstream request. pub max_retries: u32, } @@ -66,6 +70,7 @@ impl fmt::Debug for HttpBackendConfig { .field("forward_auth", &self.forward_auth) .field("extra_header_names", &self.extra_headers.keys()) .field("extra_body_keys", &self.extra_body.keys()) + .field("reasoning_effort", &self.reasoning_effort) .field("max_retries", &self.max_retries) .finish() } @@ -250,6 +255,11 @@ impl Backend { &self.config().extra_body } + /// Reasoning effort forced on outbound requests, if the target configures one. + pub fn reasoning_effort(&self) -> Option<&str> { + self.config().reasoning_effort.as_deref() + } + /// Additional attempts allowed after the initial request. pub fn max_retries(&self) -> u32 { self.config().max_retries @@ -345,6 +355,7 @@ mod tests { forward_auth: false, extra_headers: BTreeMap::new(), extra_body: BTreeMap::new(), + reasoning_effort: None, max_retries: 0, } } diff --git a/crates/libsy-llm-client/src/client.rs b/crates/libsy-llm-client/src/client.rs index 6cecd7084..2dacc6383 100644 --- a/crates/libsy-llm-client/src/client.rs +++ b/crates/libsy-llm-client/src/client.rs @@ -256,6 +256,9 @@ impl TranslatingLlmClient { strip_unsigned_thinking_blocks(&mut body); } merge_extra_body(&mut body, backend.extra_body()); + // After the merge on purpose: the effort override must win over both the caller's + // value and any `reasoning` default a target set through `extra_body`. + apply_reasoning_effort(&mut body, backend); if matches!(backend, Backend::Anthropic(_)) { enable_anthropic_prompt_caching(&mut body); } @@ -933,6 +936,44 @@ fn is_unsigned_thinking_block(block: &Value) -> bool { ) } +// Forces the target's configured reasoning effort onto the outbound body, replacing the +// caller's value. Unlike `extra_body`, this is an override: a route that sends one model at a +// higher effort than the client asked for is the point of the setting. +fn apply_reasoning_effort(body: &mut Value, backend: &Backend) { + let Some(effort) = backend.reasoning_effort() else { + return; + }; + let Value::Object(object) = body else { + return; + }; + match backend { + Backend::OpenAiResponses(_) => { + // Responses nests effort under `reasoning` next to fields the caller may have set + // (`summary`, for example), so only the `effort` key is replaced. A `reasoning` + // value that is not an object is malformed and is replaced whole. + let reasoning = object + .entry("reasoning".to_string()) + .or_insert_with(|| Value::Object(serde_json::Map::new())); + if !reasoning.is_object() { + *reasoning = Value::Object(serde_json::Map::new()); + } + if let Value::Object(reasoning) = reasoning { + reasoning.insert("effort".to_string(), Value::String(effort.to_string())); + } + } + Backend::OpenAiChat(_) => { + // Chat Completions takes effort as a top-level field. + object.insert( + "reasoning_effort".to_string(), + Value::String(effort.to_string()), + ); + } + // Anthropic has no effort field (thinking is a token budget); the runner rejects the + // setting on Anthropic clients at load time, so this arm is unreachable in practice. + Backend::Anthropic(_) => {} + } +} + // Applies target defaults without overriding fields supplied by the caller. fn merge_extra_body(body: &mut Value, extra_body: &BTreeMap) { let Value::Object(object) = body else { @@ -1051,6 +1092,7 @@ mod tests { forward_auth: false, extra_headers: BTreeMap::new(), extra_body: BTreeMap::new(), + reasoning_effort: None, max_retries: 0, } } @@ -1088,6 +1130,22 @@ mod tests { vec![ModelConfig::new("gpt", Backend::OpenAiChat(backend), None)] } + fn chat_map_with_effort(base_url: &str, effort: &str) -> Vec { + let mut backend = config(base_url); + backend.reasoning_effort = Some(effort.to_string()); + vec![ModelConfig::new("gpt", Backend::OpenAiChat(backend), None)] + } + + fn responses_map_with_effort(base_url: &str, effort: &str) -> Vec { + let mut backend = config(base_url); + backend.reasoning_effort = Some(effort.to_string()); + vec![ModelConfig::new( + "gpt", + Backend::OpenAiResponses(backend), + None, + )] + } + fn anthropic_map(base_url: &str) -> Vec { vec![ModelConfig::new( "claude", @@ -1541,6 +1599,89 @@ mod tests { Ok(()) } + /// A configured reasoning effort replaces the caller's value on both OpenAI wire formats, + /// which `extra_body` (defaults only) cannot do. + #[tokio::test] + async fn reasoning_effort_override_replaces_the_callers_effort() + -> std::result::Result<(), Box> { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/v1/chat/completions")) + .and(wiremock::matchers::body_partial_json(json!({ + "model": "gpt", + "reasoning_effort": "max" + }))) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "id": "1", + "model": "gpt", + "choices": [{ + "index": 0, + "message": {"role": "assistant", "content": "ok"}, + "finish_reason": "stop" + }], + "usage": {} + }))) + .mount(&server) + .await; + let client = TranslatingLlmClient::new(&chat_map_with_effort( + &format!("{}/v1", server.uri()), + "max", + ))?; + client + .call_rewrite_model_raw( + json!({ + "model": "client-facing", + "messages": [{"role": "user", "content": "hi"}], + "reasoning_effort": "high" + }), + None, + Some(&ModelId::from("gpt")), + WireFormat::OpenAiChat, + ) + .await?; + + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/v1/responses")) + .and(wiremock::matchers::body_partial_json(json!({ + "model": "gpt", + "reasoning": {"effort": "max", "summary": "auto"} + }))) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "id": "resp_1", + "object": "response", + "model": "gpt", + "status": "completed", + "output": [{ + "type": "message", + "id": "msg_1", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": "ok"}] + }], + "usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2} + }))) + .mount(&server) + .await; + let client = TranslatingLlmClient::new(&responses_map_with_effort( + &format!("{}/v1", server.uri()), + "max", + ))?; + client + .call_rewrite_model_raw( + json!({ + "model": "client-facing", + "input": [{"role": "user", "content": "hi"}], + "reasoning": {"effort": "high", "summary": "auto"} + }), + None, + Some(&ModelId::from("gpt")), + WireFormat::OpenAiResponses, + ) + .await?; + Ok(()) + } + #[tokio::test] async fn extra_body_adds_defaults_without_overriding_the_request() -> std::result::Result<(), Box> { diff --git a/crates/libsy-llm-client/src/run.rs b/crates/libsy-llm-client/src/run.rs index 8561816e7..9a98ea614 100644 --- a/crates/libsy-llm-client/src/run.rs +++ b/crates/libsy-llm-client/src/run.rs @@ -894,6 +894,7 @@ mod tests { forward_auth: false, extra_headers: BTreeMap::new(), extra_body: BTreeMap::new(), + reasoning_effort: None, max_retries: 2, }) }; @@ -985,6 +986,7 @@ mod tests { forward_auth: false, extra_headers: BTreeMap::new(), extra_body: BTreeMap::new(), + reasoning_effort: None, max_retries: 0, }) }; diff --git a/crates/switchyard-runner/src/config.rs b/crates/switchyard-runner/src/config.rs index a2ef9aea7..867a8e53a 100644 --- a/crates/switchyard-runner/src/config.rs +++ b/crates/switchyard-runner/src/config.rs @@ -3,7 +3,7 @@ //! Version-1 TOML deployment loading for the shared runner. -use std::collections::{BTreeMap, HashMap, HashSet}; +use std::collections::{BTreeMap, HashMap}; use std::fs; use std::path::Path; use std::sync::Arc; @@ -160,16 +160,35 @@ impl DeploymentConfig { ))); } - let mut seen_client_model_ids = HashSet::new(); + // The LLM client keeps one backend per model id, so two targets naming the same model on + // the same client share it. That is harmless when their request settings agree (an alias + // for a different system prompt, say) and silently wrong when they do not: the second + // target's reasoning_effort or extra_body would never reach the wire. + let mut seen_client_model_ids: HashMap<(&str, &str), (&String, &TargetConfig)> = + HashMap::new(); for (target_name, target) in &self.targets { validate_value("target name", target_name)?; validate_value(&format!("target {target_name} id"), &target.id)?; - if !seen_client_model_ids.insert((target.llm_client.as_str(), target.id.as_str())) { - tracing::warn!( - "target {target_name} reuses model id {} on llm client {}; only one target per id is kept and the other is dropped. Give each target a unique model id, or point both routes at one target.", - target.id, - target.llm_client - ); + match seen_client_model_ids.entry((target.llm_client.as_str(), target.id.as_str())) { + std::collections::hash_map::Entry::Vacant(slot) => { + slot.insert((target_name, target)); + } + std::collections::hash_map::Entry::Occupied(slot) => { + let (first_name, first) = slot.get(); + if first.reasoning_effort != target.reasoning_effort + || first.extra_body != target.extra_body + { + return Err(RunnerError::configuration(format!( + "targets {first_name} and {target_name} both name model {} on llm client {} but with different reasoning_effort or extra_body; one target per model id is kept, so give each its own model id or llm client", + target.id, target.llm_client + ))); + } + tracing::warn!( + "target {target_name} reuses model id {} on llm client {}; only one target per id is kept and the other is dropped. Give each target a unique model id, or point both routes at one target.", + target.id, + target.llm_client + ); + } } } @@ -232,7 +251,7 @@ impl DeploymentConfig { for (name, client_config) in &self.llm_clients { validate_value("llm client name", name)?; - build_backend(name, client_config, &BTreeMap::new())?; + build_backend(name, client_config, &BTreeMap::new(), None)?; } for (target_name, target) in &self.targets { let client_config = self.llm_clients.get(&target.llm_client).ok_or_else(|| { @@ -246,9 +265,26 @@ impl DeploymentConfig { .ok_or_else(|| { RunnerError::configuration("validated llm client was not initialized") })?; + if let Some(effort) = &target.reasoning_effort { + if effort.trim().is_empty() { + return Err(RunnerError::configuration(format!( + "target {target_name} reasoning_effort must not be empty" + ))); + } + if matches!(client_config.format, ClientFormat::AnthropicMessages) { + return Err(RunnerError::configuration(format!( + "target {target_name} reasoning_effort is only supported on openai_chat and openai_responses clients" + ))); + } + } model_configs.push(ModelConfig::new( target.id.clone(), - build_backend(&target.llm_client, client_config, &target.extra_body)?, + build_backend( + &target.llm_client, + client_config, + &target.extra_body, + target.reasoning_effort.clone(), + )?, None, )); } @@ -489,6 +525,9 @@ struct TargetConfig { #[serde(default)] extra_body: BTreeMap, system_prompt: Option, + /// Reasoning effort forced on every request to this target, replacing the caller's value. + /// Only meaningful on `openai_chat` and `openai_responses` clients. + reasoning_effort: Option, } #[derive(Clone, Copy, Debug, Deserialize)] @@ -521,6 +560,7 @@ fn build_backend( client_name: &str, config: &LlmClientConfig, extra_body: &BTreeMap, + reasoning_effort: Option, ) -> RunnerResult { if config.max_retries > MAX_CONFIGURED_RETRIES { return Err(RunnerError::configuration(format!( @@ -560,6 +600,7 @@ fn build_backend( forward_auth: config.forward_auth, extra_headers: config.extra_headers.clone(), extra_body: extra_body.clone(), + reasoning_effort, max_retries: config.max_retries, }; let backend = match config.format { @@ -949,6 +990,51 @@ new = ["send_message"] Ok(()) } + #[test] + fn a_target_reasoning_effort_parses_and_is_rejected_where_unsupported() -> RunnerResult<()> { + let strong = "[targets.strong]\nid = \"strong/model\"\nllm_client = \"responses\""; + let weak = "[targets.weak]\nid = \"weak/model\"\nllm_client = \"anthropic\""; + assert!(VALID_CONFIG.contains(strong) && VALID_CONFIG.contains(weak)); + + let forced = VALID_CONFIG.replace(strong, &format!("{strong}\nreasoning_effort = \"max\"")); + runner_from_toml(&forced)?; + + let blank = VALID_CONFIG.replace(strong, &format!("{strong}\nreasoning_effort = \" \"")); + assert!(error_message(&blank).contains("reasoning_effort must not be empty")); + + let anthropic = VALID_CONFIG.replace(weak, &format!("{weak}\nreasoning_effort = \"high\"")); + assert!( + error_message(&anthropic) + .contains("only supported on openai_chat and openai_responses") + ); + Ok(()) + } + + #[test] + fn duplicate_targets_with_conflicting_settings_are_rejected() -> RunnerResult<()> { + let strong = "[targets.strong]\nid = \"strong/model\"\nllm_client = \"responses\""; + assert!(VALID_CONFIG.contains(strong)); + // Same model, same client, different effort: the second target could never take effect. + let conflicting = VALID_CONFIG.replace( + strong, + &format!( + "{strong}\n\n[targets.strong_max]\nid = \"strong/model\"\nllm_client = \"responses\"\nreasoning_effort = \"max\"" + ), + ); + assert!( + error_message(&conflicting).contains("different reasoning_effort or extra_body"), + "{}", + error_message(&conflicting) + ); + // An alias with identical settings is still allowed (it only warns). + let alias = VALID_CONFIG.replace( + strong, + &format!("{strong}\n\n[targets.strong_alias]\nid = \"strong/model\"\nllm_client = \"responses\""), + ); + runner_from_toml(&alias)?; + Ok(()) + } + #[test] fn classifier_judge_completion_caps_are_configurable() -> RunnerResult<()> { let capability = VALID_CONFIG.replace( @@ -1348,7 +1434,7 @@ target = "azure" let Some(client) = config.llm_clients.get("primary") else { return Err(RunnerError::configuration("primary llm client is missing")); }; - let backend = build_backend("primary", client, &target.extra_body)?; + let backend = build_backend("primary", client, &target.extra_body, None)?; assert_eq!( backend.extra_body().get("service_tier"), diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rs index 99fa471da..19adfb1fb 100644 --- a/crates/switchyard-server/tests/server.rs +++ b/crates/switchyard-server/tests/server.rs @@ -513,6 +513,7 @@ fn random_state_with_retries( forward_auth: false, extra_headers: BTreeMap::new(), extra_body: BTreeMap::new(), + reasoning_effort: None, max_retries, }); let target_models = routes diff --git a/docs/reference/toml_schema.md b/docs/reference/toml_schema.md index 3df685872..92733bd57 100644 --- a/docs/reference/toml_schema.md +++ b/docs/reference/toml_schema.md @@ -90,6 +90,7 @@ calls an upstream. | `llm_client` | Yes | — | Key under `[llm_clients]`. | | `system_prompt` | No | unset | System prompt prepended when this target serves a completion. | | `extra_body` | No | `{}` | Values merged into the upstream request when the request does not already set that key. | +| `reasoning_effort` | No | unset | Reasoning effort forced on every request to this target, replacing the value the caller sent (`reasoning.effort` on `openai_responses`, `reasoning_effort` on `openai_chat`). Rejected on `anthropic_messages` clients. Use it to run one target at a different effort than the client asked for, for example a strong tier at `max` behind a client that sends `high`. Two targets for the same model id on the same `llm_client` collapse into one, so give each effort tier its own `llm_clients` entry (same endpoint, different name). | Each selected or fallback target is prepared from the routed request independently. A prompt configured for one target is therefore not carried into another target's fallback request.