From 57f841689983ee97c1c68daf997e727f3f3d2507 Mon Sep 17 00:00:00 2001 From: Lin Jia Date: Thu, 10 Sep 2026 10:17:50 -0700 Subject: [PATCH 1/2] feat(runner): let a target name its upstream model separately from its routing id The LLM client keys backends by model id, and the runner keeps one target per model id and llm client, so two targets that address the same provider model could not carry different settings: the second was dropped with a warning. An effort-tier pair for one model (high and max), or two headers or endpoints for one model, therefore could not be expressed. This adds an optional model key on [targets.]. The target's id stays the unique routing identity used by routes, affinity, and the escalation latch; model is the name sent upstream and defaults to id. The LLM client carries it as an upstream model name on the model config and substitutes it into the outbound body in place of the routing id. Tests cover the substitution against a mock upstream, two targets sharing one provider model under distinct ids, and the blank rejection. The TOML schema reference documents the key. Signed-off-by: Lin Jia --- CHANGELOG.md | 5 ++ crates/libsy-llm-client/src/client.rs | 70 +++++++++++++++++++++++++- crates/switchyard-runner/src/config.rs | 32 +++++++++++- docs/reference/toml_schema.md | 1 + 4 files changed, 104 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 995648431..66e6c6405 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,11 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). on the Responses wire, `reasoning_effort` on Chat Completions), so a strong tier can run at `max` behind a client that sends `high`. `extra_body` only fills absent keys and could not do this. Rejected on Anthropic clients. +- **Target `model` distinct from `id`** — a target may name the provider model + it sends upstream separately from its routing id, so several targets can + address one provider model with different settings (effort, headers, + endpoint). Previously the runner kept a single target per model id and + dropped the rest. - **Raw Responses stream trace** — an opt-in trace of every upstream Responses event as received, under `RUST_LOG=switchyard_translation::responses::raw=trace`, for diagnosing provider-specific event shapes. (#646) diff --git a/crates/libsy-llm-client/src/client.rs b/crates/libsy-llm-client/src/client.rs index 2dacc6383..470914077 100644 --- a/crates/libsy-llm-client/src/client.rs +++ b/crates/libsy-llm-client/src/client.rs @@ -64,6 +64,10 @@ pub struct ModelConfig { model_name: ModelId, default_backend: Backend, other_backends: Option>, + /// Model name sent upstream when it differs from `model_name`. Lets two configs + /// with different backend settings (effort, headers, endpoint) address the same + /// provider model under distinct routing ids. + upstream_model: Option, } impl ModelConfig { @@ -78,8 +82,22 @@ impl ModelConfig { model_name: model_name.into(), default_backend, other_backends, + upstream_model: None, } } + + /// Sends `upstream_model` as the provider's model name instead of `model_name`. + pub fn with_upstream_model(mut self, upstream_model: impl Into) -> Self { + self.upstream_model = Some(upstream_model.into()); + self + } + + /// The model name the provider sees for this config. + fn upstream_name(&self) -> &str { + self.upstream_model + .as_deref() + .unwrap_or_else(|| self.model_name.as_ref()) + } } /// A model-bearing provider operation outside the normal completion endpoint. @@ -244,8 +262,14 @@ impl TranslatingLlmClient { .map_err(|error| LlmClientError::RequestEncoding(error.to_string()))?; // `encode_request` round-trips a preserved same-format body verbatim, // which keeps the caller's original `model`; force the resolved model so - // the upstream always sees the target id. - set_json_model(&mut body, model); + // the upstream always sees the configured provider model name. + let upstream_model = self + .model_to_config + .get(model) + .map(ModelConfig::upstream_name) + .unwrap_or_else(|| model.as_ref()) + .to_string(); + set_json_model(&mut body, &upstream_model); if matches!(backend, Backend::OpenAiResponses(_)) { sanitize_openai_responses_provider_body(&mut body); } @@ -1146,6 +1170,13 @@ mod tests { )] } + fn chat_map_with_upstream_model(base_url: &str, upstream: &str) -> Vec { + vec![ + ModelConfig::new("gpt-tier", Backend::OpenAiChat(config(base_url)), None) + .with_upstream_model(upstream), + ] + } + fn anthropic_map(base_url: &str) -> Vec { vec![ModelConfig::new( "claude", @@ -1682,6 +1713,41 @@ mod tests { Ok(()) } + /// A config's upstream model name, not its routing id, is what the provider receives. + #[tokio::test] + async fn upstream_model_name_replaces_the_routing_id_on_the_wire() + -> std::result::Result<(), Box> { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/v1/chat/completions")) + .and(wiremock::matchers::body_partial_json(json!({"model": "gpt"}))) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "id": "1", + "model": "gpt", + "choices": [{ + "index": 0, + "message": {"role": "assistant", "content": "ok"}, + "finish_reason": "stop" + }], + "usage": {} + }))) + .mount(&server) + .await; + let client = TranslatingLlmClient::new(&chat_map_with_upstream_model( + &format!("{}/v1", server.uri()), + "gpt", + ))?; + client + .call_rewrite_model_raw( + json!({"model": "client-facing", "messages": [{"role": "user", "content": "hi"}]}), + None, + Some(&ModelId::from("gpt-tier")), + WireFormat::OpenAiChat, + ) + .await?; + Ok(()) + } + #[tokio::test] async fn extra_body_adds_defaults_without_overriding_the_request() -> std::result::Result<(), Box> { diff --git a/crates/switchyard-runner/src/config.rs b/crates/switchyard-runner/src/config.rs index 867a8e53a..7e8412fde 100644 --- a/crates/switchyard-runner/src/config.rs +++ b/crates/switchyard-runner/src/config.rs @@ -277,7 +277,7 @@ impl DeploymentConfig { ))); } } - model_configs.push(ModelConfig::new( + let mut model_config = ModelConfig::new( target.id.clone(), build_backend( &target.llm_client, @@ -286,7 +286,16 @@ impl DeploymentConfig { target.reasoning_effort.clone(), )?, None, - )); + ); + if let Some(model) = &target.model { + if model.trim().is_empty() { + return Err(RunnerError::configuration(format!( + "target {target_name} model must not be empty" + ))); + } + model_config = model_config.with_upstream_model(model.clone()); + } + model_configs.push(model_config); } let mut clients = BTreeMap::new(); @@ -520,7 +529,11 @@ struct LlmClientConfig { #[derive(Debug, Deserialize)] #[serde(deny_unknown_fields)] struct TargetConfig { + /// Routing id of the target; also the model name sent upstream unless `model` is set. id: ModelId, + /// Provider model name sent upstream when it differs from `id`, so several targets + /// (for example one per reasoning effort) can address the same provider model. + model: Option, llm_client: String, #[serde(default)] extra_body: BTreeMap, @@ -1035,6 +1048,21 @@ new = ["send_message"] Ok(()) } + #[test] + fn two_targets_can_share_an_upstream_model_under_distinct_ids() -> RunnerResult<()> { + let strong = "[targets.strong]\nid = \"strong/model\"\nllm_client = \"responses\""; + assert!(VALID_CONFIG.contains(strong)); + let aliased = VALID_CONFIG.replace( + strong, + "[targets.strong]\nid = \"strong-max\"\nmodel = \"strong/model\"\nllm_client = \"responses\"\n\n[targets.strong_low]\nid = \"strong-low\"\nmodel = \"strong/model\"\nllm_client = \"responses\"", + ); + runner_from_toml(&aliased)?; + + let blank = VALID_CONFIG.replace(strong, &format!("{strong}\nmodel = \" \"")); + assert!(error_message(&blank).contains("model must not be empty")); + Ok(()) + } + #[test] fn classifier_judge_completion_caps_are_configurable() -> RunnerResult<()> { let capability = VALID_CONFIG.replace( diff --git a/docs/reference/toml_schema.md b/docs/reference/toml_schema.md index 92733bd57..54ad88162 100644 --- a/docs/reference/toml_schema.md +++ b/docs/reference/toml_schema.md @@ -88,6 +88,7 @@ calls an upstream. |---|:---:|---|---| | `id` | Yes | — | Exact model ID sent upstream. | | `llm_client` | Yes | — | Key under `[llm_clients]`. | +| `model` | No | same as `id` | Provider model name sent upstream when it differs from `id`. Lets several targets address one provider model with different settings, for example one target per reasoning effort; the routing id stays unique. | | `system_prompt` | No | unset | System prompt prepended when this target serves a completion. | | `extra_body` | No | `{}` | Values merged into the upstream request when the request does not already set that key. | | `reasoning_effort` | No | unset | Reasoning effort forced on every request to this target, replacing the value the caller sent (`reasoning.effort` on `openai_responses`, `reasoning_effort` on `openai_chat`). Rejected on `anthropic_messages` clients. Use it to run one target at a different effort than the client asked for, for example a strong tier at `max` behind a client that sends `high`. Two targets for the same model id on the same `llm_client` collapse into one, so give each effort tier its own `llm_clients` entry (same endpoint, different name). | From 7ae48f69ea5f68f23ed68ff26a28cfe5c9f417b3 Mon Sep 17 00:00:00 2001 From: Lin Jia Date: Thu, 10 Sep 2026 12:48:15 -0700 Subject: [PATCH 2/2] docs(client): describe ModelConfig's routing and upstream identities Documents the public ModelConfig type as a whole (routing id, optional upstream model name, default and additional backends) and corrects the TOML schema row for a target's id, which is the routing identifier and only the upstream model name when model is unset. Found by review. Signed-off-by: Lin Jia --- crates/libsy-llm-client/src/client.rs | 13 ++++++++++--- docs/reference/toml_schema.md | 2 +- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/crates/libsy-llm-client/src/client.rs b/crates/libsy-llm-client/src/client.rs index 470914077..1ff42fc85 100644 --- a/crates/libsy-llm-client/src/client.rs +++ b/crates/libsy-llm-client/src/client.rs @@ -57,8 +57,13 @@ const INITIAL_RETRY_DELAY: Duration = Duration::from_millis(250); const MAX_RETRY_BACKOFF: Duration = Duration::from_secs(2); const MAX_RETRY_AFTER: Duration = Duration::from_secs(60); -/// How one model is served: the `default_backend` used when the request does not -/// pin a wire format, plus any `other_backends` reachable over additional formats. +/// How one routed model is served. +/// +/// `model_name` is the routing identity a route or target refers to, and the key the client +/// resolves a call by; `upstream_model`, when set, is the name the provider receives instead. +/// Requests go to `default_backend` unless they pin a wire format served by one of +/// `other_backends`. Several configs may point at one provider model under distinct routing +/// ids, each with its own backend settings. #[derive(Clone, Debug)] pub struct ModelConfig { model_name: ModelId, @@ -1720,7 +1725,9 @@ mod tests { let server = MockServer::start().await; Mock::given(method("POST")) .and(path("/v1/chat/completions")) - .and(wiremock::matchers::body_partial_json(json!({"model": "gpt"}))) + .and(wiremock::matchers::body_partial_json( + json!({"model": "gpt"}), + )) .respond_with(ResponseTemplate::new(200).set_body_json(json!({ "id": "1", "model": "gpt", diff --git a/docs/reference/toml_schema.md b/docs/reference/toml_schema.md index 54ad88162..342c5b5c2 100644 --- a/docs/reference/toml_schema.md +++ b/docs/reference/toml_schema.md @@ -86,7 +86,7 @@ calls an upstream. | Key | Required | Default | Meaning | |---|:---:|---|---| -| `id` | Yes | — | Exact model ID sent upstream. | +| `id` | Yes | — | Routing identifier of the target, unique per `llm_client`. Also the model ID sent upstream unless `model` is set. | | `llm_client` | Yes | — | Key under `[llm_clients]`. | | `model` | No | same as `id` | Provider model name sent upstream when it differs from `id`. Lets several targets address one provider model with different settings, for example one target per reasoning effort; the routing id stays unique. | | `system_prompt` | No | unset | System prompt prepended when this target serves a completion. |