diff --git a/README.md b/README.md index fa972f4cc..6985d9700 100644 --- a/README.md +++ b/README.md @@ -327,6 +327,7 @@ Most use an LLM as a judge. All of them pick between an **efficient** model and |---|---|---|---| | **[Capability](docs/routing_algorithms/llm_classifier_routing.md)** | The first request is judged by an LLM. | `llm_classifier` | 71.2% at $79.32 | | **[Stage](docs/routing_algorithms/stage_router_routing.md)** | Tool responses are judged by pattern matching or an LLM. | `stage_router` | 72.7% at $68.19 | +| **[Plan/Execute](docs/routing_algorithms/plan_execute_routing.md)** | A capable model plans, then an efficient model executes after the first edit. | `plan_execute` | not yet benchmarked | | **[Capability + Stage](docs/routing_algorithms/composite_routing.md)** | Combines the two above. | `composite` | not yet benchmarked | | **[Escalation](docs/routing_algorithms/escalation_router_routing.md)** | Starts efficient. Responses are judged by an LLM for issues, then escalated. | `llm_classifier` + `mode = "escalation"` | 75.7% at $85.00 | | **[Advisor Gate](docs/routing_algorithms/advisor_gate_routing.md)** | One model serves every turn; a stronger advisor approves its plans and "done" claims, or sends it back. | `advisor` | lifts a weak executor 43.8% → 54.7% | diff --git a/benchmark/README.md b/benchmark/README.md index dfaa36a52..d783a53ea 100644 --- a/benchmark/README.md +++ b/benchmark/README.md @@ -193,6 +193,22 @@ benchmark/server-configs/tb-lite-single-gpt-5-5.toml benchmark/server-configs/tb-lite-single-opus-4-7.toml ``` +To benchmark the plan/execute boundary, use +`benchmark/server-configs/tb-lite-plan-execute-opus-kimi.toml`. It starts each task on Opus and +hands the preserved trajectory to Kimi after the first edit or write tool call: + +```bash +bash benchmark/run-baseline.sh \ + --harbor-path benchmark/datasets/openthoughts-tblite-closed-book \ + --server-config benchmark/server-configs/tb-lite-plan-execute-opus-kimi.toml \ + --model switchyard \ + --agent codex \ + --reasoning-effort xhigh \ + --n-tasks 1 \ + --n-concurrent 1 \ + --max-retries 0 +``` + By default, the runner starts in the background and prints the PID, log path, and kill command. ## Book Modes diff --git a/benchmark/server-configs/tb-lite-plan-execute-opus-kimi.toml b/benchmark/server-configs/tb-lite-plan-execute-opus-kimi.toml new file mode 100644 index 000000000..fc7811fe8 --- /dev/null +++ b/benchmark/server-configs/tb-lite-plan-execute-opus-kimi.toml @@ -0,0 +1,26 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Plan/execute configuration for Harbor Terminal-Bench Lite. +# Harbor selects this route with --model switchyard. + +schema_version = 1 + +[llm_clients.openrouter] +format = "openai_chat" +base_url = "https://openrouter.ai/api/v1" +api_key_env = "OPENROUTER_API_KEY" + +[targets.capable] +id = "anthropic/claude-opus-4.7" +llm_client = "openrouter" + +[targets.efficient] +id = "moonshotai/kimi-k2.7-code" +llm_client = "openrouter" + +[routes.switchyard] +id = "switchyard" +type = "plan_execute" +capable_target = "capable" +efficient_target = "efficient" diff --git a/crates/libsy/src/algorithms.rs b/crates/libsy/src/algorithms.rs index fa6c2af5d..cd2d36b50 100644 --- a/crates/libsy/src/algorithms.rs +++ b/crates/libsy/src/algorithms.rs @@ -13,6 +13,7 @@ pub mod fall_through; pub mod llm_class; pub mod noop; pub mod passthrough; +pub mod plan_execute; pub mod rand; pub mod stage; pub mod subagent; diff --git a/crates/libsy/src/algorithms/plan_execute.rs b/crates/libsy/src/algorithms/plan_execute.rs new file mode 100644 index 000000000..49e7d1b08 --- /dev/null +++ b/crates/libsy/src/algorithms/plan_execute.rs @@ -0,0 +1,354 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Starts coding tasks on a capable planner, then hands execution to an efficient model. + +use std::collections::HashSet; +use std::sync::Arc; + +use parking_lot::Mutex; +use switchyard_protocol::{ModelId, Request}; + +use super::util::prompts::{SystemPromptProcessor, TargetPrompts}; +use super::util::tool_signals::ToolSignals; +use crate::core::algorithm::{Algorithm, Driver, RoutingIdentity}; +use crate::core::processor::{Event, Processor}; +use crate::{LibsyError, Result, RoutingOutcome}; + +/// Default instruction prepended while the capable model is planning. +pub const DEFAULT_PLANNING_PROMPT: &str = + include_str!("../prompts/plan-execute/planning-system-prompt.md"); + +/// Maximum session latches retained by one router instance. +const MAX_EXECUTING_SESSIONS: usize = 4_096; + +/// Configuration for [`PlanExecute`]. +#[derive(Clone, Debug)] +pub struct PlanExecuteConfig { + /// System instruction prepended until the first edit or write tool call. + pub planning_prompt: String, +} + +impl Default for PlanExecuteConfig { + fn default() -> Self { + Self { + planning_prompt: DEFAULT_PLANNING_PROMPT.trim().to_string(), + } + } +} + +/// Routes planning turns to a capable model and all turns after the first edit +/// to an efficient model while preserving the caller's full trajectory. +pub struct PlanExecute { + capable: ModelId, + efficient: ModelId, + planning_prompt: SystemPromptProcessor, + executing_sessions: Mutex>, +} + +impl PlanExecute { + /// Creates a plan/execute router. + /// + /// Returns an error when the planning prompt is empty. + pub fn new(capable: ModelId, efficient: ModelId, config: PlanExecuteConfig) -> Result { + if config.planning_prompt.trim().is_empty() { + return Err(LibsyError::AlgorithmError { + message: "planning_prompt must not be empty".to_string(), + }); + } + let planning_prompt = SystemPromptProcessor::new( + TargetPrompts::default().with(capable.clone(), config.planning_prompt), + ); + Ok(Self { + capable, + efficient, + planning_prompt, + executing_sessions: Mutex::new(HashSet::new()), + }) + } + + /// Whether this request is in execution, latching the transition for keyed sessions. + fn is_executing(&self, request: &Request) -> bool { + let signals = ToolSignals::from_request(request, None); + let mutation_seen = signals.edit_count > 0 || signals.write_count > 0; + let Some(identity) = RoutingIdentity::from_request(request) else { + return mutation_seen; + }; + + let mut sessions = self.executing_sessions.lock(); + let executing = if mutation_seen { + if sessions.len() >= MAX_EXECUTING_SESSIONS + && !sessions.contains(&identity) + && let Some(evicted) = sessions.iter().next().cloned() + { + sessions.remove(&evicted); + } + sessions.insert(identity.clone()); + true + } else { + sessions.contains(&identity) + }; + if request + .metadata + .as_ref() + .and_then(|metadata| metadata.session_final) + == Some(true) + { + sessions.remove(&identity); + } + executing + } +} + +#[async_trait::async_trait] +impl Algorithm for PlanExecute { + fn name(&self) -> &str { + "plan_execute" + } + + async fn route( + self: Arc, + _driver: Driver, + mut request: Request, + ) -> Result { + if self.is_executing(&request) { + tracing::info!(target = %self.efficient, phase = "execute", "plan-execute selected target"); + Ok(RoutingOutcome::route_to( + self.efficient.clone(), + Vec::new(), + request, + )) + } else { + self.planning_prompt + .process( + &mut (), + Event::Decision { + request: &mut request, + selected_model_id: &self.capable, + }, + ) + .await?; + tracing::info!(target = %self.capable, phase = "plan", "plan-execute selected target"); + Ok(RoutingOutcome::route_to( + self.capable.clone(), + Vec::new(), + request, + )) + } + } +} + +#[cfg(test)] +mod tests { + use std::sync::{Arc, Mutex}; + + use serde_json::json; + use switchyard_protocol::{ + ContentBlock, InstructionBlock, LlmRequest, Message, Metadata, Request, Role, ToolCall, + }; + + use super::*; + use crate::core::testing::{reply, test_drive}; + + fn algorithm() -> Arc { + Arc::new( + PlanExecute::new( + ModelId::from("model/capable"), + ModelId::from("model/efficient"), + PlanExecuteConfig::default(), + ) + .expect("default config should be valid"), + ) + } + + fn request(messages: Vec, session_id: Option<&str>) -> Request { + Request { + llm_request: LlmRequest { + model: Some("switchyard/plan-execute".to_string()), + messages, + ..LlmRequest::default() + }, + metadata: session_id.map(|session_id| Metadata { + session_id: Some(session_id.to_string()), + ..Metadata::default() + }), + ..Request::default() + } + } + + fn tool_call(name: &str, arguments: serde_json::Value) -> Message { + Message { + role: Role::Assistant, + content: vec![ContentBlock::ToolCall(ToolCall { + id: "call-1".to_string(), + name: name.to_string(), + arguments, + })], + } + } + + async fn route_and_capture( + algorithm: Arc, + request: Request, + ) -> (ModelId, Request) { + let captured = Arc::new(Mutex::new(None)); + let capture = Arc::clone(&captured); + let (selected, _) = test_drive(algorithm, request, move |_target, request| { + let capture = Arc::clone(&capture); + async move { + *capture.lock().expect("capture lock should be available") = Some(request); + Ok(reply("ok")) + } + }) + .await + .expect("routing should succeed"); + let request = captured + .lock() + .expect("capture lock should be available") + .take() + .expect("answer request should be captured"); + (selected, request) + } + + #[tokio::test] + async fn initial_turn_uses_capable_model_with_planning_prefix() { + let messages = vec![Message::text(Role::User, "fix the parser")]; + let (selected, routed) = + route_and_capture(algorithm(), request(messages.clone(), Some("task-1"))).await; + + assert_eq!(selected, "model/capable"); + assert_eq!(routed.llm_request.messages, messages); + assert_eq!(routed.llm_request.instructions.len(), 1); + assert_eq!(routed.llm_request.instructions[0].role, Role::System); + assert_eq!( + routed.llm_request.instructions[0].content, + vec![ContentBlock::Text { + text: DEFAULT_PLANNING_PROMPT.trim().to_string() + }] + ); + } + + #[tokio::test] + async fn read_only_tool_calls_remain_in_planning() { + let messages = vec![ + Message::text(Role::User, "fix the parser"), + tool_call("exec_command", json!({"cmd": "rg parser crates"})), + ]; + + let (selected, routed) = + route_and_capture(algorithm(), request(messages.clone(), None)).await; + + assert_eq!(selected, "model/capable"); + assert_eq!(routed.llm_request.messages, messages); + assert_eq!(routed.llm_request.instructions.len(), 1); + } + + #[tokio::test] + async fn first_edit_switches_to_efficient_and_keeps_the_trajectory() { + let messages = vec![ + Message::text(Role::User, "fix the parser"), + Message::text(Role::Assistant, "I will update the parser now."), + tool_call("apply_patch", json!({"patch": "*** Begin Patch"})), + ]; + let mut input = request(messages.clone(), Some("task-2")); + input.llm_request.instructions.push(InstructionBlock { + role: Role::Developer, + content: vec![ContentBlock::Text { + text: "keep the public API stable".to_string(), + }], + }); + + let (selected, routed) = route_and_capture(algorithm(), input).await; + + assert_eq!(selected, "model/efficient"); + assert_eq!(routed.llm_request.messages, messages); + assert_eq!(routed.llm_request.instructions.len(), 1); + assert_eq!(routed.llm_request.instructions[0].role, Role::Developer); + } + + #[tokio::test] + async fn shell_file_write_switches_to_execution() { + let messages = vec![tool_call( + "exec_command", + json!({"cmd": "python -c 'from pathlib import Path; Path(\"x\").write_text(\"y\")'"}), + )]; + + let (selected, routed) = route_and_capture(algorithm(), request(messages, None)).await; + + assert_eq!(selected, "model/efficient"); + assert!(routed.llm_request.instructions.is_empty()); + } + + #[tokio::test] + async fn shell_redirection_switches_to_execution() { + let messages = vec![tool_call( + "exec_command", + json!({"cmd": "printf 'completed\\n' > task.txt"}), + )]; + + let (selected, routed) = route_and_capture(algorithm(), request(messages, None)).await; + + assert_eq!(selected, "model/efficient"); + assert!(routed.llm_request.instructions.is_empty()); + } + + #[tokio::test] + async fn execution_latches_by_session_after_history_compaction() { + let algorithm = algorithm(); + let edit = request( + vec![tool_call("write_file", json!({"path": "src/lib.rs"}))], + Some("task-3"), + ); + let (selected, _) = route_and_capture(Arc::clone(&algorithm), edit).await; + assert_eq!(selected, "model/efficient"); + + let compacted = request( + vec![Message::text( + Role::User, + "Continue from the compacted summary", + )], + Some("task-3"), + ); + let (selected, routed) = route_and_capture(algorithm, compacted).await; + + assert_eq!(selected, "model/efficient"); + assert!(routed.llm_request.instructions.is_empty()); + } + + #[tokio::test] + async fn final_request_uses_then_releases_the_session_latch() { + let algorithm = algorithm(); + let edit = request( + vec![tool_call("write_file", json!({"path": "src/lib.rs"}))], + Some("task-4"), + ); + let (selected, _) = route_and_capture(Arc::clone(&algorithm), edit).await; + assert_eq!(selected, "model/efficient"); + + let mut final_request = request(vec![Message::text(Role::User, "Finish")], Some("task-4")); + final_request + .metadata + .as_mut() + .expect("session metadata should exist") + .session_final = Some(true); + let (selected, _) = route_and_capture(Arc::clone(&algorithm), final_request).await; + assert_eq!(selected, "model/efficient"); + + let reused = request(vec![Message::text(Role::User, "New task")], Some("task-4")); + let (selected, _) = route_and_capture(algorithm, reused).await; + assert_eq!(selected, "model/capable"); + } + + #[test] + fn empty_planning_prompt_is_rejected() { + let result = PlanExecute::new( + ModelId::from("model/capable"), + ModelId::from("model/efficient"), + PlanExecuteConfig { + planning_prompt: " ".to_string(), + }, + ); + + assert!(matches!(result, Err(LibsyError::AlgorithmError { .. }))); + } +} diff --git a/crates/libsy/src/algorithms/util/tool_signals.rs b/crates/libsy/src/algorithms/util/tool_signals.rs index 43e1bd814..1aabe2156 100644 --- a/crates/libsy/src/algorithms/util/tool_signals.rs +++ b/crates/libsy/src/algorithms/util/tool_signals.rs @@ -120,6 +120,8 @@ static BASH_WRITE_PATTERNS: &[&str] = &[ "tee ", "printf >", "printf >>", + " > ", + " >> ", "> /", ">> /", "<< 'eof'", @@ -1150,6 +1152,14 @@ mod tests { assert_eq!(sig.edit_count, 0); } + #[test] + fn bash_redirection_after_arguments_counts_as_write() { + let request = with_messages(vec![bash("printf 'completed\\n' > task.txt")]); + let sig = ToolSignals::from_request(&request, None); + assert_eq!(sig.write_count, 1); + assert_eq!(sig.edit_count, 0); + } + #[test] fn bash_sed_inplace_counts_as_edit() { let request = with_messages(vec![bash("sed -i 's/foo/bar/g' /app/file.py")]); diff --git a/crates/libsy/src/lib.rs b/crates/libsy/src/lib.rs index 7999f2428..781fbe7f7 100644 --- a/crates/libsy/src/lib.rs +++ b/crates/libsy/src/lib.rs @@ -23,6 +23,7 @@ pub use algorithms::llm_class::{ }; pub use algorithms::noop::Noop; pub use algorithms::passthrough::Passthrough; +pub use algorithms::plan_execute::{DEFAULT_PLANNING_PROMPT, PlanExecute, PlanExecuteConfig}; pub use algorithms::rand::{Random, RandomClassifier}; pub use algorithms::stage::{LlmFallback, StageRouter, StageRouterConfig}; pub use algorithms::subagent::{SubagentRouter, SubagentRouterConfig}; diff --git a/crates/libsy/src/prompts/plan-execute/planning-system-prompt.md b/crates/libsy/src/prompts/plan-execute/planning-system-prompt.md new file mode 100644 index 000000000..65e108dde --- /dev/null +++ b/crates/libsy/src/prompts/plan-execute/planning-system-prompt.md @@ -0,0 +1 @@ +You are in the planning phase. Inspect the task and the relevant code, then form a concrete implementation plan before modifying any files. Use read-only tools as needed. Do not make an edit until the plan is complete; your first edit marks the handoff to execution. diff --git a/crates/switchyard-runner/src/algorithm.rs b/crates/switchyard-runner/src/algorithm.rs index 21a8fa9a0..e49376a33 100644 --- a/crates/switchyard-runner/src/algorithm.rs +++ b/crates/switchyard-runner/src/algorithm.rs @@ -13,9 +13,9 @@ use libsy::{ AdvisorGate, AdvisorGateConfig, Algorithm, ClassifierContractConfig, ClassifierResponseFormat, ClassifyTrigger, CompositeRouter, CompositeRouterConfig, CustomClassifierConfig, CustomClassifierPolicy, EscalationJudgeConfig, GateTrigger, HandoffNoteConfig, - LlmClassifierConfig, LlmFallback, LlmTaskClassifier, Noop, Passthrough, PickerMode, Random, - StageRouter, StageRouterConfig, SubagentRouter, SubagentRouterConfig, TaskClassifierConfig, - ToolSemantics, + LlmClassifierConfig, LlmFallback, LlmTaskClassifier, Noop, Passthrough, PickerMode, + PlanExecute, PlanExecuteConfig, Random, StageRouter, StageRouterConfig, SubagentRouter, + SubagentRouterConfig, TaskClassifierConfig, ToolSemantics, }; use serde::Deserialize; use switchyard_protocol::ModelId; @@ -240,6 +240,17 @@ pub enum AlgorithmSpec { #[serde(default)] subagents: Option, }, + /// Plans with a capable model, then switches permanently to an efficient + /// model when the conversation records its first edit or write tool call. + PlanExecute { + /// Target used before the first edit. + capable_target: String, + /// Target used from the first edit onward. + efficient_target: String, + /// Replaces the built-in planning system prompt. + #[serde(default)] + planning_prompt: Option, + }, /// Asks a judge model which target should serve the request. LlmClassifier { /// Judge and tier settings, written directly in the route table. @@ -440,6 +451,11 @@ impl AlgorithmSpec { } names } + Self::PlanExecute { + capable_target, + efficient_target, + .. + } => vec![capable_target.as_str(), efficient_target.as_str()], Self::LlmClassifier { config, .. } => { match config.mode.unwrap_or(if config.escalation.is_some() { ClassifierMode::Escalation @@ -569,6 +585,7 @@ impl AlgorithmSpec { Self::Noop { .. } | Self::Random { .. } | Self::Passthrough { .. } + | Self::PlanExecute { .. } | Self::LlmClassifier { .. } | Self::StageRouter { .. } | Self::Auto { .. } @@ -909,6 +926,25 @@ fn build_algorithm( let parent: Arc = Arc::new(algorithm); attach_subagent_router(route_name, parent, subagents.as_ref(), targets) } + AlgorithmSpec::PlanExecute { + capable_target, + efficient_target, + planning_prompt, + } => { + let capable = resolve_target_model_id(route_name, capable_target, targets)?; + let efficient = resolve_target_model_id(route_name, efficient_target, targets)?; + let mut config = PlanExecuteConfig::default(); + if let Some(prompt) = planning_prompt { + config.planning_prompt = prompt.clone(); + } + let algorithm = PlanExecute::new(capable, efficient, config).map_err(|error| { + AlgorithmConfigError::with_source( + format!("plan_execute route {route_name}: {error}"), + error, + ) + })?; + Ok(Arc::new(algorithm)) + } AlgorithmSpec::LlmClassifier { config: classifier_config, .. diff --git a/crates/switchyard-runner/src/config.rs b/crates/switchyard-runner/src/config.rs index 867a8e53a..f4523dd66 100644 --- a/crates/switchyard-runner/src/config.rs +++ b/crates/switchyard-runner/src/config.rs @@ -885,6 +885,46 @@ new = ["send_message"] } } + #[test] + fn plan_execute_route_builds_and_claims_both_targets() -> RunnerResult<()> { + let config = format!( + r#"{VALID_CONFIG} + +[routes.plan_execute] +id = "switchyard/plan-execute" +type = "plan_execute" +capable_target = "strong" +efficient_target = "weak" +planning_prompt = "Inspect and plan before editing." +"# + ); + let runner = runner_from_toml(&config)?; + + assert!( + runner + .models() + .any(|model| model.id.as_str() == "switchyard/plan-execute") + ); + Ok(()) + } + + #[test] + fn plan_execute_route_rejects_an_empty_prompt() { + let config = format!( + r#"{VALID_CONFIG} + +[routes.plan_execute] +id = "switchyard/plan-execute" +type = "plan_execute" +capable_target = "strong" +efficient_target = "weak" +planning_prompt = " " +"# + ); + + assert!(error_message(&config).contains("planning_prompt must not be empty")); + } + #[test] fn passthrough_and_stage_accept_subagent_routing() -> RunnerResult<()> { let stage = stage_config(); diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rs index 19adfb1fb..77595b49f 100644 --- a/crates/switchyard-server/tests/server.rs +++ b/crates/switchyard-server/tests/server.rs @@ -1320,6 +1320,206 @@ confidence_threshold = 0.5 Ok(()) } +/// A Codex-style Responses trajectory starts on the capable planner and hands +/// the complete conversation to the efficient executor after `apply_patch`. +#[tokio::test] +async fn plan_execute_route_hands_off_after_the_first_edit() -> TestResult { + const PLANNING_PROMPT: &str = "Inspect first and make a concrete plan before editing."; + let upstream = MockUpstream::start().await?; + let state = load_test_config(&format!( + r#" +schema_version = 1 + +[llm_clients.upstream] +format = "openai_chat" +base_url = "{base_url}" + +[targets.capable] +id = "model/capable" +llm_client = "upstream" + +[targets.efficient] +id = "model/efficient" +llm_client = "upstream" + +[routes.plan_execute] +id = "switchyard/plan-execute" +type = "plan_execute" +capable_target = "capable" +efficient_target = "efficient" +planning_prompt = "{PLANNING_PROMPT}" +"#, + base_url = upstream.base_url + ))?; + let app = build_switchyard_router(state); + + let planning = send( + &app, + "POST", + "/v1/responses", + Some(json!({ + "model": "switchyard/plan-execute", + "instructions": "Keep the public API stable.", + "input": "Fix the parser." + })), + ) + .await?; + assert_eq!(planning.status, StatusCode::OK); + assert_eq!( + planning + .headers + .get("x-model-router-selected-model") + .and_then(|value| value.to_str().ok()), + Some("model/capable") + ); + + let executing = send( + &app, + "POST", + "/v1/responses", + Some(json!({ + "model": "switchyard/plan-execute", + "instructions": "Keep the public API stable.", + "input": [ + {"type": "message", "role": "user", "content": "Fix the parser."}, + {"type": "message", "role": "assistant", "content": "The plan is ready."}, + { + "type": "function_call", + "call_id": "call-edit", + "name": "apply_patch", + "arguments": "{\"patch\":\"*** Begin Patch\"}" + }, + { + "type": "function_call_output", + "call_id": "call-edit", + "output": "Success. Updated the file." + } + ] + })), + ) + .await?; + assert_eq!(executing.status, StatusCode::OK); + assert_eq!( + executing + .headers + .get("x-model-router-selected-model") + .and_then(|value| value.to_str().ok()), + Some("model/efficient") + ); + + let chat_execution = send( + &app, + "POST", + "/v1/chat/completions", + Some(json!({ + "model": "switchyard/plan-execute", + "messages": [ + {"role": "user", "content": "Fix the parser."}, + { + "role": "assistant", + "tool_calls": [{ + "id": "call-edit", + "type": "function", + "function": { + "name": "apply_patch", + "arguments": "{\"patch\":\"*** Begin Patch\"}" + } + }] + }, + {"role": "tool", "tool_call_id": "call-edit", "content": "updated"} + ] + })), + ) + .await?; + assert_eq!(chat_execution.status, StatusCode::OK); + assert_eq!( + chat_execution + .headers + .get("x-model-router-selected-model") + .and_then(|value| value.to_str().ok()), + Some("model/efficient") + ); + + let anthropic_execution = send( + &app, + "POST", + "/v1/messages", + Some(json!({ + "model": "switchyard/plan-execute", + "max_tokens": 128, + "messages": [ + {"role": "user", "content": "Fix the parser."}, + { + "role": "assistant", + "content": [{ + "type": "tool_use", + "id": "call-edit", + "name": "write_file", + "input": {"path": "src/parser.rs", "content": "fixed"} + }] + }, + { + "role": "user", + "content": [{ + "type": "tool_result", + "tool_use_id": "call-edit", + "content": "updated" + }] + } + ] + })), + ) + .await?; + assert_eq!(anthropic_execution.status, StatusCode::OK); + assert_eq!( + anthropic_execution + .headers + .get("x-model-router-selected-model") + .and_then(|value| value.to_str().ok()), + Some("model/efficient") + ); + + let calls = upstream.calls.lock().await; + assert_eq!(calls.len(), 4); + assert_eq!(calls[0]["model"], "model/capable"); + assert_eq!(calls[1]["model"], "model/efficient"); + assert_eq!(calls[2]["model"], "model/efficient"); + assert_eq!(calls[3]["model"], "model/efficient"); + let planning_messages = calls[0]["messages"] + .as_array() + .ok_or("planning request did not contain messages")?; + assert_eq!(planning_messages[0]["role"], "system"); + assert_eq!(planning_messages[0]["content"], PLANNING_PROMPT); + assert!( + planning_messages + .iter() + .any(|message| message["content"] == "Keep the public API stable.") + ); + assert!( + calls[1]["messages"] + .as_array() + .is_some_and(|messages| messages.iter().all(|message| { + message["content"] + .as_str() + .is_none_or(|content| !content.contains(PLANNING_PROMPT)) + })), + "the planning instruction must be pruned from the execution request" + ); + for call in &calls[2..] { + assert!( + !call["messages"].to_string().contains(PLANNING_PROMPT), + "the planning instruction must be absent after cross-format decoding" + ); + } + assert!( + calls[1]["messages"] + .to_string() + .contains("The plan is ready."), + "the efficient model must inherit the pre-edit trajectory" + ); + Ok(()) +} + #[tokio::test] async fn toml_config_constructs_and_serves_multiple_algorithms() -> TestResult { let upstream = MockUpstream::start().await?; diff --git a/crates/switchyard-translation/src/codecs/responses/buffered.rs b/crates/switchyard-translation/src/codecs/responses/buffered.rs index 3db153209..d37a22287 100644 --- a/crates/switchyard-translation/src/codecs/responses/buffered.rs +++ b/crates/switchyard-translation/src/codecs/responses/buffered.rs @@ -889,6 +889,7 @@ fn decode_responses_tools( .get("name") .and_then(Value::as_str) .filter(|name| !name.is_empty()); + let description = tool.get("description").and_then(Value::as_str); for mut child in decode_responses_tools(tool.get("tools"), namespaces, custom_tools) { // A nested container already qualified its own children, and the // innermost name is the one that identifies the tool. @@ -899,7 +900,10 @@ fn decode_responses_tools( let qualified = crate::codex_namespaces::qualified_tool_name(container, &child.name); crate::codex_namespaces::record_tool_namespace( - namespaces, &qualified, container, + namespaces, + &qualified, + container, + description, ); child.name = qualified; } @@ -1498,9 +1502,15 @@ fn encode_responses_tools( } } for (namespace, children) in containers { + let description = namespaces + .and_then(|namespaces| { + crate::codex_namespaces::namespace_description(namespaces, &namespace) + }) + .unwrap_or_default(); out.push(json!({ "type": "namespace", "name": namespace, + "description": description, "tools": children, })); } diff --git a/crates/switchyard-translation/src/codex_namespaces.rs b/crates/switchyard-translation/src/codex_namespaces.rs index faaff5f77..91c73bdf7 100644 --- a/crates/switchyard-translation/src/codex_namespaces.rs +++ b/crates/switchyard-translation/src/codex_namespaces.rs @@ -48,8 +48,18 @@ pub fn record_tool_namespace( namespaces: &mut Map, qualified: &str, namespace: &str, + description: Option<&str>, ) { - namespaces.insert(qualified.to_string(), Value::String(namespace.to_string())); + let value = description.map_or_else( + || Value::String(namespace.to_string()), + |description| { + serde_json::json!({ + "namespace": namespace, + "description": description, + }) + }, + ); + namespaces.insert(qualified.to_string(), value); } /// Stores a collected mapping on a request's extensions, when it has entries. @@ -79,13 +89,29 @@ pub fn split_qualified_name( namespaces: &Map, qualified: &str, ) -> Option<(String, String)> { - let namespace = namespaces.get(qualified).and_then(Value::as_str)?; + let value = namespaces.get(qualified)?; + let namespace = value + .as_str() + .or_else(|| value.get("namespace").and_then(Value::as_str))?; let tool = qualified .strip_prefix(namespace)? .strip_prefix(NAMESPACE_SEPARATOR)?; Some((tool.to_string(), namespace.to_string())) } +/// Returns a retained description for `namespace`. +pub fn namespace_description<'a>( + namespaces: &'a Map, + namespace: &str, +) -> Option<&'a str> { + namespaces.values().find_map(|value| { + let object = value.as_object()?; + (object.get("namespace").and_then(Value::as_str) == Some(namespace)) + .then(|| object.get("description").and_then(Value::as_str)) + .flatten() + }) +} + /// Reverse map from an upstream tool name to its Codex tool name and namespace. /// /// The exact qualified name is always registered. A model often returns a near @@ -179,8 +205,8 @@ mod tests { use switchyard_protocol::ProviderExtensions; use super::{ - attach_tool_namespaces, qualified_tool_name, qualified_tool_origins, record_tool_namespace, - restore_qualified_tool_names, split_qualified_name, tool_namespaces, + attach_tool_namespaces, namespace_description, qualified_tool_name, qualified_tool_origins, + record_tool_namespace, restore_qualified_tool_names, split_qualified_name, tool_namespaces, }; fn extensions(pairs: &[(&str, &str)]) -> ProviderExtensions { @@ -190,6 +216,7 @@ mod tests { &mut namespaces, &qualified_tool_name(namespace, tool), namespace, + None, ); } let mut extensions = ProviderExtensions::default(); @@ -217,6 +244,26 @@ mod tests { ); } + #[test] + fn retains_a_namespace_description_without_changing_name_lookup() { + let mut namespaces = Map::new(); + record_tool_namespace( + &mut namespaces, + "multi_agent_v1__spawn_agent", + "multi_agent_v1", + Some("Tools for managing sub-agents."), + ); + + assert_eq!( + split_qualified_name(&namespaces, "multi_agent_v1__spawn_agent"), + Some(("spawn_agent".to_string(), "multi_agent_v1".to_string())) + ); + assert_eq!( + namespace_description(&namespaces, "multi_agent_v1"), + Some("Tools for managing sub-agents.") + ); + } + // A bare name claimed by two namespaces must not be guessed: a wrong guess // dispatches the call to the wrong server. #[test] diff --git a/crates/switchyard-translation/tests/request_translation.rs b/crates/switchyard-translation/tests/request_translation.rs index 0c3565cb1..15f309ff1 100644 --- a/crates/switchyard-translation/tests/request_translation.rs +++ b/crates/switchyard-translation/tests/request_translation.rs @@ -780,6 +780,43 @@ fn responses_request_translates_codex_tool_shape_to_openai_chat() -> TestResult Ok(()) } +#[test] +fn responses_reencode_preserves_required_namespace_description() -> TestResult { + let engine = TranslationEngine::default(); + let policy = TranslationPolicy::default(); + let body = json!({ + "model": "switchyard", + "input": "Fix the parser", + "tools": [{ + "type": "namespace", + "name": "multi_agent_v1", + "description": "Tools for managing sub-agents.", + "tools": [{ + "type": "function", + "name": "spawn_agent", + "description": "Start one agent.", + "parameters": {"type": "object"} + }] + }] + }); + let mut request = engine + .decode_request(WireFormat::OpenAiResponses, &body, &policy)? + .request; + + prepare_request_for_target(&mut request, &"gpt-5.6-sol".into(), Some("Plan first.")); + let output = engine + .encode_request(WireFormat::OpenAiResponses, &request, &policy)? + .body; + + assert_eq!(output["tools"][0]["type"], "namespace"); + assert_eq!(output["tools"][0]["name"], "multi_agent_v1"); + assert_eq!( + output["tools"][0]["description"], + "Tools for managing sub-agents." + ); + Ok(()) +} + // Verifies Python-style Responses tool definitions translate into OpenAI Chat tools. #[test] fn responses_request_translates_python_compatible_tool_shape_to_openai_chat() -> TestResult { @@ -1541,6 +1578,43 @@ fn responses_empty_reasoning_without_encrypted_content_is_omitted() -> TestResul Ok(()) } +#[test] +fn responses_reencode_preserves_encrypted_reasoning_item() -> TestResult { + let engine = TranslationEngine::default(); + let policy = normalized_policy(); + let reasoning = json!({ + "type": "reasoning", + "id": "reasoning-1", + "summary": [], + "encrypted_content": "opaque-encrypted-reasoning" + }); + let body = json!({ + "model": "switchyard", + "input": [ + {"type": "message", "role": "user", "content": "Inspect the repo"}, + reasoning, + { + "type": "function_call", + "name": "shell", + "call_id": "call-ls", + "arguments": "{\"command\":\"ls\"}" + }, + {"type": "function_call_output", "call_id": "call-ls", "output": "a.py"} + ] + }); + + let mut request = engine + .decode_request(WireFormat::OpenAiResponses, &body, &policy)? + .request; + prepare_request_for_target(&mut request, &"gpt-5.6-sol".into(), Some("Plan first.")); + let output = engine + .encode_request(WireFormat::OpenAiResponses, &request, &policy)? + .body; + + assert_eq!(output["input"][1], reasoning); + Ok(()) +} + // Verifies Responses JSON schema text format maps to Chat response_format shape. #[test] fn responses_json_schema_text_format_maps_to_chat_response_format() -> TestResult { diff --git a/docs/core_concepts.md b/docs/core_concepts.md index 87a3eeb4e..6bd5ea184 100644 --- a/docs/core_concepts.md +++ b/docs/core_concepts.md @@ -78,6 +78,7 @@ route types: | `passthrough` | Sends every request to one target. | | `random` | Selects among targets using optional relative weights. | | `llm_classifier` | Uses a classifier target to choose between weak and strong targets. | +| `plan_execute` | Plans on a capable target, then switches to an efficient target after the first edit. | | `stage_router` | Uses tool-result and progress signals to choose an efficient or capable target. | Strong, weak, capable, and efficient are roles within an algorithm, not fixed diff --git a/docs/getting_started.md b/docs/getting_started.md index 5f61e8034..ca2d927c1 100644 --- a/docs/getting_started.md +++ b/docs/getting_started.md @@ -137,6 +137,7 @@ settings. The Rust server also supports: | [Random](routing_algorithms/random_routing.md) | You need a weighted split for A/B tests or baselines. | `random` | | [LLM classifier](routing_algorithms/llm_classifier_routing.md) | Request content should decide whether to use the weak or strong target. | `llm_classifier` | | [Stage router](routing_algorithms/stage_router_routing.md) | Built-in or configured tool-activity signals should select an efficient or capable target. | `stage_router` | +| [Plan/execute](routing_algorithms/plan_execute_routing.md) | A capable model should plan, then an efficient model should execute after the first edit. | `plan_execute` | A single TOML file can declare multiple routes. The table key, such as `routes.smart`, is a local configuration name; each route's `id` is exposed as a diff --git a/docs/operations/soak_test.md b/docs/operations/soak_test.md index c0ce76cb4..c9a369d77 100644 --- a/docs/operations/soak_test.md +++ b/docs/operations/soak_test.md @@ -86,6 +86,7 @@ python3.12 scripts/benchmark_routing_algorithms.py \ --direct-model mock/weak \ --model noop=switchyard/noop \ --model passthrough=switchyard/passthrough \ + --model plan_execute=switchyard/plan-execute \ --model random=switchyard/random \ --model llm_classifier=switchyard/classifier \ --model stage_router=switchyard/stage \ @@ -219,12 +220,13 @@ The script gives each tool one job: | Combined report | Joins scenario, load, oha, AIPerf, and routing-counter metrics in Markdown, CSV, JSON, and an overhead plot. It keeps resilience rows separate from throughput rows. | | `switchyard-soak` | Runs the standard scenario set while checking public API variants, server health, metrics, process use, and required results. | -`scripts/local_soak_test.toml` exercises `noop`, `random`, `passthrough`, `llm_classifier`, and -`stage_router`. It uses the accepted maximum retry count (10), a zero-weight random target, and the -upper classifier and stage thresholds (1.0). Classifier affinity is disabled so every measured -request includes the classifier call. The scenario backend returns `p_solve=1.0` for easy markers -and `p_solve=0.1` for hard markers; with the configured threshold, those requests select the weak -and strong targets respectively. The config is validated with +`scripts/local_soak_test.toml` exercises `noop`, `random`, `passthrough`, `plan_execute`, +`llm_classifier`, and `stage_router`. It uses the accepted maximum retry count (10), a zero-weight +random target, and the upper classifier and stage thresholds (1.0). Classifier affinity is disabled +so every measured request includes the classifier call. The scenario backend returns `p_solve=1.0` +for easy markers and `p_solve=0.1` for hard markers; with the configured threshold, those requests +select the weak and strong targets respectively. The `stage-transitions` scenario contains an +`apply_patch` call and therefore exercises the plan/execute handoff. The config is validated with `switchyard-server --dry-run` before either service starts. Run resilience cases separately so expected transport failures do not contaminate throughput diff --git a/docs/reference/toml_schema.md b/docs/reference/toml_schema.md index 92733bd57..73d37cc20 100644 --- a/docs/reference/toml_schema.md +++ b/docs/reference/toml_schema.md @@ -236,6 +236,18 @@ Classifier prompts must not contain `{{RESPONSE_SCHEMA}}`. Switchyard supplies the schema automatically: through the structured-output request in `json_schema` mode, or in the prompt in `json_object` mode. +### `plan_execute` + +Starts on a capable planning target and switches to an efficient execution +target after the first edit or write tool call. See +[Plan/Execute Routing](../routing_algorithms/plan_execute_routing.md). + +| Key | Required | Default | Meaning | +|---|:---:|---|---| +| `capable_target` | Yes | — | Target used before the first mutation. | +| `efficient_target` | Yes | — | Target used from the first mutation onward. | +| `planning_prompt` | No | packaged prompt | Non-empty system instruction prepended only during planning. | + ### `stage_router` Scores tool signals to pick a tier per turn. See diff --git a/docs/routing_algorithms/overview.md b/docs/routing_algorithms/overview.md index 074b3c151..bad61e010 100644 --- a/docs/routing_algorithms/overview.md +++ b/docs/routing_algorithms/overview.md @@ -17,6 +17,7 @@ configuration and tuning. For the vocabulary these pages use, see | [LLM Classifier Routing](llm_classifier_routing.md) | Request content should decide whether a turn needs the weak or strong tier. | `llm_classifier` | | [Stage-Router Routing](stage_router_routing.md) | Built-in or configured tool-activity signals should route most turns without an extra classifier call. | `stage_router` | | Auto Routing | You want a recommended default instead of picking a strategy yourself. For a deeper dive on the current default, see [Stage-Router Routing](stage_router_routing.md); for full control, pick one of the strategies above instead. | `auto` | +| [Plan/Execute Routing](plan_execute_routing.md) | A capable model should plan the task, then an efficient model should execute after the first edit. | `plan_execute` | | [Composite Routing](composite_routing.md) | Routing algorithms are composed, one setting the configuration of another before handing off. Today an LLM classifier sets a stage router's default tier. | `composite` | | [Escalation-Router Routing](escalation_router_routing.md) | Start every task on the weak tier and escalate to strong when an LLM judge detects trouble. | `llm_classifier` with `escalation` | | [Advisor-Gate Routing](advisor_gate_routing.md) | One model should serve every turn, with a stronger reviewer approving its "done" claims or sending back a redo plan. | `advisor` | diff --git a/docs/routing_algorithms/plan_execute_routing.md b/docs/routing_algorithms/plan_execute_routing.md new file mode 100644 index 000000000..16eef6c78 --- /dev/null +++ b/docs/routing_algorithms/plan_execute_routing.md @@ -0,0 +1,83 @@ +# Plan/Execute Routing + +Plan/execute routing starts a coding task on a capable model and switches to an +efficient model after the first file mutation. It makes no classifier call. + +Before the transition, Switchyard prepends a planning system instruction to the +outbound request. Read-only inspection and planning tool calls stay on the +capable target. An edit or write tool call anywhere in the normalized +conversation moves the request to the efficient target and the planning +instruction is no longer added. The efficient model receives the caller's full +conversation, including the capable model's plan and tool history. + +The transition is latched by session ID when one is available, so later context +compaction cannot move that session back into planning. Without a session ID, +Switchyard determines the phase from the conversation on every request; callers +must therefore retain the first mutation in the history to keep the efficient +target selected. A request marked as the session's final request uses the latch +for that request and then releases it. + +## Configure the route + +```toml +schema_version = 1 + +[llm_clients.openrouter] +format = "openai_chat" +base_url = "https://openrouter.ai/api/v1" +api_key_env = "OPENROUTER_API_KEY" + +[targets.planner] +id = "anthropic/claude-opus-4.7" +llm_client = "openrouter" + +[targets.executor] +id = "moonshotai/kimi-k2.7-code" +llm_client = "openrouter" + +[routes.plan_execute] +id = "switchyard/plan-execute" +type = "plan_execute" +capable_target = "planner" +efficient_target = "executor" +``` + +`planning_prompt` optionally replaces the built-in planning instruction: + +```toml +[routes.plan_execute] +id = "switchyard/plan-execute" +type = "plan_execute" +capable_target = "planner" +efficient_target = "executor" +planning_prompt = "Inspect the task and write a concrete plan before editing." +``` + +The prompt must contain non-whitespace text. It is inserted ahead of caller +system and developer instructions only during planning. Switchyard does not add +it to conversation messages, so the handoff removes the planning constraint +without deleting or rewriting any caller-owned trajectory. + +## Transition signals + +The route reuses the stage router's provider-neutral tool-signal extraction. It +recognizes dedicated edit and write tools such as `apply_patch`, `Edit`, +`Write`, and `write_file`, plus common file-mutating shell commands issued +through Claude Code, Codex, and other coding-agent shells. Read, search, test, +and planning tools do not trigger the handoff. + +Because the switch is based on the recorded tool call, a failed first edit still +begins execution. This keeps the phase boundary deterministic and lets the +efficient model diagnose and retry the attempted change from the inherited +history. + +## Benchmarking + +Use the same route ID and task set for every run. Compare it with passthrough +routes for each target to measure quality, input/output tokens, time to first +token, and end-to-end latency. A stable session ID is recommended for long tasks +that may compact their history. + +See [Soak Testing](../operations/soak_test.md) for the local scenario backend +and routing benchmark workflow. The repository also includes +`benchmark/server-configs/tb-lite-plan-execute-opus-kimi.toml` for Harbor Terminal-Bench Lite. diff --git a/mkdocs.yml b/mkdocs.yml index 3a9565fb5..cb0044ac6 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -30,6 +30,7 @@ nav: - Sub-Agent-Aware Routing: routing_algorithms/subagent_routing.md - Random Routing: routing_algorithms/random_routing.md - LLM Classifier Routing: routing_algorithms/llm_classifier_routing.md + - Plan/Execute Routing: routing_algorithms/plan_execute_routing.md - Stage-Router Routing: routing_algorithms/stage_router_routing.md - Composite Routing: routing_algorithms/composite_routing.md - Escalation-Router Routing: routing_algorithms/escalation_router_routing.md diff --git a/scripts/local_soak_test.toml b/scripts/local_soak_test.toml index 967095094..dbf240ea9 100644 --- a/scripts/local_soak_test.toml +++ b/scripts/local_soak_test.toml @@ -38,6 +38,12 @@ id = "switchyard/passthrough" type = "passthrough" target = "weak" +[routes.plan_execute] +id = "switchyard/plan-execute" +type = "plan_execute" +capable_target = "strong" +efficient_target = "weak" + [routes.classifier] id = "switchyard/classifier" type = "llm_classifier" diff --git a/scripts/run_local_soak_test.py b/scripts/run_local_soak_test.py index 32decdae3..6090332de 100755 --- a/scripts/run_local_soak_test.py +++ b/scripts/run_local_soak_test.py @@ -32,6 +32,7 @@ ("noop", "switchyard/noop"), ("random", "switchyard/random"), ("passthrough", "switchyard/passthrough"), + ("plan_execute", "switchyard/plan-execute"), ("llm_classifier", "switchyard/classifier"), ("stage_router", "switchyard/stage"), )