diff --git a/crates/libsy/src/algorithms/llm_class.rs b/crates/libsy/src/algorithms/llm_class.rs index 599e9f9e0..1efb50b3b 100644 --- a/crates/libsy/src/algorithms/llm_class.rs +++ b/crates/libsy/src/algorithms/llm_class.rs @@ -13,7 +13,6 @@ use switchyard_protocol::{ContentBlock, Message, ModelId, Role}; use super::escalation; use super::fall_through::{DefaultTarget, FallThrough}; -use super::util::DEFAULT_JUDGE_MAX_OUTPUT_TOKENS; use super::util::affinity::{AffinityRouter, ClassifyTrigger}; use super::util::classifier_contract::{ ClassifierContract, ClassifierContractConfig, ClassifierResponseFormat, @@ -24,6 +23,7 @@ use super::util::llm_judge::{ SerdeDecoder, StructuredJudge, }; use super::util::target_selector::TargetSelectorPolicy; +use super::util::{DEFAULT_JUDGE_CHAR_BUDGET, DEFAULT_JUDGE_MAX_OUTPUT_TOKENS, truncate_middle}; use crate::core::algorithm::{self, Algorithm, Driver}; use crate::core::classifier::{Classification, Classifier, Score}; use crate::core::state::State; @@ -160,9 +160,105 @@ fn task_messages(messages: &[Message]) -> Vec { } } +/// Characters one message costs a judge. +/// +/// Counts tool traffic as well as visible text: in a coding-agent conversation a single +/// tool result is routinely larger than every text block around it, so measuring text +/// alone would report a payload as small while the judge is billed for all of it. +fn message_chars(message: &Message) -> usize { + message.content.iter().map(block_chars).sum() +} + +/// Characters one content block costs a judge. +fn block_chars(block: &ContentBlock) -> usize { + match block { + ContentBlock::Text { text } + | ContentBlock::Refusal { text } + | ContentBlock::Reasoning { text, .. } => text.chars().count(), + ContentBlock::ToolCall(call) => { + call.name.chars().count() + call.arguments.to_string().chars().count() + } + ContentBlock::ToolResult(result) => result.content.iter().map(block_chars).sum(), + // Media and unknown blocks are opaque here; their wire cost is not text. + _ => 0, + } +} + +/// Whether clipping can shrink this block. +/// +/// Tool arguments and results are JSON the judge may need to read as a unit, and reasoning +/// is replayed with provider state, so none of them can be cut down to fit. +fn is_clippable(block: &ContentBlock) -> bool { + matches!( + block, + ContentBlock::Text { .. } | ContentBlock::Refusal { .. } + ) +} + +/// Total judge payload size for a selected message list. +fn payload_chars(messages: &[Message]) -> usize { + messages.iter().map(message_chars).sum() +} + +/// Selects the trailing window, narrowing it until the payload fits `budget` characters. +/// +/// A window is counted in turns, and turn size varies by orders of magnitude: four turns is +/// a few hundred characters of conversation, or tens of thousands when one turn carries a +/// large tool result. Judge cost and latency would otherwise be decided by the request +/// rather than by configuration, and a single large result can crowd out the task being +/// judged. +/// +/// Whole turns are dropped from the oldest end rather than clipping individual messages, +/// because [`trim_messages`] is what keeps tool calls paired with their results; removing +/// messages by hand would hand the judge a result whose call was never introduced. +fn window_within_budget(messages: &[Message], window: usize, budget: usize) -> Vec { + let mut window = window; + let mut kept = trim_messages(messages, window); + while window > 0 && payload_chars(&kept) > budget { + window -= 1; + kept = trim_messages(messages, window); + } + // The anchors — client instructions and the opening task — survive an empty window, so + // a task statement larger than the whole budget still has to be clipped. + if payload_chars(&kept) > budget { + clip_to_budget(&mut kept, budget); + } + kept +} + +/// Clips text blocks so an unwindowable payload still fits `budget`. +/// +/// The share is per block rather than per message: one message can carry several text +/// blocks, and a per-message allowance would let each of them spend it in full. +/// +/// Blocks that cannot be clipped are charged first, so what remains is what the text is +/// allowed to spend. When they alone exceed the budget every text block collapses to +/// nothing and the payload still overruns; cutting tool JSON to fit would hand the judge +/// malformed values, which is worse than an oversized prompt. +fn clip_to_budget(messages: &mut [Message], budget: usize) { + let blocks = || messages.iter().flat_map(|message| &message.content); + let clippable = blocks().filter(|block| is_clippable(block)).count(); + if clippable == 0 { + return; + } + let fixed: usize = blocks() + .filter(|block| !is_clippable(block)) + .map(block_chars) + .sum(); + let per_block = budget.saturating_sub(fixed) / clippable; + for block in messages.iter_mut().flat_map(|message| &mut message.content) { + if let ContentBlock::Text { text } | ContentBlock::Refusal { text } = block { + *text = truncate_middle(text, per_block); + } + } +} + /// Selects the task messages shown to capability and custom-schema classifiers. struct TaskInput { recent_turn_window: Option, + /// Character budget for the windowed payload. Unused without a window, where the + /// selection is the opening task and latest follow-up rather than conversation. + judge_char_budget: usize, } impl ClassifierInput for TaskInput { @@ -170,7 +266,14 @@ impl ClassifierInput for TaskInput { // The default preserves the whole-task anchor and latest user update. A // configured window widens that to the surrounding conversation. let mut messages = match self.recent_turn_window { - Some(window) => trim_messages(&request.llm_request.messages, window), + // The routing instruction appended below is part of what the judge is sent, so + // its cost comes out of the budget rather than on top of it. + Some(window) => window_within_budget( + &request.llm_request.messages, + window, + self.judge_char_budget + .saturating_sub(TRAILING_ROUTING_INSTRUCTION.chars().count()), + ), None => task_messages(&request.llm_request.messages), }; // Only the windowed path carries assistant turns and tool traffic for the judge @@ -264,6 +367,11 @@ pub struct TaskClassifierConfig { /// `Some(n)` widens that to the client instructions, the opening task, and /// the last `n` turns after it. pub recent_turn_window: Option, + /// Character budget for a windowed judge payload. + /// + /// Bounds what one request can spend on a judge call when `recent_turn_window` is set: + /// the window narrows until the selection fits. Ignored without a window. + pub judge_char_budget: usize, /// Prompt and verdict contract settings for the classifier judge. pub contract: ClassifierContractConfig, /// Maximum completion tokens available to the classifier verdict. @@ -283,6 +391,8 @@ struct TaskClassifierConfigWire { message_hash_fallback: bool, #[serde(default)] recent_turn_window: Option, + #[serde(default = "default_judge_char_budget")] + judge_char_budget: usize, #[serde(default)] prompt: Option, #[serde(default)] @@ -308,6 +418,7 @@ impl<'de> Deserialize<'de> for TaskClassifierConfig { classify_trigger: wire.classify_trigger, message_hash_fallback: wire.message_hash_fallback, recent_turn_window: wire.recent_turn_window, + judge_char_budget: wire.judge_char_budget, contract, max_output_tokens: wire.max_output_tokens, }) @@ -318,6 +429,21 @@ const fn default_judge_max_output_tokens() -> u64 { DEFAULT_JUDGE_MAX_OUTPUT_TOKENS } +const fn default_judge_char_budget() -> usize { + DEFAULT_JUDGE_CHAR_BUDGET +} + +/// A zero budget would clip every message to the trim marker, leaving the judge a payload +/// it cannot route. Rejecting it at construction beats serving empty verdicts. +fn validate_judge_char_budget(budget: usize) -> Result<()> { + if budget == 0 { + return Err(LibsyError::AlgorithmError { + message: "judge_char_budget must be at least 1".to_string(), + }); + } + Ok(()) +} + impl Default for TaskClassifierConfig { fn default() -> Self { Self { @@ -326,6 +452,7 @@ impl Default for TaskClassifierConfig { classify_trigger: ClassifyTrigger::default(), message_hash_fallback: false, recent_turn_window: None, + judge_char_budget: DEFAULT_JUDGE_CHAR_BUDGET, contract: ClassifierContractConfig::default(), max_output_tokens: DEFAULT_JUDGE_MAX_OUTPUT_TOKENS, } @@ -335,6 +462,7 @@ impl Default for TaskClassifierConfig { impl TaskClassifierConfig { /// Validates routing thresholds before the classifier is constructed. fn validate(&self) -> Result<()> { + validate_judge_char_budget(self.judge_char_budget)?; if !(0.0..=1.0).contains(&self.base_threshold) { return Err(LibsyError::AlgorithmError { message: format!( @@ -408,6 +536,8 @@ pub struct CustomClassifierConfig { pub message_hash_fallback: bool, /// Trailing conversation turns shown to the classifier judge. pub recent_turn_window: Option, + /// Character budget for a windowed judge payload. Ignored without a window. + pub judge_char_budget: usize, /// Maximum completion tokens available to the classifier verdict. pub max_output_tokens: u64, } @@ -426,11 +556,13 @@ impl CustomClassifierConfig { classify_trigger: ClassifyTrigger::default(), message_hash_fallback: false, recent_turn_window: None, + judge_char_budget: DEFAULT_JUDGE_CHAR_BUDGET, max_output_tokens: DEFAULT_JUDGE_MAX_OUTPUT_TOKENS, } } fn validate(&self) -> Result<()> { + validate_judge_char_budget(self.judge_char_budget)?; if self.max_output_tokens == 0 { return Err(LibsyError::AlgorithmError { message: "max_output_tokens must be at least 1".to_string(), @@ -594,6 +726,7 @@ impl LlmTaskClassifier { StructuredJudge::new( TaskInput { recent_turn_window: config.recent_turn_window, + judge_char_budget: config.judge_char_budget, }, contract, SerdeDecoder::new(), @@ -674,6 +807,7 @@ impl LlmTaskClassifier { classify_trigger, message_hash_fallback, recent_turn_window, + judge_char_budget, max_output_tokens, } = config; let contract = ClassifierContract::from_inner_schema(&prompt, response_schema)?; @@ -686,7 +820,10 @@ impl LlmTaskClassifier { }; let classifier: Arc> = Arc::new(JudgeClassifier::new( StructuredJudge::new( - TaskInput { recent_turn_window }, + TaskInput { + recent_turn_window, + judge_char_budget, + }, contract, JsonSchemaDecoder::new(), JudgeRuntimeConfig::new(max_output_tokens)?, @@ -1282,8 +1419,19 @@ mod tests { /// The text of each message a judge with `recent_turn_window` would be sent. /// The no-window case is covered by `capability_judge_builds_a_structured_request`. fn capability_judge(recent_turn_window: Option) -> Result { + capability_judge_with_budget(recent_turn_window, DEFAULT_JUDGE_CHAR_BUDGET) + } + + /// A judge whose windowed payload is capped at `judge_char_budget` characters. + fn capability_judge_with_budget( + recent_turn_window: Option, + judge_char_budget: usize, + ) -> Result { Ok(StructuredJudge::new( - TaskInput { recent_turn_window }, + TaskInput { + recent_turn_window, + judge_char_budget, + }, LlmTaskClassifier::load_capability_contract(&ClassifierContractConfig::default())?, SerdeDecoder::new(), JudgeRuntimeConfig::new(DEFAULT_JUDGE_MAX_OUTPUT_TOKENS)?, @@ -1337,6 +1485,184 @@ mod tests { Ok(()) } + /// Builds a request whose windowed selection the judge will be handed. + fn windowed_request(messages: Vec) -> Request { + Request { + llm_request: LlmRequest { + messages, + ..LlmRequest::default() + }, + raw_request: None, + metadata: None, + } + } + + /// The messages a judge with this window and budget is actually sent. + fn budgeted_messages( + messages: Vec, + window: usize, + budget: usize, + ) -> Result> { + let judge = capability_judge_with_budget(Some(window), budget)?; + Ok(judge + .build_request(&State::default(), &windowed_request(messages)) + .llm_request + .messages) + } + + /// A window is counted in turns, so one turn carrying a large tool result would + /// otherwise decide the judge's cost for a fixed configuration. The window narrows + /// from the oldest end until the payload fits, so the newest evidence survives. + #[test] + fn an_oversized_turn_narrows_the_window_to_fit_the_budget() -> Result<()> { + let messages = vec![ + Message::text(Role::System, "client instructions"), + Message::text(Role::User, "initial task"), + Message::text(Role::Assistant, "x".repeat(20_000)), + Message::text(Role::User, "recent 1"), + Message::text(Role::Assistant, "recent 2"), + ]; + let built = budgeted_messages(messages, 4, 5_000)?; + let texts: Vec = built + .iter() + .filter_map(|message| message.text_content("\n")) + .collect(); + + assert!(payload_chars(&built) <= 5_000, "{}", payload_chars(&built)); + // The anchors and the newest turns stay; only the oversized older turn goes. + assert!(texts.contains(&"client instructions".to_string())); + assert!(texts.contains(&"initial task".to_string())); + assert!(texts.contains(&"recent 2".to_string())); + assert!(!texts.iter().any(|text| text.len() > 10_000), "{texts:?}"); + Ok(()) + } + + /// Narrowing drops whole turns through `trim_messages`, so a surviving tool result + /// still has the call that introduced its id. Removing messages directly would not. + #[test] + fn narrowing_for_the_budget_keeps_tool_pairs_whole() -> Result<()> { + let mut bulky = tool_result("call-1"); + bulky.content = vec![ContentBlock::ToolResult(ToolResult { + tool_call_id: "call-1".to_string(), + content: vec![ContentBlock::Text { + text: "y".repeat(20_000), + }], + is_error: None, + })]; + let messages = vec![ + Message::text(Role::System, "client instructions"), + Message::text(Role::User, "initial task"), + tool_call("call-1"), + bulky, + tool_call("call-2"), + tool_result("call-2"), + ]; + let built = budgeted_messages(messages, 4, 5_000)?; + + assert!(payload_chars(&built) <= 5_000, "{}", payload_chars(&built)); + let calls: BTreeSet = built + .iter() + .flat_map(|message| &message.content) + .filter_map(|block| match block { + ContentBlock::ToolCall(call) => Some(call.id.clone()), + _ => None, + }) + .collect(); + for block in built.iter().flat_map(|message| &message.content) { + if let ContentBlock::ToolResult(result) = block { + assert!( + calls.contains(&result.tool_call_id), + "orphaned result {:?} in {calls:?}", + result.tool_call_id + ); + } + } + Ok(()) + } + + /// The anchors survive an empty window, so a task statement larger than the whole + /// budget cannot be dropped and has to be clipped instead. + #[test] + fn an_oversized_task_is_clipped_once_the_window_cannot_shrink() -> Result<()> { + let messages = vec![ + Message::text(Role::System, "client instructions"), + Message::text(Role::User, "z".repeat(40_000)), + Message::text(Role::Assistant, "recent"), + ]; + let built = budgeted_messages(messages, 2, 1_000)?; + let task = built + .iter() + .filter_map(|message| message.text_content("\n")) + .find(|text| text.starts_with('z')) + .ok_or_else(|| LibsyError::AlgorithmError { + message: "clipped task missing".to_string(), + })?; + + assert!(payload_chars(&built) <= 1_000, "{}", payload_chars(&built)); + // Clipping is marked, so the judge can tell a trimmed task from a short one. + assert!(task.contains("[trimmed]"), "{task}"); + Ok(()) + } + + /// The routing instruction is appended after selection, so the budget has to cover it: + /// a selection sized to the limit would otherwise ship a payload above the limit. + #[test] + fn the_budget_covers_the_appended_routing_instruction() -> Result<()> { + let messages = vec![ + Message::text(Role::System, "client instructions"), + Message::text(Role::User, "w".repeat(4_000)), + Message::text(Role::Assistant, "recent"), + ]; + let built = budgeted_messages(messages, 2, 600)?; + + // `built` includes the trailing instruction, so this measures the whole payload. + assert!( + built + .iter() + .any(|message| message.text_content("\n").as_deref() + == Some(TRAILING_ROUTING_INSTRUCTION)) + ); + assert!(payload_chars(&built) <= 600, "{}", payload_chars(&built)); + Ok(()) + } + + /// One message can carry several text blocks. A per-message allowance would let each + /// block spend it in full, so the share is per block. + #[test] + fn several_text_blocks_in_one_message_share_the_budget() -> Result<()> { + let crowded = Message { + role: Role::User, + content: vec![ + ContentBlock::Text { + text: "a".repeat(9_000), + }, + ContentBlock::Text { + text: "b".repeat(9_000), + }, + ], + }; + let messages = vec![ + Message::text(Role::System, "client instructions"), + crowded, + Message::text(Role::Assistant, "recent"), + ]; + let built = budgeted_messages(messages, 2, 900)?; + + assert!(payload_chars(&built) <= 900, "{}", payload_chars(&built)); + Ok(()) + } + + /// A zero budget would clip every message to the trim marker, so it is rejected at + /// construction rather than serving the judge an unroutable payload. + #[test] + fn a_zero_judge_char_budget_is_rejected() { + let config = TaskClassifierConfig { + judge_char_budget: 0, + ..TaskClassifierConfig::default() + }; + assert!(config.validate().is_err()); + } + fn tool_call(id: &str) -> Message { Message { role: Role::Assistant, @@ -1598,6 +1924,7 @@ mod tests { let judge: CapabilityJudge = StructuredJudge::new( TaskInput { recent_turn_window: None, + judge_char_budget: DEFAULT_JUDGE_CHAR_BUDGET, }, contract, SerdeDecoder::new(), diff --git a/crates/libsy/src/algorithms/util.rs b/crates/libsy/src/algorithms/util.rs index 3f1202e26..70ad9e06e 100644 --- a/crates/libsy/src/algorithms/util.rs +++ b/crates/libsy/src/algorithms/util.rs @@ -28,3 +28,37 @@ pub(crate) fn decisive(target: &ModelId) -> Classification { /// Default completion budget for internal classifier and escalation judge calls. pub(crate) const DEFAULT_JUDGE_MAX_OUTPUT_TOKENS: u64 = 4_096; + +/// Default character budget for a judge payload, shared by the escalation judge and the +/// windowed classifier judges so one turn carrying a large tool result cannot decide how +/// much a judge call costs. +pub(crate) const DEFAULT_JUDGE_CHAR_BUDGET: usize = 18_000; + +/// Separator marking where [`truncate_middle`] dropped a message's interior. +pub(crate) const TRIM_MARKER: &str = " ...[trimmed] "; + +/// Keeps the head and tail of `text` within `limit` characters. +/// +/// The head gets two thirds of the surviving budget: for a judge reading agent activity the +/// command or error signature that opens a message carries more signal than its trailing +/// output. Clipping is marked so the judge can tell a trimmed message from a short one. +pub(crate) fn truncate_middle(text: &str, limit: usize) -> String { + let chars: Vec = text.chars().collect(); + if chars.len() <= limit { + return text.to_string(); + } + // Below the marker's own width there is no room to say the text was clipped, so keep + // what fits and drop the marker. Marking anyway would push the result past `limit`, + // which callers budgeting a payload rely on it never doing. + let marker = TRIM_MARKER.chars().count(); + if limit <= marker { + return chars[..limit].iter().collect(); + } + let keep = limit - marker; + let head = keep * 2 / 3; + let tail = keep - head; + let mut out: String = chars[..head].iter().collect(); + out.push_str(TRIM_MARKER); + out.extend(chars[chars.len() - tail..].iter()); + out +} diff --git a/crates/libsy/src/algorithms/util/escalation.rs b/crates/libsy/src/algorithms/util/escalation.rs index 54ded690c..4dcadf2bb 100644 --- a/crates/libsy/src/algorithms/util/escalation.rs +++ b/crates/libsy/src/algorithms/util/escalation.rs @@ -15,6 +15,7 @@ use super::llm_judge::{ ClassifierInput, JudgeClassifier, JudgePolicy, JudgeRuntimeConfig, SerdeDecoder, StructuredJudge, }; +use super::truncate_middle; use crate::core::classifier::{Classification, Score}; use crate::core::state::State; use crate::{LibsyError, Result}; @@ -23,9 +24,6 @@ use switchyard_protocol::Request; const PROMPT_TEMPLATE: &str = include_str!("../../prompts/escalation/prompt.md"); const SCHEMA_TEMPLATE: &str = include_str!("../../prompts/escalation/schema.json"); -/// Separator marking where [`truncate_middle`] dropped a message's interior. -const TRIM_MARKER: &str = " ...[trimmed] "; - /// Suffix marking a transcript cut off by [`MAX_REQUEST_CHARS`]. const TRUNCATION_SUFFIX: &str = "..."; @@ -210,27 +208,6 @@ fn collect_text(content: &[ContentBlock], parts: &mut Vec) { } } -/// Keeps the head and tail of `text` within `limit` characters. -/// -/// The head gets two thirds of the surviving budget: for a trajectory judge the command or -/// error signature that opens a message carries more signal than its trailing output. -fn truncate_middle(text: &str, limit: usize) -> String { - let chars: Vec = text.chars().collect(); - if chars.len() <= limit { - return text.to_string(); - } - let keep = limit - .saturating_sub(TRIM_MARKER.chars().count()) - .max(20) - .min(chars.len()); - let head = keep * 2 / 3; - let tail = keep - head; - let mut out: String = chars[..head].iter().collect(); - out.push_str(TRIM_MARKER); - out.extend(chars[chars.len() - tail..].iter()); - out -} - /// Renders a compact role-labelled transcript for the judge. /// /// The framing anchors — system/developer messages and the first user message, where agent @@ -462,6 +439,23 @@ mod tests { assert_eq!(truncate_middle("short", 50), "short"); } + /// Callers budget a payload on the promise that clipping never exceeds the limit. Below + /// the marker's own width there is no room to mark the cut, so the marker is dropped + /// rather than pushing the result over. + #[test] + fn truncate_middle_never_exceeds_a_limit_narrower_than_the_marker() { + use crate::algorithms::util::TRIM_MARKER; + + let text = "a".repeat(100); + for limit in 0..=TRIM_MARKER.chars().count() + 2 { + let trimmed = truncate_middle(&text, limit); + assert!( + trimmed.chars().count() <= limit, + "limit {limit}: {trimmed:?}" + ); + } + } + #[test] fn summary_keeps_anchors_and_the_recent_window() { let mut messages = vec![ diff --git a/crates/switchyard-py/src/libsy_bindings.rs b/crates/switchyard-py/src/libsy_bindings.rs index 756dbc8bf..832b4cf1a 100644 --- a/crates/switchyard-py/src/libsy_bindings.rs +++ b/crates/switchyard-py/src/libsy_bindings.rs @@ -145,6 +145,7 @@ impl PyCustomClassifierConfig { session_affinity=false, message_hash_fallback=false, recent_turn_window=None, + judge_char_budget=18_000, max_output_tokens=4096 ))] #[allow(clippy::too_many_arguments)] @@ -155,6 +156,7 @@ impl PyCustomClassifierConfig { session_affinity: bool, message_hash_fallback: bool, recent_turn_window: Option, + judge_char_budget: usize, max_output_tokens: u64, ) -> PyResult { // Convert the Python schema into serde JSON and pair it with the target-selector policy; @@ -167,6 +169,7 @@ impl PyCustomClassifierConfig { inner.classify_trigger = classify_trigger(session_affinity); inner.message_hash_fallback = message_hash_fallback; inner.recent_turn_window = recent_turn_window; + inner.judge_char_budget = judge_char_budget; inner.max_output_tokens = max_output_tokens; Ok(Self { inner }) } @@ -263,6 +266,7 @@ impl PyTaskClassifierConfig { session_affinity=false, message_hash_fallback=false, recent_turn_window=None, + judge_char_budget=18_000, max_output_tokens=4096, prompt=None, response_format_type="json_schema" @@ -274,6 +278,7 @@ impl PyTaskClassifierConfig { session_affinity: bool, message_hash_fallback: bool, recent_turn_window: Option, + judge_char_budget: usize, max_output_tokens: u64, prompt: Option, response_format_type: &str, @@ -285,6 +290,7 @@ impl PyTaskClassifierConfig { classify_trigger: classify_trigger(session_affinity), message_hash_fallback, recent_turn_window, + judge_char_budget, contract: classifier_contract(prompt, response_format_type)?, max_output_tokens, }, diff --git a/crates/switchyard-runner/src/algorithm.rs b/crates/switchyard-runner/src/algorithm.rs index 0e1ede289..6b9632a68 100644 --- a/crates/switchyard-runner/src/algorithm.rs +++ b/crates/switchyard-runner/src/algorithm.rs @@ -104,6 +104,7 @@ struct CapabilityClassifierRouteConfig { classify_trigger: ClassifyTrigger, message_hash_fallback: bool, recent_turn_window: Option, + judge_char_budget: usize, prompt: Option, response_format_type: ClassifierResponseFormat, max_output_tokens: u64, @@ -130,6 +131,7 @@ struct CustomClassifierRouteConfig { classify_trigger: ClassifyTrigger, message_hash_fallback: bool, recent_turn_window: Option, + judge_char_budget: usize, max_output_tokens: u64, } @@ -160,6 +162,11 @@ pub struct LlmClassifierRouteConfig { /// How many trailing turns the judge sees. Unset shows it the opening task /// and the latest user follow-up only. pub recent_turn_window: Option, + /// Most characters a windowed judge payload may use. The window narrows from the + /// oldest turn until it fits, so one large tool result cannot decide the judge's + /// cost. Ignored without `recent_turn_window`. + #[serde(default = "default_judge_char_budget")] + pub judge_char_budget: usize, /// Replaces the packaged judge prompt. Required in custom mode. pub prompt: Option, /// How the judge is asked for structured output. Use `json_object` when the @@ -363,6 +370,9 @@ pub struct StageClassifierConfig { /// and the latest user follow-up only. #[serde(default)] pub recent_turn_window: Option, + /// Most characters a windowed judge payload may use. Ignored without a window. + #[serde(default = "default_judge_char_budget")] + pub judge_char_budget: usize, /// Replaces the packaged judge prompt. #[serde(default)] pub prompt: Option, @@ -404,6 +414,7 @@ impl StageClassifierConfig { classify_trigger: self.classify_trigger, message_hash_fallback: self.message_hash_fallback, recent_turn_window: self.recent_turn_window, + judge_char_budget: self.judge_char_budget, contract: classifier_contract(self.prompt.as_deref()) .with_response_format_type(self.response_format_type), max_output_tokens: self.max_output_tokens, @@ -579,6 +590,7 @@ impl LlmClassifierRouteConfig { classify_trigger, message_hash_fallback, recent_turn_window, + judge_char_budget, prompt, response_format_type, max_output_tokens, @@ -633,6 +645,7 @@ impl LlmClassifierRouteConfig { classify_trigger: *classify_trigger, message_hash_fallback: *message_hash_fallback, recent_turn_window: *recent_turn_window, + judge_char_budget: *judge_char_budget, prompt: prompt.clone(), response_format_type: *response_format_type, max_output_tokens: *max_output_tokens, @@ -713,6 +726,7 @@ impl LlmClassifierRouteConfig { classify_trigger: *classify_trigger, message_hash_fallback: *message_hash_fallback, recent_turn_window: *recent_turn_window, + judge_char_budget: *judge_char_budget, max_output_tokens: *max_output_tokens, }, )) @@ -810,6 +824,7 @@ fn build_subagent_router_config( config.policy.into_libsy(), ); classifier_config.recent_turn_window = config.recent_turn_window; + classifier_config.judge_char_budget = config.judge_char_budget; classifier_config.max_output_tokens = config.max_output_tokens; let subagent_targets = resolved_targets .iter() @@ -908,6 +923,7 @@ fn build_algorithm( classify_trigger: config.classify_trigger, message_hash_fallback: config.message_hash_fallback, recent_turn_window: config.recent_turn_window, + judge_char_budget: config.judge_char_budget, contract: classifier_contract(config.prompt.as_deref()) .with_response_format_type(config.response_format_type), max_output_tokens: config.max_output_tokens, @@ -960,6 +976,7 @@ fn build_algorithm( classifier_config.classify_trigger = config.classify_trigger; classifier_config.message_hash_fallback = config.message_hash_fallback; classifier_config.recent_turn_window = config.recent_turn_window; + classifier_config.judge_char_budget = config.judge_char_budget; classifier_config.max_output_tokens = config.max_output_tokens; LlmTaskClassifier::new(LlmClassifierConfig::Custom { judge_target: classifier, @@ -1172,6 +1189,10 @@ fn default_classifier_max_output_tokens() -> u64 { TaskClassifierConfig::default().max_output_tokens } +fn default_judge_char_budget() -> usize { + TaskClassifierConfig::default().judge_char_budget +} + fn resolve_targets<'a>( route_name: &str, names: impl IntoIterator, diff --git a/crates/switchyard-runner/src/config.rs b/crates/switchyard-runner/src/config.rs index a4bb309aa..fe2b8a9fc 100644 --- a/crates/switchyard-runner/src/config.rs +++ b/crates/switchyard-runner/src/config.rs @@ -68,6 +68,7 @@ struct RouteConfig { tool_calling: Option, reasoning: Option, vision: Option, + base_instructions: Option, algorithm: AlgorithmSpec, } @@ -87,6 +88,7 @@ impl<'de> Deserialize<'de> for RouteConfig { let tool_calling = take_optional(&mut table, "tool_calling")?; let reasoning = take_optional(&mut table, "reasoning")?; let vision = take_optional(&mut table, "vision")?; + let base_instructions = take_optional(&mut table, "base_instructions")?; let algorithm = AlgorithmSpec::deserialize(toml::Value::Table(table)) .map_err(serde::de::Error::custom)?; Ok(Self { @@ -95,6 +97,7 @@ impl<'de> Deserialize<'de> for RouteConfig { tool_calling, reasoning, vision, + base_instructions, algorithm, }) } @@ -127,6 +130,7 @@ impl RouteConfig { tool_calling: self.tool_calling, reasoning: self.reasoning, vision: self.vision, + base_instructions: self.base_instructions.clone(), } } diff --git a/crates/switchyard-runner/src/route.rs b/crates/switchyard-runner/src/route.rs index 89f6e722c..a584283d1 100644 --- a/crates/switchyard-runner/src/route.rs +++ b/crates/switchyard-runner/src/route.rs @@ -18,7 +18,7 @@ use crate::DecisionTarget; /// /// An unset capability is undeclared: it serializes as `null` in the OpenAI /// `data` entry, and the Codex entry falls back to a safe default for it. -#[derive(Clone, Copy, Default)] +#[derive(Clone, Default)] pub struct ModelCapabilities { pub context_window: Option, pub tool_calling: Option, @@ -36,6 +36,14 @@ pub struct ModelCapabilities { /// sending*. An undeclared vision-capable route therefore loses the image in the /// client, and the proxy never receives one to forward. pub vision: Option, + /// Base instructions this route advertises to Codex. + /// + /// Codex adopts a served value in place of its own bundled prompt, and its catalog + /// decoder rejects an entry that supplies neither `base_instructions` nor + /// `model_messages.instructions_template` — one rejected entry discards the whole + /// catalog. A proxy therefore cannot decline to answer the question, only choose the + /// answer, so the operator supplies it. Unset serves a placeholder and warns. + pub base_instructions: Option, } /// Caller credential family required by a forwarded-auth route. @@ -161,8 +169,8 @@ impl Route { } /// Returns model-list capability metadata. - pub fn capabilities(&self) -> ModelCapabilities { - self.capabilities + pub fn capabilities(&self) -> &ModelCapabilities { + &self.capabilities } /// Returns the forwarded caller credential family. diff --git a/crates/switchyard-runner/src/runner.rs b/crates/switchyard-runner/src/runner.rs index 36ecb6410..3692e7274 100644 --- a/crates/switchyard-runner/src/runner.rs +++ b/crates/switchyard-runner/src/runner.rs @@ -23,7 +23,7 @@ pub struct Runner { pub struct ModelInfo<'a> { pub id: &'a ModelId, pub algorithm: &'a str, - pub capabilities: ModelCapabilities, + pub capabilities: &'a ModelCapabilities, } /// Fully resolved routing decision. diff --git a/crates/switchyard-server/src/lib.rs b/crates/switchyard-server/src/lib.rs index 8b6f06de8..1a180f546 100644 --- a/crates/switchyard-server/src/lib.rs +++ b/crates/switchyard-server/src/lib.rs @@ -322,6 +322,12 @@ pub struct BoundServer { impl BoundServer { /// Binds the configured address and prepares the HTTP router. pub fn bind(state: ServerState, options: ServerRunOptions) -> ServerResult { + let routes: Vec<(&str, &ModelCapabilities)> = state + .runner + .models() + .map(|model| (model.id.as_str(), model.capabilities)) + .collect(); + warn_unconfigured_base_instructions(&routes); let listener = bind_tcp_listener(options.addr, options.backlog)?; let addr = listener.local_addr().map_err(server_io_error)?; Ok(Self { @@ -1469,7 +1475,7 @@ async fn not_found() -> Response { } fn model_list_payload<'a>( - entries: impl IntoIterator, + entries: impl IntoIterator, ) -> Value { let mut entries = entries.into_iter().collect::>(); entries.sort_unstable_by_key(|(model_id, _)| *model_id); @@ -1478,11 +1484,11 @@ fn model_list_payload<'a>( let last_id = model_ids.last().copied(); json!({ "object": "list", - "data": entries.iter().map(|(model, caps)| model_entry_json(model, *caps)).collect::>(), + "data": entries.iter().map(|(model, caps)| model_entry_json(model, caps)).collect::>(), "models": entries .iter() .enumerate() - .map(|(priority, (model, caps))| codex_model_entry_json(model, *caps, priority)) + .map(|(priority, (model, caps))| codex_model_entry_json(model, caps, priority)) .collect::>(), "first_id": first_id, "last_id": last_id, @@ -1492,7 +1498,7 @@ fn model_list_payload<'a>( }) } -fn model_entry_json(model: &str, capabilities: ModelCapabilities) -> Value { +fn model_entry_json(model: &str, capabilities: &ModelCapabilities) -> Value { json!({ "id": model, "object": "model", @@ -1514,6 +1520,36 @@ fn model_entry_json(model: &str, capabilities: ModelCapabilities) -> Value { }) } +/// Served to Codex when a route declares no `base_instructions`. +/// +/// Codex adopts whatever the catalog supplies, so this placeholder replaces the agent's +/// own prompt. It exists because the catalog cannot omit the field, not because it is a +/// reasonable prompt; a route serving Codex should set `base_instructions` instead. +const PLACEHOLDER_BASE_INSTRUCTIONS: &str = "You are Codex, a coding agent."; + +/// Warns once for every route that will serve [`PLACEHOLDER_BASE_INSTRUCTIONS`]. +/// +/// Codex adopting a six-word prompt changes agent behaviour on every turn and invalidates +/// any comparison between a routed session and a direct one. That is a measurement problem +/// rather than a failure, so nothing else surfaces it. +fn warn_unconfigured_base_instructions(routes: &[(&str, &ModelCapabilities)]) { + let unconfigured: Vec<&str> = routes + .iter() + .filter(|(_, capabilities)| capabilities.base_instructions.is_none()) + .map(|(model, _)| *model) + .collect(); + if unconfigured.is_empty() { + return; + } + tracing::warn!( + target: "switchyard_server", + routes = unconfigured.join(", "), + "no base_instructions configured, so Codex is served a placeholder prompt and \ + adopts it in place of its own; set base_instructions on these routes to keep a \ + routed session comparable with a direct one" + ); +} + // Builds the metadata Codex requires when it discovers models from a direct provider. // // This mirrors Codex's `ModelInfo` card. The benchmark harness builds the same card in @@ -1533,7 +1569,7 @@ fn model_entry_json(model: &str, capabilities: ModelCapabilities) -> Value { // supported_parameters — and fall back to the route's declared value. Some backends // publish nothing (the NVIDIA gateway returns id-only models and blocks /model/info), // so keep failing closed to config. -fn codex_model_entry_json(model: &str, capabilities: ModelCapabilities, priority: usize) -> Value { +fn codex_model_entry_json(model: &str, capabilities: &ModelCapabilities, priority: usize) -> Value { // Codex is non-functional without shell and apply_patch, so an undeclared tool // capability defaults to enabled here; the OpenAI `data` entry reports the raw // Option separately for clients that want the undeclared state. @@ -1553,9 +1589,17 @@ fn codex_model_entry_json(model: &str, capabilities: ModelCapabilities, priority "additional_speed_tiers": [], "availability_nux": null, "upgrade": null, - // Required `ModelInfo` string. Unlike the launcher, the server cannot read - // Codex's bundled prompt, so it sends a minimal stub. - "base_instructions": "You are Codex, a coding agent.", + // Codex adopts this in place of its own prompt, and its catalog decoder rejects + // an entry carrying neither `base_instructions` nor + // `model_messages.instructions_template` — one rejected entry discards the whole + // catalog, taking `input_modalities` and every other advertised capability with it. + // Omitting it is therefore not an option; a route says what to serve, and an + // unconfigured route gets the placeholder `warn_unconfigured_base_instructions` + // reports at startup. + "base_instructions": capabilities + .base_instructions + .as_deref() + .unwrap_or(PLACEHOLDER_BASE_INSTRUCTIONS), "supports_reasoning_summaries": reasoning, "default_reasoning_summary": "none", "support_verbosity": reasoning, diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rs index 0f6567092..4f4333b4d 100644 --- a/crates/switchyard-server/tests/server.rs +++ b/crates/switchyard-server/tests/server.rs @@ -2317,6 +2317,7 @@ id = "reasoning" type = "passthrough" target = "shared" reasoning = true +base_instructions = "You are a coding agent running in the Codex CLI." [routes.undeclared] id = "undeclared" @@ -2357,6 +2358,27 @@ target = "shared" codex_metadata["declared"]["apply_patch_tool_type"], "freeform" ); + // Codex's catalog decoder rejects an entry supplying neither `base_instructions` nor + // `model_messages.instructions_template`, and one rejected entry discards the whole + // catalog — `input_modalities` included. Every entry must therefore carry the field. + for (slug, entry) in &codex_metadata { + let instructions = entry["base_instructions"].as_str(); + assert!( + instructions.is_some_and(|text| !text.is_empty()), + "{slug}: {entry:?}" + ); + } + // A route that declares its own instructions serves them verbatim, so an operator can + // paste Codex's bundled prompt and keep a routed session comparable with a direct one. + assert_eq!( + codex_metadata["reasoning"]["base_instructions"], + "You are a coding agent running in the Codex CLI." + ); + // An undeclared route still has to answer, so it serves the placeholder. + assert_eq!( + codex_metadata["declared"]["base_instructions"], + "You are Codex, a coding agent." + ); // Constant fields Codex requires: a typo here would fail its decode, so pin them. assert_eq!(codex_metadata["declared"]["visibility"], "list"); assert_eq!(codex_metadata["declared"]["supported_in_api"], json!(true)); diff --git a/docs/reference/toml_schema.md b/docs/reference/toml_schema.md index 2708b68db..e6019e81f 100644 --- a/docs/reference/toml_schema.md +++ b/docs/reference/toml_schema.md @@ -114,6 +114,7 @@ Every route takes the common keys below, plus the keys for its type. | `tool_calling` | No | unset | Whether `GET /v1/models` advertises tool-calling support for this route. Unset values appear as `null`. | | `reasoning` | No | unset | Whether `GET /v1/models` advertises reasoning support to Codex direct-provider discovery. Unset routes are advertised as non-reasoning. | | `vision` | No | unset | Whether `GET /v1/models` advertises **image input** to Codex direct-provider discovery. Unset routes are advertised as text-only. This is not cosmetic: Codex reads `input_modalities` from the model card and, when it reads text-only, replaces an attached image with the text `image content omitted because you do not support image input` **before sending**, so a route whose target can see but which does not declare `vision = true` loses the image in the client. Declare it only when every target the route can select accepts images. | +| `base_instructions` | No | placeholder | Base instructions this route advertises to Codex. Codex **adopts a served value in place of its own bundled prompt**, and its catalog decoder rejects an entry that carries neither `base_instructions` nor `model_messages.instructions_template` — one rejected entry discards the entire catalog, `input_modalities` included. Switchyard therefore always sends the field. Unset routes send the placeholder `You are Codex, a coding agent.` and log a startup warning, which changes agent behaviour on every turn and makes a routed session incomparable with a direct one. For parity, set this to the contents of Codex's own `codex-rs/models-manager/prompt.md`. | ### `noop` @@ -201,6 +202,7 @@ Capability mode classifies before serving. See | `classify_trigger` | No | `every_request` | When the judge runs. `every_request` judges every request, tool continuations included. `user_turn` judges each new user message and retains that target across intervening tool calls only when requests carry a session ID; without a session ID, it behaves like `every_request`. `new_session` judges once and reuses that target for the session. | | `message_hash_fallback` | No | `false` | Keys affinity on the first user message. Requires `classify_trigger = "new_session"`. | | `recent_turn_window` | No | unset | When unset, the judge sees the opening task and latest user follow-up, when present. When set, it also sees trailing turns. | +| `judge_char_budget` | No | `18000` | Most characters a windowed judge payload may use. The window narrows from the oldest turn until it fits, so one large tool result cannot decide judge cost and latency. Ignored without `recent_turn_window`. | | `prompt` | No | packaged prompt | Replaces the capability prompt. The packaged schema is sent separately as structured-output configuration. | Escalation mode serves the weak target first and judges the completed turn. See @@ -230,6 +232,7 @@ policy selector, and routes to any configured target label. | `classify_trigger` | No | `every_request` | When the judge runs. `every_request` judges every request, tool continuations included. `user_turn` judges each new user message and retains that target across intervening tool calls only when requests carry a session ID; without a session ID, it behaves like `every_request`. `new_session` judges once and reuses that target for the session. | | `message_hash_fallback` | No | `false` | Keys affinity on the first user message. Requires `classify_trigger = "new_session"`. | | `recent_turn_window` | No | unset | When unset, the judge sees the opening task and latest user follow-up, when present. When set, it also sees trailing turns. | +| `judge_char_budget` | No | `18000` | Most characters a windowed judge payload may use. The window narrows from the oldest turn until it fits, so one large tool result cannot decide judge cost and latency. Ignored without `recent_turn_window`. | Classifier prompts must not contain `{{RESPONSE_SCHEMA}}`. Switchyard supplies the schema automatically: through the structured-output request in `json_schema` diff --git a/docs/routing_algorithms/llm_classifier_routing.md b/docs/routing_algorithms/llm_classifier_routing.md index 4706175e3..770e1fea4 100644 --- a/docs/routing_algorithms/llm_classifier_routing.md +++ b/docs/routing_algorithms/llm_classifier_routing.md @@ -106,6 +106,7 @@ for the server merge behavior. | `base_threshold` | required | Lowest `p_solve` that routes a supported task to `weak_target`. Must be between `0` and `1`. | | `threshold_step` | `0.0` | Amount added for each boundary step. Must be finite and non-negative, and `base_threshold + 2 * threshold_step` must not exceed `1`. | | `recent_turn_window` | unset | When unset, the judge sees the opening user task and the latest user message when they differ. When set to `N`, it sees the opening user task and the last `N` conversation messages after that task. `0` keeps only the opening task. Client system and developer instructions are not shown to the judge. | +| `judge_char_budget` | `18000` | Most characters the windowed selection may use. A window is counted in turns, and one turn carrying a large tool result can be worth tens of thousands of characters, so the window narrows from the oldest turn until the payload fits. A task statement larger than the whole budget is clipped instead, marked with `...[trimmed]`. Ignored when `recent_turn_window` is unset. | | `classify_trigger` | `every_request` | When the judge runs. `every_request` judges every request, tool continuations included. `user_turn` judges each new user message and holds that target across the tool calls between. `new_session` judges once and reuses that target for the session. | | `message_hash_fallback` | `false` | When session metadata is absent, keys affinity from the first user-message text. Requires `classify_trigger = "new_session"`. | | `prompt` | packaged capability prompt | Replaces the classifier's system prompt. The packaged verdict schema and routing policy remain active. | diff --git a/switchyard_rust/libsy.py b/switchyard_rust/libsy.py index 1589d60ff..31e03144d 100644 --- a/switchyard_rust/libsy.py +++ b/switchyard_rust/libsy.py @@ -74,6 +74,7 @@ def __init__( session_affinity: bool = False, message_hash_fallback: bool = False, recent_turn_window: int | None = None, + judge_char_budget: int = 18_000, max_output_tokens: int = 4096, ) -> None: ... @@ -149,6 +150,7 @@ def __init__( session_affinity: bool = False, message_hash_fallback: bool = False, recent_turn_window: int | None = None, + judge_char_budget: int = 18_000, max_output_tokens: int = 4096, prompt: str | None = None, response_format_type: Literal["json_schema", "json_object"] = "json_schema",