diff --git a/crates/libsy-llm-client/tests/observability.rs b/crates/libsy-llm-client/tests/observability.rs index 473658281..03d00fadc 100644 --- a/crates/libsy-llm-client/tests/observability.rs +++ b/crates/libsy-llm-client/tests/observability.rs @@ -34,9 +34,9 @@ use tracing_subscriber::layer::{Context as LayerContext, SubscriberExt}; use tracing_subscriber::registry::LookupSpan; use switchyard_libsy::{ - Algorithm, ClassifyTrigger, Driver, LibsyError, LlmClassifierConfig, LlmTaskClassifier, - PickerMode, RoutingOutcome, RuntimeModels, StageRouter, StageRouterConfig, Step, - TaskClassifierConfig, + Algorithm, ClassifierContractConfig, ClassifyTrigger, DeescalationConfig, Driver, + EscalationJudgeConfig, LibsyError, LlmClassifierConfig, LlmTaskClassifier, PickerMode, + RoutingOutcome, RuntimeModels, StageRouter, StageRouterConfig, Step, TaskClassifierConfig, }; use switchyard_llm_client::{ClientRouter, RunObservation, RunObserver}; use switchyard_protocol::{Category, ModelId}; @@ -760,6 +760,94 @@ async fn affinity_warns_once_when_request_has_no_usable_identity() -> switchyard Ok(()) } +#[tokio::test] +async fn stateful_escalation_warns_once_without_a_session_id() -> switchyard_libsy::Result<()> { + let _guard = serialize_test().lock().await; + let (store, _, _, _, _) = telemetry(); + let event_count = store.events().len(); + let router = Arc::new(LlmTaskClassifier::new(LlmClassifierConfig::Escalation { + contract: ClassifierContractConfig::default(), + config: EscalationJudgeConfig::default(), + max_output_tokens: 64, + })?) as Arc; + let client = Arc::new(JudgeClient { + judge_model: "warning-judge".into(), + outcome: JudgeOutcome::Reply(r#"{"escalate":false,"reason":"progressing"}"#), + }) as Arc; + + for _ in 0..2 { + switchyard_llm_client::run( + router.clone(), + ClientRouter::single(client.clone()), + classifier_request(), + classifier_models("warning-judge", "warning-efficient", "warning-capable"), + None, + ) + .await?; + } + + let warnings = store.events()[event_count..] + .iter() + .filter(|event| { + event.target == "libsy" + && event.level == "WARN" + && event.fields.get("message").is_some_and(|message| { + message.contains("stateful escalation has no session ID") + }) + }) + .count(); + assert_eq!(warnings, 1); + Ok(()) +} + +#[tokio::test] +async fn deescalation_evidence_stays_pending_until_confirmed() -> switchyard_libsy::Result<()> { + let router = Arc::new(LlmTaskClassifier::new(LlmClassifierConfig::Escalation { + contract: ClassifierContractConfig::default(), + config: EscalationJudgeConfig { + confirmations: 1, + deescalation: Some(DeescalationConfig { + strong_min_calls: 1, + strong_max_calls: None, + confirmations: 2, + weak_cooldown_calls: 0, + }), + ..EscalationJudgeConfig::default() + }, + max_output_tokens: 64, + })?) as Arc; + let client = |verdict| { + Arc::new(JudgeClient { + judge_model: "evidence-judge".into(), + outcome: JudgeOutcome::Reply(verdict), + }) as Arc + }; + let request = request_with_metadata("evidence-session", "evidence-correlation"); + + switchyard_llm_client::run( + router.clone(), + ClientRouter::single(client(r#"{"escalate":true,"reason":"stuck"}"#)), + request.clone(), + classifier_models("evidence-judge", "evidence-efficient", "evidence-capable"), + None, + ) + .await?; + let outcome = switchyard_llm_client::decide( + router, + ClientRouter::single(client(r#"{"escalate":false,"reason":"recovered"}"#)), + request, + classifier_models("evidence-judge", "evidence-efficient", "evidence-capable"), + ) + .await?; + + assert_eq!(outcome.selected_model_id()?.as_str(), "evidence-capable"); + assert_eq!( + outcome.metadata.and_then(|metadata| metadata.evidence), + Some(json!({"source": "escalation", "verdict": "pending"})) + ); + Ok(()) +} + #[tokio::test] async fn affinity_keeps_the_algorithm_selection_after_client_fallback() -> switchyard_libsy::Result<()> { diff --git a/crates/libsy/src/algorithms/escalation.rs b/crates/libsy/src/algorithms/escalation.rs index dfe2c7fe3..5bda02edd 100644 --- a/crates/libsy/src/algorithms/escalation.rs +++ b/crates/libsy/src/algorithms/escalation.rs @@ -4,32 +4,44 @@ //! Escalation routing that judges an efficient model's answer before selecting a serving tier. use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; use async_trait::async_trait; use switchyard_protocol::{ - AggLlmResponse, Category, LlmClientError, Message, Request, Response, Role, + AggLlmResponse, Category, LlmClientError, Message, ModelId, Request, Response, Role, }; use super::util::buffered_response::buffer_response; use super::util::classifier_contract::ClassifierContractConfig; use super::util::decisive; -use super::util::escalation::{self, EscalationJudge, EscalationJudgeConfig, EscalationPolicy}; +use super::util::escalation::{ + self, DeescalationConfig, EscalationJudge, EscalationJudgeConfig, EscalationPolicy, + EvaluationPhase, +}; use super::util::llm_judge::JudgeClassifier; use crate::core::algorithm::Driver; use crate::core::classifier::{Classification, Classifier}; use crate::core::state::{State, StateValue}; use crate::{LibsyError, Result}; -/// Session-state key holding the consecutive-escalate streak. const STREAK_KEY: &str = "escalation_streak"; +const STRONG_CALLS_KEY: &str = "escalation_strong_calls"; +const RELEASE_STREAK_KEY: &str = "escalation_release_streak"; +const WEAK_COOLDOWN_KEY: &str = "escalation_weak_cooldown"; -fn streak(state: &State) -> u32 { - match state.extra.get(STREAK_KEY) { +fn count(state: &State, key: &str) -> u32 { + match state.extra.get(key) { Some(StateValue::Count(n)) => *n, _ => 0, } } +fn set_count(state: &mut State, key: &str, value: u32) { + state + .extra + .insert(key.to_string(), StateValue::Count(value)); +} + fn assistant_message(response: &AggLlmResponse) -> Message { Message { role: Role::Assistant, @@ -44,9 +56,16 @@ fn assistant_message(response: &AggLlmResponse) -> Message { /// confirms. Returns the efficient response directly when not escalating so the caller does /// not pay for a second model call. struct EscalationClassifier { - judge: JudgeClassifier, + escalation_judge: JudgeClassifier, /// Consecutive escalate verdicts required to latch. confirmations: u32, + deescalation: Option, + missing_session_warning_emitted: AtomicBool, +} + +struct DeescalationPolicy { + config: DeescalationConfig, + judge: JudgeClassifier, } /// Builds the escalation classifier used by the shared LLM classifier route shell. @@ -56,13 +75,112 @@ pub(super) fn build_classifier( max_output_tokens: u64, ) -> Result>> { let confirmations = config.confirmations; + let deescalation = match config.deescalation { + Some(deescalation) => Some(DeescalationPolicy { + config: deescalation, + judge: escalation::build_judge( + &contract_config, + config.clone(), + Some(EvaluationPhase::Strong), + max_output_tokens, + )?, + }), + None => None, + }; + let is_phase_aware = deescalation.is_some(); let classifier: Arc> = Arc::new(EscalationClassifier { - judge: escalation::build_judge(&contract_config, config, max_output_tokens)?, + escalation_judge: escalation::build_judge( + &contract_config, + config, + is_phase_aware.then_some(EvaluationPhase::Efficient), + max_output_tokens, + )?, confirmations, + deescalation, + missing_session_warning_emitted: AtomicBool::new(false), }); Ok(classifier) } +impl EscalationClassifier { + async fn review_capable( + &self, + deescalation: &DeescalationPolicy, + state: &mut State, + request: &Request, + driver: &Driver, + capable: &ModelId, + efficient: &ModelId, + strong_calls: u32, + ) -> Result<(Classification, Option)> { + let next_strong_call = strong_calls.saturating_add(1); + + // The call that confirms escalation is the first capable call. + if next_strong_call < deescalation.config.strong_min_calls { + set_count(state, STRONG_CALLS_KEY, next_strong_call); + driver.set_evidence(serde_json::json!({"source": "escalation", "verdict": "latched"})); + return Ok((decisive(capable), None)); + } + + let capable_response = driver + .call_model(request.clone(), vec![capable.clone(), efficient.clone()]) + .await?; + if capable_response.served_model() == Some(efficient) { + set_count(state, RELEASE_STREAK_KEY, 0); + driver.set_evidence(serde_json::json!({"source": "fallback"})); + return Ok((decisive(efficient), Some(capable_response))); + } + let capable_response = match buffer_response(capable.as_str(), capable_response).await { + Ok(response) => response, + Err(LibsyError::ClientCall { + source: LlmClientError::Transport { .. }, + .. + }) => { + set_count(state, RELEASE_STREAK_KEY, 0); + driver.set_evidence( + serde_json::json!({"source": "fallback", "reason_code": "transport"}), + ); + return Ok((decisive(efficient), None)); + } + Err(error) => return Err(error), + }; + let mut judge_request = request.clone(); + judge_request + .llm_request + .messages + .push(assistant_message(&capable_response.agg)); + + let (classification, _) = deescalation + .judge + .score(state, &mut judge_request, driver) + .await?; + let best = classification.argmax(false)?; + let release_streak = match &best { + Some(score) if score.target == *efficient => { + count(state, RELEASE_STREAK_KEY).saturating_add(1) + } + _ => 0, + }; + set_count(state, STRONG_CALLS_KEY, next_strong_call); + set_count(state, RELEASE_STREAK_KEY, release_streak); + + if release_streak >= deescalation.config.confirmations { + set_count(state, STREAK_KEY, 0); + set_count(state, STRONG_CALLS_KEY, 0); + set_count(state, RELEASE_STREAK_KEY, 0); + tracing::debug!( + target = %efficient, + "de-escalation policy released session to efficient tier" + ); + } else if release_streak > 0 { + driver.set_evidence(serde_json::json!({"source": "escalation", "verdict": "pending"})); + } + + // A confirmed release applies on the next request; this turn is already complete. + Ok((decisive(capable), Some(capable_response.into_response()))) + } +} + #[async_trait] impl Classifier for EscalationClassifier { async fn score( @@ -74,8 +192,65 @@ impl Classifier for EscalationClassifier { let capable = driver.first_model_for(&Category::Capable)?.clone(); let efficient = driver.first_model_for(&Category::Efficient)?.clone(); - // A confirmed session stays capable without a judge call. - if streak(state) >= self.confirmations { + let has_session_id = request + .metadata + .as_ref() + .and_then(|metadata| metadata.session_id.as_deref()) + .is_some_and(|session_id| !session_id.is_empty()); + if (self.deescalation.is_some() || self.confirmations > 1) + && !has_session_id + && !self + .missing_session_warning_emitted + .swap(true, Ordering::Relaxed) + { + tracing::warn!( + target: "libsy", + confirmations = self.confirmations, + deescalation = self.deescalation.is_some(), + required_header = "x-switchyard-session-id", + "stateful escalation has no session ID; routing state will not persist" + ); + } + + let mut strong_calls = count(state, STRONG_CALLS_KEY); + if let Some(deescalation) = &self.deescalation + && deescalation + .config + .strong_max_calls + .is_some_and(|strong_max_calls| strong_calls >= strong_max_calls) + { + set_count(state, STREAK_KEY, 0); + set_count(state, STRONG_CALLS_KEY, 0); + set_count(state, RELEASE_STREAK_KEY, 0); + set_count( + state, + WEAK_COOLDOWN_KEY, + deescalation.config.weak_cooldown_calls, + ); + strong_calls = 0; + tracing::debug!( + target = %efficient, + "de-escalation policy reached its hard limit and returned to efficient tier" + ); + } + if let Some(deescalation) = &self.deescalation + && strong_calls > 0 + { + return self + .review_capable( + deescalation, + state, + request, + driver, + &capable, + &efficient, + strong_calls, + ) + .await; + } + + // A confirmed permanent escalation stays capable without a judge call. + if self.deescalation.is_none() && count(state, STREAK_KEY) >= self.confirmations { driver.set_evidence(serde_json::json!({ "source": "escalation", "verdict": "latched", @@ -142,25 +317,40 @@ impl Classifier for EscalationClassifier { .messages .push(assistant_message(&efficient_response.agg)); - let (classification, _) = self.judge.score(state, &mut judge_request, driver).await?; + let weak_cooldown = count(state, WEAK_COOLDOWN_KEY); + if weak_cooldown > 0 { + set_count(state, WEAK_COOLDOWN_KEY, weak_cooldown - 1); + driver + .set_evidence(serde_json::json!({"source": "deescalation", "verdict": "cooldown"})); + return Ok(( + decisive(&efficient), + Some(efficient_response.into_response()), + )); + } + + let (classification, _) = self + .escalation_judge + .score(state, &mut judge_request, driver) + .await?; - let held = streak(state); + let held = count(state, STREAK_KEY); let best = classification.argmax(false)?; let (escalate, pending) = match &best { - Some(score) if score.target == capable => (true, held + 1), + Some(score) if score.target == capable => (true, held.saturating_add(1)), Some(_) => (false, 0), None => (false, held), }; - state - .extra - .insert(STREAK_KEY.to_string(), StateValue::Count(pending)); + set_count(state, STREAK_KEY, pending); if escalate && pending >= self.confirmations { - // Streak confirmed: drop the efficient response, caller will serve capable. driver.set_evidence(serde_json::json!({ "source": "escalation", "verdict": "escalate", })); + if self.deescalation.is_some() { + set_count(state, STRONG_CALLS_KEY, 1); + set_count(state, RELEASE_STREAK_KEY, 0); + } return Ok((decisive(&capable), None)); } @@ -291,6 +481,43 @@ mod tests { )?)) } + fn deescalation_router(config: DeescalationConfig) -> Result> { + Ok(Arc::new(LlmTaskClassifier::new( + LlmClassifierConfig::Escalation { + contract: ClassifierContractConfig::default(), + config: EscalationJudgeConfig { + confirmations: 1, + deescalation: Some(config), + ..EscalationJudgeConfig::default() + }, + max_output_tokens: DEFAULT_JUDGE_MAX_OUTPUT_TOKENS, + }, + )?)) + } + + async fn selected_over( + router: Arc, + turns: usize, + model: Arc, + judge: Arc, + ) -> Result> { + let request = classify_session_request(); + let mut selected = Vec::with_capacity(turns); + for _ in 0..turns { + selected.push( + test_drive_with_models( + router.clone(), + request.clone(), + runtime_models(), + queued(Arc::clone(&model), Arc::clone(&judge)), + ) + .await? + .0, + ); + } + Ok(selected) + } + #[tokio::test] async fn serves_efficient_when_judge_declines() -> Result<()> { let judge = Queue::new([r#"{"escalate":false,"reason":"progressing"}"#]); @@ -391,6 +618,113 @@ mod tests { Ok(()) } + #[tokio::test] + async fn deescalation_holds_then_returns_to_efficient() -> Result<()> { + let judge = Queue::new([ + r#"{"escalate":true,"reason":"stuck"}"#, + r#"{"escalate":false,"reason":"recovered"}"#, + r#"{"escalate":false,"reason":"routine"}"#, + r#"{"escalate":false,"reason":"progressing"}"#, + ]); + let model = Queue::new([ + "efficient draft", + "capable t1", + "capable t2", + "capable t3", + "capable t4", + "efficient resumed", + ]); + let router = deescalation_router(DeescalationConfig { + strong_min_calls: 3, + strong_max_calls: None, + confirmations: 2, + weak_cooldown_calls: 0, + })?; + assert_eq!( + selected_over(router, 5, model, judge).await?, + ["capable", "capable", "capable", "capable", "efficient"].map(ModelId::from) + ); + Ok(()) + } + + #[tokio::test] + async fn deescalation_hard_limit_forces_a_weak_cooldown() -> Result<()> { + let judge = Queue::new([ + r#"{"escalate":true,"reason":"stuck"}"#, + r#"{"escalate":true,"reason":"still hard"}"#, + r#"{"escalate":true,"reason":"still hard"}"#, + r#"{"escalate":false,"reason":"progressing"}"#, + ]); + let model = Queue::new([ + "efficient draft", + "capable t1", + "capable t2", + "capable t3", + "efficient cooldown t1", + "efficient cooldown t2", + "efficient judged", + ]); + let router = deescalation_router(DeescalationConfig { + strong_min_calls: 2, + strong_max_calls: Some(3), + confirmations: 2, + weak_cooldown_calls: 2, + })?; + assert_eq!( + selected_over(router, 6, model, judge).await?, + [ + "capable", + "capable", + "capable", + "efficient", + "efficient", + "efficient", + ] + .map(ModelId::from) + ); + Ok(()) + } + + #[tokio::test] + async fn strong_review_returns_an_efficient_fallback_without_judging_it() -> Result<()> { + let judge = Queue::new([r#"{"escalate":true,"reason":"stuck"}"#]); + let model = Queue::new(["efficient draft", "capable t1", "efficient fallback"]); + let router = deescalation_router(DeescalationConfig { + strong_min_calls: 1, + strong_max_calls: None, + confirmations: 1, + weak_cooldown_calls: 0, + })?; + let request = classify_session_request(); + + test_drive_with_models( + router.clone(), + request.clone(), + runtime_models(), + queued(Arc::clone(&model), Arc::clone(&judge)), + ) + .await?; + let serve = move |target: ModelId, _request: Request| { + let model = Arc::clone(&model); + async move { + assert_ne!(target, "judge", "fallback answer must not be judged"); + let mut response = reply(model.take()); + response.set_served_model(&ModelId::from("efficient")); + Ok(response) + } + }; + + let (selected_model, response) = + test_drive_with_models(router, request, runtime_models(), serve).await?; + + assert_eq!(selected_model, "efficient"); + assert_eq!( + response.llm_response.as_agg().map(completion_text), + Some("efficient fallback".to_string()) + ); + Ok(()) + } + #[tokio::test] async fn falls_back_to_capable_when_efficient_overflows() -> Result<()> { for streamed in [false, true] { diff --git a/crates/libsy/src/algorithms/util/classifier_contract.rs b/crates/libsy/src/algorithms/util/classifier_contract.rs index d597ed750..6cdc42551 100644 --- a/crates/libsy/src/algorithms/util/classifier_contract.rs +++ b/crates/libsy/src/algorithms/util/classifier_contract.rs @@ -175,7 +175,7 @@ impl ClassifierContract { } } -fn validate_prompt(prompt_template: &str) -> Result<()> { +pub(super) fn validate_prompt(prompt_template: &str) -> Result<()> { if prompt_template.trim().is_empty() { return Err(algorithm_error("classifier prompt must not be empty")); } diff --git a/crates/libsy/src/algorithms/util/escalation.rs b/crates/libsy/src/algorithms/util/escalation.rs index b0979e286..bfe91d17f 100644 --- a/crates/libsy/src/algorithms/util/escalation.rs +++ b/crates/libsy/src/algorithms/util/escalation.rs @@ -11,7 +11,7 @@ use serde::Deserialize; use serde_json::Value; use switchyard_protocol::{Category, ContentBlock, Message, Role}; -use super::classifier_contract::{ClassifierContract, ClassifierContractConfig}; +use super::classifier_contract::{ClassifierContract, ClassifierContractConfig, validate_prompt}; use super::llm_judge::{ ClassifierInput, JudgeClassifier, JudgePolicy, JudgeRuntimeConfig, SerdeDecoder, StructuredJudge, @@ -23,6 +23,7 @@ use crate::{LibsyError, Result}; use switchyard_protocol::Request; const PROMPT_TEMPLATE: &str = include_str!("../../prompts/escalation/prompt.md"); +const DEESCALATION_PROMPT: &str = include_str!("../../prompts/escalation/deescalation.md"); const SCHEMA_TEMPLATE: &str = include_str!("../../prompts/escalation/schema.json"); /// Separator marking where [`truncate_middle`] dropped a message's interior. @@ -45,6 +46,47 @@ const TASK_CHARS: usize = 4_000; /// Backstop on the assembled transcript; the per-message caps normally bind first. const MAX_REQUEST_CHARS: usize = 18_000; +/// Optional policy for returning an escalated session to the efficient tier. +#[derive(Clone, Copy, Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct DeescalationConfig { + /// Minimum number of capable-tier turns before the judge may release the session. + pub strong_min_calls: u32, + /// Optional hard limit on capable-tier turns before a forced return. + #[serde(default)] + pub strong_max_calls: Option, + /// Consecutive judge declines required to return to the efficient tier. + pub confirmations: u32, + /// Efficient calls served without judging after a hard-limit return. + #[serde(default)] + pub weak_cooldown_calls: u32, +} + +impl DeescalationConfig { + fn validate(&self) -> Result<()> { + let reject = |message: &str| { + Err(LibsyError::AlgorithmError { + message: message.to_string(), + }) + }; + if self.strong_min_calls == 0 { + return reject("deescalation.strong_min_calls must be at least 1"); + } + if self.confirmations == 0 { + return reject("deescalation.confirmations must be at least 1"); + } + if self + .strong_max_calls + .is_some_and(|strong_max_calls| strong_max_calls < self.strong_min_calls) + { + return reject( + "deescalation.strong_max_calls must be at least deescalation.strong_min_calls", + ); + } + Ok(()) + } +} + /// The tuning surface for the trajectory judge. /// /// The routing settings retain their benchmarked defaults. Everything else is a fixed invariant @@ -61,6 +103,8 @@ pub struct EscalationJudgeConfig { pub recent_turn_window: usize, /// Per-message cap inside the trailing window. pub window_message_chars: usize, + /// De-escalation policy. `None` preserves permanent latching to the capable tier. + pub deescalation: Option, } impl EscalationJudgeConfig { @@ -79,6 +123,9 @@ impl EscalationJudgeConfig { self.window_message_chars )); } + if let Some(deescalation) = self.deescalation { + deescalation.validate()?; + } Ok(()) } } @@ -89,6 +136,23 @@ impl Default for EscalationJudgeConfig { confirmations: 2, recent_turn_window: 28, window_message_chars: 500, + deescalation: None, + } + } +} + +/// Router-controlled phase attached to judge input when de-escalation is enabled. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum EvaluationPhase { + Efficient, + Strong, +} + +impl EvaluationPhase { + fn marker(self) -> &'static str { + match self { + Self::Efficient => "EFFICIENT_EVALUATION", + Self::Strong => "STRONG_EVALUATION", } } } @@ -107,12 +171,18 @@ pub(crate) struct EscalationVerdict { /// Builds the condensed trajectory presented to the escalation judge. pub(crate) struct EscalationInput { config: EscalationJudgeConfig, + phase: Option, } impl ClassifierInput for EscalationInput { fn build_messages(&self, _state: &State, request: &Request) -> Vec { let messages = &request.llm_request.messages; - let summary = summarize_for_judge(messages, conversation_turn(request), &self.config); + let summary = summarize_for_judge( + messages, + conversation_turn(request), + self.phase, + &self.config, + ); vec![Message::text(Role::User, summary)] } } @@ -125,7 +195,9 @@ pub(crate) type EscalationJudge = StructuredJudge, +} impl JudgePolicy for EscalationPolicy { type Verdict = EscalationVerdict; @@ -158,15 +230,21 @@ impl JudgePolicy for EscalationPolicy { } } -/// Maps present verdicts to stable `escalate` or `continue` values; absent verdicts add nothing. +/// Maps present verdicts to phase-aware evidence; absent verdicts add nothing. fn escalation_evidence( - _policy: &EscalationPolicy, + policy: &EscalationPolicy, verdict: Option<&EscalationVerdict>, ) -> Option { verdict.map(|verdict| { + let verdict = match (policy.phase, verdict.escalate) { + (Some(EvaluationPhase::Strong), true) => "retain", + (Some(EvaluationPhase::Strong), false) => "deescalate", + (_, true) => "escalate", + (_, false) => "continue", + }; serde_json::json!({ "source": "escalation", - "verdict": if verdict.escalate { "escalate" } else { "continue" }, + "verdict": verdict, }) }) } @@ -178,23 +256,40 @@ fn escalation_evidence( pub(crate) fn build_judge( contract_config: &ClassifierContractConfig, config: EscalationJudgeConfig, + phase: Option, max_output_tokens: u64, ) -> Result> { config.validate()?; - let contract = - ClassifierContract::from_config(contract_config, PROMPT_TEMPLATE, SCHEMA_TEMPLATE)?; + let contract = build_contract(contract_config, phase.is_some())?; Ok(JudgeClassifier::new( StructuredJudge::new( - EscalationInput { config }, + EscalationInput { config, phase }, contract, SerdeDecoder::new(), JudgeRuntimeConfig::new(max_output_tokens)?, ), - EscalationPolicy, + EscalationPolicy { phase }, ) .with_evidence(escalation_evidence)) } +fn build_contract( + contract_config: &ClassifierContractConfig, + phase_aware: bool, +) -> Result { + let prompt = contract_config.prompt().unwrap_or(PROMPT_TEMPLATE); + validate_prompt(prompt)?; + let phase_aware_config = phase_aware.then(|| { + contract_config.clone().with_prompt(format!( + "{}\n\n{}", + prompt.trim_end(), + DEESCALATION_PROMPT.trim() + )) + }); + let contract_config = phase_aware_config.as_ref().unwrap_or(contract_config); + ClassifierContract::from_config(contract_config, PROMPT_TEMPLATE, SCHEMA_TEMPLATE) +} + /// The 1-indexed model invocation the transcript ends on: one per assistant reply. /// /// The judge reads the turn *including* the reply it is judging, so the newest assistant @@ -272,6 +367,7 @@ fn truncate_middle(text: &str, limit: usize) -> String { fn summarize_for_judge( messages: &[Message], turn: usize, + phase: Option, config: &EscalationJudgeConfig, ) -> String { let mut anchors: Vec = Vec::new(); @@ -316,7 +412,10 @@ fn summarize_for_judge( window.len(), messages.len(), ); - std::iter::once(header) + phase + .map(|phase| format!("Routing phase: {}", phase.marker())) + .into_iter() + .chain(std::iter::once(header)) .chain(anchors.iter().cloned()) .chain(window.iter().cloned()) .collect::>() @@ -381,16 +480,17 @@ mod tests { use super::*; use crate::algorithms::util::llm_judge::Judge; - fn escalation_judge(max_output_tokens: u64) -> Result { + fn escalation_judge( + max_output_tokens: u64, + phase: Option, + contract_config: &ClassifierContractConfig, + ) -> Result { Ok(StructuredJudge::new( EscalationInput { config: EscalationJudgeConfig::default(), + phase, }, - ClassifierContract::from_config( - &ClassifierContractConfig::default(), - PROMPT_TEMPLATE, - SCHEMA_TEMPLATE, - )?, + build_contract(contract_config, phase.is_some())?, SerdeDecoder::new(), JudgeRuntimeConfig::new(max_output_tokens)?, )) @@ -398,7 +498,11 @@ mod tests { #[test] fn judge_request_is_rubric_plus_summary_under_a_completion_cap() -> Result<()> { - let judge = escalation_judge(super::super::DEFAULT_JUDGE_MAX_OUTPUT_TOKENS)?; + let judge = escalation_judge( + super::super::DEFAULT_JUDGE_MAX_OUTPUT_TOKENS, + None, + &ClassifierContractConfig::default(), + )?; // As the classifier calls it: the turn's reply is already on the transcript. let mut judged = request_at_turn(None, 4); @@ -411,6 +515,12 @@ mod tests { // Rubric in instructions, condensed trajectory as the sole user message. assert_eq!(built.llm_request.instructions.len(), 1); assert_eq!(built.llm_request.instructions[0].role, Role::System); + assert_eq!( + built.llm_request.instructions[0].content.as_slice(), + &[ContentBlock::Text { + text: PROMPT_TEMPLATE.to_string() + }] + ); assert_eq!(built.llm_request.messages.len(), 1); assert_eq!(built.llm_request.messages[0].role, Role::User); assert!( @@ -418,6 +528,11 @@ mod tests { .text_content("") .is_some_and(|text| text.contains("Conversation turn 4")) ); + assert!( + built.llm_request.messages[0] + .text_content("") + .is_none_or(|text| !text.contains("Routing phase:")) + ); // Bounded output, so a reasoning judge cannot run away mid-verdict. assert_eq!( built.llm_request.output.max_output_tokens, @@ -429,7 +544,7 @@ mod tests { #[test] fn judge_request_uses_the_configured_completion_cap() -> Result<()> { - let judge = escalation_judge(512)?; + let judge = escalation_judge(512, None, &ClassifierContractConfig::default())?; let built = judge.build_request(&State::default(), &request_at_turn(None, 1)); @@ -496,6 +611,69 @@ mod tests { assert_eq!(truncate_middle("short", 50), "short"); } + #[test] + fn deescalation_settings_must_be_valid() { + let zero = EscalationJudgeConfig { + deescalation: Some(DeescalationConfig { + strong_min_calls: 0, + strong_max_calls: None, + confirmations: 2, + weak_cooldown_calls: 0, + }), + ..EscalationJudgeConfig::default() + }; + assert!( + zero.validate() + .is_err_and(|error| error.to_string().contains("at least 1")) + ); + + let inverted = EscalationJudgeConfig { + deescalation: Some(DeescalationConfig { + strong_min_calls: 4, + strong_max_calls: Some(3), + confirmations: 2, + weak_cooldown_calls: 0, + }), + ..EscalationJudgeConfig::default() + }; + assert!( + inverted + .validate() + .is_err_and(|error| error.to_string().contains("at least deescalation")) + ); + } + + #[test] + fn deescalation_contract_marks_both_routing_phases() -> Result<()> { + let contract = ClassifierContractConfig::default().with_prompt("Custom trajectory rubric."); + for (phase, marker) in [ + (EvaluationPhase::Efficient, "EFFICIENT_EVALUATION"), + (EvaluationPhase::Strong, "STRONG_EVALUATION"), + ] { + let judge = escalation_judge(512, Some(phase), &contract)?; + let built = judge.build_request(&State::default(), &request_at_turn(None, 1)); + let system_prompt = built.llm_request.instructions[0].content.iter().find_map( + |content| match content { + ContentBlock::Text { text } => Some(text.as_str()), + _ => None, + }, + ); + assert!(system_prompt.is_some_and(|prompt| { + prompt.starts_with("Custom trajectory rubric.") + && prompt.contains("EFFICIENT_EVALUATION") + && prompt.contains("STRONG_EVALUATION") + })); + assert!( + built.llm_request.messages[0] + .text_content("") + .is_some_and(|summary| summary.starts_with(&format!( + "Routing phase: {marker}" + ))) + ); + } + Ok(()) + } + #[test] fn summary_keeps_anchors_and_the_recent_window() { let mut messages = vec![ @@ -510,7 +688,7 @@ mod tests { ..EscalationJudgeConfig::default() }; - let summary = summarize_for_judge(&messages, 11, &config); + let summary = summarize_for_judge(&messages, 11, None, &config); assert!( summary.contains("[system] you are a coding agent"), @@ -551,7 +729,7 @@ mod tests { ..EscalationJudgeConfig::default() }; - let summary = summarize_for_judge(&messages, 40, &config); + let summary = summarize_for_judge(&messages, 40, None, &config); assert!( summary.contains("[user (task)] "), @@ -590,7 +768,7 @@ mod tests { ..EscalationJudgeConfig::default() }; - let summary = summarize_for_judge(&messages, 21, &config); + let summary = summarize_for_judge(&messages, 21, None, &config); assert!( summary.chars().count() <= MAX_REQUEST_CHARS, diff --git a/crates/libsy/src/lib.rs b/crates/libsy/src/lib.rs index 6e84c456a..f2384db74 100644 --- a/crates/libsy/src/lib.rs +++ b/crates/libsy/src/lib.rs @@ -32,7 +32,7 @@ pub use algorithms::util::affinity::{AffinityRouter, ClassifyTrigger}; pub use algorithms::util::classifier_contract::{ ClassifierContractConfig, ClassifierResponseFormat, }; -pub use algorithms::util::escalation::EscalationJudgeConfig; +pub use algorithms::util::escalation::{DeescalationConfig, EscalationJudgeConfig}; pub use algorithms::util::prompts::append_note; pub use algorithms::util::subagent::{SubagentGate, SubagentOverride}; pub use algorithms::util::tool_signals::{DEFAULT_RECENT_WINDOW, ToolSemantics, ToolSignals}; diff --git a/crates/libsy/src/prompts/escalation/deescalation.md b/crates/libsy/src/prompts/escalation/deescalation.md new file mode 100644 index 000000000..d3d770b17 --- /dev/null +++ b/crates/libsy/src/prompts/escalation/deescalation.md @@ -0,0 +1,17 @@ +# Routing phase + +When a routing phase marker is present, routing is reversible and the +phase rules below replace the earlier one-way escalation rule. + +The routing input begins with one of these router-generated markers: + +- `EFFICIENT_EVALUATION`: Review the efficient-tier response. Return + `escalate: true` when the trajectory needs the strong tier; otherwise + return `escalate: false`. +- `STRONG_EVALUATION`: Review the strong-tier response. Return + `escalate: true` when the remaining work still needs the strong tier. + Return `escalate: false` only when the difficult part is resolved and + the remaining work is routine enough for the efficient tier. + +The router, not the judge, applies confirmation counts and decides when +to change tiers. Judge only the phase named in the routing input. diff --git a/crates/switchyard-py/src/libsy_bindings.rs b/crates/switchyard-py/src/libsy_bindings.rs index 9789949a9..23c90d7ad 100644 --- a/crates/switchyard-py/src/libsy_bindings.rs +++ b/crates/switchyard-py/src/libsy_bindings.rs @@ -14,10 +14,10 @@ use pyo3::prelude::*; use serde_json::Value; use switchyard_libsy::{ Algorithm, CallModel, ClassifierContractConfig, ClassifierResponseFormat, ClassifyTrigger, - CustomClassifierConfig, CustomClassifierPolicy, EscalationJudgeConfig, HandoffNoteConfig, - LibsyError as RustLibsyError, LlmClassifierConfig, LlmFallback, LlmTaskClassifier, Noop, - PickerMode, Random, RoutingOutcome, RuntimeModels, StageRouter, StageRouterConfig, - Step as RustStep, StepStream, TaskClassifierConfig, ToolSemantics, + CustomClassifierConfig, CustomClassifierPolicy, DeescalationConfig, EscalationJudgeConfig, + HandoffNoteConfig, LibsyError as RustLibsyError, LlmClassifierConfig, LlmFallback, + LlmTaskClassifier, Noop, PickerMode, Random, RoutingOutcome, RuntimeModels, StageRouter, + StageRouterConfig, Step as RustStep, StepStream, TaskClassifierConfig, ToolSemantics, }; use switchyard_protocol::{ Category, LlmClientError, LlmResponse, LlmResponseStream, LlmResponseStreamEvent, Metadata, @@ -93,6 +93,49 @@ impl PyTaskClassifierConfig { } } +/// Settings for returning an escalated session to the efficient tier. +/// +/// `strong_min_calls` and `confirmations` must be positive; `strong_max_calls`, when set, +/// must not be lower than `strong_min_calls`. Classifier construction reports invalid values +/// as `ValueError`. +#[pyclass( + name = "DeescalationConfig", + module = "switchyard.libsy", + frozen, + skip_from_py_object +)] +#[derive(Clone)] +struct PyDeescalationConfig { + inner: DeescalationConfig, +} + +#[pymethods] +impl PyDeescalationConfig { + #[new] + #[pyo3(signature = ( + *, + strong_min_calls, + confirmations, + strong_max_calls=None, + weak_cooldown_calls=0 + ))] + fn new( + strong_min_calls: u32, + confirmations: u32, + strong_max_calls: Option, + weak_cooldown_calls: u32, + ) -> Self { + Self { + inner: DeescalationConfig { + strong_min_calls, + strong_max_calls, + confirmations, + weak_cooldown_calls, + }, + } + } +} + /// Settings for response-based escalation classification. #[pyclass( name = "EscalationClassifierConfig", @@ -115,15 +158,18 @@ impl PyEscalationClassifierConfig { confirmations=2, recent_turn_window=28, window_message_chars=500, + deescalation=None, max_output_tokens=4096, prompt=None, response_format_type="json_schema" ))] #[allow(clippy::too_many_arguments)] fn new( + py: Python<'_>, confirmations: u32, recent_turn_window: usize, window_message_chars: usize, + deescalation: Option>, max_output_tokens: u64, prompt: Option, response_format_type: &str, @@ -134,6 +180,9 @@ impl PyEscalationClassifierConfig { confirmations, recent_turn_window, window_message_chars, + deescalation: deescalation + .map(|config| config.bind(py).try_borrow().map(|config| config.inner)) + .transpose()?, }, max_output_tokens, }) @@ -881,6 +930,7 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { let libsy_module = PyModule::new(module.py(), "libsy")?; libsy_module.add_class::()?; libsy_module.add_class::()?; + libsy_module.add_class::()?; libsy_module.add_class::()?; libsy_module.add_class::()?; libsy_module.add_class::()?; diff --git a/crates/switchyard-runner/src/config.rs b/crates/switchyard-runner/src/config.rs index 2d258474f..d2805c3de 100644 --- a/crates/switchyard-runner/src/config.rs +++ b/crates/switchyard-runner/src/config.rs @@ -1111,6 +1111,12 @@ new = ["send_message"] ); runner_from_toml(&escalating)?; + let reversible = VALID_CONFIG.replace( + "base_threshold = 0.5", + "base_threshold = 0.5\nescalation = { confirmations = 2, deescalation = { strong_min_calls = 3, confirmations = 2, strong_max_calls = 6, weak_cooldown_calls = 8 } }", + ); + runner_from_toml(&reversible)?; + // A setting that would starve the judge is rejected here rather than on the first // request, the same as any other unusable route configuration. let starved = VALID_CONFIG.replace( @@ -1118,6 +1124,7 @@ new = ["send_message"] "base_threshold = 0.5\nescalation = { confirmations = 0 }", ); assert!(error_message(&starved).contains("confirmations must be at least 1")); + Ok(()) } diff --git a/docs/reference/toml_schema.md b/docs/reference/toml_schema.md index e595e7cef..d8326e33b 100644 --- a/docs/reference/toml_schema.md +++ b/docs/reference/toml_schema.md @@ -214,9 +214,14 @@ Escalation mode serves the weak target first and judges the completed turn. See | `strong_target` | Yes | — | Target used after the session latches. | | `weak_target` | Yes | — | Target served before the latch. | | `prompt` | No | packaged prompt | Replaces the trajectory-judge prompt. | -| `escalation.confirmations` | No | `2` | Consecutive escalate verdicts required to latch. Above `1` needs a session ID. | +| `escalation.confirmations` | No | `2` | Consecutive escalate verdicts required to latch. Above `1` needs a stable session ID. | | `escalation.recent_turn_window` | No | `28` | Trailing messages shown to the judge. | | `escalation.window_message_chars` | No | `500` | Per-message cap inside that window. | +| `escalation.deescalation` | No | unset | Enables phase-aware de-escalation. Requires a stable session ID. | +| `escalation.deescalation.strong_min_calls` | With de-escalation | — | Strong-tier turns before release is allowed. Must be at least `1`. | +| `escalation.deescalation.confirmations` | With de-escalation | — | Consecutive judge declines required to return to weak. Must be at least `1`. | +| `escalation.deescalation.strong_max_calls` | No | unset | Hard limit on strong-tier turns before forced de-escalation. Must be at least `strong_min_calls`. | +| `escalation.deescalation.weak_cooldown_calls` | No | `0` | Weak calls served without judging after a hard-limit return. | Existing configurations that contain `escalation` but omit `mode` remain valid. diff --git a/docs/routing_algorithms/escalation_router_routing.md b/docs/routing_algorithms/escalation_router_routing.md index 1ed9e8085..0d4adc976 100644 --- a/docs/routing_algorithms/escalation_router_routing.md +++ b/docs/routing_algorithms/escalation_router_routing.md @@ -2,7 +2,8 @@ Escalation routing starts each conversation on a cheaper weak model. An LLM judge reads how the work is going and latches the session to a strong model when it -detects sustained trouble. +detects sustained trouble. An optional de-escalation policy can later return the +session to the weak model. Use it for multi-turn agent workloads where a weak model handles routine work but may need rescue after repeated errors, loops, or drift. Unlike plain @@ -50,7 +51,8 @@ The route-level `prompt` key replaces the packaged trajectory-judge prompt. It uses the escalation verdict schema rather than the capability verdict schema. Switchyard supplies that schema according to the route's `response_format_type`: through the structured-output request in the default `json_schema` mode, or in -the prompt in `json_object` mode. +the prompt in `json_object` mode. When de-escalation is enabled, Switchyard +appends the phase-specific verdict contract to packaged and custom prompts. ## How the decision works @@ -69,7 +71,8 @@ For each turn on an unlatched session, Switchyard: streak reaches `confirmations`. That turn is billed for a weak call, a judge call, and a strong call. -A latched session routes straight to the strong target with no judge call: +By default, a latched session routes straight to the strong target with no +judge call: ```mermaid %%{init: {"flowchart": {"nodeSpacing": 18, "rankSpacing": 26}}}%% @@ -98,7 +101,7 @@ compatibility guidance as the LLM classifier judge. See ## Tuning options -The judge exposes three settings. Their defaults are the benchmarked +The judge exposes three base settings. Their defaults are the benchmarked configuration, so a bare `escalation = {}` is a valid, tuned route: | Key | Default | Meaning | @@ -107,10 +110,54 @@ configuration, so a bare `escalation = {}` is a valid, tuned route: | `recent_turn_window` | `28` | Trailing messages shown to the judge on top of the anchors. Must be at least `1`. | | `window_message_chars` | `500` | Per-message truncation cap inside that trailing window. Must be at least `50`. | +Replace the inline `escalation` value in the example with nested tables to make +escalation reversible: + +```toml +[routes.agent.escalation] +confirmations = 2 + +[routes.agent.escalation.deescalation] +strong_min_calls = 3 +confirmations = 2 +strong_max_calls = 6 +weak_cooldown_calls = 8 +``` + +| De-escalation key | Required | Default | Meaning | +|---|:---:|---|---| +| `strong_min_calls` | Yes | — | Strong-tier turns before release is allowed. Must be at least `1`. | +| `confirmations` | Yes | — | Consecutive judge declines required to return to weak. Must be at least `1`. | +| `strong_max_calls` | No | unset | Hard limit on strong-tier turns before forced de-escalation. Must be at least `strong_min_calls`. | +| `weak_cooldown_calls` | No | `0` | Weak calls served without judging after a hard-limit return. | + +With this table present, Switchyard marks judge input as either +`EFFICIENT_EVALUATION` or `STRONG_EVALUATION`. In the strong phase, +`escalate: true` keeps the strong tier. An `escalate: false` verdict can release +the next request only after `strong_min_calls` is reached and the configured +confirmation streak is complete. A timeout, error, or unparseable verdict +retains the strong tier. Omitting the table preserves the permanent latch and +does not add phase markers to judge input. + +When `strong_max_calls` is set, the request after that many strong-tier turns returns +to weak even if the judge has not released it. `weak_cooldown_calls` then +prevents immediate re-escalation and avoids turn-by-turn bouncing. + +If the strong target is unavailable during a review, the normal candidate +fallback may serve the weak target. Switchyard does not judge that fallback as a +strong answer, clears any partial release streak, and retries the strong phase +on the next turn. + `confirmations` is the main cost dial. `1` latches sooner and spends more on the strong tier. `2` or higher requires a session identity, because the streak is retained per session — without one, every turn starts from zero and the route -never latches. Clients supply it with `x-switchyard-session-id`. +never latches. De-escalation also requires a session identity to retain its +phase and confirmation counts. Clients supply it with +`x-switchyard-session-id`. + +When stateful escalation receives no session ID, Switchyard logs one warning per +route. The request still succeeds, but its temporary state cannot carry into the +next request. Anchor and transcript caps remain fixed. Set the route-level `max_output_tokens` key to change the judge's reply budget. Any decline still diff --git a/switchyard/libsy/__init__.py b/switchyard/libsy/__init__.py index 88a5aa372..7cb8480e6 100644 --- a/switchyard/libsy/__init__.py +++ b/switchyard/libsy/__init__.py @@ -7,6 +7,7 @@ Algorithm, ContextWindowExceededError, CustomClassifierConfig, + DeescalationConfig, EscalationClassifierConfig, LibsyError, LlmClassifierConfig, @@ -25,6 +26,7 @@ "Algorithm", "ContextWindowExceededError", "CustomClassifierConfig", + "DeescalationConfig", "EscalationClassifierConfig", "LibsyError", "LlmClassifierConfig", diff --git a/switchyard_rust/libsy.py b/switchyard_rust/libsy.py index fd768ba71..7bdf2cb6c 100644 --- a/switchyard_rust/libsy.py +++ b/switchyard_rust/libsy.py @@ -15,6 +15,7 @@ "Algorithm", "ContextWindowExceededError", "CustomClassifierConfig", + "DeescalationConfig", "EscalationClassifierConfig", "LibsyError", "LlmClassifierConfig", @@ -78,6 +79,25 @@ def __init__( max_output_tokens: int = 4096, ) -> None: ... + @final + class DeescalationConfig: + """Configure when an escalated session may return to the efficient tier. + + ``strong_min_calls`` and ``confirmations`` must be positive. + ``strong_max_calls``, when set, must not be lower than + ``strong_min_calls``. Values are validated when the classifier is built, + and invalid values raise ``ValueError``. + """ + + def __init__( + self, + *, + strong_min_calls: int, + confirmations: int, + strong_max_calls: int | None = None, + weak_cooldown_calls: int = 0, + ) -> None: ... + @final class EscalationClassifierConfig: """Configure response-based escalation between two targets. @@ -92,6 +112,7 @@ def __init__( confirmations: int = 2, recent_turn_window: int = 28, window_message_chars: int = 500, + deescalation: DeescalationConfig | None = None, max_output_tokens: int = 4096, prompt: str | None = None, response_format_type: Literal["json_schema", "json_object"] = "json_schema", diff --git a/tests/test_libsy_minimal_bindings.py b/tests/test_libsy_minimal_bindings.py index 3263ff638..e28ffc126 100644 --- a/tests/test_libsy_minimal_bindings.py +++ b/tests/test_libsy_minimal_bindings.py @@ -13,6 +13,7 @@ Algorithm, ContextWindowExceededError, CustomClassifierConfig, + DeescalationConfig, EscalationClassifierConfig, LlmClassifierConfig, LlmResponse, @@ -243,6 +244,20 @@ async def call(self, request: dict[str, Any]) -> dict[str, Any]: assert response["model"] == "weak" +def test_escalation_accepts_optional_deescalation_config() -> None: + config = EscalationClassifierConfig( + deescalation=DeescalationConfig( + strong_min_calls=3, + confirmations=2, + strong_max_calls=6, + weak_cooldown_calls=8, + ) + ) + algorithm = LlmClassifierConfig.escalation(config=config) + + assert isinstance(algorithms.llm_classifier(algorithm), Algorithm) + + async def test_custom_classifier_routes_across_named_targets() -> None: class JudgeClient(EchoClient): async def call(self, request: dict[str, Any]) -> dict[str, Any]: