diff --git a/crates/libsy/src/algorithms/stage.rs b/crates/libsy/src/algorithms/stage.rs index 4638f5e9b..c2f576287 100644 --- a/crates/libsy/src/algorithms/stage.rs +++ b/crates/libsy/src/algorithms/stage.rs @@ -140,6 +140,9 @@ pub struct StageRouterConfig { pub recent_window: Option, /// Exact tool-name semantics added to the built-in coding vocabulary. pub tool_semantics: ToolSemantics, + /// Requests to keep on the capable tier after an escalation. A clean test + /// pass clears the hold early. + pub capable_hold_turns: u32, /// Note handed to the model on a signal-driven escalation, and on a /// hand-back to the efficient tier when a de-escalation note is configured. pub handoff_notes: Option, @@ -161,6 +164,7 @@ impl StageRouterConfig { confidence_threshold, recent_window: None, tool_semantics: ToolSemantics::default(), + capable_hold_turns: 2, handoff_notes: None, capable_system_prompt: None, efficient_system_prompt: None, @@ -218,7 +222,8 @@ pub(crate) fn build_stage_route(config: StageRouterConfig) -> Result &'static str { match self { Self::Override => "override", - Self::TestsPassed => "tests_passed", + Self::CapableHold => "capable_hold", Self::Dimensions => "dimensions", Self::Ambiguous => "ambiguous", Self::LlmClassifier => "llm-classifier", @@ -344,54 +346,57 @@ pub fn score_signal(signal: &ToolSignals) -> ScoreResult { } } -/// Hard **escalate** — force the capable tier no matter what the scorer would -/// say. Fires on a critical error or a compacted context. -fn should_escalate(signal: &ToolSignals) -> bool { +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum OverrideReason { + RepeatedFailure, + CriticalError, + Compaction, +} + +impl OverrideReason { + fn as_str(self) -> &'static str { + match self { + Self::RepeatedFailure => "repeated_failure", + Self::CriticalError => "critical_error", + Self::Compaction => "compaction", + } + } +} + +/// Hard **escalate** — force the capable tier no matter what the scorer would say. +fn override_reason(signal: &ToolSignals) -> Option { // Compaction wipes the accumulated signals, so a task that had escalated // would snap back to efficient — a context big enough to overflow belongs capable. if signal.compacted { - return true; + return Some(OverrideReason::Compaction); } // A critical error is unambiguous. - signal.severity >= SEVERITY_CRITICAL -} - -/// Hard **de-escalate** — drop to the cheap tier on a settled turn: tests -/// passed, code was just written or edited, and nothing errored in the window. -fn should_deescalate(signal: &ToolSignals) -> bool { - signal.tests_passed - && (signal.recent_write_count + signal.recent_edit_count) >= 1 - && signal.severity <= 0.0 + if signal.severity >= SEVERITY_CRITICAL { + return Some(OverrideReason::CriticalError); + } + signal + .repeated_failure + .then_some(OverrideReason::RepeatedFailure) } /// Decide a turn's tier from its signal. /// /// The rules run in order; the first that fires wins: /// -/// 1. **Escalate** — a hard reason to go capable (critical error / compaction). -/// 2. **De-escalate** — a hard reason to go cheap (a settled turn). -/// 3. **Scorer** — no hard reason, so weigh the two axes; if confident, follow it. -/// 4. **Fall open** — not confident: hand to the classifier, else the default. +/// 1. **Escalate** — repeated failure, critical error, or compaction. +/// 2. **Scorer** — no hard reason, so weigh the two axes; if confident, follow it. +/// 3. **Fall open** — not confident: hand to the classifier, else the default. /// -/// Rules 1 and 2 are the two hard shortcuts that skip the scorer — one always -/// escalates, one always de-escalates. **Escalate is checked first**, so a -/// critical error still wins on a turn whose tests also happened to pass. -/// -/// Deterministic and pure: the async classifier lives in the caller, so rule 4 +/// Deterministic and pure: the async classifier lives in the caller, so rule 3 /// returns [`PickOutcome::ConsultClassifier`] instead of calling it here. The /// `no_signal` case (no tool activity yet) is handled one level up. pub fn pick_tier(signal: &ToolSignals, mode: PickerMode, confidence_threshold: f64) -> PickOutcome { // 1. Escalate — a hard reason to go capable, ahead of everything else. - if should_escalate(signal) { + if override_reason(signal).is_some() { return resolved(Tier::Capable, DecisionSource::Override, 0.5, Some(1.0)); } - // 2. De-escalate — a hard reason to go cheap (the turn is winding down). - if should_deescalate(signal) { - return resolved(Tier::Efficient, DecisionSource::TestsPassed, 0.5, None); - } - - // 3. Scorer — no hard reason either way, so weigh error vs production. + // 2. Scorer — no hard reason, so weigh error vs production. // Resolve outside the closed ambiguous band [0.5 - t/2, 0.5 + t/2]. let scored = score_signal(signal); let probability = scored.probability(); @@ -410,7 +415,7 @@ pub fn pick_tier(signal: &ToolSignals, mode: PickerMode, confidence_threshold: f ); } - // 4. Fall open — the signals didn't corroborate enough to be sure. Hand off + // 3. Fall open — the signals didn't corroborate enough to be sure. Hand off // to the caller's classifier; with none, land on the picker's default. PickOutcome::ConsultClassifier { probability, @@ -513,6 +518,7 @@ impl HandoffNoteConfig { pub struct StageClassifier { mode: PickerMode, confidence_threshold: f64, + capable_hold_turns: u32, handoff_notes: Option, } @@ -523,10 +529,18 @@ impl StageClassifier { Self { mode, confidence_threshold, + capable_hold_turns: 2, handoff_notes: None, } } + /// Keep the capable tier for this many requests after an escalation. A + /// clean passing test clears the hold early. Set to zero to disable. + pub fn with_capable_hold_turns(mut self, turns: u32) -> Self { + self.capable_hold_turns = turns; + self + } + /// Hand the routed model a note on a signal-driven escalation, and on a /// hand-back to the efficient tier when a de-escalation note is configured. pub fn with_handoff_notes(mut self, config: HandoffNoteConfig) -> Self { @@ -549,6 +563,42 @@ impl StageClassifier { prompts::append_note(request, note); } } + + fn capable_hold_key(request: &Request) -> String { + request + .metadata + .as_ref() + .and_then(|metadata| metadata.agent_id.as_deref()) + .map_or_else( + || CAPABLE_HOLD_KEY.to_string(), + |agent_id| format!("{CAPABLE_HOLD_KEY}:{agent_id}"), + ) + } + + fn consume_capable_hold(state: &mut State, key: &str) -> bool { + let Some(StateValue::Count(remaining)) = state.extra.get_mut(key) else { + return false; + }; + if *remaining == 0 { + state.extra.remove(key); + return false; + } + *remaining -= 1; + if *remaining == 0 { + state.extra.remove(key); + } + true + } + + fn set_capable_hold(&self, state: &mut State, key: &str) { + if self.capable_hold_turns == 0 { + state.extra.remove(key); + } else { + state + .extra + .insert(key.to_string(), StateValue::Count(self.capable_hold_turns)); + } + } } #[async_trait] @@ -559,15 +609,23 @@ impl Classifier for StageClassifier { request: &mut Request, driver: &Driver, ) -> Result<(Classification, Option)> { - let tool_signals = &state.tool_signals; - let Some(signal) = tool_signals else { + let Some(signal) = state.tool_signals.clone() else { // No tool activity yet — nothing to score, so the signals have no // opinion, same as a below-threshold turn. return Ok((Self::abstain(state), None)); }; - let outcome = pick_tier(signal, self.mode, self.confidence_threshold); - record_score_metrics(signal, &outcome); + let capable_hold_key = Self::capable_hold_key(request); + let clean_test_pass = signal.tests_passed && signal.no_error_streak > 0; + if clean_test_pass { + state.extra.remove(&capable_hold_key); + } + let outcome = if !clean_test_pass && Self::consume_capable_hold(state, &capable_hold_key) { + resolved(Tier::Capable, DecisionSource::CapableHold, 0.5, Some(1.0)) + } else { + pick_tier(&signal, self.mode, self.confidence_threshold) + }; + record_score_metrics(&signal, &outcome); match outcome { PickOutcome::Resolved { tier, @@ -580,8 +638,26 @@ impl Classifier for StageClassifier { Tier::Efficient => Category::Efficient, }; let target = driver.first_model_for(&category)?; + if tier == Tier::Capable + && matches!( + source, + DecisionSource::Override | DecisionSource::Dimensions + ) + { + self.set_capable_hold(state, &capable_hold_key); + } record_decision_source(state, source); record_routing_decision(source, target); + if source == DecisionSource::Override + && let Some(reason) = override_reason(&signal) + { + tracing::info!( + decision_source = source.as_str(), + override_reason = reason.as_str(), + target = %target, + "stage router override" + ); + } // Only a resolved turn routes on this classifier's target, so it // is the only branch whose tier the signals actually chose — an // ambiguous turn is decided further down the cascade. @@ -674,6 +750,48 @@ mod tests { )); } + #[test] + fn repeated_failure_overrides_to_capable() { + let signal = ToolSignals { + severity: HARD_SEVERITY as f32, + repeated_failure: true, + ..Default::default() + }; + assert!(matches!( + pick_tier(&signal, PickerMode::EfficientFirst, 0.5), + PickOutcome::Resolved { + tier: Tier::Capable, + source: DecisionSource::Override, + .. + } + )); + } + + #[test] + fn override_reasons_are_specific() { + assert_eq!( + override_reason(&ToolSignals { + repeated_failure: true, + ..Default::default() + }), + Some(OverrideReason::RepeatedFailure) + ); + assert_eq!( + override_reason(&ToolSignals { + severity: SEVERITY_CRITICAL, + ..Default::default() + }), + Some(OverrideReason::CriticalError) + ); + assert_eq!( + override_reason(&ToolSignals { + compacted: true, + ..Default::default() + }), + Some(OverrideReason::Compaction) + ); + } + #[test] fn one_signal_scores_below_half() { // A single full wrong signal ≈ 0.46 confidence — just under 0.5. @@ -825,11 +943,10 @@ mod tests { } #[tokio::test] - async fn classifier_deescalates_settled_turn_to_weak() -> Result<()> { - // Tests passed with recent production and no error → the settled-turn shortcut - // resolves straight to a definite efficient-tier score. + async fn passing_tests_do_not_force_the_efficient_tier() -> Result<()> { let signal = ToolSignals { tests_passed: true, + no_error_streak: 1, recent_write_count: 1, severity: 0.0, ..Default::default() @@ -839,13 +956,101 @@ mod tests { let classification = StageClassifier::new(PickerMode::EfficientFirst, 0.5) .score(&mut state, &mut Request::default(), &driver) .await?; - match classification.0 { - Classification::Scores(scores) => { - assert_eq!(scores.len(), 1); - assert_eq!(scores[0].target, "weak"); - } - _ => panic!("expected a definite classification"), + assert!(matches!(classification.0, Classification::Ambiguous(_))); + Ok(()) + } + + #[tokio::test] + async fn escalation_holds_capable_for_two_more_turns() -> Result<()> { + let classifier = StageClassifier::new(PickerMode::EfficientFirst, 0.5); + let mut state = state_with(critical()); + let driver = driver(); + + for _ in 0..3 { + let classification = classifier + .score(&mut state, &mut Request::default(), &driver) + .await?; + assert_eq!(classification.0.argmax(false)?.unwrap().target, "strong"); + state.tool_signals = Some(ToolSignals::default()); } + + let classification = classifier + .score(&mut state, &mut Request::default(), &driver) + .await?; + assert!(classification.0.argmax(false)?.is_none()); + Ok(()) + } + + #[tokio::test] + async fn passing_tests_clear_the_capable_hold() -> Result<()> { + let classifier = StageClassifier::new(PickerMode::EfficientFirst, 0.5); + let mut state = state_with(critical()); + let driver = driver(); + classifier + .score(&mut state, &mut Request::default(), &driver) + .await?; + state.tool_signals = Some(ToolSignals { + tests_passed: true, + no_error_streak: 1, + severity: HARD_SEVERITY as f32, + ..Default::default() + }); + + let classification = classifier + .score(&mut state, &mut Request::default(), &driver) + .await?; + assert!(classification.0.argmax(false)?.is_none()); + assert!(!state.extra.contains_key(CAPABLE_HOLD_KEY)); + Ok(()) + } + + #[tokio::test] + async fn capable_hold_is_isolated_per_agent() -> Result<()> { + let classifier = StageClassifier::new(PickerMode::EfficientFirst, 0.5); + let mut state = state_with(critical()); + let driver = driver(); + let mut parent = Request { + metadata: Some(Metadata { + agent_id: Some("parent".to_string()), + ..Default::default() + }), + ..Default::default() + }; + classifier.score(&mut state, &mut parent, &driver).await?; + + state.tool_signals = Some(ToolSignals::default()); + let mut child = Request { + metadata: Some(Metadata { + agent_id: Some("child".to_string()), + ..Default::default() + }), + ..Default::default() + }; + let classification = classifier.score(&mut state, &mut child, &driver).await?; + + assert!(classification.0.argmax(false)?.is_none()); + assert!(matches!( + state.extra.get("capable_hold_turns:parent"), + Some(StateValue::Count(2)) + )); + Ok(()) + } + + #[tokio::test] + async fn zero_disables_the_capable_hold() -> Result<()> { + let classifier = + StageClassifier::new(PickerMode::EfficientFirst, 0.5).with_capable_hold_turns(0); + let mut state = state_with(critical()); + let driver = driver(); + classifier + .score(&mut state, &mut Request::default(), &driver) + .await?; + state.tool_signals = Some(ToolSignals::default()); + + let classification = classifier + .score(&mut state, &mut Request::default(), &driver) + .await?; + assert!(classification.0.argmax(false)?.is_none()); Ok(()) } @@ -905,7 +1110,7 @@ mod tests { #[test] fn deescalation_note_applies_to_efficient_when_configured() { assert_eq!( - config(true).note_for(Tier::Efficient, DecisionSource::TestsPassed), + config(true).note_for(Tier::Efficient, DecisionSource::Dimensions), Some(DEESCALATION) ); } @@ -914,7 +1119,7 @@ mod tests { fn no_deescalation_note_when_unconfigured() { let config = HandoffNoteConfig::new(ESCALATION, None, true); assert_eq!( - config.note_for(Tier::Efficient, DecisionSource::TestsPassed), + config.note_for(Tier::Efficient, DecisionSource::Dimensions), None ); } @@ -972,27 +1177,28 @@ mod tests { } #[tokio::test] - async fn every_turn_the_signals_drive_carries_the_note() -> Result<()> { - // Stateless by design: the note describes this turn's signals, so a run - // of escalated turns each carries one. Nothing tracks the previous tier. + async fn held_turns_do_not_repeat_the_escalation_note() -> Result<()> { let classifier = noting_classifier(PickerMode::EfficientFirst); let mut state = state_with(critical()); let driver = driver(); - for _ in 0..3 { - let mut request = request(); - classifier.score(&mut state, &mut request, &driver).await?; - assert_eq!(trailing_text(&request), Some(format!("hi|{ESCALATION}"))); + let mut first = request(); + classifier.score(&mut state, &mut first, &driver).await?; + assert_eq!(trailing_text(&first), Some(format!("hi|{ESCALATION}"))); + + for _ in 0..2 { + let mut held = request(); + classifier.score(&mut state, &mut held, &driver).await?; + assert_eq!(trailing_text(&held), Some("hi".to_string())); } Ok(()) } #[tokio::test] - async fn a_settled_turn_carries_the_deescalation_note() -> Result<()> { - // Tests passed with recent production resolves to weak on the settled-turn - // shortcut, which is the hand-back the de-escalation note is for. + async fn a_passing_test_does_not_force_a_deescalation_note() -> Result<()> { let signal = ToolSignals { tests_passed: true, + no_error_streak: 1, recent_write_count: 1, ..Default::default() }; @@ -1004,7 +1210,7 @@ mod tests { .score(&mut state, &mut request, &driver) .await?; - assert_eq!(trailing_text(&request), Some(format!("hi|{DEESCALATION}"))); + assert_eq!(trailing_text(&request), Some("hi".to_string())); Ok(()) } diff --git a/crates/libsy/src/algorithms/util/tool_signals.rs b/crates/libsy/src/algorithms/util/tool_signals.rs index 43e1bd814..f305d5135 100644 --- a/crates/libsy/src/algorithms/util/tool_signals.rs +++ b/crates/libsy/src/algorithms/util/tool_signals.rs @@ -12,6 +12,8 @@ #![allow(dead_code)] +use std::path::Path; + use async_trait::async_trait; use serde::Deserialize; use serde_json::Value; @@ -39,7 +41,7 @@ static ERROR_PATTERNS: &[(&str, f32, &[&str])] = &[ ), ( "connection_refused", - CRITICAL, + HARD, &[ "connection refused", "connectionrefusederror", @@ -83,17 +85,14 @@ static ERROR_PATTERNS: &[(&str, f32, &[&str])] = &[ ], ), // SOFT: plain non-zero exit without a recognisable exception traceback. - ( - "exit_nonzero", - SOFT, - &[ - "exit code 1", - "exit code 2", - "exit status 1", - "returned non-zero", - "exited with code", - ], - ), + ("exit_nonzero", SOFT, &["returned non-zero"]), +]; + +static NONZERO_EXIT_PHRASES: &[&str] = &[ + "exit code", + "exit status", + "exited with code", + "exited with status", ]; static EDIT_TOOL_NAMES: &[&str] = &[ @@ -132,6 +131,13 @@ static BASH_WRITE_PATTERNS: &[&str] = &[ /// interpreter is running them rather than a search looking for them. static PYTHON_WRITE_PATTERNS: &[&str] = &["write_text(", "writelines(", ".write("]; +static JAVASCRIPT_WRITE_PATTERNS: &[&str] = &[ + "writefilesync(", + "writefile(", + "appendfilesync(", + "appendfile(", +]; + static BASH_EDIT_PATTERNS: &[&str] = &[ "sed -i", "sed --in-place", @@ -151,6 +157,31 @@ static BASH_READ_PATTERNS: &[&str] = &[ "diff ", "which ", "ps ", "df ", "du ", "stat ", "file ", "less ", "more ", ]; +/// Read-only shell programs seen in Codex trajectories. Matching is limited to +/// command-segment starts so prose and arguments do not masquerade as actions. +static BASH_READ_COMMANDS: &[&str] = &[ + "cat", "rg", "nl", "jq", "pwd", "tree", "sed", "grep", "ls", "find", "head", "tail", "wc", + "diff", "which", "ps", "df", "du", "stat", "file", "less", "more", "readlink", "realpath", + "basename", "dirname", "printenv", +]; + +static GIT_READ_SUBCOMMANDS: &[&str] = &[ + "status", + "diff", + "log", + "show", + "show-ref", + "rev-parse", + "ls-files", + "ls-remote", + "ls-tree", + "grep", + "blame", + "merge-base", + "check-ignore", + "tag", +]; + static READ_TOOL_NAMES: &[&str] = &["read", "view", "read_file", "search_files"]; // Planning / scratchpad tool calls — investigative (non-producing) activity. @@ -170,8 +201,8 @@ static BASH_TOOL_NAMES: &[&str] = &[ "exec_command", // codex ]; -// Prefer false negatives: tests_passed routes the picker to EFFICIENT, so a false -// positive would drop tier on an unfinished task. +// Prefer false negatives: tests_passed clears a capable hold, so a false positive +// could hand an unfinished task back too early. static TEST_PASS_PHRASES: &[&str] = &[ " passed", "passed in", @@ -296,6 +327,9 @@ pub struct ToolSignals { /// Windowed so an error persists through the recovery turns instead of clearing /// the instant the next result is clean. pub severity: f32, + /// The same hard-or-critical failure appeared at least twice in the recent + /// tool-result window. + pub repeated_failure: bool, /// Consecutive clean tool results back from the most recent. `0` if the last failed. pub no_error_streak: u32, /// Total edit-style tool calls in the request. @@ -322,7 +356,7 @@ pub struct ToolSignals { /// Consecutive trailing tool calls in the `Unknown` category (no Write/Edit/Read/ /// Plan match). Surfaced in the classifier state summary; not scored directly. pub pure_bash_streak: u32, - /// At least one of the last three tool results matched a test-pass pattern. + /// A tool result after the latest recent failure matched a test-pass pattern. pub tests_passed: bool, /// Total `ToolResult` blocks, counted per block (a message batching N /// results contributes N) and including empty-content results. @@ -450,16 +484,23 @@ fn classify_tool_call_with_semantics( && let Some(cmd) = command { // Write/edit redirection trumps read-like operands. - if BASH_WRITE_PATTERNS.iter().any(|p| cmd.contains(p)) { + if BASH_WRITE_PATTERNS.iter().any(|p| cmd.contains(p)) || shell_command_is_write(cmd) { return ToolSemantic::Mutate(MutationKind::Write); } if cmd.contains("python") && PYTHON_WRITE_PATTERNS.iter().any(|p| cmd.contains(p)) { return ToolSemantic::Mutate(MutationKind::Write); } - if BASH_EDIT_PATTERNS.iter().any(|p| cmd.contains(p)) { + if shell_invokes_program(cmd, "node") + && JAVASCRIPT_WRITE_PATTERNS + .iter() + .any(|pattern| cmd.contains(pattern)) + { + return ToolSemantic::Mutate(MutationKind::Write); + } + if BASH_EDIT_PATTERNS.iter().any(|p| cmd.contains(p)) || shell_command_is_edit(cmd) { return ToolSemantic::Mutate(MutationKind::Edit); } - if BASH_READ_PATTERNS.iter().any(|p| cmd.contains(p)) { + if BASH_READ_PATTERNS.iter().any(|p| cmd.contains(p)) || shell_command_is_read(cmd) { return ToolSemantic::Observe; } } @@ -474,6 +515,176 @@ fn is_builtin_tool_name(lower: &str) -> bool { || BASH_TOOL_NAMES.contains(&lower) } +/// Split a shell line at unquoted command separators. This intentionally avoids +/// pretending to be a full shell parser; only the leading program and flags of +/// each segment are inspected below. +fn shell_segments(command: &str) -> impl Iterator { + let mut chars = command.char_indices(); + let mut start = 0usize; + let mut quote = None; + let mut escaped = false; + let mut finished = false; + + std::iter::from_fn(move || { + loop { + for (index, character) in chars.by_ref() { + if escaped { + escaped = false; + } else if character == '\\' && quote != Some('\'') { + escaped = true; + } else if quote == Some(character) { + quote = None; + } else if quote.is_none() && matches!(character, '\'' | '"') { + quote = Some(character); + } else if quote.is_none() && matches!(character, '\n' | ';' | '|' | '&') { + let segment = command[start..index].trim(); + start = index + character.len_utf8(); + if !segment.is_empty() { + return Some(segment); + } + } + } + + if finished { + return None; + } + finished = true; + let segment = command[start..].trim(); + if !segment.is_empty() { + return Some(segment); + } + } + }) +} + +fn shell_words(segment: &str) -> std::iter::Peekable> { + let mut words = segment.split_ascii_whitespace().peekable(); + + if words.peek().copied() == Some("env") { + words.next(); + while words.peek().is_some_and(|word| word.starts_with('-')) { + words.next(); + } + } + while words + .peek() + .is_some_and(|word| word.contains('=') && !word.starts_with('=')) + { + words.next(); + } + + words +} + +fn program_name(word: &str) -> &str { + Path::new(word) + .file_name() + .and_then(|name| name.to_str()) + .unwrap_or(word) +} + +fn shell_invokes_program(command: &str, expected: &str) -> bool { + shell_segments(command).any(|segment| { + shell_words(segment) + .next() + .is_some_and(|word| program_name(word) == expected) + }) +} + +fn shell_command_is_write(command: &str) -> bool { + shell_segments(command).any(|segment| { + let mut words = shell_words(segment); + let Some(program) = words.next().map(program_name) else { + return false; + }; + if matches!(program, "cp" | "mkdir" | "touch" | "install") { + return true; + } + + let redirects_output = words.any(|word| matches!(word, ">" | ">>")); + redirects_output + && (matches!(program, "echo" | "printf" | "git") + || BASH_READ_COMMANDS.contains(&program)) + }) +} + +fn shell_command_is_edit(command: &str) -> bool { + shell_segments(command).any(|segment| { + let mut words = shell_words(segment); + let Some(program) = words.next().map(program_name) else { + return false; + }; + let has_arg = |arg: &str| words.clone().any(|word| word == arg); + + match program { + "mv" | "rm" => true, + "perl" => words + .take_while(|word| word.starts_with('-')) + .any(|option| { + option + .trim_start_matches('-') + .chars() + .any(|flag| flag == 'i') + }), + "git" => words + .next() + .is_some_and(|subcommand| matches!(subcommand, "apply" | "am" | "restore")), + "gofmt" => has_arg("-w"), + "cargo" => words.clone().next() == Some("fmt") && !has_arg("--check"), + "ruff" => { + let subcommand = words.clone().next(); + (subcommand == Some("format") && !has_arg("--check")) + || (subcommand == Some("check") && has_arg("--fix")) + } + "prettier" => has_arg("--write"), + "black" => !has_arg("--check"), + _ => { + (words.clone().any(|word| program_name(word) == "prettier") && has_arg("--write")) + || (words.clone().any(|word| program_name(word) == "ruff") + && ((has_arg("format") && !has_arg("--check")) + || (has_arg("check") && has_arg("--fix")))) + } + } + }) +} + +fn shell_command_is_read(command: &str) -> bool { + shell_segments(command).any(|segment| { + if segment == "env" { + return true; + } + let mut words = shell_words(segment); + let Some(program) = words.next().map(program_name) else { + return false; + }; + + if BASH_READ_COMMANDS.contains(&program) { + return true; + } + if program == "command" && words.next() == Some("-v") { + return true; + } + if program == "type" { + return true; + } + if program != "git" { + return false; + } + + match words.next() { + Some("branch") => words.next().is_none_or(|arg| arg.starts_with('-')), + Some("remote") => words + .next() + .is_none_or(|arg| arg.starts_with('-') || arg == "get-url"), + Some("config") => words + .next() + .is_some_and(|arg| matches!(arg, "--get" | "--get-all" | "--list" | "-l")), + Some(subcommand) => GIT_READ_SUBCOMMANDS.contains(&subcommand), + None => false, + } + }) +} + // ─── extraction entry point ─────────────────────────────────────────────────── /// Extract all tool-execution signals from a normalized [`Request`]. @@ -609,11 +820,17 @@ fn build_signal( // error signal instead of the router flapping straight back to the weak tier. let sev_start = tool_texts.len().saturating_sub(recent_window.max(1)); let mut severity = 0.0f32; + let mut failure_fingerprints = Vec::new(); + let mut repeated_failure = false; for text in &tool_texts[sev_start..] { let (sev, _patterns) = classify_text(text); if sev > severity { severity = sev; } + if let Some(fingerprint) = failure_fingerprint(text) { + repeated_failure |= failure_fingerprints.contains(&fingerprint); + failure_fingerprints.push(fingerprint); + } } let no_error_streak = compute_no_error_streak(&tool_texts); @@ -682,6 +899,7 @@ fn build_signal( ToolSignals { severity, + repeated_failure, no_error_streak, edit_count, write_count, @@ -744,9 +962,171 @@ pub(crate) fn classify_text(text: &str) -> (f32, Vec) { severity = severity.max(*sev); } } + if has_nonzero_exit_status(&lower) && !patterns.iter().any(|p| p == "exit_nonzero") { + patterns.push("exit_nonzero".to_string()); + severity = severity.max(SOFT); + } + for (name, matched) in [ + ("compile_error", has_compiler_diagnostic(&lower)), + ("runtime_exception", has_runtime_exception(&lower)), + ("runtime_panic", has_runtime_panic(&lower)), + ("patch_error", has_patch_failure(&lower)), + ] { + if matched && !patterns.iter().any(|pattern| pattern == name) { + patterns.push(name.to_string()); + severity = severity.max(HARD); + } + } (severity, patterns) } +/// Stable identity for a material failure. Soft non-zero exits are excluded: +/// they are too generic to prove that an agent is repeating the same mistake. +fn failure_fingerprint(text: &str) -> Option { + let (severity, patterns) = classify_text(text); + if severity < HARD { + return None; + } + + let lower = text.to_lowercase(); + let diagnostic = lower + .lines() + .find(|line| is_failure_diagnostic(line)) + .or_else(|| lower.lines().find(|line| !line.trim().is_empty())) + .unwrap_or_default(); + let normalized = normalize_failure_text(diagnostic); + Some(format!("{}|{normalized}", patterns.join(","))) +} + +fn is_failure_diagnostic(line: &str) -> bool { + let line = line.trim(); + [ + "error", + "exception", + "panic", + "failed", + "timed out", + "timeout", + "connection refused", + "cannot allocate memory", + "out of memory", + "not found", + ] + .iter() + .any(|marker| line.contains(marker)) +} + +/// Removes values that normally change between retries while retaining the +/// diagnostic wording that distinguishes one failure from another. +fn normalize_failure_text(text: &str) -> String { + let mut normalized = String::new(); + for word in text.split_whitespace() { + if !normalized.is_empty() { + normalized.push(' '); + } + let mut in_digits = false; + if word.starts_with('/') || word.contains("/src/") || word.contains("/tmp/") { + normalized.push_str(""); + continue; + } + for character in word.chars() { + if character.is_ascii_digit() { + if !in_digits { + normalized.push('#'); + in_digits = true; + } + } else { + normalized.push(character); + in_digits = false; + } + } + } + normalized.chars().take(240).collect() +} + +fn has_compiler_diagnostic(lower: &str) -> bool { + lower.lines().any(|line| { + let line = line.trim_start(); + if matches!( + line, + "compilation failed" | "error: compilation failed" | "error: could not compile" + ) || line.starts_with("error: could not compile ") + { + return true; + } + + let Some(rest) = line.strip_prefix("error[e") else { + return false; + }; + let Some((code, _)) = rest.split_once("]:") else { + return false; + }; + !code.is_empty() && code.chars().all(|character| character.is_ascii_digit()) + }) +} + +fn has_runtime_exception(lower: &str) -> bool { + let has_exception_line = lower.lines().any(|line| { + let line = line.trim_start(); + [ + "typeerror:", + "referenceerror:", + "rangeerror:", + "runtimeerror:", + "keyerror:", + "attributeerror:", + ] + .iter() + .any(|prefix| line.starts_with(prefix)) + }); + has_exception_line && (lower.contains("\n at ") || lower.contains("\n at ")) +} + +fn has_runtime_panic(lower: &str) -> bool { + lower + .lines() + .any(|line| line.trim_start().starts_with("panic: runtime error:")) + && (lower.contains("\ngoroutine ") || lower.contains("[signal sig")) +} + +fn has_patch_failure(lower: &str) -> bool { + lower.lines().any(|line| { + let line = line.trim_start(); + line.starts_with("error: patch failed:") + || line.starts_with("patch failed:") + || line.contains(": patch does not apply") + || line.starts_with("invalid context") + }) +} + +/// Detects `exit_nonzero` only when a supported exit phrase is followed by a +/// nonzero decimal status. +/// +/// Codex includes "Process exited with code 0" on clean tool results, so exit +/// phrases must parse their numeric status instead of matching the phrase alone. +fn has_nonzero_exit_status(lower: &str) -> bool { + NONZERO_EXIT_PHRASES + .iter() + .any(|phrase| phrase_followed_by_nonzero_integer(lower, phrase)) +} + +/// Matches common "exit code/status N" spellings after optional separators. +fn phrase_followed_by_nonzero_integer(lower: &str, phrase: &str) -> bool { + let mut cursor = 0usize; + while let Some(rel) = lower[cursor..].find(phrase) { + let value_start = cursor + rel + phrase.len(); + let rest = lower[value_start..].trim_start_matches(|c: char| { + c.is_ascii_whitespace() || matches!(c, ':' | '=' | '\'' | '"' | '`') + }); + let digits: String = rest.chars().take_while(|c| c.is_ascii_digit()).collect(); + if !digits.is_empty() && digits.chars().any(|d| d != '0') { + return true; + } + cursor = value_start; + } + false +} + fn compute_no_error_streak(tool_texts: &[String]) -> u32 { let mut streak = 0u32; for text in tool_texts.iter().rev() { @@ -761,7 +1141,12 @@ fn compute_no_error_streak(tool_texts: &[String]) -> u32 { fn detect_tests_passed(tool_texts: &[String], recent_window: usize) -> bool { let start = tool_texts.len().saturating_sub(recent_window.max(1)); - tool_texts[start..].iter().any(|text| { + let recent = &tool_texts[start..]; + let after_latest_failure = recent + .iter() + .rposition(|text| classify_text(text).0 > 0.0) + .map_or(recent, |index| &recent[index + 1..]); + after_latest_failure.iter().any(|text| { let lower = text.to_lowercase(); TEST_PASS_PHRASES.iter().any(|p| lower.contains(p)) && !TEST_FAILURE_LITERAL.iter().any(|p| lower.contains(p)) @@ -811,7 +1196,9 @@ mod tests { use super::*; use crate::algorithms::util::stage::score_signal; use serde_json::json; - use switchyard_protocol::{ContentBlock, LlmRequest, Message, Role, ToolCall, ToolResult}; + use switchyard_protocol::{ + ContentBlock, LlmRequest, Message, Metadata, Role, ToolCall, ToolResult, + }; fn with_messages(messages: Vec) -> Request { Request { @@ -882,6 +1269,36 @@ mod tests { assert_eq!(sev, CRITICAL); } + #[test] + fn connection_refused_is_hard() { + let (severity, _) = classify_text("Connection refused on port 8000"); + assert_eq!(severity, HARD); + } + + #[test] + fn repeated_failure_ignores_volatile_paths_and_numbers() { + let request = with_messages(vec![ + tr("error[E0308]: mismatched types at /tmp/a/src/lib.rs:12"), + tr("error[E0308]: mismatched types at /tmp/b/src/lib.rs:47"), + ]); + assert!(ToolSignals::from_request(&request, None).repeated_failure); + } + + #[test] + fn different_failures_are_not_repeated() { + let request = with_messages(vec![ + tr("error[E0308]: mismatched types"), + tr("error[E0509]: cannot move out"), + ]); + assert!(!ToolSignals::from_request(&request, None).repeated_failure); + } + + #[test] + fn one_material_failure_is_not_repeated() { + let request = with_messages(vec![tr("Connection refused on port 8000")]); + assert!(!ToolSignals::from_request(&request, None).repeated_failure); + } + #[test] fn severity_is_max_across_patterns() { // exit_nonzero (SOFT) + traceback (HARD) → HARD. @@ -889,6 +1306,64 @@ mod tests { assert_eq!(sev, HARD); } + #[test] + fn codex_process_exit_zero_stays_clean() { + let (sev, patterns) = + classify_text("Chunk ID: abc\nProcess exited with code 0\nOutput:\nok"); + assert_eq!(sev, 0.0); + assert!(!patterns.contains(&"exit_nonzero".to_string())); + } + + #[test] + fn nonzero_exit_codes_are_soft_errors() { + let cases = [ + "Process exited with code 1", + "Process exited with code 127", + "exit code: 2", + "exit status 3", + "exited with status 9", + ]; + for case in cases { + let (sev, patterns) = classify_text(case); + assert_eq!(sev, SOFT, "expected soft severity for {case}"); + assert!(patterns.contains(&"exit_nonzero".to_string())); + } + } + + #[test] + fn partial_process_failures_are_hard_errors() { + let cases = [ + ( + "Process running with session ID 12\nOutput:\nerror[E0509]: cannot move out", + "compile_error", + ), + ( + "Process exited with code 0\nOutput:\nTypeError: value is undefined\n at main.js:1:2", + "runtime_exception", + ), + ( + "Process running with session ID 13\nOutput:\npanic: runtime error: index out of range\n\ngoroutine 6 [running]:", + "runtime_panic", + ), + ( + "Process exited with code 0\nOutput:\nerror: patch failed: src/lib.rs:4\nerror: src/lib.rs: patch does not apply", + "patch_error", + ), + ]; + for (text, expected_pattern) in cases { + let (severity, patterns) = classify_text(text); + assert_eq!(severity, HARD, "expected hard severity for {text}"); + assert!(patterns.iter().any(|pattern| pattern == expected_pattern)); + } + } + + #[test] + fn source_text_that_names_exceptions_stays_clean() { + let text = + "pub enum TypeError: this is documentation\nlet sample = 'panic: runtime error:';"; + assert_eq!(classify_text(text).0, 0.0); + } + #[test] fn file_does_not_exist_is_hard() { // Claude Code Read-tool miss. Trace-mined addition (22 true / 2 false positives). @@ -938,6 +1413,25 @@ mod tests { )); } + #[test] + fn tests_passed_must_follow_the_latest_failure() { + assert!(!detect_tests_passed( + &[ + "5 passed in 0.12s".to_string(), + "Traceback (most recent call last):\nValueError".to_string(), + "edit applied".to_string(), + ], + DEFAULT_RECENT_WINDOW + )); + assert!(detect_tests_passed( + &[ + "Traceback (most recent call last):\nValueError".to_string(), + "5 passed in 0.12s".to_string(), + ], + DEFAULT_RECENT_WINDOW + )); + } + #[test] fn severity_is_windowed_over_recent_results() { // An error two results back, then two clean results. @@ -1129,6 +1623,17 @@ mod tests { assert!(ToolSignals::from_request(&request, None).compacted); } + #[test] + fn codex_compaction_metadata_stays_on_parent_route() { + let mut request = with_messages(vec![bash("ls")]); + request.metadata = Some(Metadata { + is_subagent: true, + agent_kind: Some("compact".to_string()), + ..Default::default() + }); + assert!(!ToolSignals::from_request(&request, None).compacted); + } + #[test] fn no_compaction_marker_stays_uncompacted() { let request = with_messages(vec![ @@ -1338,6 +1843,102 @@ mod tests { } } + #[test] + fn codex_inspection_commands_classify_as_read() { + let cases = [ + "sed -n '1,80p' src/lib.rs", + "rg -n 'needle' src", + "nl -ba src/lib.rs", + "cat package.json", + "jq '.scripts' package.json", + "git status --short", + "git log --oneline -5", + "git show HEAD:src/lib.rs", + "git branch --show-current", + "git remote -v", + "git config --get remote.origin.url", + ]; + for command in cases { + assert_eq!( + classify_tool_call("exec_command", Some(command)), + ToolSemantic::Observe, + "expected Read for {command}" + ); + } + } + + #[test] + fn quoted_shell_separators_do_not_create_commands() { + for command in ["rg 'foo|rm obsolete.rs'", "rg \"foo; rm obsolete.rs\""] { + assert_eq!( + classify_tool_call("exec_command", Some(command)), + ToolSemantic::Observe, + "quoted text must not be parsed as a command: {command}" + ); + } + } + + #[test] + fn codex_shell_mutations_classify_as_production() { + let writes = [ + "cp source.rs destination.rs", + "mkdir -p src/generated", + "touch src/generated/mod.rs", + "git show HEAD:file.rs > file.rs", + "node <<'node'\nfs.writefilesync('file.js', text)\nnode", + ]; + for command in writes { + assert_eq!( + classify_tool_call("exec_command", Some(command)), + ToolSemantic::Mutate(MutationKind::Write), + "expected Write for {command}" + ); + } + + let edits = [ + "mv old.rs new.rs", + "rm obsolete.rs", + "gofmt -w main.go", + "cargo fmt", + "ruff check --fix src", + "perl -0pi -e 's/old/new/' src/lib.rs", + "npx prettier --write src/lib.ts", + "uv run ruff format src", + "git apply fix.patch", + ]; + for command in edits { + assert_eq!( + classify_tool_call("exec_command", Some(command)), + ToolSemantic::Mutate(MutationKind::Edit), + "expected Edit for {command}" + ); + } + } + + #[test] + fn formatter_checks_are_not_edits() { + for command in [ + "cargo fmt --check", + "ruff format --check src", + "black --check src", + ] { + assert_ne!( + classify_tool_call("exec_command", Some(command)), + ToolSemantic::Mutate(MutationKind::Edit), + "read-only formatter check must not be Edit: {command}" + ); + } + } + + #[test] + fn embedded_comparison_is_not_a_shell_write() { + let command = "node <<'node'\nif (index > 0) console.log(index)\nnode"; + assert_eq!( + classify_tool_call("exec_command", Some(command)), + ToolSemantic::Unknown + ); + } + #[test] fn bash_write_precedence_over_read() { // `cat /file > out` contains both `cat /` (read) and ` > ` (write); diff --git a/crates/switchyard-runner/src/algorithm.rs b/crates/switchyard-runner/src/algorithm.rs index 626c150af..b3935a04c 100644 --- a/crates/switchyard-runner/src/algorithm.rs +++ b/crates/switchyard-runner/src/algorithm.rs @@ -495,6 +495,9 @@ pub struct StageTierConfig { /// Exact tool-name semantics added to the built-in stage vocabulary. #[serde(default)] pub tool_semantics: ToolSemantics, + /// Requests to keep on the capable tier after an escalation. + #[serde(default)] + pub capable_hold_turns: Option, /// Notes handed to a tier when the router switches to it. #[serde(default)] pub handoff_notes: Option, @@ -1237,6 +1240,7 @@ fn build_algorithm( confidence_threshold, recent_turn_window, tool_semantics, + capable_hold_turns, handoff_notes, .. } = tiers; @@ -1247,6 +1251,9 @@ fn build_algorithm( } let mut config = StageRouterConfig::new(*picker, *confidence_threshold); config.recent_window = *recent_turn_window; + if let Some(turns) = capable_hold_turns { + config.capable_hold_turns = *turns; + } config.tool_semantics = tool_semantics.clone(); config.handoff_notes = handoff_notes.clone(); // The judge is called through its own target, so it is not a routing @@ -1281,6 +1288,9 @@ fn build_algorithm( let mut stage_config = StageRouterConfig::new(PickerMode::EfficientFirst, stage.confidence_threshold); stage_config.recent_window = stage.recent_turn_window; + if let Some(turns) = stage.capable_hold_turns { + stage_config.capable_hold_turns = turns; + } stage_config.tool_semantics = stage.tool_semantics.clone(); stage_config.handoff_notes = stage.handoff_notes.clone(); let config = CompositeRouterConfig { diff --git a/crates/switchyard-runner/src/config.rs b/crates/switchyard-runner/src/config.rs index e2e1c7b14..2095638e1 100644 --- a/crates/switchyard-runner/src/config.rs +++ b/crates/switchyard-runner/src/config.rs @@ -895,6 +895,7 @@ capable_target = "strong" efficient_target = "weak" picker = "efficient_first" confidence_threshold = 1.0 +capable_hold_turns = 2 [routes.stage.tool_semantics] observe = ["lookup_customer"] @@ -929,6 +930,7 @@ classify_trigger = "user_turn" capable_target = "strong" efficient_target = "weak" confidence_threshold = 0.5 +capable_hold_turns = 2 [routes.composed.stage.tool_semantics] new = ["send_message"] diff --git a/docs/reference/toml_schema.md b/docs/reference/toml_schema.md index b482ecb6e..d505cca28 100644 --- a/docs/reference/toml_schema.md +++ b/docs/reference/toml_schema.md @@ -266,6 +266,7 @@ optional `handoff_notes` and `classifier` tables and for tuning. | `picker` | Yes | — | `efficient_first`, or `capable_first` (experimental, unbenchmarked). Tier used when the signals are not confident. | | `confidence_threshold` | Yes | — | Corroboration a decisive pick needs. In `[0, 1]`. | | `recent_turn_window` | No | `3` | Trailing tool results the signals are computed over. | +| `capable_hold_turns` | No | `2` | Requests kept on the capable tier after escalation. A clean test pass clears the hold early; `0` disables it. | | `tool_semantics.observe` | No | `[]` | Exact ASCII case-insensitive domain tool names that count as read-only investigation. | | `tool_semantics.mutate` | No | `[]` | Exact ASCII case-insensitive domain tool names that count as state-changing production. | | `tool_semantics.plan` | No | `[]` | Exact ASCII case-insensitive domain tool names that count as planning or task decomposition. | @@ -305,6 +306,7 @@ configuration. Today a classifier sets the tier a stage router falls open to whe | `stage.efficient_target` | Yes | — | Efficient tier. | | `stage.confidence_threshold` | Yes | — | Corroboration a decisive signal needs. In `[0, 1]`. | | `stage.recent_turn_window` | No | `3` | Trailing tool results the signals are computed over. | +| `stage.capable_hold_turns` | No | `2` | Requests kept on the capable tier after escalation. A clean test pass clears the hold early; `0` disables it. | | `stage.tool_semantics.observe` | No | `[]` | Additional exact ASCII case-insensitive tool names that count as observation. | | `stage.tool_semantics.mutate` | No | `[]` | Additional exact ASCII case-insensitive tool names that count as mutation. | | `stage.tool_semantics.plan` | No | `[]` | Additional exact ASCII case-insensitive tool names that count as planning. | diff --git a/docs/routing_algorithms/stage_router_routing.md b/docs/routing_algorithms/stage_router_routing.md index 6e7ff9ac1..f9f600af1 100644 --- a/docs/routing_algorithms/stage_router_routing.md +++ b/docs/routing_algorithms/stage_router_routing.md @@ -31,7 +31,7 @@ For each LLM call, stage-router estimates which stage the agent is in from the The axes are **corroborative**: the signed score is `tanh`-squashed to a confidence in `[0, 1]`, so one full signal alone scores ~`0.46` and a second corroborating signal is what pushes it decisively past a `0.5` threshold. A -critical-error severity is a hard override that escalates on its own. The router +repeated failures and critical-error severity are hard overrides that escalate. The router then routes: - the **capable** tier for uncertain, exploratory, or error-recovery turns, and @@ -199,6 +199,7 @@ efficient_target = "weak" picker = "efficient_first" confidence_threshold = 0.5 recent_turn_window = 3 # optional, defaults to 3 +capable_hold_turns = 2 # optional, defaults to 2 ``` Save as `routes.toml` and start the server: @@ -242,8 +243,8 @@ rules are outside this exact-name configuration. Add a `[routes.stage.handoff_notes]` section to pass a contextual note to the model the router switches to. The escalation note is sent to the capable tier on -a signal-driven escalation; the de-escalation note is sent back to the efficient -tier when a settled signal drops the turn there. +a signal-driven escalation; the de-escalation note is sent when the scorer +decisively picks the efficient tier. ```toml [routes.stage.handoff_notes] @@ -302,8 +303,8 @@ paths through its cascade: | Source | When | |---|---| -| `override` | A critical-error severity (or a context-compaction marker) forced the capable tier. | -| `tests_passed` | A settled run — a recent test pass with a recent write and no windowed error — landed the turn on the efficient tier. | +| `override` | A repeated failure, critical-error severity, or context-compaction marker forced the capable tier. Structured logs set `override_reason` to `repeated_failure`, `critical_error`, or `compaction`. | +| `capable_hold` | A recent escalation kept this recovery turn on the capable tier. | | `dimensions` | The corroborative scorer crossed `confidence_threshold` and picked the tier by the sign of the score. | | `llm-classifier` | The signals were ambiguous and the classifier returned a verdict. | | `fall_open` | The signals were ambiguous and the classifier failed or wasn't configured; the default tier was used. | diff --git a/examples/litellm/tests/unit/test_routing_plugin_configuration.py b/examples/litellm/tests/unit/test_routing_plugin_configuration.py index 123fb7681..73fbbcfc4 100644 --- a/examples/litellm/tests/unit/test_routing_plugin_configuration.py +++ b/examples/litellm/tests/unit/test_routing_plugin_configuration.py @@ -48,7 +48,7 @@ async def test_loads_stage_prompt_and_handoff_rewrites(tmp_path: Path) -> None: config_path.write_text( 'algorithm = "stage"\n' 'picker = "capable_first"\n' - "confidence_threshold = 0.5\n" + "confidence_threshold = 0.4\n" 'escalation_note = "The efficient tier failed."\n' 'deescalation_note = "The capable tier recovered."\n' 'capable_system_prompt = "Use the capable tier."\n'