From b04238fd041184e3245a3329290b0e6085034cf4 Mon Sep 17 00:00:00 2001 From: Lin Jia Date: Sun, 6 Sep 2026 09:14:29 -0700 Subject: [PATCH 1/2] Log escalation judge verdicts at debug The escalation judge already returns a reason with every verdict, but the router discarded it after reading the boolean, so an operator tuning the judge prompt could not see why sessions were held on the efficient tier. Keep the reason on the verdict and emit one debug event per verdict under the util::escalation target; it is silent unless that target is enabled. Co-Authored-By: Claude Fable 5.1 (cherry picked from commit 5e6cf24ed4a86b223c2aba4940f43c655049a58e) Signed-off-by: Lin Jia --- crates/libsy/src/algorithms/util/escalation.rs | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/crates/libsy/src/algorithms/util/escalation.rs b/crates/libsy/src/algorithms/util/escalation.rs index 54ded690c..bdc6ac95e 100644 --- a/crates/libsy/src/algorithms/util/escalation.rs +++ b/crates/libsy/src/algorithms/util/escalation.rs @@ -88,11 +88,14 @@ impl Default for EscalationJudgeConfig { } /// The judge's verdict. The schema also requires a `reason`, which makes the judge state its -/// case and measurably sharpens the verdict — routing reads only the boolean, so it is -/// deserialized away rather than carried. +/// case and measurably sharpens the verdict. Routing reads only the boolean; the reason is +/// kept solely so an operator can see why the judge held or escalated when the +/// `switchyard_libsy::algorithms::util::escalation` target is enabled at `debug`. #[derive(Deserialize)] pub(crate) struct EscalationVerdict { escalate: bool, + #[serde(default)] + reason: String, } /// Builds the condensed trajectory presented to the escalation judge. @@ -125,6 +128,13 @@ impl JudgePolicy for EscalationPolicy { type Verdict = EscalationVerdict; fn to_classification(&self, verdict: Option<&EscalationVerdict>) -> Classification { + if let Some(verdict) = verdict { + tracing::debug!( + escalate = verdict.escalate, + reason = %verdict.reason, + "escalation judge verdict" + ); + } match verdict { Some(verdict) if verdict.escalate => Classification::Scores(vec![Score { target: self.capable.clone(), From c5cbfc90326bff9213134fefbf6aa548fa688bc5 Mon Sep 17 00:00:00 2001 From: Lin Jia Date: Sun, 6 Sep 2026 10:39:32 -0700 Subject: [PATCH 2/2] Anchor every pre-reply user message as task framing for the escalation judge The trajectory summary handed to the escalation judge pinned only the first user message as the task statement. Codex sends an environment context block as its first user message and the task as the second, so the judge saw shell and cwd boilerplate as the task while the real task sat in the rolling window, truncated to the per-message cap, and scrolled out after about thirty messages. From then on every task-aware pattern in the rubric (drift, unverified completion, violated constraints) had nothing to compare against, and only friction patterns could fire. Treat every user message that precedes the first assistant reply as task framing and anchor them all, with a wider per-message budget so a multi-thousand-character feature specification survives intact. Co-Authored-By: Claude Fable 5.1 (cherry picked from commit fc61f715ed165e73e0664e4a866ec3b99aa38742) Signed-off-by: Lin Jia --- .../libsy/src/algorithms/util/escalation.rs | 74 ++++++++++++++++--- 1 file changed, 63 insertions(+), 11 deletions(-) diff --git a/crates/libsy/src/algorithms/util/escalation.rs b/crates/libsy/src/algorithms/util/escalation.rs index bdc6ac95e..d29a4cb37 100644 --- a/crates/libsy/src/algorithms/util/escalation.rs +++ b/crates/libsy/src/algorithms/util/escalation.rs @@ -33,8 +33,12 @@ const TRUNCATION_SUFFIX: &str = "..."; /// which coding-agent harnesses make very large. const SYSTEM_CHARS: usize = 1_000; -/// Cap for the first user message — the task statement, so it gets the widest anchor budget. -const FIRST_USER_CHARS: usize = 2_000; +/// Per-message cap for task-framing user messages — every user message that precedes the first +/// assistant reply. Coding-agent harnesses often send environment boilerplate as the first user +/// message and the task itself as the second, so anchoring only the first would pin the +/// boilerplate and let the task scroll out of the window. Feature specifications run to several +/// thousand characters, so this gets the widest anchor budget. +const TASK_CHARS: usize = 4_000; /// Backstop on the assembled transcript; the per-message caps normally bind first. const MAX_REQUEST_CHARS: usize = 18_000; @@ -257,7 +261,7 @@ fn summarize_for_judge( ) -> String { let mut anchors: Vec = Vec::new(); let mut window: Vec = Vec::new(); - let mut first_user_seen = false; + let mut assistant_seen = false; for message in messages { let text = message_text(message); @@ -267,18 +271,23 @@ fn summarize_for_judge( role_label(message.role), truncate_middle(&text, SYSTEM_CHARS) )), - Role::User if !first_user_seen => { - first_user_seen = true; + // Everything the user said before the agent first replied is task framing. + Role::User if !assistant_seen => { anchors.push(format!( "[user (task)] {}", - truncate_middle(&text, FIRST_USER_CHARS) + truncate_middle(&text, TASK_CHARS) + )); + } + role => { + if role == Role::Assistant { + assistant_seen = true; + } + window.push(format!( + "[{}] {}", + role_label(role), + truncate_middle(&text, config.window_message_chars) )); } - role => window.push(format!( - "[{}] {}", - role_label(role), - truncate_middle(&text, config.window_message_chars) - )), } } @@ -503,6 +512,49 @@ mod tests { assert!(!summary.contains("step 6"), "{summary}"); } + #[test] + fn summary_anchors_every_user_message_before_the_first_reply() { + // Codex sends environment boilerplate as the first user message and the task as the + // second. Both are framing; the task must stay visible after the window has moved on. + let mut messages = vec![ + Message::text( + Role::Developer, + "...", + ), + Message::text( + Role::User, + "/app", + ), + Message::text(Role::User, "Implement RFC 5545 timezone interop in rrule."), + ]; + for i in 0..40 { + messages.push(Message::text(Role::Assistant, format!("step {i}"))); + messages.push(Message::text(Role::User, format!("later user note {i}"))); + } + let config = EscalationJudgeConfig { + recent_turn_window: 3, + ..EscalationJudgeConfig::default() + }; + + let summary = summarize_for_judge(&messages, 40, &config); + + assert!( + summary.contains("[user (task)] "), + "{summary}" + ); + assert!( + summary.contains("[user (task)] Implement RFC 5545 timezone interop in rrule."), + "{summary}" + ); + // User messages after the first reply are ordinary window entries, not anchors. + assert!( + !summary.contains("[user (task)] later user note"), + "{summary}" + ); + assert!(summary.contains("[user] later user note 39"), "{summary}"); + assert!(!summary.contains("later user note 0\n"), "{summary}"); + } + #[test] fn summary_drops_oldest_window_lines_under_the_char_cap() { // MAX_REQUEST_CHARS is a backstop, not a dial: at default settings the window caps