From faa0a76e5db3f5d7750f9ee0bc27bf6079b2712c Mon Sep 17 00:00:00 2001 From: nachiketb Date: Wed, 9 Sep 2026 12:57:50 -0700 Subject: [PATCH 1/3] feat(libsy): populate bounded outcome evidence Signed-off-by: nachiketb --- crates/libsy/src/algorithms/advisor_gate.rs | 47 ++++++++++++++----- crates/libsy/src/algorithms/composite.rs | 3 ++ crates/libsy/src/algorithms/escalation.rs | 27 ++++++++++- crates/libsy/src/algorithms/fall_through.rs | 5 +- crates/libsy/src/algorithms/llm_class.rs | 25 +++++++++- crates/libsy/src/algorithms/stage.rs | 14 +++++- crates/libsy/src/algorithms/util/affinity.rs | 7 ++- .../libsy/src/algorithms/util/escalation.rs | 16 ++++++- crates/libsy/src/algorithms/util/llm_judge.rs | 46 +++++++++++++----- crates/libsy/src/algorithms/util/stage.rs | 13 ++++- crates/libsy/src/algorithms/util/subagent.rs | 5 +- crates/libsy/src/core/algorithm.rs | 31 ++++++++++-- crates/libsy/src/core/outcome_metadata.rs | 6 ++- 13 files changed, 206 insertions(+), 39 deletions(-) diff --git a/crates/libsy/src/algorithms/advisor_gate.rs b/crates/libsy/src/algorithms/advisor_gate.rs index 87c15bd1f..72fc2a230 100644 --- a/crates/libsy/src/algorithms/advisor_gate.rs +++ b/crates/libsy/src/algorithms/advisor_gate.rs @@ -270,14 +270,34 @@ impl AdvisorGate { .consult(driver, &request, review_tail.as_deref(), trigger_label) .await { - Ok(ConsultOutcome::Approve) => Ok(RoutingOutcome::answered( - self.executor.clone(), - request, - turn.into_response(), - )), - Ok(ConsultOutcome::Redo { plan }) => Ok(self.redo(request, turn, &plan)), - Ok(ConsultOutcome::Failed) => { + Ok(ConsultOutcome::Approve) => { + driver.set_evidence(serde_json::json!({ + "source": "advisor", + "verdict": "approve", + "trigger": trigger_label, + })); + Ok(RoutingOutcome::answered( + self.executor.clone(), + request, + turn.into_response(), + )) + } + Ok(ConsultOutcome::Redo { plan }) => { + driver.set_evidence(serde_json::json!({ + "source": "advisor", + "verdict": "redo", + "trigger": trigger_label, + })); + Ok(self.redo(request, turn, &plan)) + } + Ok(ConsultOutcome::Failed { reason }) => { self.budget.refund_failure(scope); + driver.set_evidence(serde_json::json!({ + "source": "advisor", + "verdict": "fail_open", + "trigger": trigger_label, + "reason_code": reason, + })); Ok(RoutingOutcome::answered( self.executor.clone(), request, @@ -361,9 +381,8 @@ impl AdvisorGate { let agg = match reply { Ok(agg) => agg, Err(error) => { - record_consult_failure(crate::algorithms::util::llm_judge::libsy_error_reason( - &error, - )); + let reason = crate::algorithms::util::llm_judge::libsy_error_reason(&error); + record_consult_failure(reason); if !self.config.fail_open { // Surface as an algorithm failure (5xx), never as the // advisor's own client error: a typed ContextWindowExceeded @@ -386,7 +405,7 @@ impl AdvisorGate { reply_head: None, usage: None, }); - return Ok(ConsultOutcome::Failed); + return Ok(ConsultOutcome::Failed { reason }); } }; let reply_text = advisor_reply_text(&agg); @@ -426,7 +445,9 @@ impl AdvisorGate { reply_head: Some(reply_head), usage: Some(&agg.usage), }); - Ok(ConsultOutcome::Failed) + Ok(ConsultOutcome::Failed { + reason: "parse_error", + }) } } } @@ -485,7 +506,7 @@ impl Algorithm for AdvisorGate { enum ConsultOutcome { Approve, Redo { plan: String }, - Failed, + Failed { reason: &'static str }, } fn algorithm_error(message: impl Into) -> LibsyError { diff --git a/crates/libsy/src/algorithms/composite.rs b/crates/libsy/src/algorithms/composite.rs index 9d1916446..af75f7b64 100644 --- a/crates/libsy/src/algorithms/composite.rs +++ b/crates/libsy/src/algorithms/composite.rs @@ -90,6 +90,9 @@ impl Processor for TierSetter { if let Some(tier) = identity.and_then(|identity| self.tiers.lock().get(&identity).copied()) { set_fall_open(state, tier); + if let Some(driver) = driver { + driver.set_evidence_if_empty(serde_json::json!({"source": "retained"})); + } } Ok(()) } diff --git a/crates/libsy/src/algorithms/escalation.rs b/crates/libsy/src/algorithms/escalation.rs index 593c22925..47523dc13 100644 --- a/crates/libsy/src/algorithms/escalation.rs +++ b/crates/libsy/src/algorithms/escalation.rs @@ -92,6 +92,10 @@ impl Classifier for EscalationClassifier { // A confirmed session stays capable without a judge call. if streak(state) >= self.confirmations { + driver.set_evidence(serde_json::json!({ + "source": "escalation", + "verdict": "latched", + })); return Ok((decisive(&self.capable), None)); } @@ -111,7 +115,13 @@ impl Classifier for EscalationClassifier { Err(LibsyError::ClientCall { source: LlmClientError::ContextWindowExceeded { .. }, .. - }) => return Ok((decisive(&self.capable), None)), + }) => { + driver.set_evidence(serde_json::json!({ + "source": "fallback", + "reason_code": "context_window", + })); + return Ok((decisive(&self.capable), None)); + } Err(e) => return Err(e), }; // The call resolves when its stream handle arrives; transport can still fail while @@ -119,6 +129,10 @@ impl Classifier for EscalationClassifier { let agg = match efficient_response.llm_response.into_agg().await { Ok(agg) => agg, Err(LlmClientError::Transport { .. }) => { + driver.set_evidence(serde_json::json!({ + "source": "fallback", + "reason_code": "transport", + })); return Ok((decisive(&self.capable), None)); } Err(source) => { @@ -158,9 +172,20 @@ impl Classifier for EscalationClassifier { if escalate && pending >= self.confirmations { // Streak confirmed: drop the efficient response, caller will serve capable. + driver.set_evidence(serde_json::json!({ + "source": "escalation", + "verdict": "escalate", + })); return Ok((decisive(&self.capable), None)); } + if escalate { + driver.set_evidence(serde_json::json!({ + "source": "escalation", + "verdict": "pending", + })); + } + Ok((decisive(&self.efficient), Some(efficient_response))) } } diff --git a/crates/libsy/src/algorithms/fall_through.rs b/crates/libsy/src/algorithms/fall_through.rs index 2146d96fb..c2cebe68a 100644 --- a/crates/libsy/src/algorithms/fall_through.rs +++ b/crates/libsy/src/algorithms/fall_through.rs @@ -73,9 +73,12 @@ impl Classifier for DefaultTarget { &self, _state: &mut S, _request: &mut Request, - _driver: Option<&Driver>, + driver: Option<&Driver>, ) -> Result<(Classification, Option)> { // Zero confidence: this is a fallback, not a judgement. + if let Some(driver) = driver { + driver.set_evidence_if_empty(serde_json::json!({"source": "fall_open"})); + } Ok(( Classification::Scores(vec![Score { target: self.target.clone(), diff --git a/crates/libsy/src/algorithms/llm_class.rs b/crates/libsy/src/algorithms/llm_class.rs index 599e9f9e0..b78d20ad5 100644 --- a/crates/libsy/src/algorithms/llm_class.rs +++ b/crates/libsy/src/algorithms/llm_class.rs @@ -243,6 +243,28 @@ impl JudgePolicy for TaskClassifierPolicy { } } +fn capability_evidence( + policy: &TaskClassifierPolicy, + verdict: Option<&TaskClassifierVerdict>, +) -> Option { + let verdict = verdict?; + let Some(threshold) = verdict + .is_valid() + .then(|| policy.threshold(verdict)) + .flatten() + else { + return Some(serde_json::json!({ + "source": "fail_open", + "reason_code": "invalid_verdict", + })); + }; + Some(serde_json::json!({ + "source": "llm_classifier", + "score": verdict.p_solve, + "threshold": threshold, + })) +} + #[derive(Clone, Debug)] /// Settings that control capability classifier prompting and routing. pub struct TaskClassifierConfig { @@ -605,7 +627,8 @@ impl LlmTaskClassifier { capable_target.clone(), &config, ), - ), + ) + .with_evidence(capability_evidence), capable_target: capable_target.clone(), }); let inner: Arc> = classifier.clone(); diff --git a/crates/libsy/src/algorithms/stage.rs b/crates/libsy/src/algorithms/stage.rs index 483a46282..e30dc38f4 100644 --- a/crates/libsy/src/algorithms/stage.rs +++ b/crates/libsy/src/algorithms/stage.rs @@ -56,6 +56,15 @@ impl Classifier for SourceStamp { if let Some(winner) = classification.argmax(false)? { record_decision_source(state, self.source); record_routing_decision(self.source, &winner.target); + if let Some(driver) = driver { + let source = match self.source { + DecisionSource::LlmClassifier => "llm_classifier", + source => source.as_str(), + }; + driver.set_evidence_if_empty(serde_json::json!({ + "source": source, + })); + } } Ok((classification, served)) } @@ -73,10 +82,13 @@ impl Classifier for FallOpen { &self, state: &mut State, _request: &mut Request, - _driver: Option<&Driver>, + driver: Option<&Driver>, ) -> Result<(Classification, Option)> { let tier = fall_open_tier(state).unwrap_or(self.default_tier); let target = self.targets.name(tier).clone(); + if let Some(driver) = driver { + driver.set_evidence_if_empty(serde_json::json!({"source": "fall_open"})); + } Ok(( Classification::Scores(vec![Score { target, diff --git a/crates/libsy/src/algorithms/util/affinity.rs b/crates/libsy/src/algorithms/util/affinity.rs index 7c696aa4e..f557a28d6 100644 --- a/crates/libsy/src/algorithms/util/affinity.rs +++ b/crates/libsy/src/algorithms/util/affinity.rs @@ -232,7 +232,7 @@ where &self, _state: &mut S, request: &mut Request, - _driver: Option<&Driver>, + driver: Option<&Driver>, ) -> crate::Result<(Classification, Option)> { let Some(key) = self.affinity_key(request) else { return Ok((Classification::Scores(Vec::new()), None)); @@ -243,6 +243,11 @@ where return Ok((Classification::Scores(Vec::new()), None)); } let assigned = self.assignments.lock().get(&key).cloned(); + if assigned.is_some() + && let Some(driver) = driver + { + driver.set_evidence(serde_json::json!({"source": "retained"})); + } Ok(( Classification::Scores(match assigned { Some(target) => vec![Score { diff --git a/crates/libsy/src/algorithms/util/escalation.rs b/crates/libsy/src/algorithms/util/escalation.rs index 54ded690c..92b02ce22 100644 --- a/crates/libsy/src/algorithms/util/escalation.rs +++ b/crates/libsy/src/algorithms/util/escalation.rs @@ -8,6 +8,7 @@ //! lives with the assembled algorithm in [`crate::algorithms::escalation`]. use serde::Deserialize; +use serde_json::Value; use switchyard_protocol::{ContentBlock, Message, ModelId, Role}; use super::classifier_contract::{ClassifierContract, ClassifierContractConfig}; @@ -139,6 +140,18 @@ impl JudgePolicy for EscalationPolicy { } } +fn escalation_evidence( + _policy: &EscalationPolicy, + verdict: Option<&EscalationVerdict>, +) -> Option { + verdict.map(|verdict| { + serde_json::json!({ + "source": "escalation", + "verdict": if verdict.escalate { "escalate" } else { "continue" }, + }) + }) +} + /// Builds the trajectory judge over `judge_target`, scoring `capable` when it escalates. /// /// Loads the packaged prompt and schema, so an unusable asset or an unusable `config` value @@ -163,7 +176,8 @@ pub(crate) fn build_judge( ), judge_target, EscalationPolicy { capable, efficient }, - )) + ) + .with_evidence(escalation_evidence)) } /// The 1-indexed model invocation the transcript ends on: one per assistant reply. diff --git a/crates/libsy/src/algorithms/util/llm_judge.rs b/crates/libsy/src/algorithms/util/llm_judge.rs index 0210bda27..9c5e16849 100644 --- a/crates/libsy/src/algorithms/util/llm_judge.rs +++ b/crates/libsy/src/algorithms/util/llm_judge.rs @@ -202,11 +202,18 @@ pub trait JudgePolicy: Send + Sync { fn to_classification(&self, verdict: Option<&Self::Verdict>) -> Classification; } +type EvidenceFn = fn(&P, Option<&V>) -> Option; + /// A classifier that calls one judge target and routes through its verdict policy. -pub struct JudgeClassifier { +pub struct JudgeClassifier +where + J: Judge, + P: JudgePolicy, +{ judge: J, target: ModelId, policy: P, + evidence: Option>, } impl JudgeClassifier @@ -220,6 +227,23 @@ where judge, target, policy, + evidence: None, + } + } + + /// Enables bounded evidence for built-in judges without widening the public policy trait. + pub(crate) fn with_evidence(mut self, evidence: EvidenceFn) -> Self { + self.evidence = Some(evidence); + self + } + + fn report_fail_open(&self, driver: &Driver, error: String, reason: &'static str) { + report_fail_open(self.target.as_str(), error, reason); + if self.evidence.is_some() { + driver.set_evidence(serde_json::json!({ + "source": "fail_open", + "reason_code": reason, + })); } } @@ -246,11 +270,7 @@ where ) .await .inspect_err(|error| { - report_fail_open( - judge_model, - safe_error_summary(error), - libsy_error_reason(error), - ) + self.report_fail_open(driver, safe_error_summary(error), libsy_error_reason(error)); }) .ok()?; let aggregate = response @@ -258,17 +278,13 @@ where .into_agg() .await .inspect_err(|error| { - report_fail_open( - judge_model, - safe_client_error(error), - client_error_reason(error), - ) + self.report_fail_open(driver, safe_client_error(error), client_error_reason(error)); }) .ok()?; self.judge .parse(&aggregate) .inspect_err(|error| { - report_fail_open(judge_model, safe_error_summary(error), "parse_error") + self.report_fail_open(driver, safe_error_summary(error), "parse_error"); }) .ok() } @@ -333,6 +349,12 @@ where }); }; let verdict = self.verdict(state, request, driver).await; + if let Some(evidence) = self + .evidence + .and_then(|evidence| evidence(&self.policy, verdict.as_ref())) + { + driver.set_evidence(evidence); + } // A judge consultation is a side call, never the turn's answer. Ok((self.policy.to_classification(verdict.as_ref()), None)) } diff --git a/crates/libsy/src/algorithms/util/stage.rs b/crates/libsy/src/algorithms/util/stage.rs index 44098293c..d5da79596 100644 --- a/crates/libsy/src/algorithms/util/stage.rs +++ b/crates/libsy/src/algorithms/util/stage.rs @@ -604,7 +604,7 @@ impl Classifier for StageClassifier { &self, state: &mut State, request: &mut Request, - _driver: Option<&Driver>, + driver: Option<&Driver>, ) -> Result<(Classification, Option)> { let tool_signals = &state.tool_signals; let Some(signal) = tool_signals else { @@ -629,6 +629,17 @@ impl Classifier for StageClassifier { // is the only branch whose tier the signals actually chose — an // ambiguous turn is decided further down the cascade. self.apply_handoff_note(request, tier, source); + if let Some(driver) = driver { + let evidence = match (source, confidence) { + (DecisionSource::Dimensions, Some(confidence)) => serde_json::json!({ + "source": source.as_str(), + "confidence": confidence, + "threshold": self.confidence_threshold, + }), + _ => serde_json::json!({"source": source.as_str()}), + }; + driver.set_evidence(evidence); + } // Prefer pick_tier's own confidence (e.g. 1.0 for an Override) // over re-deriving it from the neutral 0.5 placeholder. let conf = confidence.unwrap_or_else(|| 2.0 * (probability - 0.5).abs()); diff --git a/crates/libsy/src/algorithms/util/subagent.rs b/crates/libsy/src/algorithms/util/subagent.rs index 0294dc158..10447b65e 100644 --- a/crates/libsy/src/algorithms/util/subagent.rs +++ b/crates/libsy/src/algorithms/util/subagent.rs @@ -123,7 +123,7 @@ where &self, _state: &mut S, request: &mut Request, - _driver: Option<&Driver>, + driver: Option<&Driver>, ) -> Result<(Classification, Option)> { // Delegated *work* only. A harness maintenance turn (e.g. Codex `compact`) carries // sub-agent lineage but is not delegated work, so it abstains and routes normally. @@ -131,6 +131,9 @@ where .metadata .as_ref() .is_some_and(Metadata::is_subagent_work); + if is_delegated_work && let Some(driver) = driver { + driver.set_evidence(serde_json::json!({"source": "subagent"})); + } Ok(( Classification::Scores(if is_delegated_work { vec![Score { diff --git a/crates/libsy/src/core/algorithm.rs b/crates/libsy/src/core/algorithm.rs index 56e56f5fd..cecab86af 100644 --- a/crates/libsy/src/core/algorithm.rs +++ b/crates/libsy/src/core/algorithm.rs @@ -8,6 +8,8 @@ use std::{future::Future, panic::AssertUnwindSafe, pin::Pin, sync::Arc, time::In use async_trait::async_trait; use futures::{FutureExt, Stream, StreamExt}; +use parking_lot::Mutex; +use serde_json::Value; use tokio::sync::{mpsc, oneshot}; use tokio_stream::wrappers::ReceiverStream; use tracing::Instrument; @@ -121,6 +123,8 @@ pub struct Driver { step_tx: mpsc::Sender>, /// The owning algorithm's telemetry label, stamped onto every call this driver publishes. algorithm: String, + /// Run-scoped evidence shared by driver clones and attached only to a successful outcome. + evidence: Arc>>, } impl Driver { @@ -136,11 +140,25 @@ impl Driver { Self { step_tx, algorithm: algorithm.to_string(), + evidence: Arc::new(Mutex::new(None)), }, step_rx, ) } + /// Replace the current run's evidence when a component makes the final decision. + pub(crate) fn set_evidence(&self, evidence: Value) { + *self.evidence.lock() = Some(evidence); + } + + /// Supply fallback evidence without replacing a decision made earlier in the cascade. + pub(crate) fn set_evidence_if_empty(&self, evidence: Value) { + let mut current = self.evidence.lock(); + if current.is_none() { + *current = Some(evidence); + } + } + /// Publish a model call and await the consumer's response. /// /// Errors if the stream is closed or the call failed. @@ -204,9 +222,9 @@ impl Driver { /// when the algorithm finishes. pub(crate) async fn finish(&self, result: Result) -> Result<()> { let result = result.map(|mut outcome| { - let metadata = outcome - .metadata - .get_or_insert_with(|| crate::OutcomeMetadata::new(self.algorithm.clone(), None)); + let metadata = outcome.metadata.get_or_insert_with(|| { + crate::OutcomeMetadata::new(self.algorithm.clone(), self.evidence.lock().take()) + }); tracing::Span::current().record("outcome_id", metadata.outcome_id()); outcome }); @@ -466,6 +484,8 @@ mod tests { let response = driver .call_model(request.clone(), vec![target.clone()]) .await?; + driver.set_evidence(serde_json::json!({"source": "test"})); + driver.set_evidence_if_empty(serde_json::json!({"source": "ignored"})); Ok(RoutingOutcome::answered(target, request, response)) } } @@ -743,7 +763,10 @@ mod tests { .get_version_num(), 7 ); - assert!(metadata.evidence.is_none()); + assert_eq!( + metadata.evidence, + Some(serde_json::json!({"source": "test"})) + ); let response = outcome .response .ok_or_else(|| test_error("expected an answered outcome"))?; diff --git a/crates/libsy/src/core/outcome_metadata.rs b/crates/libsy/src/core/outcome_metadata.rs index 595e09961..8362944c5 100644 --- a/crates/libsy/src/core/outcome_metadata.rs +++ b/crates/libsy/src/core/outcome_metadata.rs @@ -3,6 +3,8 @@ //! Metadata describing a routing outcome. +use serde_json::Value; + /// Identity and optional algorithm evidence attached to a successful routing outcome. #[derive(Clone, Debug, PartialEq, Eq)] pub struct OutcomeMetadata { @@ -10,12 +12,12 @@ pub struct OutcomeMetadata { /// Stable name of the algorithm that produced the outcome. pub algorithm: String, /// Optional bounded, machine-readable evidence produced by the algorithm. - pub evidence: Option, + pub evidence: Option, } impl OutcomeMetadata { /// Creates outcome metadata with a new UUIDv7 identifier. - pub fn new(algorithm: String, evidence: Option) -> Self { + pub fn new(algorithm: String, evidence: Option) -> Self { Self { outcome_id: uuid::Uuid::now_v7().to_string(), algorithm, From 6b9c16c6dbfa031f817b969eac253f0151e3b0b7 Mon Sep 17 00:00:00 2001 From: nachiketb Date: Wed, 9 Sep 2026 13:06:49 -0700 Subject: [PATCH 2/3] docs(libsy): clarify evidence helper contracts Signed-off-by: nachiketb --- crates/libsy/src/algorithms/llm_class.rs | 1 + crates/libsy/src/algorithms/util/escalation.rs | 1 + crates/libsy/src/algorithms/util/llm_judge.rs | 1 + 3 files changed, 3 insertions(+) diff --git a/crates/libsy/src/algorithms/llm_class.rs b/crates/libsy/src/algorithms/llm_class.rs index b78d20ad5..fbc474067 100644 --- a/crates/libsy/src/algorithms/llm_class.rs +++ b/crates/libsy/src/algorithms/llm_class.rs @@ -243,6 +243,7 @@ impl JudgePolicy for TaskClassifierPolicy { } } +/// Maps valid verdicts to scores, invalid verdicts to a reason, and leaves absent verdicts alone. fn capability_evidence( policy: &TaskClassifierPolicy, verdict: Option<&TaskClassifierVerdict>, diff --git a/crates/libsy/src/algorithms/util/escalation.rs b/crates/libsy/src/algorithms/util/escalation.rs index 92b02ce22..00b47f98e 100644 --- a/crates/libsy/src/algorithms/util/escalation.rs +++ b/crates/libsy/src/algorithms/util/escalation.rs @@ -140,6 +140,7 @@ impl JudgePolicy for EscalationPolicy { } } +/// Maps present verdicts to stable `escalate` or `continue` values; absent verdicts add nothing. fn escalation_evidence( _policy: &EscalationPolicy, verdict: Option<&EscalationVerdict>, diff --git a/crates/libsy/src/algorithms/util/llm_judge.rs b/crates/libsy/src/algorithms/util/llm_judge.rs index 9c5e16849..0c53d21ec 100644 --- a/crates/libsy/src/algorithms/util/llm_judge.rs +++ b/crates/libsy/src/algorithms/util/llm_judge.rs @@ -237,6 +237,7 @@ where self } + /// Replaces run evidence only for judges that opted into structured evidence. fn report_fail_open(&self, driver: &Driver, error: String, reason: &'static str) { report_fail_open(self.target.as_str(), error, reason); if self.evidence.is_some() { From 86a0a41ed7f9234806afb2b8fe3ad252c56bfa8e Mon Sep 17 00:00:00 2001 From: nachiketb Date: Wed, 9 Sep 2026 13:50:07 -0700 Subject: [PATCH 3/3] fix(libsy): preserve deciding outcome evidence Signed-off-by: nachiketb --- crates/libsy/src/algorithms/llm_class.rs | 2 +- crates/libsy/src/algorithms/stage.rs | 11 ++--------- crates/libsy/src/algorithms/util/llm_judge.rs | 14 ++++++++++---- crates/libsy/src/core/outcome_metadata.rs | 6 +++++- 4 files changed, 18 insertions(+), 15 deletions(-) diff --git a/crates/libsy/src/algorithms/llm_class.rs b/crates/libsy/src/algorithms/llm_class.rs index fbc474067..652d647cf 100644 --- a/crates/libsy/src/algorithms/llm_class.rs +++ b/crates/libsy/src/algorithms/llm_class.rs @@ -260,7 +260,7 @@ fn capability_evidence( })); }; Some(serde_json::json!({ - "source": "llm_classifier", + "source": "llm-classifier", "score": verdict.p_solve, "threshold": threshold, })) diff --git a/crates/libsy/src/algorithms/stage.rs b/crates/libsy/src/algorithms/stage.rs index e30dc38f4..3fd4acc1e 100644 --- a/crates/libsy/src/algorithms/stage.rs +++ b/crates/libsy/src/algorithms/stage.rs @@ -57,12 +57,8 @@ impl Classifier for SourceStamp { record_decision_source(state, self.source); record_routing_decision(self.source, &winner.target); if let Some(driver) = driver { - let source = match self.source { - DecisionSource::LlmClassifier => "llm_classifier", - source => source.as_str(), - }; driver.set_evidence_if_empty(serde_json::json!({ - "source": source, + "source": self.source.as_str(), })); } } @@ -82,13 +78,10 @@ impl Classifier for FallOpen { &self, state: &mut State, _request: &mut Request, - driver: Option<&Driver>, + _driver: Option<&Driver>, ) -> Result<(Classification, Option)> { let tier = fall_open_tier(state).unwrap_or(self.default_tier); let target = self.targets.name(tier).clone(); - if let Some(driver) = driver { - driver.set_evidence_if_empty(serde_json::json!({"source": "fall_open"})); - } Ok(( Classification::Scores(vec![Score { target, diff --git a/crates/libsy/src/algorithms/util/llm_judge.rs b/crates/libsy/src/algorithms/util/llm_judge.rs index 0c53d21ec..a99fb82e5 100644 --- a/crates/libsy/src/algorithms/util/llm_judge.rs +++ b/crates/libsy/src/algorithms/util/llm_judge.rs @@ -237,11 +237,11 @@ where self } - /// Replaces run evidence only for judges that opted into structured evidence. + /// Adds fail-open evidence only for evidence-enabled judges and preserves an earlier decision. fn report_fail_open(&self, driver: &Driver, error: String, reason: &'static str) { report_fail_open(self.target.as_str(), error, reason); if self.evidence.is_some() { - driver.set_evidence(serde_json::json!({ + driver.set_evidence_if_empty(serde_json::json!({ "source": "fail_open", "reason_code": reason, })); @@ -350,14 +350,20 @@ where }); }; let verdict = self.verdict(state, request, driver).await; + let classification = self.policy.to_classification(verdict.as_ref()); if let Some(evidence) = self .evidence .and_then(|evidence| evidence(&self.policy, verdict.as_ref())) { - driver.set_evidence(evidence); + match &classification { + Classification::Scores(scores) if !scores.is_empty() => { + driver.set_evidence(evidence); + } + _ => driver.set_evidence_if_empty(evidence), + } } // A judge consultation is a side call, never the turn's answer. - Ok((self.policy.to_classification(verdict.as_ref()), None)) + Ok((classification, None)) } } diff --git a/crates/libsy/src/core/outcome_metadata.rs b/crates/libsy/src/core/outcome_metadata.rs index 8362944c5..ee8277727 100644 --- a/crates/libsy/src/core/outcome_metadata.rs +++ b/crates/libsy/src/core/outcome_metadata.rs @@ -11,7 +11,11 @@ pub struct OutcomeMetadata { outcome_id: String, /// Stable name of the algorithm that produced the outcome. pub algorithm: String, - /// Optional bounded, machine-readable evidence produced by the algorithm. + /// Optional algorithm-defined JSON evidence. + /// + /// Built-in algorithms emit an object with a stable `source` string and only the + /// relevant `score`, `confidence`, `threshold`, `verdict`, `trigger`, or `reason_code` + /// fields. Not every algorithm or decision produces evidence. pub evidence: Option, }