From 0d1b4858bc9c3562492424d29ef3eed5e572a00e Mon Sep 17 00:00:00 2001 From: nachiketb Date: Wed, 9 Sep 2026 15:16:38 -0700 Subject: [PATCH 1/2] feat(libsy): expose outcome metadata in OpenTelemetry and Python Signed-off-by: nachiketb --- .../libsy-llm-client/tests/observability.rs | 139 ++++++++++++------ crates/libsy/src/core/algorithm.rs | 18 ++- crates/libsy/src/observability.rs | 127 +++++++++------- crates/switchyard-py/src/libsy_bindings.rs | 41 +++++- docs/getting_started.md | 11 ++ switchyard/libsy/__init__.py | 2 + switchyard_rust/libsy.py | 17 +++ tests/test_libsy_minimal_bindings.py | 13 ++ 8 files changed, 260 insertions(+), 108 deletions(-) diff --git a/crates/libsy-llm-client/tests/observability.rs b/crates/libsy-llm-client/tests/observability.rs index 4d3921afa..95e505763 100644 --- a/crates/libsy-llm-client/tests/observability.rs +++ b/crates/libsy-llm-client/tests/observability.rs @@ -488,6 +488,7 @@ impl RoutedLlmClient for UsageClient { struct SingleCallAlgo { name: String, target_set: Vec, + metadata: Option, } #[async_trait] @@ -507,7 +508,18 @@ impl Algorithm for SingleCallAlgo { .ok_or(LibsyError::NoTargets)? .clone(); tracing::info!("picked '{target}'"); - Ok(RoutingOutcome::route_to(target.into(), Vec::new(), request)) + let mut outcome = RoutingOutcome::route_to( + target.into(), + self.target_set + .iter() + .skip(1) + .cloned() + .map(ModelId::from) + .collect(), + request, + ); + outcome.metadata = self.metadata.clone(); + Ok(outcome) } } @@ -561,6 +573,7 @@ fn algo(name: &str, model: &str) -> Arc { Arc::new(SingleCallAlgo { name: name.to_string(), target_set: vec![model.to_string()], + metadata: None, }) } @@ -748,7 +761,8 @@ async fn affinity_keeps_the_algorithm_selection_after_client_fallback() } #[tokio::test] -async fn successful_run_records_metrics_spans_and_decision_log() -> switchyard_libsy::Result<()> { +async fn successful_run_records_metrics_spans_and_outcome_metadata() -> switchyard_libsy::Result<()> +{ let _guard = serialize_test().lock().await; let (store, exporter, provider, span_exporter, _) = telemetry(); const ALGO: &str = "obs-success-algo"; @@ -775,7 +789,20 @@ async fn successful_run_records_metrics_spans_and_decision_log() -> switchyard_l request.llm_request.output.max_output_tokens = Some(512); request.llm_request.output.response_format = Some(json!({"type": "json_schema"})); request.llm_request.reasoning.effort = Some("high".to_string()); - let (selected_model, _response) = run(algo(ALGO, MODEL), client, request).await?; + let metadata = switchyard_libsy::OutcomeMetadata::new( + ALGO.to_string(), + Some(json!({ + "source": "llm-classifier", "score": 0.9, "threshold": 0.5, + "verdict": "continue", "trigger": "turn", "reason_code": "test", + "confidence": "wrong type", "unknown": LEAKED_CONTENT, + })), + ); + let algorithm = Arc::new(SingleCallAlgo { + name: ALGO.to_string(), + target_set: vec![MODEL.to_string(), "obs-fallback-model".to_string()], + metadata: Some(metadata.clone()), + }); + let (selected_model, _response) = run(algorithm, client, request).await?; assert_eq!(selected_model, MODEL); // Metrics: run/call counters and latency histograms keyed by algorithm, @@ -843,6 +870,51 @@ async fn successful_run_records_metrics_spans_and_decision_log() -> switchyard_l // llm_call span carrying the selection, outcome, and token counts. let spans = store.spans(); let run_span = find_span(&spans, "libsy.run", "algorithm", ALGO); + assert_eq!( + spans + .iter() + .filter(|span| span.name == "libsy.run" + && span.fields.get("algorithm").map(String::as_str) == Some(ALGO)) + .count(), + 1 + ); + for (field, expected) in [ + ("outcome_id", metadata.outcome_id()), + ("evidence.source", "llm-classifier"), + ("evidence.score", "0.9"), + ("evidence.threshold", "0.5"), + ("evidence.verdict", "continue"), + ("evidence.trigger", "turn"), + ("evidence.reason_code", "test"), + ] { + assert_eq!( + run_span.fields.get(field).map(String::as_str), + Some(expected), + "{field}" + ); + } + assert!(!run_span.fields.contains_key("evidence.confidence")); + assert!(!format!("{run_span:?}").contains(LEAKED_CONTENT)); + let exported = span_exporter.get_finished_spans().expect("exported spans"); + let exported_run = exported + .iter() + .find(|span| { + span.name == "libsy.run" + && otel_attribute(span, "outcome_id") + .is_some_and(|id| id.as_str() == metadata.outcome_id()) + }) + .expect("outcome span exported"); + assert_eq!( + otel_attribute(exported_run, "evidence.score"), + Some(&OtelValue::F64(0.9)) + ); + assert_eq!( + otel_attribute(exported_run, "selected_model_ids"), + Some(&OtelValue::Array(OtelArray::String(vec![ + MODEL.into(), + "obs-fallback-model".into() + ]),)) + ); assert_eq!(run_span.parent, None); assert_eq!( run_span.fields.get("session_id").map(String::as_str), @@ -880,13 +952,7 @@ async fn successful_run_records_metrics_spans_and_decision_log() -> switchyard_l run_span.fields.get("agent_role").map(String::as_str), Some("reviewer") ); - // Host-defined labels ride in generically via Metadata.extra_metadata. - assert!( - run_span - .fields - .get("extra_metadata") - .is_some_and(|extra| extra.contains("tenant") && extra.contains("obs-tenant-1")) - ); + assert!(!run_span.fields.contains_key("extra_metadata")); // The default-client serve inside `run` gets its own client-call span. let client_span = find_span(&spans, "libsy.client_call", "selected_model", MODEL); @@ -950,6 +1016,12 @@ async fn successful_run_records_metrics_spans_and_decision_log() -> switchyard_l // The algorithm logs why it made the decision. let events = store.events(); + assert!( + !events.iter().any( + |event| event.fields.get("algorithm").map(String::as_str) == Some(ALGO) + && event.fields.get("message").map(String::as_str) == Some("routing decision") + ) + ); assert!( events.iter().any(|event| { event.level == "INFO" @@ -1250,7 +1322,7 @@ async fn upstream_body_is_redacted_from_the_client_call_span() -> switchyard_lib } #[tokio::test] -async fn failed_call_records_error_outcome_and_warn_logs() -> switchyard_libsy::Result<()> { +async fn failed_call_records_metrics_without_error_details() -> switchyard_libsy::Result<()> { let _guard = serialize_test().lock().await; let (store, exporter, provider, _, _) = telemetry(); const ALGO: &str = "obs-failure-algo"; @@ -1272,7 +1344,10 @@ async fn failed_call_records_error_outcome_and_warn_logs() -> switchyard_libsy:: Ok(Step::Done(_)) => { return Err(test_error("expected the failed call to fail the run")); } - Err(_) => saw_error_step = true, + Err(error) => { + assert!(error.to_string().contains("synthetic upstream failure")); + saw_error_step = true; + } } } assert!( @@ -1307,51 +1382,27 @@ async fn failed_call_records_error_outcome_and_warn_logs() -> switchyard_libsy:: None ); - // Spans: both spans carry outcome=error and the propagated error text. + // Error details remain with the caller, while operational status is preserved. let spans = store.spans(); let run_span = find_span(&spans, "libsy.run", "algorithm", ALGO); assert_eq!( run_span.fields.get("outcome").map(String::as_str), Some("error") ); - assert!( - run_span - .fields - .get("error") - .is_some_and(|error| error.contains("synthetic upstream failure")) - ); + assert!(!run_span.fields.contains_key("error")); + assert!(!run_span.fields.contains_key("outcome_id")); let call_span = find_span(&spans, "libsy.llm_call", "selected_model", MODEL); assert_eq!( call_span.fields.get("outcome").map(String::as_str), Some("error") ); - // Structured logs warn once for the failed call and failed run. + assert_eq!(call_span.parent.as_deref(), Some("libsy.run")); + assert!(!call_span.fields.contains_key("error")); + let events = store.events(); - assert!( - events.iter().any(|event| { - event.target == "libsy" - && event.level == "WARN" - && event.fields.get("selected_model").map(String::as_str) == Some(MODEL) - && event - .fields - .get("message") - .is_some_and(|message| message.contains("model call failed")) - }), - "no call-failure log for {MODEL} in {events:?}" - ); - assert!( - events.iter().any(|event| { - event.target == "libsy" - && event.level == "WARN" - && event.fields.get("algorithm").map(String::as_str) == Some(ALGO) - && event - .fields - .get("message") - .is_some_and(|message| message.contains("algorithm run failed")) - }), - "no run-failure log for {ALGO} in {events:?}" - ); + assert!(!events.iter().any(|event| event.target == "libsy" + && event.fields.get("algorithm").map(String::as_str) == Some(ALGO))); Ok(()) } diff --git a/crates/libsy/src/core/algorithm.rs b/crates/libsy/src/core/algorithm.rs index cecab86af..a108aca66 100644 --- a/crates/libsy/src/core/algorithm.rs +++ b/crates/libsy/src/core/algorithm.rs @@ -176,7 +176,6 @@ impl Driver { selected_model = %models.first().map(ModelId::as_str).unwrap_or("NoTargets"), openinference.span.kind = "CHAIN", outcome = tracing::field::Empty, - error = tracing::field::Empty, input_tokens = tracing::field::Empty, output_tokens = tracing::field::Empty, total_tokens = tracing::field::Empty, @@ -225,7 +224,7 @@ impl Driver { let metadata = outcome.metadata.get_or_insert_with(|| { crate::OutcomeMetadata::new(self.algorithm.clone(), self.evidence.lock().take()) }); - tracing::Span::current().record("outcome_id", metadata.outcome_id()); + observability::record_outcome(metadata, &outcome.selected_model_ids); outcome }); let selected_model = result @@ -383,9 +382,18 @@ impl RoutingIdentity { /// # Observability /// /// [`run_stream`](Self::run_stream) creates a `libsy.run` span, and each offloaded model -/// call creates a `libsy.llm_call` span. Routing decisions and failures are emitted through -/// `tracing`; metrics use the global OpenTelemetry meter provider. The provider call -/// itself belongs to the host, and is instrumented by whoever makes it. +/// call creates a nested `libsy.llm_call` span. Successful outcomes record their +/// [`OutcomeMetadata`](crate::OutcomeMetadata) on `libsy.run`: `outcome_id`, `algorithm`, +/// `switchyard.algorithm`, and `selected_model_ids` (an ordered OpenTelemetry string array). +/// Optional `evidence.source`, `evidence.verdict`, `evidence.trigger`, and +/// `evidence.reason_code` are strings; `evidence.score`, `evidence.confidence`, and +/// `evidence.threshold` are numbers. Unknown evidence fields are not exported. +/// These fields are span attributes, never metric labels. +/// +/// Runs and calls retain their `outcome` status and operational metrics, but do not +/// export error details or arbitrary request extra metadata. Errors still reach the +/// caller unchanged. The host controls the tracing subscriber and global OpenTelemetry +/// meter provider; libsy installs no exporter and performs no telemetry network I/O. #[async_trait] pub trait Algorithm: Send + Sync + 'static { /// Stable, low-cardinality name identifying this algorithm — the diff --git a/crates/libsy/src/observability.rs b/crates/libsy/src/observability.rs index 763d31153..b95b257db 100644 --- a/crates/libsy/src/observability.rs +++ b/crates/libsy/src/observability.rs @@ -38,8 +38,9 @@ use std::time::{Duration, Instant}; use opentelemetry::metrics::Meter; use opentelemetry::{KeyValue, global}; use tracing::Span; +use tracing_opentelemetry::OpenTelemetrySpanExt; -use crate::Result; +use crate::{OutcomeMetadata, Result}; use switchyard_protocol::{ModelId, Request, Response}; const METRICS_SCOPE: &str = "switchyard"; @@ -61,16 +62,22 @@ pub(crate) fn outcome_value(result: &Result) -> &'static str { /// Span covering one algorithm run (the whole `route` execution). /// /// Correlation ids from the request [`switchyard_protocol::Metadata`] are recorded as span fields -/// when present. `tracing` spans cannot grow field names at runtime, so -/// arbitrary host labels ride in via [`switchyard_protocol::Metadata::extra_metadata`], recorded -/// whole into the `extra_metadata` field. `outcome` and `error` are filled in -/// by [`record_run`] when the run ends. +/// when present. Arbitrary extra metadata and error details are not exported. +/// [`record_outcome`] fills in successful outcome fields; [`record_run`] records +/// whether the run succeeded. pub(crate) fn run_span(algorithm: &str, request: &Request) -> Span { let span = tracing::info_span!( target: TRACING_TARGET, "libsy.run", algorithm, outcome_id = tracing::field::Empty, + evidence.source = tracing::field::Empty, + evidence.score = tracing::field::Empty, + evidence.confidence = tracing::field::Empty, + evidence.threshold = tracing::field::Empty, + evidence.verdict = tracing::field::Empty, + evidence.trigger = tracing::field::Empty, + evidence.reason_code = tracing::field::Empty, switchyard.algorithm = algorithm, openinference.span.kind = "CHAIN", switchyard.route = tracing::field::Empty, @@ -81,9 +88,7 @@ pub(crate) fn run_span(algorithm: &str, request: &Request) -> Span { task_kind = tracing::field::Empty, agent_role = tracing::field::Empty, correlation_id = tracing::field::Empty, - extra_metadata = tracing::field::Empty, outcome = tracing::field::Empty, - error = tracing::field::Empty, ); if let Some(route) = request.model_id() { span.record("switchyard.route", route.as_ref()); @@ -104,13 +109,50 @@ pub(crate) fn run_span(algorithm: &str, request: &Request) -> Span { if let Some(session_id) = &metadata.session_id { span.record("session.id", session_id.as_str()); } - if let Some(extra) = &metadata.extra_metadata { - span.record("extra_metadata", tracing::field::debug(extra)); - } } span } +/// Projects a successful outcome onto the existing run span. Model IDs are an +/// ordered OpenTelemetry string array, preserving fallback order. Evidence uses typed fields; +/// unknown keys and values of the wrong type are omitted. +pub(crate) fn record_outcome(metadata: &OutcomeMetadata, models: &[ModelId]) { + let span = Span::current(); + span.record("outcome_id", metadata.outcome_id()); + span.record("algorithm", metadata.algorithm.as_str()); + span.record("switchyard.algorithm", metadata.algorithm.as_str()); + span.set_attribute( + "selected_model_ids", + opentelemetry::Value::Array(opentelemetry::Array::String( + models + .iter() + .map(|model| model.to_string().into()) + .collect(), + )), + ); + if let Some(evidence) = &metadata.evidence { + for (key, field) in [ + ("source", "evidence.source"), + ("verdict", "evidence.verdict"), + ("trigger", "evidence.trigger"), + ("reason_code", "evidence.reason_code"), + ] { + if let Some(value) = evidence.get(key).and_then(serde_json::Value::as_str) { + span.record(field, value); + } + } + for (key, field) in [ + ("score", "evidence.score"), + ("confidence", "evidence.confidence"), + ("threshold", "evidence.threshold"), + ] { + if let Some(value) = evidence.get(key).and_then(serde_json::Value::as_f64) { + span.record(field, value); + } + } + } +} + /// Holds `switchyard.algorithms_in_flight` up by one for as long as it lives. struct InFlightRun { algorithm: String, @@ -141,7 +183,7 @@ fn record_algorithms_in_flight(algorithm: &str, delta: i64) { } /// Runs one algorithm task to completion, recording the run counter, duration -/// histogram, span outcome, and failure log when it resolves. Counts the run as +/// histogram and span outcome when it resolves. Counts the run as /// in flight for its whole duration. /// Executes inside the `libsy.run` span its caller instruments the task with. pub(crate) async fn observe_run( @@ -158,20 +200,10 @@ pub(crate) async fn observe_run( } /// Records the end of one algorithm run: the run counter and duration -/// histogram, the `outcome`/`error` fields on `span`, and a warn log when the -/// run failed. +/// histogram and the `outcome` field on `span`, without error details. fn record_run(algorithm: &str, duration: Duration, result: &Result, span: &Span) { let outcome = outcome_value(result); span.record("outcome", outcome); - if let Err(error) = result { - span.record("error", tracing::field::display(error)); - tracing::warn!( - target: TRACING_TARGET, - algorithm, - error = %error, - "algorithm run failed" - ); - } let attributes = [ KeyValue::new("algorithm", algorithm.to_string()), @@ -203,8 +235,7 @@ pub(crate) fn record_classifier_fail_open(judge_model: &str, reason: &'static st } /// Records the resolution of one offloaded model call: the call counter and -/// latency histogram, the `outcome`/`error`/token fields on `span`, and a warn -/// log when the call failed. +/// latency histogram and the outcome/token fields on `span`, without error details. pub(crate) fn record_llm_call( algorithm: &str, selected_model: &str, @@ -230,45 +261,27 @@ pub(crate) fn record_llm_call( .build() .record(duration.as_secs_f64() * 1000.0, &call_attributes); - match result { - Ok(response) => { - // Token usage exists only once a response is buffered; a streamed - // response resolves before its usage is known, so none is recorded. - let Some(usage) = response.llm_response.as_agg().map(|agg| &agg.usage) else { - return; - }; - for (field, value) in [ - ("input_tokens", usage.input_tokens), - ("output_tokens", usage.output_tokens), - ("total_tokens", usage.total_tokens), - ("reasoning_tokens", usage.reasoning_tokens), - ] { - if let Some(value) = value { - span.record(field, value); - } + if let Ok(response) = result { + // Token usage exists only once a response is buffered; a streamed + // response resolves before its usage is known, so none is recorded. + let Some(usage) = response.llm_response.as_agg().map(|agg| &agg.usage) else { + return; + }; + for (field, value) in [ + ("input_tokens", usage.input_tokens), + ("output_tokens", usage.output_tokens), + ("total_tokens", usage.total_tokens), + ("reasoning_tokens", usage.reasoning_tokens), + ] { + if let Some(value) = value { + span.record(field, value); } } - Err(error) => { - span.record("error", tracing::field::display(error)); - tracing::warn!( - target: TRACING_TARGET, - algorithm, - selected_model, - error = %error, - "model call failed" - ); - } } } -/// Records one published routing decision: the decision counter plus a structured debug event. +/// Counts one published routing decision. pub(crate) fn record_decision(algorithm: &str, selected_model: &ModelId) { - tracing::debug!( - target: TRACING_TARGET, - algorithm, - selected_model = %selected_model, - "routing decision" - ); meter().u64_counter("switchyard.decisions").build().add( 1, &[ diff --git a/crates/switchyard-py/src/libsy_bindings.rs b/crates/switchyard-py/src/libsy_bindings.rs index 756dbc8bf..79adca287 100644 --- a/crates/switchyard-py/src/libsy_bindings.rs +++ b/crates/switchyard-py/src/libsy_bindings.rs @@ -514,12 +514,45 @@ impl PyModelCall { } } -/// The terminal routing selection, rewritten request, and optional existing response. +/// Identity and optional JSON evidence from the Rust routing outcome. +#[pyclass(name = "OutcomeMetadata", module = "switchyard.libsy", frozen)] +struct PyOutcomeMetadata { + inner: switchyard_libsy::OutcomeMetadata, +} + +#[pymethods] +impl PyOutcomeMetadata { + /// UUIDv7 generated for this outcome. + #[getter] + fn outcome_id(&self) -> &str { + self.inner.outcome_id() + } + + /// Name of the algorithm that produced this outcome. + #[getter] + fn algorithm(&self) -> &str { + &self.inner.algorithm + } + + /// Optional evidence converted to ordinary Python JSON values. + #[getter] + fn evidence(&self, py: Python<'_>) -> PyResult>> { + self.inner + .evidence + .as_ref() + .map(|value| to_python(py, value)) + .transpose() + } +} + +/// The terminal routing selection, rewritten request, optional response, and metadata. #[pyclass(name = "RoutingOutcome", module = "switchyard.libsy", frozen)] struct PyRoutingOutcome { selected_model_ids: Vec, request: Py, response: Option>, + #[pyo3(get)] + metadata: Option>, } #[pymethods] @@ -675,13 +708,16 @@ fn step_to_python(step: RustStep) -> PyResult { selected_model_ids, request, response, - metadata: _, + metadata, } = *outcome; Python::attach(|py| { Ok(PyStep::Done { outcome: Py::new( py, PyRoutingOutcome { + metadata: metadata + .map(|inner| Py::new(py, PyOutcomeMetadata { inner })) + .transpose()?, selected_model_ids: selected_model_ids .iter() .map(ToString::to_string) @@ -849,6 +885,7 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { libsy_module.add_class::()?; libsy_module.add_class::()?; libsy_module.add_class::()?; + libsy_module.add_class::()?; libsy_module.add_class::()?; libsy_module.add_class::()?; libsy_module.add_class::()?; diff --git a/docs/getting_started.md b/docs/getting_started.md index b6070c865..05ec3be0d 100644 --- a/docs/getting_started.md +++ b/docs/getting_started.md @@ -213,6 +213,17 @@ response when routing already produced the answer. Otherwise the host makes the call from that outcome. Serving these calls yourself is what lets libsy embed in a host that already owns its HTTP stack, retries, and credentials. +Successful runs also include `OutcomeMetadata`: a unique `outcome_id`, the algorithm +name, and optional JSON evidence. In Python, read `outcome.metadata.outcome_id`, +`outcome.metadata.algorithm`, and `outcome.metadata.evidence` after checking that +`outcome.metadata` is present. Evidence is a normal Python value, usually a dictionary; +algorithms without evidence return `None`. + +With a host-installed OpenTelemetry subscriber, the existing `libsy.run` span records +the same identity, selected models, and supported evidence fields. See +the `Algorithm` observability section in the [Rust API reference](reference/rust_api.md) +for the field names. libsy does not install an exporter or send telemetry itself. + For the request, response, and streaming types the steps carry, see [`switchyard-protocol`](../crates/protocol/README.md). diff --git a/switchyard/libsy/__init__.py b/switchyard/libsy/__init__.py index b4817dc87..88a5aa372 100644 --- a/switchyard/libsy/__init__.py +++ b/switchyard/libsy/__init__.py @@ -13,6 +13,7 @@ LlmFallback, LlmResponse, ModelCall, + OutcomeMetadata, RoutingOutcome, Step, TaskClassifierConfig, @@ -30,6 +31,7 @@ "LlmFallback", "LlmResponse", "ModelCall", + "OutcomeMetadata", "RoutingOutcome", "Step", "TaskClassifierConfig", diff --git a/switchyard_rust/libsy.py b/switchyard_rust/libsy.py index 1589d60ff..9756b1c6b 100644 --- a/switchyard_rust/libsy.py +++ b/switchyard_rust/libsy.py @@ -21,6 +21,7 @@ "LlmFallback", "LlmResponse", "ModelCall", + "OutcomeMetadata", "RoutingOutcome", "Step", "TaskClassifierConfig", @@ -111,8 +112,24 @@ def respond(self, response: LlmResponse.Agg | LlmResponse.Stream) -> None: ... def fail(self, error: BaseException) -> None: ... + @final + class OutcomeMetadata: + """Read-only outcome identity and optional algorithm evidence.""" + + @property + def outcome_id(self) -> str: ... + + @property + def algorithm(self) -> str: ... + + @property + def evidence(self) -> Any | None: ... + @final class RoutingOutcome: + @property + def metadata(self) -> OutcomeMetadata | None: ... + @property def selected_model_ids(self) -> list[str]: ... diff --git a/tests/test_libsy_minimal_bindings.py b/tests/test_libsy_minimal_bindings.py index 02ac614e4..1d6076801 100644 --- a/tests/test_libsy_minimal_bindings.py +++ b/tests/test_libsy_minimal_bindings.py @@ -5,6 +5,7 @@ from collections.abc import AsyncIterator from typing import Any +from uuid import UUID import pytest @@ -14,6 +15,7 @@ CustomClassifierConfig, LlmClassifierConfig, LlmResponse, + OutcomeMetadata, RoutingOutcome, Step, TaskClassifierConfig, @@ -77,6 +79,13 @@ async def run_algorithm( call.respond(LlmResponse.Agg(response)) break case Step.Done(outcome): + assert isinstance(outcome.metadata, OutcomeMetadata) + assert UUID(outcome.metadata.outcome_id).version == 7 + evidence = outcome.metadata.evidence + assert evidence is None or isinstance(evidence, dict) + if evidence is not None and evidence.get("source") == "llm-classifier": + assert evidence["score"] == pytest.approx(0.9) + assert evidence["threshold"] == pytest.approx(0.5) if outcome.response is not None: match outcome.response: case LlmResponse.Agg(response): @@ -113,6 +122,10 @@ async def test_random_streams_complex_steps_and_accepts_a_dictionary_response() assert outcome is not None assert outcome.selected_model_ids == ["fast"] assert outcome.response is None + assert outcome.metadata is not None + assert UUID(outcome.metadata.outcome_id).version == 7 + assert outcome.metadata.algorithm == "random" + assert outcome.metadata.evidence is None response = await client.call(outcome.request) assert client.calls[0]["model"] == "fast" assert client.calls[0]["messages"][0]["content"] == [ From e8ea45bfeedd9a6d3176e14430422740e4a12bab Mon Sep 17 00:00:00 2001 From: nachiketb Date: Wed, 9 Sep 2026 15:27:08 -0700 Subject: [PATCH 2/2] fix(libsy): preserve span identity and verify Python evidence Signed-off-by: nachiketb --- .../libsy-llm-client/tests/observability.rs | 1 + crates/libsy/src/core/algorithm.rs | 12 ++++---- crates/libsy/src/observability.rs | 2 -- tests/test_libsy_minimal_bindings.py | 30 +++++++++++-------- 4 files changed, 25 insertions(+), 20 deletions(-) diff --git a/crates/libsy-llm-client/tests/observability.rs b/crates/libsy-llm-client/tests/observability.rs index 95e505763..8ae0a551e 100644 --- a/crates/libsy-llm-client/tests/observability.rs +++ b/crates/libsy-llm-client/tests/observability.rs @@ -508,6 +508,7 @@ impl Algorithm for SingleCallAlgo { .ok_or(LibsyError::NoTargets)? .clone(); tracing::info!("picked '{target}'"); + // Include configured fallbacks so the exported model order can be checked. let mut outcome = RoutingOutcome::route_to( target.into(), self.target_set diff --git a/crates/libsy/src/core/algorithm.rs b/crates/libsy/src/core/algorithm.rs index a108aca66..6e01c20c7 100644 --- a/crates/libsy/src/core/algorithm.rs +++ b/crates/libsy/src/core/algorithm.rs @@ -383,16 +383,18 @@ impl RoutingIdentity { /// /// [`run_stream`](Self::run_stream) creates a `libsy.run` span, and each offloaded model /// call creates a nested `libsy.llm_call` span. Successful outcomes record their -/// [`OutcomeMetadata`](crate::OutcomeMetadata) on `libsy.run`: `outcome_id`, `algorithm`, -/// `switchyard.algorithm`, and `selected_model_ids` (an ordered OpenTelemetry string array). +/// [`OutcomeMetadata::outcome_id`](crate::OutcomeMetadata::outcome_id) on `libsy.run`, +/// alongside `selected_model_ids` (an ordered OpenTelemetry string array). +/// `algorithm` and `switchyard.algorithm` retain the run's [`Algorithm::name`]. /// Optional `evidence.source`, `evidence.verdict`, `evidence.trigger`, and /// `evidence.reason_code` are strings; `evidence.score`, `evidence.confidence`, and /// `evidence.threshold` are numbers. Unknown evidence fields are not exported. /// These fields are span attributes, never metric labels. /// -/// Runs and calls retain their `outcome` status and operational metrics, but do not -/// export error details or arbitrary request extra metadata. Errors still reach the -/// caller unchanged. The host controls the tracing subscriber and global OpenTelemetry +/// The run/call observability helpers retain `outcome` status and operational metrics, +/// but omit error details and arbitrary request extra metadata. Algorithms and hosts +/// may emit their own logs. Errors still reach the caller unchanged. +/// The host controls the tracing subscriber and global OpenTelemetry /// meter provider; libsy installs no exporter and performs no telemetry network I/O. #[async_trait] pub trait Algorithm: Send + Sync + 'static { diff --git a/crates/libsy/src/observability.rs b/crates/libsy/src/observability.rs index b95b257db..69c6d9295 100644 --- a/crates/libsy/src/observability.rs +++ b/crates/libsy/src/observability.rs @@ -119,8 +119,6 @@ pub(crate) fn run_span(algorithm: &str, request: &Request) -> Span { pub(crate) fn record_outcome(metadata: &OutcomeMetadata, models: &[ModelId]) { let span = Span::current(); span.record("outcome_id", metadata.outcome_id()); - span.record("algorithm", metadata.algorithm.as_str()); - span.record("switchyard.algorithm", metadata.algorithm.as_str()); span.set_attribute( "selected_model_ids", opentelemetry::Value::Array(opentelemetry::Array::String( diff --git a/tests/test_libsy_minimal_bindings.py b/tests/test_libsy_minimal_bindings.py index 1d6076801..b693d4d87 100644 --- a/tests/test_libsy_minimal_bindings.py +++ b/tests/test_libsy_minimal_bindings.py @@ -81,11 +81,6 @@ async def run_algorithm( case Step.Done(outcome): assert isinstance(outcome.metadata, OutcomeMetadata) assert UUID(outcome.metadata.outcome_id).version == 7 - evidence = outcome.metadata.evidence - assert evidence is None or isinstance(evidence, dict) - if evidence is not None and evidence.get("source") == "llm-classifier": - assert evidence["score"] == pytest.approx(0.9) - assert evidence["threshold"] == pytest.approx(0.5) if outcome.response is not None: match outcome.response: case LlmResponse.Agg(response): @@ -211,14 +206,23 @@ async def call(self, request: dict[str, Any]) -> dict[str, Any]: ), ) - _, response = await run_algorithm( - algorithm, - { - "judge": judge, - "weak": weak, - "strong": EchoClient("strong"), - }, - ) + outcome: RoutingOutcome | None = None + async for step in algorithm.run_stream(request_body()): + match step: + case Step.CallModel(call): + call.respond(LlmResponse.Agg(await judge.call(call.request))) + case Step.Done(done): + outcome = done + + assert outcome is not None + assert outcome.selected_model_ids[0] == "weak" + assert outcome.metadata is not None + assert outcome.metadata.evidence == { + "source": "llm-classifier", + "score": pytest.approx(0.9), + "threshold": pytest.approx(0.5), + } + response = await weak.call(outcome.request) prompt = judge.calls[0]["instructions"][0]["content"][0]["text"] assert prompt == "Custom capability rubric."