diff --git a/crates/libsy-llm-client/tests/observability.rs b/crates/libsy-llm-client/tests/observability.rs index 8ae0a551e..bd0bc7033 100644 --- a/crates/libsy-llm-client/tests/observability.rs +++ b/crates/libsy-llm-client/tests/observability.rs @@ -795,7 +795,10 @@ async fn successful_run_records_metrics_spans_and_outcome_metadata() -> switchya Some(json!({ "source": "llm-classifier", "score": 0.9, "threshold": 0.5, "verdict": "continue", "trigger": "turn", "reason_code": "test", - "confidence": "wrong type", "unknown": LEAKED_CONTENT, + "confidence": "algorithm-defined", "custom": { + "candidates": ["fast", "quality"], "accepted": true, "optional": null, + "note": "quoted \"value\"\nnext line", + }, })), ); let algorithm = Arc::new(SingleCallAlgo { @@ -879,22 +882,22 @@ async fn successful_run_records_metrics_spans_and_outcome_metadata() -> switchya .count(), 1 ); - for (field, expected) in [ - ("outcome_id", metadata.outcome_id()), - ("evidence.source", "llm-classifier"), - ("evidence.score", "0.9"), - ("evidence.threshold", "0.5"), - ("evidence.verdict", "continue"), - ("evidence.trigger", "turn"), - ("evidence.reason_code", "test"), - ] { - assert_eq!( - run_span.fields.get(field).map(String::as_str), - Some(expected), - "{field}" - ); - } - assert!(!run_span.fields.contains_key("evidence.confidence")); + assert_eq!( + run_span.fields.get("outcome_id").map(String::as_str), + Some(metadata.outcome_id()) + ); + let evidence = metadata.evidence.as_ref().expect("outcome evidence"); + assert_eq!( + serde_json::from_str::(&run_span.fields["evidence"]) + .expect("JSON evidence"), + *evidence + ); + assert!( + !run_span + .fields + .keys() + .any(|key| key.starts_with("evidence.")) + ); assert!(!format!("{run_span:?}").contains(LEAKED_CONTENT)); let exported = span_exporter.get_finished_spans().expect("exported spans"); let exported_run = exported @@ -906,8 +909,14 @@ async fn successful_run_records_metrics_spans_and_outcome_metadata() -> switchya }) .expect("outcome span exported"); assert_eq!( - otel_attribute(exported_run, "evidence.score"), - Some(&OtelValue::F64(0.9)) + otel_attribute(exported_run, "evidence"), + Some(&OtelValue::String(evidence.to_string().into())) + ); + assert!( + !exported_run + .attributes + .iter() + .any(|attr| attr.key.as_str().starts_with("evidence.")) ); assert_eq!( otel_attribute(exported_run, "selected_model_ids"), @@ -1213,6 +1222,11 @@ async fn streamed_usage_updates_the_client_call_span() -> switchyard_libsy::Resu let spans = store.spans(); let client_span = find_span(&spans, "libsy.client_call", "selected_model", MODEL); + assert!( + !find_span(&spans, "libsy.run", "algorithm", ALGO) + .fields + .contains_key("evidence") + ); for (field, value) in [ ("otel.name", "chat obs-stream-model"), ("gen_ai.request.stream", "true"), diff --git a/crates/libsy/src/core/algorithm.rs b/crates/libsy/src/core/algorithm.rs index 6e01c20c7..44a38a350 100644 --- a/crates/libsy/src/core/algorithm.rs +++ b/crates/libsy/src/core/algorithm.rs @@ -386,9 +386,8 @@ impl RoutingIdentity { /// [`OutcomeMetadata::outcome_id`](crate::OutcomeMetadata::outcome_id) on `libsy.run`, /// alongside `selected_model_ids` (an ordered OpenTelemetry string array). /// `algorithm` and `switchyard.algorithm` retain the run's [`Algorithm::name`]. -/// Optional `evidence.source`, `evidence.verdict`, `evidence.trigger`, and -/// `evidence.reason_code` are strings; `evidence.score`, `evidence.confidence`, and -/// `evidence.threshold` are numbers. Unknown evidence fields are not exported. +/// Optional `evidence` is the full JSON serialized as a string for consumers to parse. +/// Keep it small and free of private data; nothing is filtered or redacted. /// These fields are span attributes, never metric labels. /// /// The run/call observability helpers retain `outcome` status and operational metrics, diff --git a/crates/libsy/src/observability.rs b/crates/libsy/src/observability.rs index 69c6d9295..269f3bf9b 100644 --- a/crates/libsy/src/observability.rs +++ b/crates/libsy/src/observability.rs @@ -71,13 +71,7 @@ pub(crate) fn run_span(algorithm: &str, request: &Request) -> Span { "libsy.run", algorithm, outcome_id = tracing::field::Empty, - evidence.source = tracing::field::Empty, - evidence.score = tracing::field::Empty, - evidence.confidence = tracing::field::Empty, - evidence.threshold = tracing::field::Empty, - evidence.verdict = tracing::field::Empty, - evidence.trigger = tracing::field::Empty, - evidence.reason_code = tracing::field::Empty, + evidence = tracing::field::Empty, switchyard.algorithm = algorithm, openinference.span.kind = "CHAIN", switchyard.route = tracing::field::Empty, @@ -114,8 +108,7 @@ pub(crate) fn run_span(algorithm: &str, request: &Request) -> Span { } /// Projects a successful outcome onto the existing run span. Model IDs are an -/// ordered OpenTelemetry string array, preserving fallback order. Evidence uses typed fields; -/// unknown keys and values of the wrong type are omitted. +/// ordered OpenTelemetry string array, preserving fallback order. Evidence is a JSON string. pub(crate) fn record_outcome(metadata: &OutcomeMetadata, models: &[ModelId]) { let span = Span::current(); span.record("outcome_id", metadata.outcome_id()); @@ -129,25 +122,7 @@ pub(crate) fn record_outcome(metadata: &OutcomeMetadata, models: &[ModelId]) { )), ); if let Some(evidence) = &metadata.evidence { - for (key, field) in [ - ("source", "evidence.source"), - ("verdict", "evidence.verdict"), - ("trigger", "evidence.trigger"), - ("reason_code", "evidence.reason_code"), - ] { - if let Some(value) = evidence.get(key).and_then(serde_json::Value::as_str) { - span.record(field, value); - } - } - for (key, field) in [ - ("score", "evidence.score"), - ("confidence", "evidence.confidence"), - ("threshold", "evidence.threshold"), - ] { - if let Some(value) = evidence.get(key).and_then(serde_json::Value::as_f64) { - span.record(field, value); - } - } + span.record("evidence", tracing::field::display(evidence)); } }