Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 15 additions & 11 deletions crates/libsy-llm-client/src/client.rs
Original file line number Diff line number Diff line change
Expand Up @@ -464,17 +464,9 @@ impl TranslatingLlmClient {
// Adapt the reqwest body stream to plain bytes; the SSE-decode itself is
// transport-agnostic and lives in `switchyard-translation`.
let bytes = http_response.bytes_stream().map(|chunk| {
chunk.map(|bytes| bytes.to_vec()).map_err(|error| {
if error.is_timeout() {
LlmClientError::Timeout {
source: Box::new(error),
}
} else {
LlmClientError::Transport {
source: Box::new(error),
}
}
})
chunk
.map(|bytes| bytes.to_vec())
.map_err(convert_reqwest_error)
});
let mut chunks = decode_stream(bytes, wire_format)?;
// Providers reject an over-ceiling streaming request with an in-band
Expand Down Expand Up @@ -707,6 +699,7 @@ fn record_gen_ai_request(url: &str, model: &str, streaming: bool) {
fn convert_reqwest_error(error: reqwest::Error) -> LlmClientError {
// Reqwest labels truncated or otherwise unreadable response bodies as decode
// errors, so distinguish them from serde JSON failures at the call site.
let error = error.without_url();
if error.is_timeout() {
LlmClientError::Timeout {
source: Box::new(error),
Expand Down Expand Up @@ -1047,6 +1040,17 @@ mod tests {
}
}

#[tokio::test]
async fn transport_errors_drop_the_upstream_url() {
let error = reqwest::Client::new()
.post("http://127.0.0.1:1/v1?key=CANARY")
.send()
.await
.expect_err("closed port");

assert!(!convert_reqwest_error(error).to_string().contains("CANARY"));
}

#[test]
fn anthropic_prompt_caching_marks_final_message() {
let mut body = json!({
Expand Down
2 changes: 1 addition & 1 deletion crates/switchyard-server/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -610,7 +610,7 @@ async fn proxy_unmatched(State(state): State<ServerState>, request: HttpRequest)
}
Err(error) => error_response(
StatusCode::BAD_GATEWAY,
error.to_string(),
error.without_url().to_string(),
"upstream_error",
"upstream_error",
),
Expand Down
54 changes: 54 additions & 0 deletions crates/switchyard-server/tests/server.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1996,6 +1996,60 @@ target = "weak"
Ok(())
}

#[tokio::test]
async fn transport_errors_hide_credential_bearing_upstream_urls() -> TestResult {
const CANARY: &str = "CANARY_ADMIN_QUERY_KEY";

let listener = TcpListener::bind("127.0.0.1:0").await?;
let base_url = format!("http://{}/v1?key={CANARY}", listener.local_addr()?);
drop(listener);

let routed = build_switchyard_router(random_state(&base_url, &[(ROUTE_MODEL, &["model/a"])])?);
let routed_response = send(
&routed,
"POST",
"/v1/chat/completions",
Some(json!({
"model": ROUTE_MODEL,
"messages": [{"role": "user", "content": "hello"}]
})),
)
.await?;

let fallback = load_test_config(&format!(
r#"
schema_version = 1
fallback_client = "upstream"

[llm_clients.upstream]
format = "openai_chat"
base_url = "{base_url}"

[targets.model]
id = "model/a"
llm_client = "upstream"

[routes.model]
id = "switchyard/model"
type = "passthrough"
target = "model"
"#
))?;
let fallback = build_switchyard_router(fallback);
let fallback_response = send(&fallback, "POST", "/unmatched", None).await?;

for response in [routed_response, fallback_response] {
assert_eq!(response.status, StatusCode::BAD_GATEWAY);
let body = response.json()?;
let message = body["error"]["message"].as_str().unwrap_or_default();
assert!(
!message.contains(CANARY),
"credential leaked in {message:?}"
);
}
Ok(())
}

#[tokio::test]
async fn anthropic_client_forwards_oauth_when_configured() -> TestResult {
let upstream = MockUpstream::start().await?;
Expand Down
Loading