From da26360b6ec674c9c93e2d0844339cb1104be92e Mon Sep 17 00:00:00 2001 From: ChethanUK Date: Sat, 5 Sep 2026 06:50:01 +0200 Subject: [PATCH] feat(libsy): map graded classifier verdicts onto routing targets (#348) A custom classifier's `targets` list served two jobs at once: it was both the set of models the route may dispatch to and the vocabulary the judge had to answer with. A rubric verdict like `complex` could not route, because making it sayable meant adding it to `targets`, where a second label resolving to an already-used ModelId is rejected -- correctly, since a repeated ModelId would sit twice in the fallback chain. Give the verdict vocabulary its own build-time map. `target_selector` takes an optional `labels` table from verdict value to configured target name, so several grades may share one target and one rubric can serve several operating points. Both failure modes are caught at load time because they are silent at runtime: an empty table, and a value that names no configured target. A verdict outside the map is unroutable exactly as an unknown target label already is, so `default_target` decides -- now with a warning naming the value that missed. Adds coverage for the mapping: libsy accepts many verdicts resolving to one target and rejects an empty or unknown-target `labels` map, the runner parses `policy.labels`, and the server routes one graded label end to end. Refs: https://github.com/NVIDIA-NeMo/Switchyard/issues/348 Signed-off-by: ChethanUK --- CHANGELOG.md | 7 ++ crates/libsy/src/algorithms/llm_class.rs | 101 +++++++++++++++- .../src/algorithms/util/target_selector.rs | 14 ++- crates/switchyard-runner/src/algorithm.rs | 8 +- crates/switchyard-runner/src/config.rs | 28 +++++ crates/switchyard-server/tests/server.rs | 113 ++++++++++++++++++ docs/reference/toml_schema.md | 1 + .../llm_classifier_routing.md | 47 ++++++++ 8 files changed, 311 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 76884e283..eb580239a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -72,6 +72,13 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ### Changed +- **Graded custom-classifier verdicts** — the `target_selector` policy takes an + optional `labels` map from verdict value to configured target, so a rubric can + grade a request (`simple` / `complex`) instead of naming a model and several + grades may share one target. Omitting `labels` keeps today's behavior, but when + it is set it is the whole verdict vocabulary and a verdict outside it falls back + to `default_target`. `CustomClassifierPolicy::TargetSelector` gains a field, so + external code matching that variant by its fields must be updated. (#348) - **`Algorithm::route` returns `Result`** — instead of the bare final `Result`, so callers observe the full routing outcome (see #458 for the design). (#459) diff --git a/crates/libsy/src/algorithms/llm_class.rs b/crates/libsy/src/algorithms/llm_class.rs index 599e9f9e0..00e6f911c 100644 --- a/crates/libsy/src/algorithms/llm_class.rs +++ b/crates/libsy/src/algorithms/llm_class.rs @@ -381,6 +381,12 @@ pub enum CustomClassifierPolicy { TargetSelector { /// JSON Pointer evaluated against each schema-validated verdict. selector: String, + /// Maps each verdict value the judge may return onto a configured target label. + /// + /// `None` keeps the direct lookup, where the verdict is itself a target label. + /// When set, this map is the whole verdict vocabulary: several verdicts may share + /// one target, and a verdict outside the map falls back to `default_target`. + labels: Option>, }, } @@ -389,6 +395,7 @@ impl CustomClassifierPolicy { pub fn target_selector(selector: impl Into) -> Self { Self::TargetSelector { selector: selector.into(), + labels: None, } } } @@ -678,9 +685,38 @@ impl LlmTaskClassifier { } = config; let contract = ClassifierContract::from_inner_schema(&prompt, response_schema)?; let policy = match policy { - CustomClassifierPolicy::TargetSelector { selector } => { + CustomClassifierPolicy::TargetSelector { + selector, + labels: verdict_labels, + } => { + // Without `labels` a verdict is a target label. With them, the map is the + // whole vocabulary, so several grades may resolve to one target even though + // the target list itself still rejects a repeated model. + let verdict_map = match verdict_labels { + None => target_map, + Some(verdict_labels) if verdict_labels.is_empty() => { + return Err(LibsyError::AlgorithmError { + message: "custom classifier policy labels must not be empty" + .to_string(), + }); + } + Some(verdict_labels) => verdict_labels + .into_iter() + .map(|(verdict, label)| { + let target = target_map.get(&label).cloned().ok_or_else(|| { + LibsyError::AlgorithmError { + message: format!( + "custom classifier policy label {label:?} must be one of the configured targets" + ), + } + })?; + Ok((verdict, target)) + }) + .collect::>>()?, + }; CustomPolicyRuntime::TargetSelector(TargetSelectorPolicy::new( - selector, target_map, + selector, + verdict_map, )?) } }; @@ -1649,4 +1685,65 @@ mod tests { ); Ok(()) } + + fn custom_config_with_labels(labels: Option>) -> LlmClassifierConfig { + LlmClassifierConfig::Custom { + judge_target: ModelId::from("judge"), + targets: vec![ + ("worker".to_string(), ModelId::from("worker")), + ("reviewer".to_string(), ModelId::from("reviewer")), + ], + default_target: "worker".to_string(), + config: CustomClassifierConfig::new( + "classify the delegated task", + serde_json::json!({ + "type": "object", + "properties": { + "grade": {"type": "string", "enum": ["simple", "hard", "brutal"]} + }, + "required": ["grade"], + "additionalProperties": false + }), + CustomClassifierPolicy::TargetSelector { + selector: "/grade".to_string(), + labels, + }, + ), + } + } + + #[test] + fn graded_verdict_labels_may_share_one_routing_target() { + // Two grades routing to one target is the whole point of a rubric verdict: the + // target list still rejects a repeated model, but the verdict vocabulary must not. + let config = custom_config_with_labels(Some(BTreeMap::from([ + ("simple".to_string(), "worker".to_string()), + ("hard".to_string(), "reviewer".to_string()), + ("brutal".to_string(), "reviewer".to_string()), + ]))); + + assert!(LlmTaskClassifier::new(config).is_ok()); + } + + #[test] + fn graded_verdict_labels_are_checked_against_the_configured_targets() { + let empty = LlmTaskClassifier::new(custom_config_with_labels(Some(BTreeMap::new()))); + assert!( + matches!(&empty, Err(LibsyError::AlgorithmError { message }) + if message.contains("labels must not be empty")), + "{:?}", + empty.err() + ); + + let unknown = LlmTaskClassifier::new(custom_config_with_labels(Some(BTreeMap::from([( + "simple".to_string(), + "nope".to_string(), + )])))); + assert!( + matches!(&unknown, Err(LibsyError::AlgorithmError { message }) + if message.contains("policy label \"nope\"")), + "{:?}", + unknown.err() + ); + } } diff --git a/crates/libsy/src/algorithms/util/target_selector.rs b/crates/libsy/src/algorithms/util/target_selector.rs index 4e36b648a..6f32dc812 100644 --- a/crates/libsy/src/algorithms/util/target_selector.rs +++ b/crates/libsy/src/algorithms/util/target_selector.rs @@ -42,16 +42,20 @@ impl JudgePolicy for TargetSelectorPolicy { type Verdict = Value; fn to_classification(&self, verdict: Option<&Self::Verdict>) -> Classification { - let target = verdict + let label = verdict .and_then(|verdict| self.selector.resolve(verdict).ok()) - .and_then(Value::as_str) - .and_then(|label| self.targets.get(label)); - match target { + .and_then(Value::as_str); + match label.and_then(|label| self.targets.get(label)) { Some(target) => Classification::Scores(vec![Score { target: target.clone(), confidence: 1.0, }]), - None => Classification::Ambiguous(vec![]), + None => { + // Abstaining hands the request to `default_target`. Say which value missed: + // a verdict outside a configured `labels` map is the likeliest misconfiguration. + tracing::warn!(?label, "verdict is not a configured target or label"); + Classification::Ambiguous(vec![]) + } } } } diff --git a/crates/switchyard-runner/src/algorithm.rs b/crates/switchyard-runner/src/algorithm.rs index d3e5f78ac..6f5282c1c 100644 --- a/crates/switchyard-runner/src/algorithm.rs +++ b/crates/switchyard-runner/src/algorithm.rs @@ -65,6 +65,10 @@ pub enum ClassifierPolicyConfig { TargetSelector { /// JSON Pointer to the name, such as `/decision/target`. selector: String, + /// Maps each verdict value onto a configured target name. Omit it and the verdict + /// is the target name; set it and several verdicts may share one target. + #[serde(default)] + labels: Option>, }, } @@ -84,7 +88,9 @@ pub enum ClassifierMode { impl ClassifierPolicyConfig { fn into_libsy(self) -> CustomClassifierPolicy { match self { - Self::TargetSelector { selector } => CustomClassifierPolicy::target_selector(selector), + Self::TargetSelector { selector, labels } => { + CustomClassifierPolicy::TargetSelector { selector, labels } + } } } } diff --git a/crates/switchyard-runner/src/config.rs b/crates/switchyard-runner/src/config.rs index c1fea1464..a4f138780 100644 --- a/crates/switchyard-runner/src/config.rs +++ b/crates/switchyard-runner/src/config.rs @@ -662,6 +662,17 @@ classify_trigger = "new_session""#, configured } + /// The subagent custom classifier above, with a graded `policy.labels` map added. + fn with_subagent_classifier_labels(labels: &str) -> String { + let configured = with_subagent_llm_classifier(VALID_CONFIG, "passthrough", ""); + let policy = "selector = \"/target\" }"; + assert!(configured.contains(policy)); + configured.replace( + policy, + &format!("selector = \"/target\", labels = {labels} }}"), + ) + } + fn with_subagent_passthrough(config: &str, route: &str) -> String { format!("{config}\n[routes.{route}.subagents]\ntype = \"passthrough\"\ntarget = \"strong\"") } @@ -1058,6 +1069,14 @@ classifier_magic = true ), "route random context_window must be greater than zero", ), + ( + with_subagent_classifier_labels("{}"), + "labels must not be empty", + ), + ( + with_subagent_classifier_labels("{ simple = \"nope\" }"), + "policy label \"nope\"", + ), ]; for (toml, expected) in cases { @@ -1069,6 +1088,15 @@ classifier_magic = true } } + #[test] + fn accepts_graded_classifier_policy_labels() -> RunnerResult<()> { + // Two verdict grades may name the same target; only the target list stays unique. + runner_from_toml(&with_subagent_classifier_labels( + "{ simple = \"weak\", complex = \"strong\" }", + ))?; + Ok(()) + } + #[test] fn accepts_duplicate_target_model_ids_on_one_client() -> RunnerResult<()> { // Two targets share one model id on one client. The client keeps one and drops the diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rs index 43e6abf7f..796beb02b 100644 --- a/crates/switchyard-server/tests/server.rs +++ b/crates/switchyard-server/tests/server.rs @@ -1513,6 +1513,119 @@ selector = "/decision/target" Ok(()) } +#[tokio::test] +async fn custom_classifier_maps_graded_verdict_labels_onto_targets() -> TestResult { + let upstream = MockUpstream::start().await?; + let state = load_test_config(&format!( + r#" +schema_version = 1 + +[llm_clients.upstream] +format = "openai_chat" +base_url = "{base_url}" + +[targets.classifier] +id = "model/classifier" +llm_client = "upstream" + +[targets.strong] +id = "model/strong" +llm_client = "upstream" + +[targets.middle] +id = "model/middle" +llm_client = "upstream" + +[targets.premium] +id = "model/premium" +llm_client = "upstream" + +[targets.weak] +id = "model/weak" +llm_client = "upstream" + +[routes.custom] +id = "switchyard/custom" +type = "llm_classifier" +mode = "custom" +classifier_target = "classifier" +targets = ["weak", "strong"] +default_target = "weak" +prompt = "CUSTOM MULTI TARGET" +response_schema = ''' +{{ + "type": "object", + "properties": {{ + "decision": {{ + "type": "object", + "properties": {{ + "target": {{"type": "string", "enum": ["weak", "middle", "strong", "premium"]}} + }}, + "required": ["target"], + "additionalProperties": false + }} + }}, + "required": ["decision"], + "additionalProperties": false +}} +''' + +[routes.custom.policy] +type = "target_selector" +selector = "/decision/target" +labels = {{ premium = "strong" }} +"#, + base_url = upstream.base_url + ))?; + let app = build_switchyard_router(state); + + for (task, selected) in [ + ("route this task", "model/strong"), + ("return an invalid verdict", "model/weak"), + ] { + let response = send( + &app, + "POST", + "/v1/chat/completions", + Some(json!({ + "model": "switchyard/custom", + "messages": [{"role": "user", "content": task}] + })), + ) + .await?; + assert_eq!(response.status, StatusCode::OK); + assert_eq!( + response + .headers + .get("x-model-router-selected-model") + .and_then(|value| value.to_str().ok()), + Some(selected) + ); + } + + let calls = upstream.calls.lock().await; + let judge_call = calls + .iter() + .find(|call| call["model"] == "model/classifier") + .ok_or("custom classifier target was not called")?; + let prompt = judge_call["messages"][0]["content"] + .as_str() + .ok_or("custom classifier prompt was not text")?; + assert_eq!(prompt, "CUSTOM MULTI TARGET"); + assert_eq!(judge_call["response_format"]["type"], "json_schema"); + assert_eq!( + judge_call["response_format"]["json_schema"]["name"], + "switchyard_classifier_response" + ); + assert_eq!(judge_call["response_format"]["json_schema"]["strict"], true); + assert_eq!( + judge_call["response_format"]["json_schema"]["schema"]["properties"]["decision"]["properties"] + ["target"]["enum"], + json!(["weak", "middle", "strong", "premium"]) + ); + Ok(()) +} + #[tokio::test] async fn classifier_contract_overrides_reach_every_server_mode() -> TestResult { let upstream = MockUpstream::start().await?; diff --git a/docs/reference/toml_schema.md b/docs/reference/toml_schema.md index 0193ca680..ee0c0a247 100644 --- a/docs/reference/toml_schema.md +++ b/docs/reference/toml_schema.md @@ -215,6 +215,7 @@ policy selector, and routes to any configured target label. | `prompt` | Yes | — | Judge system prompt. The configured inner schema is sent separately as structured-output configuration. | | `response_schema` | Yes | — | Inner JSON Schema encoded as a TOML string. Switchyard adds the provider wrapper. | | `policy` | Yes | — | Policy table. `target_selector` accepts a JSON Pointer such as `/decision/target`. | +| `policy.labels` | No | unset | Maps each verdict value onto a configured target name, so several verdicts may share one target. Every value must name a configured target and the table must not be empty. When set it is the whole verdict vocabulary; a verdict outside it falls back to `default_target`. | | `classify_trigger` | No | `every_request` | When the judge runs. `every_request` judges every request, tool continuations included. `user_turn` judges each new user message and retains that target across intervening tool calls only when requests carry a session ID; without a session ID, it behaves like `every_request`. `new_session` judges once and reuses that target for the session. | | `message_hash_fallback` | No | `false` | Keys affinity on the first user message. Requires `classify_trigger = "new_session"`. | | `recent_turn_window` | No | unset | When unset, the judge sees the opening task and latest user follow-up, when present. When set, it also sees trailing turns. | diff --git a/docs/routing_algorithms/llm_classifier_routing.md b/docs/routing_algorithms/llm_classifier_routing.md index 4706175e3..ce459a2af 100644 --- a/docs/routing_algorithms/llm_classifier_routing.md +++ b/docs/routing_algorithms/llm_classifier_routing.md @@ -184,6 +184,53 @@ schema to the provider in a strict structured-output wrapper and validates the returned JSON again. `jsonptr` resolves the selector against that verdict. A missing, non-string, or unknown target falls back to `default_target`. +### Grading rubrics with `policy.labels` + +A rubric that asks for a judgement — difficulty, complexity, risk — rather than +a target name routes through an optional `labels` map. Several grades may share +one target, which `targets` cannot express because two entries resolving to one +model are rejected: + +```toml +[routes.assistant] +id = "assistant" +type = "llm_classifier" +mode = "custom" +classifier_target = "classifier" +targets = ["weak", "strong"] +default_target = "strong" +prompt = """ +Grade this request's difficulty. +Return JSON matching the response schema supplied with the request. +""" +response_schema = ''' +{ + "type": "object", + "properties": { + "recommended_tier": { + "type": "string", + "enum": ["simple", "medium", "complex", "reasoning"] + } + }, + "required": ["recommended_tier"], + "additionalProperties": false +} +''' + +[routes.assistant.policy] +type = "target_selector" +selector = "/recommended_tier" +labels = { simple = "weak", medium = "weak", complex = "strong", reasoning = "strong" } +``` + +Every value in `labels` must name a configured target and the table must not be +empty; both are checked when the deployment loads. One rubric then serves +several operating points, because only the map changes between them. + +With `labels` set, `response_schema` must enumerate the **verdict** values, not +the target names. Nothing validates that pairing, and getting it wrong makes +every verdict unroutable, so all traffic silently reaches `default_target`. + This separation applies to every classifier mode. Prompts containing the legacy `{{RESPONSE_SCHEMA}}` placeholder are rejected during configuration validation.