From 34190d463c2c5d8683aaf885eef85e4de45b4b0c Mon Sep 17 00:00:00 2001 From: 1bcMax Date: Tue, 18 Aug 2026 21:30:01 -0700 Subject: [PATCH] =?UTF-8?q?fix(router):=20imperativeVerbs=20was=20weighted?= =?UTF-8?q?=20zero=20=E2=80=94=20restore=20the=20dimension=20key?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The transpile that produced router_core/config.py from config.ts snake_cased config field names. `imperativeVerbs` is both a keyword-list field and one of the 15 dimension names, so its weight landed under `imperative_verbs` while rules.py emitted `imperativeVerbs`. `weights.get(name, 0)` then scored that dimension at zero on every request. Effect: build/deploy-shaped prompts were under-classified. 3 of 8 sampled imperative prompts — "Create and deploy the service", "Set up the config and deploy it", "Develop a CLI that generates reports" — classified SIMPLE where the correct score leaves them ambiguous, which the strategy defaults up to MEDIUM. The 24-shape cross-SDK check missed it because none of those prompts sat within 0.03 of a tier boundary; it still reports 24/24 after the fix. Guarded by two tests: the weight keys and the dimension names the classifier emits must be the same set, and the weight table must match config.ts at 18bf4ab verbatim. Folded into the unreleased 1.12.0 — no published artifact carries the bug. --- CHANGELOG.md | 12 +++++++++ blockrun_llm/router_core/config.py | 2 +- tests/unit/test_router_core.py | 43 ++++++++++++++++++++++++++++++ 3 files changed, 56 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 09edb4c..a18ea44 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -38,6 +38,18 @@ All notable changes to blockrun-llm will be documented in this file. never retried, so a second model cannot sign a second transfer for one call. ### Fixed +- **One scoring dimension was silently weighted zero.** The config transpile that + produced `router_core/config.py` snake_cased key names, and `imperativeVerbs` + is both a keyword-list field *and* a dimension name — so its 0.03 weight + landed under `imperative_verbs` while the classifier emitted `imperativeVerbs`, + and `weights.get(name, 0)` scored it zero. Build/deploy-shaped requests were + under-classified: 3 of 8 sampled imperative prompts ("Create and deploy the + service", "Set up the config and deploy it", "Develop a CLI that generates + reports") landed in SIMPLE where they should have been ambiguous and defaulted + up to MEDIUM. Now guarded by a test asserting the weight keys and the emitted + dimension names are the same set, plus the verbatim upstream weight table. + Cross-SDK parity re-verified at 24/24 after the fix. + - **A 429 now walks the fallback chain instead of failing the call.** Both clients treated only 5xx as retriable, so a saturated upstream ended the request even with capable models left in the chain. Found live: a rate-limited diff --git a/blockrun_llm/router_core/config.py b/blockrun_llm/router_core/config.py index 2e27588..77e5081 100644 --- a/blockrun_llm/router_core/config.py +++ b/blockrun_llm/router_core/config.py @@ -1048,7 +1048,7 @@ "simpleIndicators": 0.02, # Reduced from 0.12 to make room for agenticTask "multiStepPatterns": 0.12, "questionComplexity": 0.05, - "imperative_verbs": 0.03, + "imperativeVerbs": 0.03, "constraintCount": 0.04, "outputFormat": 0.03, "referenceComplexity": 0.02, diff --git a/tests/unit/test_router_core.py b/tests/unit/test_router_core.py index 191a3f1..482b2dd 100644 --- a/tests/unit/test_router_core.py +++ b/tests/unit/test_router_core.py @@ -1235,3 +1235,46 @@ def test_does_not_treat_host_tool_descriptions_as_a_per_turn_requirement(self): ) assert not infer_tool_requirement("What is 17 times 9?", system_prompt) + + +class TestDimensionWeightKeys: + """Every scored dimension must find its weight. + + The config transpile that produced ``config.py`` snake_cased key names, and + ``imperativeVerbs`` is both a keyword-list field *and* a dimension name — so + the weight landed under ``imperative_verbs`` while the classifier emitted + ``imperativeVerbs``. ``weights.get(name, 0)`` then silently scored that + dimension at zero, diverging from the TypeScript SDK on any prompt whose + imperative verbs would have crossed a tier boundary. + """ + + def test_every_emitted_dimension_has_a_weight(self): + from blockrun_llm.router_core.rules import classify_by_rules + + scoring = DEFAULT_ROUTING_CONFIG["scoring"] + result = classify_by_rules("Build and deploy the service", None, 10, scoring) + emitted = {dimension["name"] for dimension in result["dimensions"]} + weighted = set(scoring["dimension_weights"]) + + assert emitted - weighted == set(), "scored dimensions with no weight" + assert weighted - emitted == set(), "weights that match no scored dimension" + + def test_the_weights_match_the_upstream_values(self): + # Ported verbatim from router-core config.ts at 18bf4ab. + assert DEFAULT_ROUTING_CONFIG["scoring"]["dimension_weights"] == { + "tokenCount": 0.08, + "codePresence": 0.15, + "reasoningMarkers": 0.18, + "technicalTerms": 0.1, + "creativeMarkers": 0.05, + "simpleIndicators": 0.02, + "multiStepPatterns": 0.12, + "questionComplexity": 0.05, + "imperativeVerbs": 0.03, + "constraintCount": 0.04, + "outputFormat": 0.03, + "referenceComplexity": 0.02, + "negationComplexity": 0.01, + "domainSpecificity": 0.02, + "agenticTask": 0.04, + }