diff --git a/CHANGELOG.md b/CHANGELOG.md index 09edb4c..a18ea44 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -38,6 +38,18 @@ All notable changes to blockrun-llm will be documented in this file. never retried, so a second model cannot sign a second transfer for one call. ### Fixed +- **One scoring dimension was silently weighted zero.** The config transpile that + produced `router_core/config.py` snake_cased key names, and `imperativeVerbs` + is both a keyword-list field *and* a dimension name — so its 0.03 weight + landed under `imperative_verbs` while the classifier emitted `imperativeVerbs`, + and `weights.get(name, 0)` scored it zero. Build/deploy-shaped requests were + under-classified: 3 of 8 sampled imperative prompts ("Create and deploy the + service", "Set up the config and deploy it", "Develop a CLI that generates + reports") landed in SIMPLE where they should have been ambiguous and defaulted + up to MEDIUM. Now guarded by a test asserting the weight keys and the emitted + dimension names are the same set, plus the verbatim upstream weight table. + Cross-SDK parity re-verified at 24/24 after the fix. + - **A 429 now walks the fallback chain instead of failing the call.** Both clients treated only 5xx as retriable, so a saturated upstream ended the request even with capable models left in the chain. Found live: a rate-limited diff --git a/blockrun_llm/router_core/config.py b/blockrun_llm/router_core/config.py index 2e27588..77e5081 100644 --- a/blockrun_llm/router_core/config.py +++ b/blockrun_llm/router_core/config.py @@ -1048,7 +1048,7 @@ "simpleIndicators": 0.02, # Reduced from 0.12 to make room for agenticTask "multiStepPatterns": 0.12, "questionComplexity": 0.05, - "imperative_verbs": 0.03, + "imperativeVerbs": 0.03, "constraintCount": 0.04, "outputFormat": 0.03, "referenceComplexity": 0.02, diff --git a/tests/unit/test_router_core.py b/tests/unit/test_router_core.py index 191a3f1..482b2dd 100644 --- a/tests/unit/test_router_core.py +++ b/tests/unit/test_router_core.py @@ -1235,3 +1235,46 @@ def test_does_not_treat_host_tool_descriptions_as_a_per_turn_requirement(self): ) assert not infer_tool_requirement("What is 17 times 9?", system_prompt) + + +class TestDimensionWeightKeys: + """Every scored dimension must find its weight. + + The config transpile that produced ``config.py`` snake_cased key names, and + ``imperativeVerbs`` is both a keyword-list field *and* a dimension name — so + the weight landed under ``imperative_verbs`` while the classifier emitted + ``imperativeVerbs``. ``weights.get(name, 0)`` then silently scored that + dimension at zero, diverging from the TypeScript SDK on any prompt whose + imperative verbs would have crossed a tier boundary. + """ + + def test_every_emitted_dimension_has_a_weight(self): + from blockrun_llm.router_core.rules import classify_by_rules + + scoring = DEFAULT_ROUTING_CONFIG["scoring"] + result = classify_by_rules("Build and deploy the service", None, 10, scoring) + emitted = {dimension["name"] for dimension in result["dimensions"]} + weighted = set(scoring["dimension_weights"]) + + assert emitted - weighted == set(), "scored dimensions with no weight" + assert weighted - emitted == set(), "weights that match no scored dimension" + + def test_the_weights_match_the_upstream_values(self): + # Ported verbatim from router-core config.ts at 18bf4ab. + assert DEFAULT_ROUTING_CONFIG["scoring"]["dimension_weights"] == { + "tokenCount": 0.08, + "codePresence": 0.15, + "reasoningMarkers": 0.18, + "technicalTerms": 0.1, + "creativeMarkers": 0.05, + "simpleIndicators": 0.02, + "multiStepPatterns": 0.12, + "questionComplexity": 0.05, + "imperativeVerbs": 0.03, + "constraintCount": 0.04, + "outputFormat": 0.03, + "referenceComplexity": 0.02, + "negationComplexity": 0.01, + "domainSpecificity": 0.02, + "agenticTask": 0.04, + }