Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,18 @@ All notable changes to blockrun-llm will be documented in this file.
never retried, so a second model cannot sign a second transfer for one call.

### Fixed
- **One scoring dimension was silently weighted zero.** The config transpile that
produced `router_core/config.py` snake_cased key names, and `imperativeVerbs`
is both a keyword-list field *and* a dimension name — so its 0.03 weight
landed under `imperative_verbs` while the classifier emitted `imperativeVerbs`,
and `weights.get(name, 0)` scored it zero. Build/deploy-shaped requests were
under-classified: 3 of 8 sampled imperative prompts ("Create and deploy the
service", "Set up the config and deploy it", "Develop a CLI that generates
reports") landed in SIMPLE where they should have been ambiguous and defaulted
up to MEDIUM. Now guarded by a test asserting the weight keys and the emitted
dimension names are the same set, plus the verbatim upstream weight table.
Cross-SDK parity re-verified at 24/24 after the fix.

- **A 429 now walks the fallback chain instead of failing the call.** Both
clients treated only 5xx as retriable, so a saturated upstream ended the
request even with capable models left in the chain. Found live: a rate-limited
Expand Down
2 changes: 1 addition & 1 deletion blockrun_llm/router_core/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -1048,7 +1048,7 @@
"simpleIndicators": 0.02, # Reduced from 0.12 to make room for agenticTask
"multiStepPatterns": 0.12,
"questionComplexity": 0.05,
"imperative_verbs": 0.03,
"imperativeVerbs": 0.03,
"constraintCount": 0.04,
"outputFormat": 0.03,
"referenceComplexity": 0.02,
Expand Down
43 changes: 43 additions & 0 deletions tests/unit/test_router_core.py
Original file line number Diff line number Diff line change
Expand Up @@ -1235,3 +1235,46 @@ def test_does_not_treat_host_tool_descriptions_as_a_per_turn_requirement(self):
)

assert not infer_tool_requirement("What is 17 times 9?", system_prompt)


class TestDimensionWeightKeys:
"""Every scored dimension must find its weight.

The config transpile that produced ``config.py`` snake_cased key names, and
``imperativeVerbs`` is both a keyword-list field *and* a dimension name — so
the weight landed under ``imperative_verbs`` while the classifier emitted
``imperativeVerbs``. ``weights.get(name, 0)`` then silently scored that
dimension at zero, diverging from the TypeScript SDK on any prompt whose
imperative verbs would have crossed a tier boundary.
"""

def test_every_emitted_dimension_has_a_weight(self):
from blockrun_llm.router_core.rules import classify_by_rules

scoring = DEFAULT_ROUTING_CONFIG["scoring"]
result = classify_by_rules("Build and deploy the service", None, 10, scoring)
emitted = {dimension["name"] for dimension in result["dimensions"]}
weighted = set(scoring["dimension_weights"])

assert emitted - weighted == set(), "scored dimensions with no weight"
assert weighted - emitted == set(), "weights that match no scored dimension"

def test_the_weights_match_the_upstream_values(self):
# Ported verbatim from router-core config.ts at 18bf4ab.
assert DEFAULT_ROUTING_CONFIG["scoring"]["dimension_weights"] == {
"tokenCount": 0.08,
"codePresence": 0.15,
"reasoningMarkers": 0.18,
"technicalTerms": 0.1,
"creativeMarkers": 0.05,
"simpleIndicators": 0.02,
"multiStepPatterns": 0.12,
"questionComplexity": 0.05,
"imperativeVerbs": 0.03,
"constraintCount": 0.04,
"outputFormat": 0.03,
"referenceComplexity": 0.02,
"negationComplexity": 0.01,
"domainSpecificity": 0.02,
"agenticTask": 0.04,
}
Loading