From 78c25bf4b16506f25ce0b769c2ae4b891d014340 Mon Sep 17 00:00:00 2001 From: Akshaya Shanbhogue Date: Tue, 25 Aug 2026 23:32:02 -0700 Subject: [PATCH 1/9] feat(eval-routing): decouple judge/agent_judge backend+model from the agent's own route MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds `checker_context.api_route: {route, model}` on TaskDefinition (4-layer merged like `agent`/`simulation`), letting a task/variant pick which backend and model the evaluation side (llm_judge/agent_judge/simulator) uses, independent of the agent under test — enabling cross-vendor grading and cheaper judge models without editing every task YAML. The override is baked into the resolved ApiRoute's own `model` field before any criterion runs, so criteria stay task-blind (only ever reading CheckContext.route.model, never checker_context/TaskDefinition directly). checker_context validates its shape (unknown namespace/key/backend name raises) both at task-load time and after the experiment-layer merge. Separately, BedrockRoute/LiteLLMRoute no longer carry bearer_token/auth_token fields — those secrets now flow only through the coder_eval.config.settings singleton, read directly by each consumer (ClaudeCodeAgent._build_sdk_env, judge_bedrock.invoke_bedrock_judge_async) at the point of use, so a route object flowing through CheckContext/environment_info/logging never carries a credential. Includes fixes from a code review pass: Bedrock model_override is now region-qualified when reusing the agent's own route (previously shipped a bare alias to the Bedrock API); a missing bearer token now raises JudgeInfrastructureError instead of an assert that handle_criterion_errors was silently downgrading to a scored 0.0; eval_model is now recorded in environment_info; the two orchestrator route-resolution call sites are deduplicated into one helper. Co-Authored-By: Claude Sonnet 5 --- docs/TASK_DEFINITION_GUIDE.md | 21 +++ src/coder_eval/agents/claude_code_agent.py | 12 +- src/coder_eval/criteria/base.py | 6 + src/coder_eval/criteria/llm_judge.py | 34 +++-- src/coder_eval/evaluation/checker.py | 2 + src/coder_eval/evaluation/judge_bedrock.py | 15 ++- src/coder_eval/models/__init__.py | 2 + src/coder_eval/models/experiment.py | 16 +++ src/coder_eval/models/routing.py | 150 ++++++++++++++++----- src/coder_eval/models/tasks.py | 70 +++++++++- src/coder_eval/orchestration/experiment.py | 43 ++++++ src/coder_eval/orchestrator.py | 67 +++++++-- tests/test_config_precedence.py | 1 - tests/test_judge_bedrock.py | 9 +- tests/test_judge_burn_in_live.py | 2 +- tests/test_litellm_cost.py | 6 +- tests/test_litellm_route.py | 44 +++--- tests/test_llm_judge_criterion.py | 14 +- tests/test_orchestrator.py | 10 +- tests/test_route_seam_exhaustiveness.py | 6 +- tests/test_routing.py | 33 ++--- 21 files changed, 447 insertions(+), 116 deletions(-) diff --git a/docs/TASK_DEFINITION_GUIDE.md b/docs/TASK_DEFINITION_GUIDE.md index a16edb43..af2607df 100644 --- a/docs/TASK_DEFINITION_GUIDE.md +++ b/docs/TASK_DEFINITION_GUIDE.md @@ -35,6 +35,7 @@ Complete reference for defining evaluation tasks in Coder Eval. - [llm_judge](#llm_judge) - [agent_judge](#agent_judge) - [skill_triggered](#skill_triggered) + - [Checker Context](#checker-context) - [Reference Solutions](#reference-solutions) - [Pre-Run Commands](#pre-run-commands) - [Post-Run Commands](#post-run-commands) @@ -1295,6 +1296,26 @@ Observed label is `"yes"` when either signal is found, else `"no"`. Expected lab **Typical pattern.** Label each dataset row with its true skill (`expected_skill`, `""` for negatives) and stack one `skill_triggered` criterion per skill against the same dataset — each gets its own confusion matrix from the same agent traces. This is the natural companion to a skill A/B experiment (skill plugin on vs. off); see the [A/B Experiment Guide](AB_EXPERIMENTS.md#recipe-ab-a-skill). +### Checker Context + +`checker_context` carries task-authored config for the success-checking side, namespaced by reserved key. Currently the only recognized namespace is **`api_route`**: + +```yaml +success_criteria: + - type: llm_judge + prompt: "Grade the fix for correctness." + +checker_context: + api_route: + route: bedrock # which backend the whole eval side (llm_judge/agent_judge/simulator) uses + model: claude-haiku-4-5 # model override for that route +``` + +- `route` selects which backend the WHOLE evaluation side calls (`llm_judge`, `agent_judge`, and the simulator all share one resolved eval route per run — this isn't per-criterion), **decoupled from the agent's own route** (a Claude agent can be graded by a differently-backed judge, or vice versa). This is a backend *name* (`direct` / `bedrock` / `litellm`), not a route object: credentials are never read from the task, always from the matching environment variables (`ANTHROPIC_API_KEY` for `direct`, `AWS_BEARER_TOKEN_BEDROCK`/`AWS_REGION` for `bedrock`, `LITELLM_BASE_URL`/`LITELLM_AUTH_TOKEN` for `litellm`). An unconfigured or unknown backend name raises at dispatch rather than silently falling back. **`route: litellm` resolves successfully but `llm_judge` has no OpenAI-compatible transport yet** — it fails with a clear "not implemented yet" error at grading time; use `bedrock` or `direct` for `llm_judge` today. +- `model` overrides the model that resolved route uses — e.g. `llm_judge`'s judge model, when the criterion itself leaves `model:` unset (an explicit per-criterion `model:` always wins; below that, `checker_context.api_route.model`; below that, the backend's own env-configured default, e.g. `BEDROCK_MODEL`/`LITELLM_MODEL`). This works because every `ApiRoute` (`DirectRoute`/`BedrockRoute`/`LiteLLMRoute`) carries its own `model` field; the orchestrator bakes the override into the resolved route's `model` before any criterion runs, so `llm_judge` just reads `context.route.model` — it never reads `checker_context` directly. **`agent_judge` does not currently honor this override** — its sub-agent's model comes from the criterion's own `agent:` block (defaulted to a fixed judge model), independent of `checker_context.api_route.model`. + +`checker_context` merges shallow-per-namespace across `default_experiment.defaults.checker_context` → `experiment.defaults.checker_context` → `task.checker_context` → `variant.checker_context` (same 4-layer precedence as `agent`/`simulation`). So a judge-model A/B, or a judge-backend A/B, is a variant-level config change, not an edit to every task YAML. + ## Reference Solutions A reference solution is always a **directory**, given relative to the task YAML's own directory: diff --git a/src/coder_eval/agents/claude_code_agent.py b/src/coder_eval/agents/claude_code_agent.py index e23f69ea..e8a1726c 100644 --- a/src/coder_eval/agents/claude_code_agent.py +++ b/src/coder_eval/agents/claude_code_agent.py @@ -36,6 +36,7 @@ from coder_eval.agents._logging import PrefixedAdapter, log_raw_sdk_event from coder_eval.agents.registry import AgentRegistry from coder_eval.agents.watchdog import ThreadedWatchdog +from coder_eval.config import settings from coder_eval.errors import ( TurnTimeoutError, format_timeout_reason, @@ -792,9 +793,16 @@ def _build_sdk_env( match route: case BedrockRoute() as br: + # `or ""` rather than asserting non-None here (unlike judge_bedrock.py's + # invoke_bedrock_judge_async): reaching a BedrockRoute at all already implies + # validate_api_keys()/resolve_route() confirmed the token upstream, and this + # is a pure env-dict builder with no error-reporting seam of its own — an + # empty token still produces a clear downstream SDK auth failure rather than + # a crash here. Kept deliberately lenient; do not "fix" to assert without + # also deciding how the resulting AssertionError should surface to the caller. env: dict[str, str] = { "CLAUDE_CODE_USE_BEDROCK": "1", - "AWS_BEARER_TOKEN_BEDROCK": br.bearer_token, + "AWS_BEARER_TOKEN_BEDROCK": settings.aws_bearer_token_bedrock or "", "AWS_REGION": br.region, } if br.disable_attribution_header: @@ -816,7 +824,7 @@ def _build_sdk_env( # them here wins over the parent environment. env = { "ANTHROPIC_BASE_URL": cr.base_url, - "ANTHROPIC_AUTH_TOKEN": cr.auth_token, + "ANTHROPIC_AUTH_TOKEN": settings.litellm_auth_token or "", # Neutralize any inherited ANTHROPIC_API_KEY: auth on this # route is the bearer ANTHROPIC_AUTH_TOKEN, and a stray # x-api-key (e.g. a real Anthropic key exported from .env) diff --git a/src/coder_eval/criteria/base.py b/src/coder_eval/criteria/base.py index c825ed2d..4f82ee47 100644 --- a/src/coder_eval/criteria/base.py +++ b/src/coder_eval/criteria/base.py @@ -63,6 +63,12 @@ class CheckContext: ``reference_comparison`` is entirely dependent on it and scores 0.0 without one. Checkers that consume neither field receive the context anyway (uniform ``_check_impl`` signature) and ignore it. + + A judge model override (``checker_context.api_route.model``) is deliberately + NOT a separate field here — it's baked into ``route.model`` by the orchestrator + before this object is built (see ``resolve_evaluation_route``), so criteria + read one thing (``route.model``) regardless of whether the value came from an + env-configured backend default or a task-authored override. """ route: "ApiRoute | None" = None diff --git a/src/coder_eval/criteria/llm_judge.py b/src/coder_eval/criteria/llm_judge.py index 390676ef..b64c7c17 100644 --- a/src/coder_eval/criteria/llm_judge.py +++ b/src/coder_eval/criteria/llm_judge.py @@ -70,6 +70,14 @@ async def _check_impl_async( ctx = context or CheckContext() route = ctx.route reference_dir = ctx.reference_dir + # criterion.model always carries a concrete default (DEFAULT_JUDGE_MODEL), so an + # explicit per-criterion `model:` is only distinguishable from "unset" via + # model_fields_set — a checker_context.api_route.model override (baked into + # route.model by resolve_evaluation_route) must not clobber a value the task + # author actually wrote. + judge_model = criterion.model + if "model" not in criterion.model_fields_set and route is not None and route.model: + judge_model = route.model # Master enablement gate. Skipped criteria don't make an LLM call and don't # affect cost; weighted score includes them as 1.0 so they don't penalize. @@ -138,6 +146,7 @@ async def _check_impl_async( # from the usage the backend reported in its response. verdict, parse_error, raw_verdict_text, response_usage = await _invoke_tool_channel( criterion=criterion, + model=judge_model, route=route, system_msg=_SYSTEM_PROMPT, user_msg=user_msg, @@ -186,12 +195,18 @@ def _maybe_transcript() -> JudgeTranscript | None: async def _invoke_tool_channel( *, criterion: LLMJudgeCriterion, + model: str, route: "ApiRoute | None", system_msg: str, user_msg: str, ) -> tuple[JudgeVerdict | None, str | None, str, TokenUsage | None]: """Dispatch the tool-channel invocation by route, via non-blocking async clients. + ``model`` is the resolved judge model — ``criterion.model`` unless the task + left it unset and ``route.model`` (from ``checker_context.api_route.model``) + supplied a default (see ``_check_impl_async``); every backend call below + uses ``model``, never ``criterion.model`` directly. + Returns ``(verdict, parse_error, raw_verdict_text, response_usage)``. ``raw_verdict_text`` is the JSON-dumped verdict for the transcript when present, or a fallback marker when the model failed to call the tool — @@ -204,7 +219,7 @@ async def _invoke_tool_channel( case BedrockRoute(): response = await invoke_bedrock_judge_async( route=route, - model=criterion.model, + model=model, system=system_msg, user=user_msg, temperature=criterion.temperature, @@ -212,10 +227,10 @@ async def _invoke_tool_channel( tool_spec=SUBMIT_VERDICT_ANTHROPIC_TOOL, ) verdict, err = extract_verdict_from_anthropic_response(response) - response_usage = token_usage_from_anthropic_dict(response, model=criterion.model) + response_usage = token_usage_from_anthropic_dict(response, model=model) case DirectRoute(): anthropic_response = await invoke_anthropic_judge_async( - model=criterion.model, + model=model, system=system_msg, user=user_msg, temperature=criterion.temperature, @@ -223,13 +238,14 @@ async def _invoke_tool_channel( tool_spec=SUBMIT_VERDICT_ANTHROPIC_TOOL, ) verdict, err = extract_verdict_from_anthropic_response(anthropic_response) - response_usage = token_usage_from_anthropic_dict(anthropic_response, model=criterion.model) + response_usage = token_usage_from_anthropic_dict(anthropic_response, model=model) case LiteLLMRoute(): - # Defensive: the evaluation route is pinned to Bedrock/Direct by - # resolve_evaluation_route, so a LiteLLM route should never reach the - # judge. Fail loudly rather than silently scoring 0.0. (Explicit arm - # keeps the match exhaustive so pyright flags any future route member.) - return None, "llm_judge: evaluation route must be Bedrock/Direct, got LiteLLM", "(litellm route)", None + # Reachable now via an explicit `checker_context.api_route.route: litellm` + # override (see resolve_evaluation_route) — but there is no OpenAI-compatible + # transport yet (tracked separately), so fail loudly rather than silently + # scoring 0.0. (Explicit arm also keeps the match exhaustive so pyright + # flags any future route member.) + return None, "llm_judge: LiteLLM judge transport is not implemented yet", "(litellm route)", None case None: # Handled by the unconfigured-arm guard in _check_impl_async before # dispatch; defensive only. diff --git a/src/coder_eval/evaluation/checker.py b/src/coder_eval/evaluation/checker.py index 52f94c0d..0e7c2c4d 100644 --- a/src/coder_eval/evaluation/checker.py +++ b/src/coder_eval/evaluation/checker.py @@ -81,6 +81,8 @@ def __init__( ``agent_judge``) can route through the same backend (Direct / Bedrock) as the main coding agent. ``None`` is acceptable for non-sub-agent criteria; ``agent_judge`` requires a route. + Any ``checker_context.api_route.model`` override is already + baked into ``route.model`` by ``resolve_evaluation_route``. """ self.sandbox = sandbox self._checker_instances: dict[str, BaseCriterion[Any]] = {} diff --git a/src/coder_eval/evaluation/judge_bedrock.py b/src/coder_eval/evaluation/judge_bedrock.py index ca201645..e9f5c474 100644 --- a/src/coder_eval/evaluation/judge_bedrock.py +++ b/src/coder_eval/evaluation/judge_bedrock.py @@ -29,6 +29,7 @@ import httpx2 +from coder_eval.config import settings from coder_eval.errors import JudgeInfrastructureError from coder_eval.errors.categories import RetryConfig from coder_eval.errors.retry import compute_backoff @@ -67,9 +68,17 @@ async def invoke_bedrock_judge_async( Raises: ValueError: ``model`` empty. - JudgeInfrastructureError: retries exhausted, non-retryable HTTP failure - (e.g. 400/401/403), or a non-dict JSON body. + JudgeInfrastructureError: no bearer token configured; retries exhausted; + non-retryable HTTP failure (e.g. 400/401/403); or a non-dict JSON body. """ + # Raise (not assert): this call runs inside LLMJudgeChecker's + # handle_criterion_errors(_async) wrapper, which catches plain Exception + # (including AssertionError) and downgrades it to a scored 0.0 — the + # opposite of the intended "internal-contract violation escalates to + # FinalStatus.ERROR" behavior. JudgeInfrastructureError is in + # _ESCALATING_EXCEPTIONS, so it propagates instead of being scored. + if settings.aws_bearer_token_bedrock is None: + raise JudgeInfrastructureError("Bedrock requires aws_bearer_token_bedrock") qualified = to_bedrock_model(model, route.region) url = f"https://bedrock-runtime.{route.region}.amazonaws.com/model/{qualified}/invoke" body = { @@ -82,7 +91,7 @@ async def invoke_bedrock_judge_async( "tool_choice": {"type": "tool", "name": tool_spec["name"]}, } headers = { - "Authorization": f"Bearer {route.bearer_token}", + "Authorization": f"Bearer {settings.aws_bearer_token_bedrock}", "Content-Type": "application/json", "Accept": "application/json", } diff --git a/src/coder_eval/models/__init__.py b/src/coder_eval/models/__init__.py index d181241d..f524b2bb 100644 --- a/src/coder_eval/models/__init__.py +++ b/src/coder_eval/models/__init__.py @@ -187,6 +187,7 @@ ReferenceSource, SimulationConfig, TaskDefinition, + validate_checker_context_shape, ) # Telemetry @@ -368,6 +369,7 @@ "PreRunCommand", "ReferenceSource", "SimulationConfig", + "validate_checker_context_shape", # Mutations "PromptPrefix", "PromptSuffix", diff --git a/src/coder_eval/models/experiment.py b/src/coder_eval/models/experiment.py index 847a8475..fe6ae6ff 100644 --- a/src/coder_eval/models/experiment.py +++ b/src/coder_eval/models/experiment.py @@ -38,6 +38,14 @@ class ExperimentVariant(BaseModel): ge=1, description="Number of replicates for each task under this variant. None = inherit.", ) + checker_context: dict[str, dict[str, Any]] | None = Field( + default=None, + description=( + "Partial TaskDefinition.checker_context overrides for this variant — currently the " + "reserved `api_route` namespace (`route`/`model`). Shallow-merged per-namespace onto the " + "task's (and/or experiment defaults') checker_context." + ), + ) template_sources: list[TemplateSource] | None = Field( default=None, description="Additional template sources appended after task's base templates" ) @@ -98,6 +106,14 @@ class ExperimentDefaults(BaseModel): description="Default number of replicates across all variants. None = 1 (no repetition).", ) agent: dict[str, Any] | None = Field(default=None, description="Partial agent config defaults") + checker_context: dict[str, dict[str, Any]] | None = Field( + default=None, + description=( + "Default TaskDefinition.checker_context applied to all variants — currently the reserved " + "`api_route` namespace (e.g. `{api_route: {model: gpt-5}}`). Shallow-merged per-namespace " + "with the task's and the variant's checker_context." + ), + ) simulation: dict[str, Any] | None = Field( default=None, description=( diff --git a/src/coder_eval/models/routing.py b/src/coder_eval/models/routing.py index dfdbd627..1d3d8247 100644 --- a/src/coder_eval/models/routing.py +++ b/src/coder_eval/models/routing.py @@ -2,7 +2,7 @@ from __future__ import annotations -from dataclasses import dataclass +from dataclasses import dataclass, replace from typing import TYPE_CHECKING, Literal from coder_eval.models.enums import ApiBackend @@ -87,13 +87,26 @@ class DirectRoute: """ judge_transport: JudgeTransport | None = "anthropic" + # Unlike BedrockRoute/LiteLLMRoute, the AGENT never reads this — the Claude Agent + # SDK picks its own default when unset. It exists so ``checker_context.api_route.model`` + # (see resolve_evaluation_route) has somewhere to land when the eval side is on Direct. + model: str | None = None @dataclass(frozen=True) class BedrockRoute: - """Route through AWS Bedrock with bearer token authentication.""" + """Route through AWS Bedrock with bearer token authentication. + + Deliberately carries NO credential field: the bearer token is a secret, and + a route object flows through orchestrator state (``CheckContext``, + ``environment_info`` recording, logging) that has no business handling one. + Every consumer that actually needs the token (``ClaudeCodeAgent._build_sdk_env`` + for the agent subprocess, ``judge_bedrock.invoke_bedrock_judge_async`` for the + judge's HTTP call) reads ``settings.aws_bearer_token_bedrock`` itself, via the + shared ``coder_eval.config.settings`` singleton — the same source ``resolve_route`` + validated before constructing this route in the first place. + """ - bearer_token: str region: str model: str | None = None # Cross-region model ID, e.g. "eu.anthropic.claude-sonnet-4-6" small_model: str | None = None # Cross-region small model ID @@ -108,13 +121,15 @@ class LiteLLMRoute: gateway fronting Bedrock open-weight models). The Claude Code SDK is pointed at ``base_url`` via ``ANTHROPIC_BASE_URL`` and - authenticates with ``auth_token`` via ``ANTHROPIC_AUTH_TOKEN`` (bearer). The - ``model``/``small_model`` ids are passed **verbatim** (no Bedrock - inference-profile qualification) — the gateway maps them to its backend. + authenticates via ``ANTHROPIC_AUTH_TOKEN`` (bearer). The ``model``/``small_model`` + ids are passed **verbatim** (no Bedrock inference-profile qualification) — the + gateway maps them to its backend. + + Deliberately carries NO credential field — see ``BedrockRoute``'s docstring for + why. ``ClaudeCodeAgent._build_sdk_env`` reads ``settings.litellm_auth_token`` itself. """ base_url: str - auth_token: str model: str | None = None small_model: str | None = None @@ -130,6 +145,16 @@ class LiteLLMRoute: } +def _bedrock_model_pair(model: str | None, small_model: str | None, region: str) -> tuple[str | None, str | None]: + """Resolve ``(model, small_model)`` into Bedrock inference-profile ids, defaulting + ``small_model`` to ``model`` when unset. Shared by every ``BedrockRoute`` construction + site (``resolve_route``, ``_resolve_backend_route``, ``resolve_evaluation_route``'s + pin-to-Claude branch) so the qualification logic can't drift between them. + """ + resolved_small = small_model or model + return to_bedrock_inference_profile(model, region), to_bedrock_inference_profile(resolved_small, region) + + def resolve_route(settings: Settings) -> ApiRoute: """Resolve an ``ApiRoute`` from static settings. @@ -155,13 +180,10 @@ def resolve_route(settings: Settings) -> ApiRoute: # ClaudeCodeAgent._build_sdk_env). Leaving it unset made every # WebFetch fail with "model issues" under the Bedrock backend. The main # model is always a valid fallback, so default to it. - small_model = settings.bedrock_small_model or settings.bedrock_model - return BedrockRoute( - bearer_token=settings.aws_bearer_token_bedrock, - region=settings.aws_region, - model=to_bedrock_inference_profile(settings.bedrock_model, settings.aws_region), - small_model=to_bedrock_inference_profile(small_model, settings.aws_region), + model, small_model = _bedrock_model_pair( + settings.bedrock_model, settings.bedrock_small_model, settings.aws_region ) + return BedrockRoute(region=settings.aws_region, model=model, small_model=small_model) case ApiBackend.DIRECT: return DirectRoute(judge_transport=_resolve_direct_judge_transport(settings)) case ApiBackend.LITELLM: @@ -177,39 +199,107 @@ def resolve_route(settings: Settings) -> ApiRoute: small_model = settings.litellm_small_model or settings.litellm_model return LiteLLMRoute( base_url=settings.litellm_base_url, - auth_token=settings.litellm_auth_token, model=settings.litellm_model, small_model=small_model, ) -def resolve_evaluation_route(settings: Settings, agent_route: ApiRoute) -> ApiRoute: +def _resolve_backend_route(settings: Settings, backend: ApiBackend, *, model_override: str | None = None) -> ApiRoute: + """Build the ``ApiRoute`` for an EXPLICITLY-requested backend, from the same + env-sourced ``Settings`` fields ``resolve_route`` reads for the agent — + credentials always come from the environment, never from a task/variant. + + Used only by the ``checker_context.api_route`` override path (see + ``resolve_evaluation_route``): raises ``ValueError`` naming the missing env + var when that backend isn't configured, rather than silently falling back + to a different backend — an explicit override that can't be honored must + fail loudly, not degrade to a backend the task author didn't ask for. + + ``model_override`` (``checker_context.api_route.model``) wins over the + backend's own env-configured default model when set. + """ + match backend: + case ApiBackend.BEDROCK: + if not settings.aws_bearer_token_bedrock or not settings.aws_region: + raise ValueError( + "checker_context route 'bedrock' requires AWS_BEARER_TOKEN_BEDROCK and AWS_REGION to be set" + ) + judge_model = model_override or settings.bedrock_model or DEFAULT_JUDGE_MODEL + model, small_model = _bedrock_model_pair(judge_model, settings.bedrock_small_model, settings.aws_region) + return BedrockRoute(region=settings.aws_region, model=model, small_model=small_model) + case ApiBackend.DIRECT: + if not settings.anthropic_api_key: + raise ValueError("checker_context route 'direct' requires ANTHROPIC_API_KEY to be set") + return DirectRoute(judge_transport="anthropic", model=model_override) + case ApiBackend.LITELLM: + if not settings.litellm_base_url or not settings.litellm_auth_token: + raise ValueError( + "checker_context route 'litellm' requires LITELLM_BASE_URL and LITELLM_AUTH_TOKEN to be set" + ) + judge_model = model_override or settings.litellm_model + small_model = settings.litellm_small_model or judge_model + return LiteLLMRoute( + base_url=settings.litellm_base_url, + model=judge_model, + small_model=small_model, + ) + + +def resolve_evaluation_route( + settings: Settings, + agent_route: ApiRoute, + *, + backend_override: str | None = None, + model_override: str | None = None, +) -> ApiRoute: """Resolve the route used by the *evaluation* side — the ``llm_judge`` / ``agent_judge`` criteria and the simulated user — which must stay on a constant Claude backend regardless of the agent under test, so grading and simulation stay comparable across models. - - Agent on Bedrock/Direct: the judge already runs on Claude via that route, - so reuse it unchanged (no behavior change for existing runs). - - Agent on LiteLLM (open-weight): the agent route cannot serve a Claude - judge, so pin evaluation to Bedrock (preferred, from the AWS bearer token) - or Direct (``ANTHROPIC_API_KEY``). If neither is configured, fall back to a + Both overrides come from the reserved ``checker_context.api_route`` namespace + (see ``TaskDefinition.checker_context``) — ``route`` (``backend_override``) + picks the backend, ``model`` (``model_override``) picks the model on + whichever route is resolved. Criteria never read either directly; they only + ever see the resulting ``CheckContext.route.model``. + + - ``backend_override`` set: build that backend's route from env, regardless + of ``agent_route`` — an explicit task/variant choice always wins. Raises + ``ValueError`` if the string isn't a known ``ApiBackend`` or that backend + isn't configured (see ``_resolve_backend_route``). + - Agent on Bedrock/Direct (no ``backend_override``): the judge already runs + on Claude via that route, so reuse it unchanged — except ``model_override``, + if set, still replaces its ``model`` (the route object itself, e.g. a + shared ``BedrockRoute``, is otherwise reused as-is). + - Agent on LiteLLM (open-weight, no ``backend_override``): the agent route + cannot serve a Claude judge, so pin evaluation to Bedrock (preferred, from + the AWS bearer token) or Direct (``ANTHROPIC_API_KEY``), honoring + ``model_override`` there too. If neither is configured, fall back to a ``DirectRoute`` with no judge transport so ``llm_judge`` fails with its clean "unconfigured" error rather than silently scoring 0.0. """ + if backend_override is not None: + try: + backend = ApiBackend(backend_override) + except ValueError as e: + valid = ", ".join(b.value for b in ApiBackend) + raise ValueError(f"checker_context route {backend_override!r} is not a known backend ({valid})") from e + return _resolve_backend_route(settings, backend, model_override=model_override) if isinstance(agent_route, BedrockRoute | DirectRoute): - return agent_route + if not model_override: + return agent_route + if isinstance(agent_route, BedrockRoute): + # Bedrock model ids must be region-qualified — reusing the agent's route + # verbatim would ship a bare alias straight to the Bedrock API (400). + qualified_model, _ = _bedrock_model_pair(model_override, None, agent_route.region) + return replace(agent_route, model=qualified_model) + return replace(agent_route, model=model_override) # agent_route is LiteLLMRoute → pin evaluation to a constant Claude backend. if settings.aws_bearer_token_bedrock and settings.aws_region: - judge_model = settings.bedrock_model or DEFAULT_JUDGE_MODEL - qualified = to_bedrock_inference_profile(judge_model, settings.aws_region) - return BedrockRoute( - bearer_token=settings.aws_bearer_token_bedrock, - region=settings.aws_region, - model=qualified, - small_model=qualified, - ) - return DirectRoute(judge_transport=_resolve_direct_judge_transport(settings)) + judge_model = model_override or settings.bedrock_model or DEFAULT_JUDGE_MODEL + model, small_model = _bedrock_model_pair(judge_model, None, settings.aws_region) + return BedrockRoute(region=settings.aws_region, model=model, small_model=small_model) + return DirectRoute(judge_transport=_resolve_direct_judge_transport(settings), model=model_override) def _resolve_direct_judge_transport(settings: Settings) -> JudgeTransport | None: diff --git a/src/coder_eval/models/tasks.py b/src/coder_eval/models/tasks.py index b3034727..50b32b2d 100644 --- a/src/coder_eval/models/tasks.py +++ b/src/coder_eval/models/tasks.py @@ -17,7 +17,7 @@ RunCommandCriterion, SuccessCriterion, ) -from coder_eval.models.enums import AgentKind +from coder_eval.models.enums import AgentKind, ApiBackend from coder_eval.models.judge_defaults import DEFAULT_JUDGE_MODEL from coder_eval.models.limits import RunLimits from coder_eval.models.merge_strategy import MergeField @@ -78,6 +78,46 @@ class UnknownTaskFieldWarning(DeprecationWarning): :data:`NORMALIZED_CRITERION_ALIASES`.""" +def validate_checker_context_shape(value: dict[str, dict[str, Any]]) -> None: + """Reject an unknown ``checker_context`` namespace/key rather than silently + no-op'ing a typo. ``checker_context`` has no Pydantic schema of its own (it's + an open, namespaced bag — see ``TaskDefinition.checker_context``'s field + description), so a misspelled ``api_rotue`` or ``rotue:`` would otherwise + pass through, get merged across every experiment layer, and simply never be + read by ``Orchestrator._eval_route_overrides`` — an override silently doing + nothing, with no error anywhere. Called both from + ``TaskDefinition.validate_checker_context`` (catches a typo on the task's own + YAML) and from ``orchestration/experiment.py::_resolve_checker_context`` + (catches one introduced only at the experiment-defaults/variant layer, which + bypasses the field validator since ``model_copy`` doesn't re-validate). + """ + known_namespaces = {"api_route"} + unknown_namespaces = set(value) - known_namespaces + if unknown_namespaces: + msg = ( + f"checker_context has unknown namespace(s) {sorted(unknown_namespaces)}; " + f"known namespaces: {sorted(known_namespaces)}" + ) + raise ValueError(msg) + api_route = value.get("api_route") + if api_route is not None: + known_keys = {"route", "model"} + unknown_keys = set(api_route) - known_keys + if unknown_keys: + msg = ( + f"checker_context.api_route has unknown key(s) {sorted(unknown_keys)}; known keys: {sorted(known_keys)}" + ) + raise ValueError(msg) + route = api_route.get("route") + if route is not None: + try: + ApiBackend(route) + except ValueError as e: + valid = sorted(b.value for b in ApiBackend) + msg = f"checker_context.api_route.route {route!r} is not a known backend ({valid})" + raise ValueError(msg) from e + + class SimulationConfig(BaseModel): """Configuration for multi-turn user simulation. @@ -438,6 +478,20 @@ class TaskDefinition(BaseModel): # noqa: CE009 -- soft-launch: see _warn_on_unk strategy="replace", # not layer-merged today; replace = the engine default if it ever is description="List of criteria that must all pass for task success", ) + checker_context: dict[str, dict[str, Any]] = Field( + default_factory=dict, + description=( + "Task-authored config for the success-checking side, namespaced by reserved key. Currently " + "the only recognized namespace is `api_route`, e.g. `{api_route: {route: litellm, model: " + "gpt-5}}`: `route` selects the backend the WHOLE evaluation side (llm_judge, agent_judge, the " + "simulator) calls, decoupled from the agent's own route; `model` overrides the model that " + "route uses. Both are consumed by the orchestrator (`resolve_evaluation_route`) BEFORE " + "`CheckContext` is built and baked into the resolved route's own `model` field — no criterion " + "ever reads `checker_context` directly, only `CheckContext.route.model`. Credentials are " + "always resolved from environment variables, never from this field. Merged shallow-per-" + "namespace across default -> experiment-defaults -> task -> variant." + ), + ) run_limits: RunLimits | None = Field( default=None, description=( @@ -715,3 +769,17 @@ def validate_success_criteria(cls, v: Any) -> Any: if not v: raise ValueError("At least one success criterion must be defined") return v + + @field_validator("checker_context") + @classmethod + def validate_checker_context(cls, v: dict[str, dict[str, Any]]) -> dict[str, dict[str, Any]]: + """Reject an unknown namespace/key rather than silently no-op'ing a typo. + + Only catches a typo already present on the TASK's own YAML — a typo + introduced solely at the experiment-defaults/variant layer bypasses this + (``model_copy`` doesn't re-validate), so ``_resolve_checker_context`` + (``orchestration/experiment.py``) calls the same shared checker after + merging, to catch those too. + """ + validate_checker_context_shape(v) + return v diff --git a/src/coder_eval/orchestration/experiment.py b/src/coder_eval/orchestration/experiment.py index d103436b..d852315f 100644 --- a/src/coder_eval/orchestration/experiment.py +++ b/src/coder_eval/orchestration/experiment.py @@ -35,6 +35,7 @@ VariantAggregate, VariantResult, apply_prompt_mutations, + validate_checker_context_shape, ) from ..path_utils import build_task_run_dir from .config import BatchRunConfig @@ -172,6 +173,46 @@ def _resolve_simulation( return resolved +def _resolve_checker_context( + default_experiment: ExperimentDefinition, + experiment: ExperimentDefinition, + task: TaskDefinition, + variant: ExperimentVariant, +) -> dict[str, dict[str, Any]]: + """Merge ``checker_context`` across the 4-layer precedence chain. + + Precedence (lowest to highest): + 1. default_experiment.defaults.checker_context + 2. experiment.defaults.checker_context + 3. task.checker_context + 4. variant.checker_context + + Unlike ``agent``/``simulation``, ``checker_context`` is an open, namespaced bag + (not a fixed model) — a later layer's namespace merges shallowly onto the same + namespace from an earlier layer (per-key overwrite within the namespace), + rather than replacing the whole namespace. A namespace absent from a layer is + left untouched by that layer. Currently the only recognized namespace is + ``api_route`` (``route``/``model``). + """ + layers: list[dict[str, dict[str, Any]] | None] = [ + default_experiment.defaults.checker_context if default_experiment.defaults else None, + experiment.defaults.checker_context if experiment.defaults else None, + task.checker_context or None, + variant.checker_context, + ] + merged: dict[str, dict[str, Any]] = {} + for layer in layers: + if not layer: + continue + for namespace, patch in layer.items(): + merged[namespace] = {**merged.get(namespace, {}), **patch} + # Catches a typo introduced only at the experiment-defaults/variant layer — + # TaskDefinition's own field validator only ever sees the task's raw YAML, + # not this merged result (model_copy doesn't re-validate). + validate_checker_context_shape(merged) + return merged + + def _resolve_repeats( default_experiment: ExperimentDefinition, experiment: ExperimentDefinition, @@ -446,6 +487,7 @@ def _add_rl(rl: RunLimits | None, source: ConfigSource) -> None: # Mirrors agent merge semantics — a later layer's keys overwrite earlier ones, and # the final dict is validated by building a SimulationConfig from it. resolved_simulation = _resolve_simulation(default_experiment, experiment, task, variant, lineage) + resolved_checker_context = _resolve_checker_context(default_experiment, experiment, task, variant) # Build resolved task (copy with overrides) resolved_task = task.model_copy( @@ -456,6 +498,7 @@ def _add_rl(rl: RunLimits | None, source: ConfigSource) -> None: "post_run": resolved_post_run, "pre_run": resolved_pre_run, "simulation": resolved_simulation, + "checker_context": resolved_checker_context, } ) diff --git a/src/coder_eval/orchestrator.py b/src/coder_eval/orchestrator.py index 63a643f1..8caee878 100644 --- a/src/coder_eval/orchestrator.py +++ b/src/coder_eval/orchestrator.py @@ -12,7 +12,7 @@ from datetime import datetime from inspect import isawaitable from pathlib import Path -from typing import Any +from typing import Any, NamedTuple from urllib.parse import urlparse from .agent import Agent @@ -150,6 +150,15 @@ async def _pump_stream( _UTTERANCE_TAG_RE = re.compile(r"^\[(ASSISTANT|RESULT - SUCCESS|RESULT - ERROR|TOOL USE)\](?: (.*))?$") +class EvalRouteOverrides(NamedTuple): + """``checker_context.api_route``'s ``(backend, model)`` pair. Named fields + (rather than a bare tuple) so a future transposition at a call site is a + typo'd attribute, not a silent positional swap of backend vs. model.""" + + backend: str | None + model: str | None + + def _format_routing(route: ApiRoute, effective_model: str | None = None) -> str: """Format the route name for the ``API routing:`` log line. @@ -1276,12 +1285,7 @@ async def _setup(self) -> None: self.sandbox.reference_dir = self._reference_dir self.result.sandbox_path = str(self.sandbox.sandbox_dir) - self.route = resolve_route(settings) - self.eval_route = resolve_evaluation_route(settings, self.route) - logger.info( - "API routing: %s", _format_routing(self.route, self.task.agent.model if self.task.agent else None) - ) - self.success_checker = SuccessChecker(self.sandbox, route=self.eval_route) + self._resolve_routes() self._record_route_environment_info() return @@ -1347,10 +1351,7 @@ async def _setup_sandbox() -> Any: self.result.sandbox_path = str(sandbox_dir) # Determine API routing from settings.api_backend enum - self.route = resolve_route(settings) - self.eval_route = resolve_evaluation_route(settings, self.route) - logger.info("API routing: %s", _format_routing(self.route, self.task.agent.model if self.task.agent else None)) - self.success_checker = SuccessChecker(self.sandbox, route=self.eval_route) + self._resolve_routes() # Create and start the agent. For a no-op (type: none) task this dispatches # to NoOpAgent, whose start/communicate/stop are no-ops — the orchestrator @@ -1456,6 +1457,43 @@ def _sync_sandbox_command_path_with_agent(self) -> None: if isinstance(path, str) and path: self.sandbox.set_command_base_path(path) + def _eval_route_overrides(self) -> EvalRouteOverrides: + """The ``(backend, model)`` pair from ``task.checker_context.api_route``, if any. + + A task/variant-authored choice for the WHOLE evaluation side (``llm_judge``, + ``agent_judge``, the simulator all share one ``eval_route``), decoupled from + the agent's own route/model — resolved into a credentialed ``ApiRoute`` (from + env vars) by ``resolve_evaluation_route``, which bakes ``model`` into the + resolved route's own ``model`` field. Reserved under one ``api_route`` + namespace (not per-criterion-type): there is exactly one eval route per run, + not one per criterion. Either element is ``None`` when unset, in which case + ``resolve_evaluation_route`` falls back to its existing pin-to-Claude / + env-configured-default behavior. NOT currently ``-D``-reachable — task/variant + YAML only. + """ + api_route = self.task.checker_context.get("api_route", {}) + backend = api_route.get("route") + model = api_route.get("model") + return EvalRouteOverrides( + backend=str(backend) if backend is not None else None, + model=str(model) if model is not None else None, + ) + + def _resolve_routes(self) -> None: + """Resolve ``self.route``/``self.eval_route``, log routing, and build + ``self.success_checker``. Shared by the evaluate-only and normal setup + paths in ``_setup`` — identical logic, previously duplicated at each call + site. Requires ``self.sandbox`` to already be set. + """ + assert self.sandbox is not None + self.route = resolve_route(settings) + overrides = self._eval_route_overrides() + self.eval_route = resolve_evaluation_route( + settings, self.route, backend_override=overrides.backend, model_override=overrides.model + ) + logger.info("API routing: %s", _format_routing(self.route, self.task.agent.model if self.task.agent else None)) + self.success_checker = SuccessChecker(self.sandbox, route=self.eval_route) + def _record_route_environment_info(self) -> None: """Persist resolved route + judge transport into ``result.environment_info``. @@ -1472,6 +1510,13 @@ def _record_route_environment_info(self) -> None: # it, distinct from the agent's api_routing. if self.eval_route is not None: self.result.environment_info["eval_routing"] = ROUTE_NAMES[type(self.eval_route)] + # bedrock_model/litellm_model below are sourced from self.route (the + # AGENT's route) — record the judge/simulator's own model separately so + # a checker_context.api_route.model override (or the LiteLLM-agent + # pinned-to-Bedrock default) is visible in run artifacts, not just + # inferable from the agent's model. + if self.eval_route.model: + self.result.environment_info["eval_model"] = self.eval_route.model if isinstance(self.route, BedrockRoute): self.result.environment_info["aws_region"] = self.route.region if self.route.model: diff --git a/tests/test_config_precedence.py b/tests/test_config_precedence.py index b726786c..d968fda3 100644 --- a/tests/test_config_precedence.py +++ b/tests/test_config_precedence.py @@ -381,7 +381,6 @@ def test_resolve_route_bedrock(): ) route = resolve_route(s) assert isinstance(route, BedrockRoute) - assert route.bearer_token == "tok-123" assert route.region == "us-east-1" assert route.model == "eu.anthropic.claude-sonnet-4-6" diff --git a/tests/test_judge_bedrock.py b/tests/test_judge_bedrock.py index a3ea103a..d0a0aeee 100644 --- a/tests/test_judge_bedrock.py +++ b/tests/test_judge_bedrock.py @@ -27,6 +27,13 @@ def _make_response(*, status_code: int = 200, json_data: Any = None, text: str = return response +@pytest.fixture(autouse=True) +def _bearer_token(monkeypatch: pytest.MonkeyPatch) -> None: + """invoke_bedrock_judge_async reads the bearer token from settings, not the + route, so every test needs one set regardless of whether it inspects it.""" + monkeypatch.setattr(judge_bedrock.settings, "aws_bearer_token_bedrock", "test-token") + + @pytest.fixture def no_sleep(monkeypatch: pytest.MonkeyPatch) -> list[float]: """Capture backoff sleeps instead of actually sleeping.""" @@ -40,7 +47,7 @@ async def fake_sleep(s: float) -> None: def _route() -> BedrockRoute: - return BedrockRoute(bearer_token="test-token", region="eu-north-1") + return BedrockRoute(region="eu-north-1") def _tool_use_response(score: float = 0.5, rationale: str = "ok") -> dict[str, Any]: diff --git a/tests/test_judge_burn_in_live.py b/tests/test_judge_burn_in_live.py index 67cfc3f1..bf780fac 100644 --- a/tests/test_judge_burn_in_live.py +++ b/tests/test_judge_burn_in_live.py @@ -110,7 +110,7 @@ def test_llm_judge_bedrock_tool_channel(hello_sandbox: Sandbox) -> None: result = SuccessChecker( hello_sandbox, init_registry=False, - route=BedrockRoute(bearer_token=bearer, region=region), + route=BedrockRoute(region=region), ).check(criterion) assert result.error is None, f"Bedrock judge failed: {result.error}\n{result.details}" diff --git a/tests/test_litellm_cost.py b/tests/test_litellm_cost.py index 2e5c77b5..7f4c220c 100644 --- a/tests/test_litellm_cost.py +++ b/tests/test_litellm_cost.py @@ -252,7 +252,7 @@ def test_joins_on_litellm_route(self, tmp_path, monkeypatch): ) monkeypatch.setattr(orch_mod.settings, "litellm_cost_log", str(log)) fake = SimpleNamespace( - route=LiteLLMRoute(base_url="http://x:4000", auth_token="k", model="deepseek/deepseek-v4-pro"), + route=LiteLLMRoute(base_url="http://x:4000", model="deepseek/deepseek-v4-pro"), result=_result([_turn(0, static_cost=0.5)]), _cost_correlation_run_id=run_id, _cost_attempt_nonce="att1", @@ -265,7 +265,7 @@ def test_joins_on_litellm_route(self, tmp_path, monkeypatch): def test_join_never_raises_on_bad_log(self, tmp_path, monkeypatch): monkeypatch.setattr(orch_mod.settings, "litellm_cost_log", str(tmp_path / "does-not-exist.jsonl")) fake = SimpleNamespace( - route=LiteLLMRoute(base_url="http://x:4000", auth_token="k"), + route=LiteLLMRoute(base_url="http://x:4000"), result=_result([_turn(0, static_cost=0.5)]), _cost_correlation_run_id="R", _cost_attempt_nonce="att1", @@ -289,7 +289,7 @@ def test_run_total_rederives_from_actual_after_join(self, tmp_path, monkeypatch) ) monkeypatch.setattr(orch_mod.settings, "litellm_cost_log", str(log)) fake = SimpleNamespace( - route=LiteLLMRoute(base_url="http://x:4000", auth_token="k", model="deepseek/deepseek-v4-pro"), + route=LiteLLMRoute(base_url="http://x:4000", model="deepseek/deepseek-v4-pro"), result=_result([_turn(0, static_cost=0.5), _turn(1, static_cost=0.5)]), _cost_correlation_run_id=run_id, _cost_attempt_nonce="att1", diff --git a/tests/test_litellm_route.py b/tests/test_litellm_route.py index 3d7a7a8c..d009dc19 100644 --- a/tests/test_litellm_route.py +++ b/tests/test_litellm_route.py @@ -47,7 +47,7 @@ def _isolated_settings(monkeypatch, **kwargs): return Settings(_env_file=None, **kwargs) def test_bedrock_agent_route_is_reused_unchanged(self, monkeypatch): - route = BedrockRoute(bearer_token="tok", region="eu-north-1", model="eu.anthropic.claude-sonnet-4-6") + route = BedrockRoute(region="eu-north-1", model="eu.anthropic.claude-sonnet-4-6") settings = self._isolated_settings(monkeypatch, api_backend=ApiBackend.BEDROCK) assert resolve_evaluation_route(settings, route) is route @@ -57,7 +57,7 @@ def test_direct_agent_route_is_reused_unchanged(self, monkeypatch): assert resolve_evaluation_route(settings, route) is route def test_litellm_agent_pins_evaluation_to_bedrock_when_aws_creds_present(self, monkeypatch): - agent = LiteLLMRoute(base_url="http://x:4000", auth_token="sk-1", model="zai.glm-5") + agent = LiteLLMRoute(base_url="http://x:4000", model="zai.glm-5") settings = self._isolated_settings( monkeypatch, api_backend=ApiBackend.LITELLM, @@ -66,13 +66,12 @@ def test_litellm_agent_pins_evaluation_to_bedrock_when_aws_creds_present(self, m ) ev = resolve_evaluation_route(settings, agent) assert isinstance(ev, BedrockRoute) - assert ev.bearer_token == "aws-tok" assert ev.region == "eu-north-1" # Judge + simulator run on a Claude model, region-qualified. assert ev.model == "eu.anthropic.claude-sonnet-4-6" def test_litellm_agent_falls_back_to_direct_when_only_anthropic_key(self, monkeypatch): - agent = LiteLLMRoute(base_url="http://x:4000", auth_token="sk-1", model="zai.glm-5") + agent = LiteLLMRoute(base_url="http://x:4000", model="zai.glm-5") settings = self._isolated_settings(monkeypatch, api_backend=ApiBackend.LITELLM, anthropic_api_key="sk-ant") ev = resolve_evaluation_route(settings, agent) assert isinstance(ev, DirectRoute) @@ -81,7 +80,7 @@ def test_litellm_agent_falls_back_to_direct_when_only_anthropic_key(self, monkey def test_litellm_agent_unconfigured_yields_direct_with_no_transport(self, monkeypatch): # No Bedrock creds and no ANTHROPIC_API_KEY → DirectRoute(None), which makes # llm_judge fail with its clean "unconfigured" error rather than scoring 0.0. - agent = LiteLLMRoute(base_url="http://x:4000", auth_token="sk-1", model="zai.glm-5") + agent = LiteLLMRoute(base_url="http://x:4000", model="zai.glm-5") settings = self._isolated_settings(monkeypatch, api_backend=ApiBackend.LITELLM) ev = resolve_evaluation_route(settings, agent) assert isinstance(ev, DirectRoute) @@ -100,8 +99,8 @@ async def test_simulator_receives_eval_route_not_agent_route(self, monkeypatch): from coder_eval import orchestrator as orch_mod from coder_eval.orchestrator import Orchestrator - eval_route = BedrockRoute(bearer_token="t", region="eu-north-1", model="eu.anthropic.claude-sonnet-4-6") - agent_route = LiteLLMRoute(base_url="http://x:4000", auth_token="k", model="zai.glm-5") + eval_route = BedrockRoute(region="eu-north-1", model="eu.anthropic.claude-sonnet-4-6") + agent_route = LiteLLMRoute(base_url="http://x:4000", model="zai.glm-5") captured: dict = {} class _SpySimulator: @@ -141,7 +140,6 @@ def test_resolves_custom_route_with_all_fields(self): route = resolve_route(settings) assert isinstance(route, LiteLLMRoute) assert route.base_url == "http://localhost:4000" - assert route.auth_token == "sk-master" assert route.model == "deepseek.v3.2" def test_rejects_scheme_less_base_url(self): @@ -253,10 +251,12 @@ def test_none_agent_skips_custom_validation(self): class TestBuildSdkEnvCustom: """_build_sdk_env() for the LiteLLM route.""" - def test_custom_route_env_has_anthropic_vars_only(self): + def test_custom_route_env_has_anthropic_vars_only(self, monkeypatch): + from coder_eval.agents import claude_code_agent as claude_code_agent_mod + + monkeypatch.setattr(claude_code_agent_mod.settings, "litellm_auth_token", "sk-1") route = LiteLLMRoute( base_url="http://x:4000", - auth_token="sk-1", model="deepseek.v3.2", small_model="deepseek.v3.2", ) @@ -273,7 +273,7 @@ def test_custom_route_env_has_anthropic_vars_only(self): assert "AWS_REGION" not in env def test_custom_route_no_model_omits_model_vars(self): - route = LiteLLMRoute(base_url="http://x:4000", auth_token="sk-1") + route = LiteLLMRoute(base_url="http://x:4000") env, model = ClaudeCodeAgent._build_sdk_env(route) assert model is None assert "ANTHROPIC_MODEL" not in env @@ -284,7 +284,7 @@ def test_custom_route_forwards_path(self, monkeypatch): custom_path = f"/custom/bin{os.pathsep}/usr/bin" monkeypatch.setenv("PATH", custom_path) - env, _ = ClaudeCodeAgent._build_sdk_env(LiteLLMRoute(base_url="http://x:4000", auth_token="sk-1")) + env, _ = ClaudeCodeAgent._build_sdk_env(LiteLLMRoute(base_url="http://x:4000")) assert env["PATH"] == custom_path def test_custom_route_neutralizes_inherited_anthropic_api_key(self, monkeypatch): @@ -292,20 +292,20 @@ def test_custom_route_neutralizes_inherited_anthropic_api_key(self, monkeypatch) empty in options.env (not merely omitted) — else it would fight the bearer auth_token against the gateway.""" monkeypatch.setenv("ANTHROPIC_API_KEY", "leaked-key") - env, _ = ClaudeCodeAgent._build_sdk_env(LiteLLMRoute(base_url="http://x:4000", auth_token="sk-1")) + env, _ = ClaudeCodeAgent._build_sdk_env(LiteLLMRoute(base_url="http://x:4000")) assert env["ANTHROPIC_API_KEY"] == "" def test_cost_log_tags_become_custom_headers(self): """cost_log_tags → ANTHROPIC_CUSTOM_HEADERS as newline-separated `Name: Value` pairs (the format Claude Code forwards verbatim), so the proxy-side cost log can join each call back to the run/task/turn.""" - route = LiteLLMRoute(base_url="http://x:4000", auth_token="sk-1", model="deepseek/deepseek-v4-pro") + route = LiteLLMRoute(base_url="http://x:4000", model="deepseek/deepseek-v4-pro") tags = {"x-ce-run-id": "abc123", "x-ce-task-id": "calc/v1", "x-ce-iteration": "2"} env, _ = ClaudeCodeAgent._build_sdk_env(route, cost_log_tags=tags) assert env["ANTHROPIC_CUSTOM_HEADERS"] == "x-ce-run-id: abc123\nx-ce-task-id: calc/v1\nx-ce-iteration: 2" def test_no_cost_log_tags_omits_custom_headers(self): - route = LiteLLMRoute(base_url="http://x:4000", auth_token="sk-1") + route = LiteLLMRoute(base_url="http://x:4000") env, _ = ClaudeCodeAgent._build_sdk_env(route) assert "ANTHROPIC_CUSTOM_HEADERS" not in env env2, _ = ClaudeCodeAgent._build_sdk_env(route, cost_log_tags={}) @@ -314,7 +314,7 @@ def test_no_cost_log_tags_omits_custom_headers(self): def test_cost_log_tags_ignored_on_non_litellm_routes(self): """The tag is a LiteLLM-only concern; Bedrock/Direct must not emit it.""" tags = {"x-ce-run-id": "abc123"} - bedrock = BedrockRoute(bearer_token="t", region="eu-north-1", model="x") + bedrock = BedrockRoute(region="eu-north-1", model="x") env_b, _ = ClaudeCodeAgent._build_sdk_env(bedrock, cost_log_tags=tags) assert "ANTHROPIC_CUSTOM_HEADERS" not in env_b env_d, _ = ClaudeCodeAgent._build_sdk_env(DirectRoute(), cost_log_tags=tags) @@ -335,7 +335,7 @@ def test_cost_log_tags_gated_on_agent_capability_not_route(self): assert AgentRegistry.get(AgentKind.CLAUDE_CODE).agent_class.supports_cost_log_tags is True assert AgentRegistry.get(AgentKind.NONE).agent_class.supports_cost_log_tags is False - route = LiteLLMRoute(base_url="http://x:4000", auth_token="sk-1", model="deepseek/deepseek-v4-pro") + route = LiteLLMRoute(base_url="http://x:4000", model="deepseek/deepseek-v4-pro") # A none-agent constructs fine on a LiteLLM route (the gate omits the kwarg)... assert create_agent(AgentKind.NONE, NoneAgentConfig(type=AgentKind.NONE), route=route) is not None # ...and it WOULD crash if the kwarg were forwarded — exactly what the gate prevents. @@ -347,7 +347,7 @@ def test_cost_log_tags_gated_on_agent_capability_not_route(self): def test_cost_log_tags_reject_header_injection(self): # A task_id/variant_id carrying a CR/LF would inject extra headers into every # SDK->proxy request; the seam must reject it (single-line ASCII only). - route = LiteLLMRoute(base_url="http://x:4000", auth_token="sk-1") + route = LiteLLMRoute(base_url="http://x:4000") with pytest.raises(ValueError, match="single-line ASCII"): ClaudeCodeAgent._build_sdk_env(route, cost_log_tags={"x-ce-task-id": "ok\nAuthorization: Bearer forged"}) @@ -356,7 +356,7 @@ class TestResolveEffectiveModelCustom: """_resolve_effective_model() on the LiteLLM route — no prefixing.""" def test_config_model_synced_verbatim(self): - route = LiteLLMRoute(base_url="http://x:4000", auth_token="sk-1", model="deepseek.v3.2") + route = LiteLLMRoute(base_url="http://x:4000", model="deepseek.v3.2") env, route_model = ClaudeCodeAgent._build_sdk_env(route) agent = _make_agent(route, config_model="zai.glm-5") effective = agent._resolve_effective_model("zai.glm-5", env, route_model) @@ -364,14 +364,14 @@ def test_config_model_synced_verbatim(self): assert env["ANTHROPIC_MODEL"] == "zai.glm-5" # no eu./anthropic. prefix def test_route_model_used_when_config_none(self): - route = LiteLLMRoute(base_url="http://x:4000", auth_token="sk-1", model="deepseek.v3.2") + route = LiteLLMRoute(base_url="http://x:4000", model="deepseek.v3.2") env, route_model = ClaudeCodeAgent._build_sdk_env(route) agent = _make_agent(route) effective = agent._resolve_effective_model(None, env, route_model) assert effective == "deepseek.v3.2" def test_both_none_returns_none(self): - route = LiteLLMRoute(base_url="http://x:4000", auth_token="sk-1") + route = LiteLLMRoute(base_url="http://x:4000") env, route_model = ClaudeCodeAgent._build_sdk_env(route) agent = _make_agent(route) effective = agent._resolve_effective_model(None, env, route_model) @@ -444,7 +444,7 @@ def _usage_after_finalize(self, effective_model: str | None) -> TokenUsage: from coder_eval.agents.claude_code_agent import _ClaudeTurnState agent = _make_agent( - LiteLLMRoute(base_url="http://x:4000", auth_token="k", model="zai.glm-5"), + LiteLLMRoute(base_url="http://x:4000", model="zai.glm-5"), config_model="zai.glm-5", ) stub = SimpleNamespace( diff --git a/tests/test_llm_judge_criterion.py b/tests/test_llm_judge_criterion.py index cf9ac941..a0e69a84 100644 --- a/tests/test_llm_judge_criterion.py +++ b/tests/test_llm_judge_criterion.py @@ -574,7 +574,7 @@ def _tool_use_block(score: float, rationale: str = "ok") -> dict: def test_judge_bedrock_route_uses_bedrock_invoker(sandbox: Sandbox) -> None: from coder_eval.models.routing import BedrockRoute - route = BedrockRoute(bearer_token="t", region="eu-north-1") + route = BedrockRoute(region="eu-north-1") criterion = LLMJudgeCriterion(description="x", prompt="grade") with ( patch( @@ -615,7 +615,7 @@ def test_judge_direct_route_uses_anthropic_invoker(sandbox: Sandbox) -> None: def test_judge_bedrock_invoke_runtime_error_maps_to_score_zero(sandbox: Sandbox) -> None: from coder_eval.models.routing import BedrockRoute - route = BedrockRoute(bearer_token="t", region="eu-north-1") + route = BedrockRoute(region="eu-north-1") criterion = LLMJudgeCriterion(description="x", prompt="grade") with patch( "coder_eval.criteria.llm_judge.invoke_bedrock_judge_async", @@ -706,7 +706,7 @@ def test_judge_bedrock_route_threads_model_unchanged(sandbox: Sandbox) -> None: """Translation happens INSIDE the helper, not at the dispatch site.""" from coder_eval.models.routing import BedrockRoute - route = BedrockRoute(bearer_token="t", region="eu-north-1") + route = BedrockRoute(region="eu-north-1") criterion = LLMJudgeCriterion(description="x", prompt="grade", model="anthropic.claude-opus-4-6-v1") with patch( "coder_eval.criteria.llm_judge.invoke_bedrock_judge_async", @@ -1021,9 +1021,7 @@ def test_llm_judge_tool_channel_bedrock(sandbox: Sandbox) -> None: with patch( "coder_eval.criteria.llm_judge.invoke_bedrock_judge_async", new=AsyncMock(return_value=bedrock_response) ) as mock_invoke: - result = SuccessChecker( - sandbox, init_registry=False, route=BedrockRoute(bearer_token="t", region="us-east-1") - ).check(criterion) + result = SuccessChecker(sandbox, init_registry=False, route=BedrockRoute(region="us-east-1")).check(criterion) assert result.score == 0.81 # Confirm we passed the Anthropic-native tool spec. kwargs = mock_invoke.call_args.kwargs @@ -1100,9 +1098,7 @@ def test_judge_usage_bedrock_from_response(sandbox: Sandbox) -> None: score=0.6, usage={"input_tokens": 900, "output_tokens": 40, "cache_read_input_tokens": 100} ) with patch("coder_eval.criteria.llm_judge.invoke_bedrock_judge_async", new=AsyncMock(return_value=resp)): - result = SuccessChecker( - sandbox, init_registry=False, route=BedrockRoute(bearer_token="t", region="us-east-1") - ).check(criterion) + result = SuccessChecker(sandbox, init_registry=False, route=BedrockRoute(region="us-east-1")).check(criterion) assert isinstance(result, JudgeCriterionResult) assert result.token_usage is not None assert result.token_usage.uncached_input_tokens == 900 diff --git a/tests/test_orchestrator.py b/tests/test_orchestrator.py index 2df31856..eadfb526 100644 --- a/tests/test_orchestrator.py +++ b/tests/test_orchestrator.py @@ -45,11 +45,11 @@ def test_format_routing_direct_judge_transport_none_renders_as_none(): def test_format_routing_non_direct_routes_unchanged(): """BedrockRoute keeps the original bare-name format — judge transport is a Direct-only concern.""" - assert _format_routing(BedrockRoute(bearer_token="t", region="us-east-1")) == "aws_bedrock" + assert _format_routing(BedrockRoute(region="us-east-1")) == "aws_bedrock" def test_format_routing_litellm_shows_model(): - out = _format_routing(LiteLLMRoute(base_url="http://localhost:4000", auth_token="k", model="zai.glm-5")) + out = _format_routing(LiteLLMRoute(base_url="http://localhost:4000", model="zai.glm-5")) assert out.startswith("litellm") assert "zai.glm-5" in out @@ -57,7 +57,7 @@ def test_format_routing_litellm_shows_model(): def test_format_routing_litellm_effective_model_wins_over_route_default(): """The --model override (effective_model) must be logged, not the route's LITELLM_MODEL default.""" out = _format_routing( - LiteLLMRoute(base_url="http://localhost:4000", auth_token="k", model="zai.glm-5"), + LiteLLMRoute(base_url="http://localhost:4000", model="zai.glm-5"), effective_model="deepseek.v3.2", ) assert "deepseek.v3.2" in out @@ -107,7 +107,7 @@ def test_record_route_environment_info_direct_none_serialized_as_string(tmp_path def test_record_route_environment_info_bedrock(tmp_path): orchestrator = _make_orchestrator_with_route( - tmp_path, BedrockRoute(bearer_token="t", region="eu-north-1", model="eu.anthropic.claude-sonnet-4-6") + tmp_path, BedrockRoute(region="eu-north-1", model="eu.anthropic.claude-sonnet-4-6") ) orchestrator._record_route_environment_info() assert orchestrator.result is not None @@ -121,7 +121,7 @@ def test_record_route_environment_info_litellm_records_host_only_no_secret(tmp_p """LiteLLM route records host + model, but NEVER the auth token or full base_url.""" orchestrator = _make_orchestrator_with_route( tmp_path, - LiteLLMRoute(base_url="http://localhost:4000", auth_token="sk-super-secret", model="zai.glm-5"), + LiteLLMRoute(base_url="http://localhost:4000", model="zai.glm-5"), ) orchestrator._record_route_environment_info() assert orchestrator.result is not None diff --git a/tests/test_route_seam_exhaustiveness.py b/tests/test_route_seam_exhaustiveness.py index b58a1307..6621be96 100644 --- a/tests/test_route_seam_exhaustiveness.py +++ b/tests/test_route_seam_exhaustiveness.py @@ -29,8 +29,8 @@ # forces this dict to grow whenever the union does. _INSTANCES: list[object] = [ DirectRoute(), - BedrockRoute(bearer_token="t", region="eu-north-1", model="x"), - LiteLLMRoute(base_url="http://localhost:4000", auth_token="k", model="m"), + BedrockRoute(region="eu-north-1", model="x"), + LiteLLMRoute(base_url="http://localhost:4000", model="m"), ] @@ -75,7 +75,7 @@ async def _stub_anthropic(**_: object) -> dict[str, object]: monkeypatch.setattr(llm_judge, "token_usage_from_anthropic_dict", lambda _resp, **_kwargs: None) criterion = MagicMock() for r in _INSTANCES: - result = await _invoke_tool_channel(criterion=criterion, route=r, system_msg="s", user_msg="u") # type: ignore[arg-type] + result = await _invoke_tool_channel(criterion=criterion, model="m", route=r, system_msg="s", user_msg="u") # type: ignore[arg-type] # (verdict, parse_error, raw_text, response_usage) — a 4-tuple means the # route matched an explicit arm rather than falling through. assert isinstance(result, tuple) and len(result) == 4 diff --git a/tests/test_routing.py b/tests/test_routing.py index 35bf11eb..35b7b94e 100644 --- a/tests/test_routing.py +++ b/tests/test_routing.py @@ -5,6 +5,7 @@ from claude_agent_sdk import ClaudeAgentOptions +from coder_eval.agents import claude_code_agent as claude_code_agent_mod from coder_eval.agents.claude_code_agent import ClaudeCodeAgent from coder_eval.config import Settings from coder_eval.models import AgentKind, BedrockRoute, DirectRoute, parse_agent_config @@ -35,7 +36,8 @@ def test_bedrock_basic_env(self, monkeypatch): """BedrockRoute produces CLAUDE_CODE_USE_BEDROCK, token, region, and forwards PATH.""" custom_path = f"/bedrock/bin{os.pathsep}/usr/bin" monkeypatch.setenv("PATH", custom_path) - route = BedrockRoute(bearer_token="tok-123", region="us-east-1") + monkeypatch.setattr(claude_code_agent_mod.settings, "aws_bearer_token_bedrock", "tok-123") + route = BedrockRoute(region="us-east-1") env, _ = ClaudeCodeAgent._build_sdk_env(route) assert env["CLAUDE_CODE_USE_BEDROCK"] == "1" assert env["AWS_BEARER_TOKEN_BEDROCK"] == "tok-123" @@ -44,26 +46,26 @@ def test_bedrock_basic_env(self, monkeypatch): def test_bedrock_attribution_header_disabled(self): """disable_attribution_header=True sets CLAUDE_CODE_ATTRIBUTION_HEADER=0.""" - route = BedrockRoute(bearer_token="t", region="r", disable_attribution_header=True) + route = BedrockRoute(region="r", disable_attribution_header=True) env, _ = ClaudeCodeAgent._build_sdk_env(route) assert env["CLAUDE_CODE_ATTRIBUTION_HEADER"] == "0" def test_bedrock_attribution_header_enabled(self): """disable_attribution_header=False omits the header key.""" - route = BedrockRoute(bearer_token="t", region="r", disable_attribution_header=False) + route = BedrockRoute(region="r", disable_attribution_header=False) env, _ = ClaudeCodeAgent._build_sdk_env(route) assert "CLAUDE_CODE_ATTRIBUTION_HEADER" not in env def test_bedrock_model_override(self): """BedrockRoute.model returned as effective_model.""" - route = BedrockRoute(bearer_token="t", region="r", model="eu.anthropic.claude-sonnet-4-6") + route = BedrockRoute(region="r", model="eu.anthropic.claude-sonnet-4-6") env, model = ClaudeCodeAgent._build_sdk_env(route) assert model == "eu.anthropic.claude-sonnet-4-6" assert env["ANTHROPIC_MODEL"] == "eu.anthropic.claude-sonnet-4-6" def test_bedrock_no_model_returns_none(self): """BedrockRoute without model returns None (use task config).""" - route = BedrockRoute(bearer_token="t", region="r") + route = BedrockRoute(region="r") env, model = ClaudeCodeAgent._build_sdk_env(route) assert model is None assert "ANTHROPIC_MODEL" not in env @@ -80,7 +82,7 @@ def test_propagates_plugin_tools_dir_when_set(self, monkeypatch): def test_propagates_plugin_tools_dir_on_bedrock_route(self, monkeypatch): """Pin must reach the SDK regardless of routing — bedrock edition.""" monkeypatch.setenv("PLUGIN_TOOLS_DIR", "/pinned/tools/@uipath") - env, _ = ClaudeCodeAgent._build_sdk_env(BedrockRoute(bearer_token="t", region="r")) + env, _ = ClaudeCodeAgent._build_sdk_env(BedrockRoute(region="r")) assert env["PLUGIN_TOOLS_DIR"] == "/pinned/tools/@uipath" def test_omits_plugin_tools_dir_when_unset(self, monkeypatch): @@ -103,13 +105,13 @@ def test_external_plugin_tools_dir_beats_fallback(self, monkeypatch): def test_bedrock_default_disables_attribution_header(self): """Default BedrockRoute (no explicit disable_attribution_header) sets CLAUDE_CODE_ATTRIBUTION_HEADER=0.""" - route = BedrockRoute(bearer_token="t", region="r") + route = BedrockRoute(region="r") env, _ = ClaudeCodeAgent._build_sdk_env(route) assert env["CLAUDE_CODE_ATTRIBUTION_HEADER"] == "0" def test_bedrock_small_model(self): """BedrockRoute.small_model appears in env as ANTHROPIC_SMALL_FAST_MODEL.""" - route = BedrockRoute(bearer_token="t", region="r", small_model="eu.anthropic.claude-haiku-4-5") + route = BedrockRoute(region="r", small_model="eu.anthropic.claude-haiku-4-5") env, _ = ClaudeCodeAgent._build_sdk_env(route) assert env["ANTHROPIC_SMALL_FAST_MODEL"] == "eu.anthropic.claude-haiku-4-5" @@ -279,7 +281,7 @@ class TestResolveEffectiveModel: def test_config_model_overrides_bedrock_route_model(self): """Task/CLI model wins over BEDROCK_MODEL and is synced into env.""" - route = BedrockRoute(bearer_token="t", region="us-east-2", model="us.anthropic.claude-sonnet-4-6") + route = BedrockRoute(region="us-east-2", model="us.anthropic.claude-sonnet-4-6") env, route_model = ClaudeCodeAgent._build_sdk_env(route) agent = _make_agent(route, config_model="global.anthropic.claude-sonnet-4-6") effective = agent._resolve_effective_model("global.anthropic.claude-sonnet-4-6", env, route_model) @@ -288,7 +290,7 @@ def test_config_model_overrides_bedrock_route_model(self): def test_route_model_used_when_config_none(self): """BEDROCK_MODEL is the fallback when no task/CLI model is set.""" - route = BedrockRoute(bearer_token="t", region="us-east-2", model="us.anthropic.claude-sonnet-4-6") + route = BedrockRoute(region="us-east-2", model="us.anthropic.claude-sonnet-4-6") env, route_model = ClaudeCodeAgent._build_sdk_env(route) agent = _make_agent(route) effective = agent._resolve_effective_model(None, env, route_model) @@ -297,7 +299,7 @@ def test_route_model_used_when_config_none(self): def test_bare_config_model_auto_prefixes_for_bedrock(self): """A bare --model on a Bedrock route gets the region's inference-profile prefix.""" - route = BedrockRoute(bearer_token="t", region="eu-north-1", model="eu.anthropic.claude-sonnet-4-6") + route = BedrockRoute(region="eu-north-1", model="eu.anthropic.claude-sonnet-4-6") env, route_model = ClaudeCodeAgent._build_sdk_env(route) agent = _make_agent(route, config_model="anthropic.claude-sonnet-4-6") effective = agent._resolve_effective_model("anthropic.claude-sonnet-4-6", env, route_model) @@ -306,7 +308,7 @@ def test_bare_config_model_auto_prefixes_for_bedrock(self): def test_both_none_returns_none(self): """No model anywhere → None, no env mutation.""" - route = BedrockRoute(bearer_token="t", region="us-east-2") + route = BedrockRoute(region="us-east-2") env, route_model = ClaudeCodeAgent._build_sdk_env(route) agent = _make_agent(route) effective = agent._resolve_effective_model(None, env, route_model) @@ -320,7 +322,7 @@ def test_config_model_injects_anthropic_model_when_route_has_none(self): by every Bedrock code path, so config_model must be injected into env on Bedrock even when _build_sdk_env left ANTHROPIC_MODEL absent. """ - route = BedrockRoute(bearer_token="t", region="eu-north-1") # no model + route = BedrockRoute(region="eu-north-1") # no model env, route_model = ClaudeCodeAgent._build_sdk_env(route) assert "ANTHROPIC_MODEL" not in env # precondition agent = _make_agent(route, config_model="claude-sonnet-4-6") @@ -340,9 +342,10 @@ def test_direct_route_does_not_inject_or_prefix(self): class TestSdkOptionsDumpRedaction: """Test that _dump_sdk_options redacts sensitive values.""" - def test_bedrock_token_redacted_in_dump(self): + def test_bedrock_token_redacted_in_dump(self, monkeypatch): """AWS_BEARER_TOKEN_BEDROCK must not appear in plain text in sdk_options dump.""" - route = BedrockRoute(bearer_token="SECRET_TOKEN_123", region="us-east-1") + monkeypatch.setattr(claude_code_agent_mod.settings, "aws_bearer_token_bedrock", "SECRET_TOKEN_123") + route = BedrockRoute(region="us-east-1") env, _ = ClaudeCodeAgent._build_sdk_env(route) opts = ClaudeAgentOptions(cwd=tempfile.gettempdir(), env=env) dump = dump_dataclass(opts) From ba036608c24f848b1993d17a5e466ca640937151 Mon Sep 17 00:00:00 2001 From: Akshaya Shanbhogue Date: Wed, 26 Aug 2026 10:59:28 -0700 Subject: [PATCH 2/9] fix(routing): make _resolve_backend_route's match exhaustive Addresses a CodeQL finding on PR #137: mixing explicit returns per case with an implicit fall-through return (None) reads as a possible bug. Add a raising wildcard arm so every path returns explicitly. Co-Authored-By: Claude Sonnet 5 --- src/coder_eval/models/routing.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/coder_eval/models/routing.py b/src/coder_eval/models/routing.py index 1d3d8247..6bb508a9 100644 --- a/src/coder_eval/models/routing.py +++ b/src/coder_eval/models/routing.py @@ -243,6 +243,11 @@ def _resolve_backend_route(settings: Settings, backend: ApiBackend, *, model_ove model=judge_model, small_model=small_model, ) + case _: + # ApiBackend covers exactly BEDROCK/DIRECT/LITELLM above; this arm is + # unreachable but makes the match exhaustive so every path returns + # explicitly (CodeQL: mixed explicit/implicit returns, PR #137 review). + raise AssertionError(f"unhandled ApiBackend: {backend!r}") def resolve_evaluation_route( From 3c4f0de15641c098a803b5b48939d8a946cfea64 Mon Sep 17 00:00:00 2001 From: Akshaya Shanbhogue Date: Wed, 26 Aug 2026 12:21:06 -0700 Subject: [PATCH 3/9] fix(eval-routing): restore DEFAULT_JUDGE_MODEL floor + add litellm judge transport MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses the PR #137 review blockers and adds the litellm-library-backed LiteLLMRoute judge transport: - resolve_evaluation_route no longer lets the agent's own env-sourced model (e.g. BEDROCK_MODEL) leak into the eval route's `model` on the no-override reuse/pin paths — route.model now means "an explicit checker_context override was given", restoring DEFAULT_JUDGE_MODEL as the judge's floor. - LLMJudgeCriterion.model is now `str | None = None` (was a materialized DEFAULT_JUDGE_MODEL default gated by model_fields_set, which doesn't survive the docker driver's model_dump/reload round trip). Precedence is now `criterion.model or route.model or DEFAULT_JUDGE_MODEL`, computed at check time. - _build_sdk_env's DirectRoute arm now neutralizes inherited Bedrock creds (AWS_BEARER_TOKEN_BEDROCK/CLAUDE_CODE_USE_BEDROCK), matching the LiteLLM arm, so an explicit `route: direct` can't silently spend the operator's Bedrock token. - Implement the `checker_context.api_route.route: litellm` judge transport via the `litellm` library (new `coder-eval[litellm]` extra), with a LiteLLMRoute.include_temperature flag to avoid a live round-trip on gateways that reject `temperature`. - Added ~15 tests covering backend_override/model_override resolution, the judge-model floor regression, docker-serialization round-tripping, and DirectRoute env neutralization — routing.py coverage 69% -> 96.5%. - Doc fixes: corrected stale claims about the judge-model fallback chain, the simulator sharing the agent's ApiRoute, and checker_context's placement/example in the guide and AB_EXPERIMENTS.md. Co-Authored-By: Claude Sonnet 5 --- docs/AB_EXPERIMENTS.md | 1 + docs/DIALOG_MODE.md | 8 +- docs/TASK_DEFINITION_GUIDE.md | 11 +- plugins/coder-eval/reference/criteria.md | 2 +- pyproject.toml | 13 + src/coder_eval/agents/claude_code_agent.py | 17 +- src/coder_eval/criteria/llm_judge.py | 44 ++- src/coder_eval/evaluation/judge_litellm.py | 145 ++++++++ src/coder_eval/evaluation/judge_usage.py | 42 +++ src/coder_eval/evaluation/verdict_tool.py | 45 +++ src/coder_eval/models/criteria.py | 9 +- src/coder_eval/models/routing.py | 38 +- tests/test_judge_litellm.py | 207 +++++++++++ tests/test_litellm_route.py | 164 ++++++++- tests/test_llm_judge_criterion.py | 88 ++++- tests/test_models.py | 8 +- tests/test_route_seam_exhaustiveness.py | 4 + tests/test_routing.py | 14 + uv.lock | 401 ++++++++++++++++++++- 19 files changed, 1211 insertions(+), 50 deletions(-) create mode 100644 src/coder_eval/evaluation/judge_litellm.py create mode 100644 tests/test_judge_litellm.py diff --git a/docs/AB_EXPERIMENTS.md b/docs/AB_EXPERIMENTS.md index 4392d7f6..0e73d6a3 100644 --- a/docs/AB_EXPERIMENTS.md +++ b/docs/AB_EXPERIMENTS.md @@ -130,6 +130,7 @@ From `ExperimentVariant` (`coder_eval/models/experiment.py`): | `initial_prompt_file` | str | Prompt replacement loaded from a file | | `run_limits` | block | Per-key cap overrides (`max_turns`, `task_timeout`, token/USD budgets) | | `driver` | `tempdir`/`docker` | Sandbox driver — enables tempdir-vs-docker arms | +| `checker_context` | dict | Backend/model override for the evaluation side (judge, simulator) — see [Checker Context](TASK_DEFINITION_GUIDE.md#checker-context); **not** currently `-D`-reachable | The `agent` dict is the lever for most A/B tests. Anything on `AgentConfig` is fair game: `model`, `permission_mode`, `allowed_tools`, `disallowed_tools`, diff --git a/docs/DIALOG_MODE.md b/docs/DIALOG_MODE.md index 0b4bad29..622c3c3b 100644 --- a/docs/DIALOG_MODE.md +++ b/docs/DIALOG_MODE.md @@ -60,9 +60,11 @@ The mechanics: - The simulator is a **tools-disabled Claude Code agent** with `allowed_tools: []`, an explicit deny-list, and no plugins or settings sources. It is pure text-in / text-out, and it **cannot see the sandbox** — no files, no terminal, no agent reasoning. Only what the agent writes in the chat. -- The simulator shares the coding agent's resolved `ApiRoute`, so backend and model come from the - run's routing (`--backend direct` / `--backend bedrock`) rather than from the `simulation:` block. - There is no model field here to set. +- The simulator runs on the run's resolved *evaluation* `ApiRoute` — the coding agent's own route + (`--backend direct` / `--backend bedrock`) unless `checker_context.api_route.route` overrides it + (see [Checker Context](TASK_DEFINITION_GUIDE.md#checker-context)). The **model** is separately + pinned by `simulation.model` (see [Simulation](TASK_DEFINITION_GUIDE.md#simulation)), not inherited + from the route — so an A/B varying the subject model doesn't silently vary the simulated user too. **Agent-kind constraint.** The *subject* agent can be any registered kind — the dialog driver only calls the agent's `communicate()`, so Codex and plugin agents work. The *simulator*, however, is diff --git a/docs/TASK_DEFINITION_GUIDE.md b/docs/TASK_DEFINITION_GUIDE.md index af2607df..db09401f 100644 --- a/docs/TASK_DEFINITION_GUIDE.md +++ b/docs/TASK_DEFINITION_GUIDE.md @@ -35,7 +35,7 @@ Complete reference for defining evaluation tasks in Coder Eval. - [llm_judge](#llm_judge) - [agent_judge](#agent_judge) - [skill_triggered](#skill_triggered) - - [Checker Context](#checker-context) +- [Checker Context](#checker-context) - [Reference Solutions](#reference-solutions) - [Pre-Run Commands](#pre-run-commands) - [Post-Run Commands](#post-run-commands) @@ -64,6 +64,7 @@ reference: { ... } # Optional reference solution (a directory pre_run: [ ... ] # Optional pre-run commands (before agent starts) post_run: [ ... ] # Optional post-run commands dataset: { ... } # Optional dataset fan-out (one task -> N row-tasks) +checker_context: { ... } # Optional: backend/model for the evaluation side (judge, simulator) ``` ### `dataset` @@ -1296,7 +1297,7 @@ Observed label is `"yes"` when either signal is found, else `"no"`. Expected lab **Typical pattern.** Label each dataset row with its true skill (`expected_skill`, `""` for negatives) and stack one `skill_triggered` criterion per skill against the same dataset — each gets its own confusion matrix from the same agent traces. This is the natural companion to a skill A/B experiment (skill plugin on vs. off); see the [A/B Experiment Guide](AB_EXPERIMENTS.md#recipe-ab-a-skill). -### Checker Context +## Checker Context `checker_context` carries task-authored config for the success-checking side, namespaced by reserved key. Currently the only recognized namespace is **`api_route`**: @@ -1311,8 +1312,8 @@ checker_context: model: claude-haiku-4-5 # model override for that route ``` -- `route` selects which backend the WHOLE evaluation side calls (`llm_judge`, `agent_judge`, and the simulator all share one resolved eval route per run — this isn't per-criterion), **decoupled from the agent's own route** (a Claude agent can be graded by a differently-backed judge, or vice versa). This is a backend *name* (`direct` / `bedrock` / `litellm`), not a route object: credentials are never read from the task, always from the matching environment variables (`ANTHROPIC_API_KEY` for `direct`, `AWS_BEARER_TOKEN_BEDROCK`/`AWS_REGION` for `bedrock`, `LITELLM_BASE_URL`/`LITELLM_AUTH_TOKEN` for `litellm`). An unconfigured or unknown backend name raises at dispatch rather than silently falling back. **`route: litellm` resolves successfully but `llm_judge` has no OpenAI-compatible transport yet** — it fails with a clear "not implemented yet" error at grading time; use `bedrock` or `direct` for `llm_judge` today. -- `model` overrides the model that resolved route uses — e.g. `llm_judge`'s judge model, when the criterion itself leaves `model:` unset (an explicit per-criterion `model:` always wins; below that, `checker_context.api_route.model`; below that, the backend's own env-configured default, e.g. `BEDROCK_MODEL`/`LITELLM_MODEL`). This works because every `ApiRoute` (`DirectRoute`/`BedrockRoute`/`LiteLLMRoute`) carries its own `model` field; the orchestrator bakes the override into the resolved route's `model` before any criterion runs, so `llm_judge` just reads `context.route.model` — it never reads `checker_context` directly. **`agent_judge` does not currently honor this override** — its sub-agent's model comes from the criterion's own `agent:` block (defaulted to a fixed judge model), independent of `checker_context.api_route.model`. +- `route` selects which backend the WHOLE evaluation side calls (`llm_judge`, `agent_judge`, and the simulator all share one resolved eval route per run — this isn't per-criterion), **decoupled from the agent's own route** (a Claude agent can be graded by a differently-backed judge, or vice versa). This is a backend *name* (`direct` / `bedrock` / `litellm`), not a route object: credentials are never read from the task, always from the matching environment variables (`ANTHROPIC_API_KEY` for `direct`, `AWS_BEARER_TOKEN_BEDROCK`/`AWS_REGION` for `bedrock`, `LITELLM_BASE_URL`/`LITELLM_AUTH_TOKEN` for `litellm`). An unconfigured or unknown backend name raises at dispatch rather than silently falling back. **`route: litellm` dispatches `llm_judge` through the `litellm` library** (the `coder-eval[litellm]` extra, `litellm.acompletion`) rather than assuming one wire protocol — a gateway-routed judge model (e.g. an Azure AI `/openai/v1` deployment) rarely speaks Anthropic Messages, so this lets `model` carry its own provider hint (e.g. `azure_ai/gpt-5.6-luna`) and get that provider's actual request/response shape handled by the library. +- `model` overrides the model that resolved route uses for **`llm_judge` only** — when the criterion itself leaves `model:` unset (precedence: an explicit per-criterion `model:` always wins; below that, `checker_context.api_route.model`; below that, the built-in `DEFAULT_JUDGE_MODEL`). This floor is deliberate and never the agent's own model — an unpinned judge must grade identically regardless of which model the agent under test is using, so `resolve_evaluation_route` never lets the agent's env-configured model (e.g. `BEDROCK_MODEL`) leak into `route.model` on its own; `route.model` is set only when this override was actually given. This works because every `ApiRoute` (`DirectRoute`/`BedrockRoute`/`LiteLLMRoute`) carries its own `model` field; the orchestrator bakes the override into the resolved route's `model` before any criterion runs, so `llm_judge` just reads `context.route.model` — it never reads `checker_context` directly. **`agent_judge` and the simulator do not honor this override** — `agent_judge`'s sub-agent model comes from the criterion's own `agent:` block (defaulted to a fixed judge model), and the simulator's model is pinned by `SimulationConfig.model` (see [Simulation](#simulation) below) — both independent of `checker_context.api_route.model` by design, for the same "measuring instrument stays fixed" reason. `checker_context` merges shallow-per-namespace across `default_experiment.defaults.checker_context` → `experiment.defaults.checker_context` → `task.checker_context` → `variant.checker_context` (same 4-layer precedence as `agent`/`simulation`). So a judge-model A/B, or a judge-backend A/B, is a variant-level config change, not an edit to every task YAML. @@ -1598,7 +1599,7 @@ simulation: | `check_criteria` | `end_of_dialog` | `end_of_dialog`, `every_turn`, or `both`. | | `model` | `anthropic.claude-sonnet-4-6` | Model that plays the simulated user. Auto-translated to the run's backend (Bedrock inference profile / bare Anthropic alias), the same way [`llm_judge`](#llm_judge)'s `model` is. | -The simulator runs as a tools-disabled Claude Code agent sharing the coding agent's `ApiRoute`, so temperature and sampling are resolved at the route level (same `-b` flag as the coding agent) and are not configured on this block. The **model is not**: it is pinned by `model` above. Inheriting it from the route meant `BEDROCK_MODEL` decided who the simulated user was, so an A/B varying the subject model silently varied its interlocutor too. Hold `model` fixed across variants for the same reason you hold a judge model fixed — the simulator is part of the measuring instrument, not the thing being measured. +The simulator runs as a tools-disabled Claude Code agent on the run's resolved *evaluation* `ApiRoute` (the coding agent's own route unless [`checker_context.api_route.route`](#checker-context) overrides it), so temperature and sampling are resolved at the route level (same `-b` flag as the coding agent by default) and are not configured on this block. The **model is not**: it is pinned by `model` above. Inheriting it from the route meant `BEDROCK_MODEL` decided who the simulated user was, so an A/B varying the subject model silently varied its interlocutor too. Hold `model` fixed across variants for the same reason you hold a judge model fixed — the simulator is part of the measuring instrument, not the thing being measured. **Semantics:** diff --git a/plugins/coder-eval/reference/criteria.md b/plugins/coder-eval/reference/criteria.md index ba5441d4..a946bc07 100644 --- a/plugins/coder-eval/reference/criteria.md +++ b/plugins/coder-eval/reference/criteria.md @@ -209,7 +209,7 @@ Optional: | `include_tool_calls` | When true, include a summary of the latest agent turn's tool calls (via summarize_commands). No-op when turn_records is unavailable. | | `include_dialog` | When true, include the full user<->agent conversation across all turns in the judge prompt. In simulation mode the user side is generated by an LLM simulator and may invent premises — the judge should treat any claim made only by the simulated user as possibly fabricated, and not penalize the agent for going along with it unless the task description contradicts it. | | `max_dialog_chars` | Aggregate cap on dialog text rendered into the judge prompt. Prevents an N-turn simulation from blowing out the judge's context window. Per-message truncation uses max_file_chars; trailing turns are dropped when this aggregate budget is exceeded (a degraded note is recorded). | -| `model` | Judge model id (e.g. 'anthropic.claude-sonnet-4-6'). On a BedrockRoute / DirectRoute the value is auto-translated: trailing '-vN[:M]' suffixes and the 'anthropic.' prefix are stripped where the backend doesn't accept them; on Bedrock the cross-region inference-profile prefix is added based on AWS_REGION. | +| `model` | Judge model id (e.g. 'anthropic.claude-sonnet-4-6'). Leave unset to fall back to checker_context.api_route.model when set, else the built-in default ('anthropic.claude-sonnet-4-6') — the fallback is never the agent's own model, so an unpinned judge grades identically across agent-model A/Bs. On a BedrockRoute / DirectRoute the value is auto-translated: trailing '-vN[:M]' suffixes and the 'anthropic.' prefix are stripped where the backend doesn't accept them; on Bedrock the cross-region inference-profile prefix is added based on AWS_REGION. | | `temperature` | Sampling temperature for the judge model. 0.0 keeps grading deterministic. | | `max_tokens` | Output token cap. Defaults to 2000 — large enough for the verbose verdict (score + rationale + a handful of findings) without runaway. | | `max_file_chars` | Per-file content truncation applied before building the prompt. | diff --git a/pyproject.toml b/pyproject.toml index a438018d..ee3e280e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -89,6 +89,19 @@ dev = [ uipath = [ "uipath>=2.10.31", ] +# Optional extra that enables the `checker_context.api_route.route: litellm` +# judge backend (llm_judge only) via the `litellm` library's `acompletion` — +# it normalizes provider-specific quirks (Azure AI's api_base/api_key shape, +# max_tokens vs max_completion_tokens naming, unsupported-param drops, ...) +# so the judge transport doesn't hand-roll per-provider HTTP. NOT the same +# thing as the LiteLLM PROXY (litellm/start-litellm.sh) the AGENT's own +# `route: litellm` points at over HTTP — this extra calls the library +# in-process. Without this extra, the framework still installs and runs; +# the litellm-route judge path fails at dispatch with a clear hint pointing +# back here. +litellm = [ + "litellm>=1.95.0,<2.0.0", +] # Optional extra that enables Codex agent support: # - CodexAgent implementation using official openai-codex SDK # Without this extra, the framework still installs and runs; Codex-dependent diff --git a/src/coder_eval/agents/claude_code_agent.py b/src/coder_eval/agents/claude_code_agent.py index e8a1726c..57b8b382 100644 --- a/src/coder_eval/agents/claude_code_agent.py +++ b/src/coder_eval/agents/claude_code_agent.py @@ -814,8 +814,21 @@ def _build_sdk_env( env["ANTHROPIC_SMALL_FAST_MODEL"] = br.small_model return {**base_env, **env}, br.model - case DirectRoute(): - return base_env, None + case DirectRoute() as dr: + # Neutralize inherited Bedrock creds: the CLI auto-selects Bedrock + # DIRECT when AWS_BEARER_TOKEN_BEDROCK is present in the inherited + # environment (same auto-selection the LiteLLM arm above guards + # against), so an explicit `route: direct` (e.g. via + # checker_context.api_route.route on a run whose agent is on + # Bedrock) would otherwise silently spend the operator's Bedrock + # bearer token instead of ANTHROPIC_API_KEY (PR #137 review). + env = { + "AWS_BEARER_TOKEN_BEDROCK": "", + "CLAUDE_CODE_USE_BEDROCK": "", + } + if dr.model: + env["ANTHROPIC_MODEL"] = dr.model + return {**base_env, **env}, dr.model case LiteLLMRoute() as cr: # Point the SDK at the custom Anthropic-compatible endpoint (e.g. diff --git a/src/coder_eval/criteria/llm_judge.py b/src/coder_eval/criteria/llm_judge.py index b64c7c17..3e0661de 100644 --- a/src/coder_eval/criteria/llm_judge.py +++ b/src/coder_eval/criteria/llm_judge.py @@ -4,6 +4,7 @@ import logging from typing import TYPE_CHECKING +from coder_eval.config import settings from coder_eval.criteria.base import BaseCriterion, CheckContext, register_criterion from coder_eval.evaluation.judge_anthropic import invoke_anthropic_judge_async from coder_eval.evaluation.judge_bedrock import invoke_bedrock_judge_async @@ -15,14 +16,18 @@ format_details, scrub_reference, ) +from coder_eval.evaluation.judge_litellm import invoke_litellm_judge_async from coder_eval.evaluation.judge_usage import ( token_usage_from_anthropic_dict, + token_usage_from_openai_dict, ) from coder_eval.evaluation.verdict_tool import ( SUBMIT_VERDICT_ANTHROPIC_TOOL, extract_verdict_from_anthropic_response, + extract_verdict_from_openai_response, ) from coder_eval.models import ( + DEFAULT_JUDGE_MODEL, BedrockRoute, CriterionResult, DirectRoute, @@ -70,14 +75,14 @@ async def _check_impl_async( ctx = context or CheckContext() route = ctx.route reference_dir = ctx.reference_dir - # criterion.model always carries a concrete default (DEFAULT_JUDGE_MODEL), so an - # explicit per-criterion `model:` is only distinguishable from "unset" via - # model_fields_set — a checker_context.api_route.model override (baked into - # route.model by resolve_evaluation_route) must not clobber a value the task - # author actually wrote. - judge_model = criterion.model - if "model" not in criterion.model_fields_set and route is not None and route.model: - judge_model = route.model + # Precedence: an explicit per-criterion `model:` always wins; otherwise fall + # back to `checker_context.api_route.model` (baked into route.model by + # resolve_evaluation_route — set only when a real override was given, never + # the agent's own model); otherwise DEFAULT_JUDGE_MODEL. `criterion.model` is + # `None` (not a materialized default) when unset, so this precedence survives + # a `model_dump(mode="json")` / reload round trip (e.g. the docker driver's + # task-serialization step) unlike a `model_fields_set` check would. + judge_model = criterion.model or (route.model if route is not None else None) or DEFAULT_JUDGE_MODEL # Master enablement gate. Skipped criteria don't make an LLM call and don't # affect cost; weighted score includes them as 1.0 so they don't penalize. @@ -240,12 +245,23 @@ async def _invoke_tool_channel( verdict, err = extract_verdict_from_anthropic_response(anthropic_response) response_usage = token_usage_from_anthropic_dict(anthropic_response, model=model) case LiteLLMRoute(): - # Reachable now via an explicit `checker_context.api_route.route: litellm` - # override (see resolve_evaluation_route) — but there is no OpenAI-compatible - # transport yet (tracked separately), so fail loudly rather than silently - # scoring 0.0. (Explicit arm also keeps the match exhaustive so pyright - # flags any future route member.) - return None, "llm_judge: LiteLLM judge transport is not implemented yet", "(litellm route)", None + # Reachable via an explicit `checker_context.api_route.route: litellm` + # override (see resolve_evaluation_route). Dispatches through the + # `litellm` library (see invoke_litellm_judge_async's module docstring) + # rather than assuming one wire protocol — task authors point this at + # whatever gateway their judge model actually lives behind. + litellm_response = await invoke_litellm_judge_async( + route=route, + auth_token=settings.litellm_auth_token, + model=model, + system=system_msg, + user=user_msg, + temperature=criterion.temperature, + max_tokens=criterion.max_tokens, + tool_spec=SUBMIT_VERDICT_ANTHROPIC_TOOL, + ) + verdict, err = extract_verdict_from_openai_response(litellm_response) + response_usage = token_usage_from_openai_dict(litellm_response, model=model) case None: # Handled by the unconfigured-arm guard in _check_impl_async before # dispatch; defensive only. diff --git a/src/coder_eval/evaluation/judge_litellm.py b/src/coder_eval/evaluation/judge_litellm.py new file mode 100644 index 00000000..7c42db0b --- /dev/null +++ b/src/coder_eval/evaluation/judge_litellm.py @@ -0,0 +1,145 @@ +"""Single-completion invoker for the LiteLLM judge backend, via the ``litellm`` +library (the ``coder-eval[litellm]`` extra) rather than a hand-rolled HTTP call. + +``LiteLLMRoute``'s docstring frames it as an Anthropic-compatible proxy — true +for the AGENT side (``ClaudeCodeAgent`` points ``ANTHROPIC_BASE_URL`` at the +local ``litellm/start-litellm.sh`` proxy). The checker side reuses the same +route/env vars (``LITELLM_BASE_URL``/``LITELLM_AUTH_TOKEN``) for a different +purpose: task authors point this at whatever gateway their judge model lives +behind (an Azure AI ``/openai/v1`` deployment, a multi-model marketplace, ...), +which is rarely that same Anthropic-passthrough proxy. Calling through +``litellm.acompletion`` — rather than assuming one specific wire protocol — +lets ``model`` carry its own provider hint (e.g. ``azure_ai/gpt-5.6-luna``) +and get that provider's actual request/response shape handled by the library, +including per-provider quirks (Azure AI's ``api_base``/``api_key`` shape, +``max_tokens`` vs ``max_completion_tokens`` naming, unsupported-parameter +drops via ``drop_params``) instead of this module hand-coding them. + +``litellm.acompletion`` always returns an OpenAI-shaped ``ModelResponse`` +regardless of the underlying provider, so the caller reuses +``extract_verdict_from_openai_response``/``token_usage_from_openai_dict`` +unchanged. + +Async on purpose: mirrors ``invoke_anthropic_judge_async`` / +``invoke_bedrock_judge_async`` — the judge's only network call, no sync twin. +""" + +from __future__ import annotations + +import logging +from typing import Any + +from coder_eval.errors import JudgeInfrastructureError +from coder_eval.models import LiteLLMRoute + + +logger = logging.getLogger(__name__) + + +async def invoke_litellm_judge_async( + *, + route: LiteLLMRoute, + auth_token: str | None, + model: str, + system: str, + user: str, + temperature: float, + max_tokens: int, + tool_spec: dict[str, Any], + timeout_seconds: float = 120.0, +) -> dict[str, Any]: + """One completion call via ``litellm.acompletion`` with a forced tool call. + + Returns the OpenAI-shaped response converted to a dict via ``model_dump`` + so the caller can reuse ``extract_verdict_from_openai_response``. + + Raises: + ValueError: ``model`` empty. + JudgeInfrastructureError: the ``litellm`` extra isn't installed; no + auth token configured; or the call fails (an eval-infra fault, + not the agent's fault — CE039). + """ + if not model: + raise ValueError("invoke_litellm_judge_async: model must not be empty") + # Raise (not assert): this call runs inside LLMJudgeChecker's + # handle_criterion_errors(_async) wrapper, which catches plain Exception + # (including AssertionError) and downgrades it to a scored 0.0 — the + # opposite of the intended "internal-contract violation escalates to + # FinalStatus.ERROR" behavior. + if not auth_token: + raise JudgeInfrastructureError("checker_context route 'litellm' requires LITELLM_AUTH_TOKEN to be set") + + try: + from litellm.exceptions import APIError, BadRequestError + from litellm.types.utils import ModelResponse + + import litellm + except ImportError as e: + raise JudgeInfrastructureError( + "checker_context route 'litellm' needs the litellm library. Install with: pip install 'coder-eval[litellm]'" + ) from e + + openai_tool = { + "type": "function", + "function": { + "name": tool_spec["name"], + "description": tool_spec["description"], + "parameters": tool_spec["input_schema"], + }, + } + + def _call_kwargs(*, include_temperature: bool) -> dict[str, Any]: + kwargs: dict[str, Any] = { + "model": model, + "api_base": route.base_url, + "api_key": auth_token, + "messages": [ + {"role": "system", "content": system}, + {"role": "user", "content": user}, + ], + "tools": [openai_tool], + "tool_choice": {"type": "function", "function": {"name": tool_spec["name"]}}, + "max_completion_tokens": max_tokens, + "timeout": timeout_seconds, + # `drop_params` only covers params litellm's own static model-cost map + # KNOWS a model rejects; a custom/gateway-routed model id (e.g. one + # behind an Azure AI deployment) isn't in that map, so an actual + # unsupported-parameter rejection still round-trips to the provider — + # handled below by retrying once without `temperature`. + "drop_params": True, + } + if include_temperature: + kwargs["temperature"] = temperature + return kwargs + + def _rejects_temperature(e: BadRequestError) -> bool: + body = e.body if isinstance(e.body, dict) else {} + # The OpenAI SDK (which litellm's Azure path calls under the hood) + # unwraps the provider's `{"error": {...}}` envelope before attaching + # `.body` to the exception it raises — so `body` here is normally + # already the inner object (`{"param": "temperature", ...}`). Handle a + # still-wrapped shape too (a different provider path, or a future + # litellm/openai version) rather than assuming one or the other. + wrapped = body.get("error") + inner = wrapped if isinstance(wrapped, dict) else body + if inner.get("param") == "temperature": + return True + return "temperature" in str(e) and "not supported" in str(e).lower() + + try: + try: + response = await litellm.acompletion(**_call_kwargs(include_temperature=route.include_temperature)) + except BadRequestError as e: + if not (route.include_temperature and _rejects_temperature(e)): + raise + logger.info("LiteLLM judge model %r rejects temperature; retrying without it", model) + response = await litellm.acompletion(**_call_kwargs(include_temperature=False)) + except APIError as e: + raise JudgeInfrastructureError(f"LiteLLM judge API error: {e}") from e + except Exception as e: + raise JudgeInfrastructureError(f"LiteLLM judge call failed: {e}") from e + if not isinstance(response, ModelResponse): + # Never actually streamed (no `stream=True` above) -- defensive only, + # keeps pyright's ModelResponse | CustomStreamWrapper union honest. + raise JudgeInfrastructureError(f"LiteLLM judge returned an unexpected response type: {type(response)}") + return response.model_dump() diff --git a/src/coder_eval/evaluation/judge_usage.py b/src/coder_eval/evaluation/judge_usage.py index dfe51dd0..e3ab0aab 100644 --- a/src/coder_eval/evaluation/judge_usage.py +++ b/src/coder_eval/evaluation/judge_usage.py @@ -64,3 +64,45 @@ def token_usage_from_anthropic_dict(resp: dict[str, Any], *, model: str | None = cache_read_tokens=tu.cache_read_input_tokens, ) return tu + + +def token_usage_from_openai_dict(resp: dict[str, Any], *, model: str | None = None) -> TokenUsage | None: + """Extract usage from an OpenAI Chat-Completions-shaped response dict. + + ``invoke_litellm_judge_async`` (parsed ``/chat/completions`` JSON) carries an + OpenAI-shaped ``usage`` block: ``prompt_tokens``/``completion_tokens``, with + the cached prefix (if any) nested under ``prompt_tokens_details.cached_tokens``. + Returns ``None`` when usage is missing or carries no tokens. + + Follows the OpenAI/Codex cache-bucket convention documented on + ``TokenUsage``: ``prompt_tokens`` is the FULL prompt inclusive of the cached + prefix, so the fresh (uncached) slice is ``prompt_tokens - cached_tokens``; + OpenAI bills no separate cache-write fee, so ``cache_creation_input_tokens`` + is always 0 (mirrors ``CodexAgent._token_usage_from_sdk``). + + ``model`` prices the call from the rate card, same as the Anthropic-shaped + extractor above. + """ + u = resp.get("usage") + if not isinstance(u, dict): + return None + prompt_tokens = _coerce_int(u.get("prompt_tokens")) + details = u.get("prompt_tokens_details") + cached = _coerce_int(details.get("cached_tokens")) if isinstance(details, dict) else 0 + tu = TokenUsage( + uncached_input_tokens=max(prompt_tokens - cached, 0), + output_tokens=_coerce_int(u.get("completion_tokens")), + cache_creation_input_tokens=0, + cache_read_input_tokens=cached, + ) + if tu.is_empty(): + return None + if model: + tu.total_cost_usd = calculate_cost( + model, + uncached_input_tokens=tu.uncached_input_tokens, + output_tokens=tu.output_tokens, + cache_creation_tokens=tu.cache_creation_input_tokens, + cache_read_tokens=tu.cache_read_input_tokens, + ) + return tu diff --git a/src/coder_eval/evaluation/verdict_tool.py b/src/coder_eval/evaluation/verdict_tool.py index 815cc4c6..16186a48 100644 --- a/src/coder_eval/evaluation/verdict_tool.py +++ b/src/coder_eval/evaluation/verdict_tool.py @@ -14,6 +14,7 @@ from __future__ import annotations +import json from dataclasses import dataclass from typing import Any @@ -163,6 +164,50 @@ def extract_verdict_from_anthropic_response( return None, _format_validation_error(e) +def extract_verdict_from_openai_response( + response: dict[str, Any], +) -> tuple[JudgeVerdict | None, str | None]: + """Read the LAST ``submit_verdict`` tool call from an OpenAI Chat-Completions-shaped response. + + Walks ``response["choices"][0]["message"]["tool_calls"]`` for entries named + ``submit_verdict``, JSON-decodes the last one's ``function.arguments`` + string (OpenAI, unlike Anthropic, delivers tool args as a JSON string, not + a dict), and validates it against ``JudgeVerdict``. Used by + ``invoke_litellm_judge_async``. + + LAST-call discipline + non-dict/undecodable guard, mirroring + ``extract_verdict_from_anthropic_response``: a final ``submit_verdict`` + call whose arguments are missing, not valid JSON, or not an object is + treated as an invalid-args failure, not "did not call". + """ + choices = response.get("choices") or [] + message = choices[0].get("message") if choices and isinstance(choices[0], dict) else None + tool_calls = message.get("tool_calls") if isinstance(message, dict) else None + saw_submit_verdict = False + last_input: dict[str, Any] | None = None + for call in tool_calls or []: + if not isinstance(call, dict): + continue + function = call.get("function") + if not isinstance(function, dict) or function.get("name") != SUBMIT_VERDICT_TOOL_NAME: + continue + saw_submit_verdict = True + arguments = function.get("arguments") + try: + decoded = json.loads(arguments) if isinstance(arguments, str) else None + except ValueError: + decoded = None + last_input = decoded if isinstance(decoded, dict) else None + if not saw_submit_verdict: + return None, "Judge did not call submit_verdict" + if last_input is None: + return None, "submit_verdict input must be an object" + try: + return JudgeVerdict.model_validate(last_input), None + except ValidationError as e: + return None, _format_validation_error(e) + + def _format_validation_error(e: ValidationError) -> str: """Normalize a Pydantic ``ValidationError`` into the legacy vocabulary. diff --git a/src/coder_eval/models/criteria.py b/src/coder_eval/models/criteria.py index 7e54637c..b0822638 100644 --- a/src/coder_eval/models/criteria.py +++ b/src/coder_eval/models/criteria.py @@ -1327,10 +1327,13 @@ class LLMJudgeCriterion(BaseSuccessCriterion): "aggregate budget is exceeded (a degraded note is recorded)." ), ) - model: str = Field( - default=DEFAULT_JUDGE_MODEL, + model: str | None = Field( + default=None, description=( - "Judge model id (e.g. 'anthropic.claude-sonnet-4-6'). " + "Judge model id (e.g. 'anthropic.claude-sonnet-4-6'). Leave unset to fall back to " + "checker_context.api_route.model when set, else the built-in default " + f"({DEFAULT_JUDGE_MODEL!r}) — the fallback is never the agent's own model, so an " + "unpinned judge grades identically across agent-model A/Bs. " "On a BedrockRoute / DirectRoute the value is auto-translated: " "trailing '-vN[:M]' suffixes and the 'anthropic.' prefix are stripped where " "the backend doesn't accept them; on Bedrock the cross-region inference-profile " diff --git a/src/coder_eval/models/routing.py b/src/coder_eval/models/routing.py index 6bb508a9..c8c4ad3a 100644 --- a/src/coder_eval/models/routing.py +++ b/src/coder_eval/models/routing.py @@ -127,11 +127,21 @@ class LiteLLMRoute: Deliberately carries NO credential field — see ``BedrockRoute``'s docstring for why. ``ClaudeCodeAgent._build_sdk_env`` reads ``settings.litellm_auth_token`` itself. + + ``include_temperature`` (checker side only — ``invoke_litellm_judge_async``): + whether the judge call sends ``temperature`` at all. Defaults to ``False`` + because a gateway-routed model id (e.g. an Azure AI deployment) isn't in + ``litellm``'s static param-support table, so an unsupported ``temperature`` + isn't caught by ``drop_params`` — it round-trips to the provider and back as + a live rejection before the judge retries without it. Defaulting to omitted + skips that wasted round trip for the common case; set ``True`` for a + gateway/model known to accept it. """ base_url: str model: str | None = None small_model: str | None = None + include_temperature: bool = False ApiRoute = DirectRoute | BedrockRoute | LiteLLMRoute @@ -273,15 +283,21 @@ def resolve_evaluation_route( ``ValueError`` if the string isn't a known ``ApiBackend`` or that backend isn't configured (see ``_resolve_backend_route``). - Agent on Bedrock/Direct (no ``backend_override``): the judge already runs - on Claude via that route, so reuse it unchanged — except ``model_override``, - if set, still replaces its ``model`` (the route object itself, e.g. a - shared ``BedrockRoute``, is otherwise reused as-is). + on Claude via that route, so reuse it — except its ``model`` is always + reset to ``model_override`` (``None`` when unset). The agent's own + env-sourced model (e.g. ``BEDROCK_MODEL``) must NOT leak into the judge's + default: ``route.model`` must mean "an explicit override was given", not + "whatever the agent happens to be using" — otherwise an unpinned judge + silently starts grading with a different model whenever the agent's + model changes, breaking before/after comparability (PR #137 review: + "the judge loses DEFAULT_JUDGE_MODEL as its floor"). - Agent on LiteLLM (open-weight, no ``backend_override``): the agent route cannot serve a Claude judge, so pin evaluation to Bedrock (preferred, from the AWS bearer token) or Direct (``ANTHROPIC_API_KEY``), honoring - ``model_override`` there too. If neither is configured, fall back to a - ``DirectRoute`` with no judge transport so ``llm_judge`` fails with its - clean "unconfigured" error rather than silently scoring 0.0. + ``model_override`` there too — same "no override, no baked-in model" rule + as above. If neither backend is configured, fall back to a ``DirectRoute`` + with no judge transport so ``llm_judge`` fails with its clean + "unconfigured" error rather than silently scoring 0.0. """ if backend_override is not None: try: @@ -291,9 +307,7 @@ def resolve_evaluation_route( raise ValueError(f"checker_context route {backend_override!r} is not a known backend ({valid})") from e return _resolve_backend_route(settings, backend, model_override=model_override) if isinstance(agent_route, BedrockRoute | DirectRoute): - if not model_override: - return agent_route - if isinstance(agent_route, BedrockRoute): + if isinstance(agent_route, BedrockRoute) and model_override: # Bedrock model ids must be region-qualified — reusing the agent's route # verbatim would ship a bare alias straight to the Bedrock API (400). qualified_model, _ = _bedrock_model_pair(model_override, None, agent_route.region) @@ -301,8 +315,10 @@ def resolve_evaluation_route( return replace(agent_route, model=model_override) # agent_route is LiteLLMRoute → pin evaluation to a constant Claude backend. if settings.aws_bearer_token_bedrock and settings.aws_region: - judge_model = model_override or settings.bedrock_model or DEFAULT_JUDGE_MODEL - model, small_model = _bedrock_model_pair(judge_model, None, settings.aws_region) + if model_override: + model, small_model = _bedrock_model_pair(model_override, None, settings.aws_region) + else: + model, small_model = None, None return BedrockRoute(region=settings.aws_region, model=model, small_model=small_model) return DirectRoute(judge_transport=_resolve_direct_judge_transport(settings), model=model_override) diff --git a/tests/test_judge_litellm.py b/tests/test_judge_litellm.py new file mode 100644 index 00000000..5be0bd1a --- /dev/null +++ b/tests/test_judge_litellm.py @@ -0,0 +1,207 @@ +"""Tests for the LiteLLM judge invoker, which calls through the ``litellm`` +library (``litellm.acompletion``) rather than a hand-rolled HTTP client — +``litellm`` normalizes provider-specific request/response shapes so the judge +transport doesn't have to. +""" + +from __future__ import annotations + +import sys +from typing import Any +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +from coder_eval.errors import JudgeInfrastructureError +from coder_eval.evaluation.judge_litellm import invoke_litellm_judge_async +from coder_eval.evaluation.verdict_tool import SUBMIT_VERDICT_ANTHROPIC_TOOL +from coder_eval.models.routing import LiteLLMRoute + + +def _make_response(*, score: float = 0.5, rationale: str = "ok") -> MagicMock: + """Mimic litellm's OpenAI-shaped ``ModelResponse``: ``model_dump()`` returns + an OpenAI Chat-Completions-native tool_calls dict, regardless of the + underlying provider litellm actually routed to. + + ``spec=ModelResponse`` so ``invoke_litellm_judge_async``'s defensive + ``isinstance(response, ModelResponse)`` guard (against the + ``ModelResponse | CustomStreamWrapper`` union ``acompletion`` is typed to + return) passes against the mock the same as it would the real object.""" + import json + + from litellm.types.utils import ModelResponse + + response = MagicMock(spec=ModelResponse) + response.model_dump.return_value = { + "choices": [ + { + "message": { + "tool_calls": [ + { + "function": { + "name": "submit_verdict", + "arguments": json.dumps({"score": score, "rationale": rationale}), + } + } + ] + } + } + ] + } + return response + + +def _route(*, include_temperature: bool = False) -> LiteLLMRoute: + return LiteLLMRoute(base_url="http://gateway:4000", model="gpt-5.6-luna", include_temperature=include_temperature) + + +async def _invoke(**overrides): + defaults = { + "route": _route(), + "auth_token": "sk-master", + "model": "azure_ai/gpt-5.6-luna", + "system": "s", + "user": "u", + "temperature": 0.0, + "max_tokens": 10, + "tool_spec": SUBMIT_VERDICT_ANTHROPIC_TOOL, + } + defaults.update(overrides) + return await invoke_litellm_judge_async(**defaults) + + +async def test_invoke_litellm_judge_calls_acompletion() -> None: + acompletion = AsyncMock(return_value=_make_response(score=0.42)) + with patch("litellm.acompletion", new=acompletion): + result = await _invoke() + acompletion.assert_called_once() + kwargs = acompletion.call_args.kwargs + # Model id travels verbatim, provider prefix and all -- litellm routes on it. + assert kwargs["model"] == "azure_ai/gpt-5.6-luna" + assert kwargs["api_base"] == "http://gateway:4000" + assert kwargs["api_key"] == "sk-master" + assert kwargs["drop_params"] is True + assert kwargs["tools"][0]["function"]["name"] == "submit_verdict" + assert kwargs["tool_choice"] == {"type": "function", "function": {"name": "submit_verdict"}} + assert result["choices"][0]["message"]["tool_calls"][0]["function"]["name"] == "submit_verdict" + + +async def test_invoke_litellm_judge_omits_temperature_by_default() -> None: + """LiteLLMRoute.include_temperature defaults to False: a gateway-routed model + id isn't in litellm's static param table, so an unsupported `temperature` + isn't caught by `drop_params` -- it round-trips to the provider and back as + a live rejection. Skip sending it at all unless the route opts in.""" + acompletion = AsyncMock(return_value=_make_response()) + with patch("litellm.acompletion", new=acompletion): + await _invoke(temperature=0.7, max_tokens=321, system="sys", user="usr") + kwargs: dict[str, Any] = dict(acompletion.call_args.kwargs) + assert "temperature" not in kwargs + assert kwargs["max_completion_tokens"] == 321 + assert kwargs["messages"] == [ + {"role": "system", "content": "sys"}, + {"role": "user", "content": "usr"}, + ] + + +async def test_invoke_litellm_judge_sends_temperature_when_route_opts_in() -> None: + acompletion = AsyncMock(return_value=_make_response()) + with patch("litellm.acompletion", new=acompletion): + await _invoke(route=_route(include_temperature=True), temperature=0.7) + kwargs: dict[str, Any] = dict(acompletion.call_args.kwargs) + assert kwargs["temperature"] == 0.7 + + +async def test_invoke_litellm_judge_retries_without_temperature_when_rejected() -> None: + """A gateway-routed model litellm has no static param metadata for (so + `drop_params` can't preflight it) can still reject `temperature` live even + when the route opted in — observed against a real Azure AI deployment. + Must retry once without it rather than failing the whole judge call.""" + from litellm.exceptions import BadRequestError + + rejection = BadRequestError( + message="Unsupported parameter: 'temperature' is not supported with this model.", + model="azure_ai/gpt-5.6-luna", + llm_provider="azure_ai", + body={"error": {"message": "...", "param": "temperature", "code": None}}, + ) + acompletion = AsyncMock(side_effect=[rejection, _make_response(score=0.9)]) + with patch("litellm.acompletion", new=acompletion): + result = await _invoke(route=_route(include_temperature=True), temperature=0.3) + assert acompletion.call_count == 2 + first_kwargs, second_kwargs = (c.kwargs for c in acompletion.call_args_list) + assert first_kwargs["temperature"] == 0.3 + assert "temperature" not in second_kwargs + assert result["choices"][0]["message"]["tool_calls"][0]["function"]["name"] == "submit_verdict" + + +async def test_invoke_litellm_judge_reraises_unrelated_bad_request() -> None: + from litellm.exceptions import BadRequestError + + rejection = BadRequestError( + message="Unsupported parameter: 'foo'.", + model="m", + llm_provider="azure_ai", + body={"error": {"message": "...", "param": "foo", "code": None}}, + ) + acompletion = AsyncMock(side_effect=rejection) + with ( + patch("litellm.acompletion", new=acompletion), + pytest.raises(JudgeInfrastructureError, match="LiteLLM judge call failed"), + ): + await _invoke(route=_route(include_temperature=True)) + acompletion.assert_called_once() + + +async def test_invoke_litellm_judge_reraises_bad_request_when_route_did_not_opt_in() -> None: + """No point retrying-without-temperature when temperature was never sent.""" + from litellm.exceptions import BadRequestError + + rejection = BadRequestError( + message="Unsupported parameter: 'temperature' is not supported with this model.", + model="m", + llm_provider="azure_ai", + body={"error": {"message": "...", "param": "temperature", "code": None}}, + ) + acompletion = AsyncMock(side_effect=rejection) + with ( + patch("litellm.acompletion", new=acompletion), + pytest.raises(JudgeInfrastructureError, match="LiteLLM judge call failed"), + ): + await _invoke() + acompletion.assert_called_once() + + +async def test_invoke_litellm_judge_raises_on_empty_model() -> None: + with pytest.raises(ValueError): + await _invoke(model="") + + +async def test_invoke_litellm_judge_raises_on_missing_auth_token() -> None: + with pytest.raises(JudgeInfrastructureError, match="LITELLM_AUTH_TOKEN"): + await _invoke(auth_token=None) + + +async def test_invoke_litellm_judge_raises_when_library_not_installed(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setitem(sys.modules, "litellm", None) + with pytest.raises(JudgeInfrastructureError, match=r"pip install 'coder-eval\[litellm\]'"): + await _invoke() + + +async def test_invoke_litellm_judge_wraps_api_error() -> None: + from litellm.exceptions import APIError + + acompletion = AsyncMock(side_effect=APIError(status_code=500, message="boom", llm_provider="azure_ai", model="m")) + with ( + patch("litellm.acompletion", new=acompletion), + pytest.raises(JudgeInfrastructureError, match="LiteLLM judge API error"), + ): + await _invoke() + + +async def test_invoke_litellm_judge_escalates_on_signature_break() -> None: + acompletion = AsyncMock(side_effect=TypeError("acompletion() got an unexpected keyword argument 'drop_params'")) + with ( + patch("litellm.acompletion", new=acompletion), + pytest.raises(JudgeInfrastructureError, match="LiteLLM judge call failed"), + ): + await _invoke() diff --git a/tests/test_litellm_route.py b/tests/test_litellm_route.py index d009dc19..6e323625 100644 --- a/tests/test_litellm_route.py +++ b/tests/test_litellm_route.py @@ -46,15 +46,21 @@ def _isolated_settings(monkeypatch, **kwargs): monkeypatch.delenv(var, raising=False) return Settings(_env_file=None, **kwargs) - def test_bedrock_agent_route_is_reused_unchanged(self, monkeypatch): + def test_bedrock_agent_route_is_reused_with_model_reset(self, monkeypatch): + # Same region/other fields reused verbatim, but `model` is reset to None + # (no checker_context override) rather than carrying over the agent's own + # env-sourced model — see TestResolveEvaluationRouteJudgeFloor. route = BedrockRoute(region="eu-north-1", model="eu.anthropic.claude-sonnet-4-6") settings = self._isolated_settings(monkeypatch, api_backend=ApiBackend.BEDROCK) - assert resolve_evaluation_route(settings, route) is route + ev = resolve_evaluation_route(settings, route) + assert ev is not route + assert ev == BedrockRoute(region="eu-north-1", model=None) - def test_direct_agent_route_is_reused_unchanged(self, monkeypatch): - route = DirectRoute(judge_transport="anthropic") + def test_direct_agent_route_is_reused_with_model_reset(self, monkeypatch): + route = DirectRoute(judge_transport="anthropic", model="gpt-4") settings = self._isolated_settings(monkeypatch, api_backend=ApiBackend.DIRECT) - assert resolve_evaluation_route(settings, route) is route + ev = resolve_evaluation_route(settings, route) + assert ev == DirectRoute(judge_transport="anthropic", model=None) def test_litellm_agent_pins_evaluation_to_bedrock_when_aws_creds_present(self, monkeypatch): agent = LiteLLMRoute(base_url="http://x:4000", model="zai.glm-5") @@ -67,8 +73,9 @@ def test_litellm_agent_pins_evaluation_to_bedrock_when_aws_creds_present(self, m ev = resolve_evaluation_route(settings, agent) assert isinstance(ev, BedrockRoute) assert ev.region == "eu-north-1" - # Judge + simulator run on a Claude model, region-qualified. - assert ev.model == "eu.anthropic.claude-sonnet-4-6" + # No checker_context override -> model is None, so llm_judge falls back + # to DEFAULT_JUDGE_MODEL rather than an env-sourced value. + assert ev.model is None def test_litellm_agent_falls_back_to_direct_when_only_anthropic_key(self, monkeypatch): agent = LiteLLMRoute(base_url="http://x:4000", model="zai.glm-5") @@ -87,6 +94,149 @@ def test_litellm_agent_unconfigured_yields_direct_with_no_transport(self, monkey assert ev.judge_transport is None +class TestResolveEvaluationRouteJudgeFloor: + """Regression coverage for PR #137 review Axis 8 ('the judge loses + DEFAULT_JUDGE_MODEL as its floor'): resolve_evaluation_route must never bake + the agent's own env-sourced model into the eval route's `model` unless a real + checker_context.api_route.model override was given — otherwise an unpinned + llm_judge silently starts grading with a different model whenever the + agent's model changes (BEDROCK_MODEL), breaking before/after comparability. + """ + + @staticmethod + def _isolated_settings(monkeypatch, **kwargs): + for var in ("AWS_BEARER_TOKEN_BEDROCK", "AWS_REGION", "ANTHROPIC_API_KEY", "BEDROCK_MODEL"): + monkeypatch.delenv(var, raising=False) + return Settings(_env_file=None, **kwargs) + + def test_bedrock_agent_route_no_override_strips_agent_model(self, monkeypatch): + # Simulates an agent route built with a real BEDROCK_MODEL (e.g. opus), + # reused for the eval side with no checker_context override -> the eval + # route's model must be None so llm_judge falls back to DEFAULT_JUDGE_MODEL, + # not the agent's model. + agent_route = BedrockRoute(region="eu-north-1", model="eu.anthropic.claude-opus-4-1") + settings = self._isolated_settings(monkeypatch, api_backend=ApiBackend.BEDROCK) + ev = resolve_evaluation_route(settings, agent_route) + assert isinstance(ev, BedrockRoute) + assert ev.model is None + + def test_bedrock_agent_route_with_override_qualifies_model(self, monkeypatch): + agent_route = BedrockRoute(region="eu-north-1", model="eu.anthropic.claude-opus-4-1") + settings = self._isolated_settings(monkeypatch, api_backend=ApiBackend.BEDROCK) + ev = resolve_evaluation_route(settings, agent_route, model_override="claude-haiku-4-5") + assert isinstance(ev, BedrockRoute) + assert ev.model == "eu.anthropic.claude-haiku-4-5" + + def test_direct_agent_route_no_override_strips_agent_model(self, monkeypatch): + agent_route = DirectRoute(judge_transport="anthropic", model="gpt-4") + settings = self._isolated_settings(monkeypatch, api_backend=ApiBackend.DIRECT) + ev = resolve_evaluation_route(settings, agent_route) + assert isinstance(ev, DirectRoute) + assert ev.model is None + + def test_litellm_agent_pin_to_bedrock_no_override_strips_bedrock_model(self, monkeypatch): + # Agent on LiteLLM (open-weight); AWS creds present with a real + # BEDROCK_MODEL configured for some unrelated purpose. No override -> + # the pinned eval route must NOT inherit BEDROCK_MODEL. + agent_route = LiteLLMRoute(base_url="http://x:4000", model="zai.glm-5") + settings = self._isolated_settings( + monkeypatch, + api_backend=ApiBackend.LITELLM, + aws_bearer_token_bedrock="aws-tok", + aws_region="eu-north-1", + bedrock_model="claude-opus-4-1", + ) + ev = resolve_evaluation_route(settings, agent_route) + assert isinstance(ev, BedrockRoute) + assert ev.model is None + + def test_litellm_agent_pin_to_bedrock_with_override(self, monkeypatch): + agent_route = LiteLLMRoute(base_url="http://x:4000", model="zai.glm-5") + settings = self._isolated_settings( + monkeypatch, + api_backend=ApiBackend.LITELLM, + aws_bearer_token_bedrock="aws-tok", + aws_region="eu-north-1", + ) + ev = resolve_evaluation_route(settings, agent_route, model_override="claude-haiku-4-5") + assert isinstance(ev, BedrockRoute) + assert ev.model == "eu.anthropic.claude-haiku-4-5" + + +class TestBackendOverride: + """resolve_evaluation_route(backend_override=...) — the checker_context.api_route.route + override path, dispatched through _resolve_backend_route. Zero coverage before this PR + review (Axis 3 blocker).""" + + @staticmethod + def _isolated_settings(monkeypatch, **kwargs): + for var in ( + "AWS_BEARER_TOKEN_BEDROCK", + "AWS_REGION", + "ANTHROPIC_API_KEY", + "LITELLM_BASE_URL", + "LITELLM_AUTH_TOKEN", + ): + monkeypatch.delenv(var, raising=False) + return Settings(_env_file=None, **kwargs) + + def test_override_to_bedrock_builds_route_from_env(self, monkeypatch): + agent_route = DirectRoute() + settings = self._isolated_settings( + monkeypatch, + api_backend=ApiBackend.DIRECT, + aws_bearer_token_bedrock="aws-tok", + aws_region="eu-north-1", + ) + ev = resolve_evaluation_route(settings, agent_route, backend_override="bedrock") + assert isinstance(ev, BedrockRoute) + assert ev.region == "eu-north-1" + + def test_override_to_bedrock_without_creds_raises(self, monkeypatch): + agent_route = DirectRoute() + settings = self._isolated_settings(monkeypatch, api_backend=ApiBackend.DIRECT) + with pytest.raises(ValueError, match="requires AWS_BEARER_TOKEN_BEDROCK and AWS_REGION"): + resolve_evaluation_route(settings, agent_route, backend_override="bedrock") + + def test_override_to_direct_builds_route_from_env(self, monkeypatch): + agent_route = BedrockRoute(region="eu-north-1") + settings = self._isolated_settings(monkeypatch, api_backend=ApiBackend.BEDROCK, anthropic_api_key="sk-ant") + ev = resolve_evaluation_route(settings, agent_route, backend_override="direct") + assert isinstance(ev, DirectRoute) + assert ev.judge_transport == "anthropic" + + def test_override_to_direct_without_key_raises(self, monkeypatch): + agent_route = BedrockRoute(region="eu-north-1") + settings = self._isolated_settings(monkeypatch, api_backend=ApiBackend.BEDROCK) + with pytest.raises(ValueError, match="requires ANTHROPIC_API_KEY"): + resolve_evaluation_route(settings, agent_route, backend_override="direct") + + def test_override_to_litellm_builds_route_from_env(self, monkeypatch): + agent_route = BedrockRoute(region="eu-north-1") + settings = self._isolated_settings( + monkeypatch, + api_backend=ApiBackend.BEDROCK, + litellm_base_url="http://gateway:4000", + litellm_auth_token="sk-master", + ) + ev = resolve_evaluation_route(settings, agent_route, backend_override="litellm", model_override="gpt-5.6-luna") + assert isinstance(ev, LiteLLMRoute) + assert ev.base_url == "http://gateway:4000" + assert ev.model == "gpt-5.6-luna" + + def test_override_to_litellm_without_creds_raises(self, monkeypatch): + agent_route = BedrockRoute(region="eu-north-1") + settings = self._isolated_settings(monkeypatch, api_backend=ApiBackend.BEDROCK) + with pytest.raises(ValueError, match="requires LITELLM_BASE_URL and LITELLM_AUTH_TOKEN"): + resolve_evaluation_route(settings, agent_route, backend_override="litellm") + + def test_unknown_backend_raises(self, monkeypatch): + agent_route = DirectRoute() + settings = self._isolated_settings(monkeypatch, api_backend=ApiBackend.DIRECT) + with pytest.raises(ValueError, match="is not a known backend"): + resolve_evaluation_route(settings, agent_route, backend_override="not-a-backend") + + class TestEvalRouteWiring: """The orchestrator must hand the simulated user the eval_route (constant Claude), never the agent's (possibly open-weight) route — guards the diff --git a/tests/test_llm_judge_criterion.py b/tests/test_llm_judge_criterion.py index a0e69a84..8083c369 100644 --- a/tests/test_llm_judge_criterion.py +++ b/tests/test_llm_judge_criterion.py @@ -20,6 +20,7 @@ LLMJudgeCriterion, TurnRecord, ) +from coder_eval.models.judge_defaults import DEFAULT_JUDGE_MODEL from coder_eval.sandbox import Sandbox @@ -571,6 +572,42 @@ def _tool_use_block(score: float, rationale: str = "ok") -> dict: } +def _openai_tool_call_block(score: float, rationale: str = "ok") -> dict: + import json + + return { + "choices": [ + { + "message": { + "tool_calls": [ + { + "function": { + "name": "submit_verdict", + "arguments": json.dumps({"score": score, "rationale": rationale}), + } + } + ] + } + } + ] + } + + +def test_llm_judge_criterion_model_survives_json_round_trip() -> None: + """Regression test for PR #137 review Axis 2: an unset LLMJudgeCriterion.model + must stay unset (None) after a model_dump(mode="json") / reload round trip — + the exact shape the docker driver's task serialization performs + (isolation/docker_runner.py -> cli/run_task_internal_command.py::load_task). + A `model_fields_set` sentinel would NOT survive this (every field is materialized + by model_dump), silently making the judge-model override inert under --driver docker.""" + import json + + criterion = LLMJudgeCriterion(description="x", prompt="grade") + assert criterion.model is None + reloaded = LLMJudgeCriterion.model_validate(json.loads(json.dumps(criterion.model_dump(mode="json")))) + assert reloaded.model is None + + def test_judge_bedrock_route_uses_bedrock_invoker(sandbox: Sandbox) -> None: from coder_eval.models.routing import BedrockRoute @@ -587,13 +624,62 @@ def test_judge_bedrock_route_uses_bedrock_invoker(sandbox: Sandbox) -> None: m_bedrock.assert_called_once() kwargs = m_bedrock.call_args.kwargs assert kwargs["route"] is route - assert kwargs["model"] == criterion.model + # No explicit criterion.model and no checker_context override on the route -> + # falls back to DEFAULT_JUDGE_MODEL, never the agent's own model. + assert kwargs["model"] == DEFAULT_JUDGE_MODEL assert kwargs["temperature"] == criterion.temperature assert kwargs["max_tokens"] == criterion.max_tokens assert kwargs["tool_spec"]["name"] == "submit_verdict" assert m_anthropic.call_count == 0 +def test_judge_bedrock_route_with_explicit_route_model_still_wins(sandbox: Sandbox) -> None: + """A route.model that IS set (a real checker_context.api_route.model override, + as resolve_evaluation_route only ever produces) is honored — contrast with + the routing-level regression test in test_litellm_route.py, which asserts + resolve_evaluation_route itself never bakes the agent's own model into + route.model absent such an override.""" + from coder_eval.models.routing import BedrockRoute + + route = BedrockRoute(region="eu-north-1", model="eu.anthropic.claude-opus-4-1") + criterion = LLMJudgeCriterion(description="x", prompt="grade") + with ( + patch( + "coder_eval.criteria.llm_judge.invoke_bedrock_judge_async", new=AsyncMock(return_value=_tool_use_block(0.7)) + ) as m_bedrock, + patch("coder_eval.criteria.llm_judge.invoke_anthropic_judge_async", new=AsyncMock()), + ): + SuccessChecker(sandbox, init_registry=False, route=route).check(criterion) + assert m_bedrock.call_args.kwargs["model"] == "eu.anthropic.claude-opus-4-1" + + +def test_judge_litellm_route_uses_litellm_invoker(sandbox: Sandbox) -> None: + from coder_eval.models.routing import LiteLLMRoute + + route = LiteLLMRoute(base_url="http://gateway:4000", model="gpt-5-luna") + criterion = LLMJudgeCriterion(description="x", prompt="grade") + with ( + patch( + "coder_eval.criteria.llm_judge.invoke_litellm_judge_async", + new=AsyncMock(return_value=_openai_tool_call_block(0.9)), + ) as m_litellm, + patch("coder_eval.criteria.llm_judge.invoke_bedrock_judge_async", new=AsyncMock()) as m_bedrock, + patch("coder_eval.criteria.llm_judge.invoke_anthropic_judge_async", new=AsyncMock()) as m_anthropic, + ): + result = SuccessChecker(sandbox, init_registry=False, route=route).check(criterion) + assert result.score == 0.9 + m_litellm.assert_called_once() + kwargs = m_litellm.call_args.kwargs + assert kwargs["route"] is route + # No explicit criterion.model set -> falls back to route.model (checker_context override). + assert kwargs["model"] == "gpt-5-luna" + assert kwargs["temperature"] == criterion.temperature + assert kwargs["max_tokens"] == criterion.max_tokens + assert kwargs["tool_spec"]["name"] == "submit_verdict" + assert m_bedrock.call_count == 0 + assert m_anthropic.call_count == 0 + + def test_judge_direct_route_uses_anthropic_invoker(sandbox: Sandbox) -> None: from coder_eval.models.routing import DirectRoute diff --git a/tests/test_models.py b/tests/test_models.py index 3f533557..60fa21a2 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -187,11 +187,15 @@ class TestLLMJudgeCriterion: def test_llm_judge_criterion_defaults(self): """Constructing with the minimum required fields yields documented defaults.""" - from coder_eval.models import DEFAULT_JUDGE_MODEL, LLMJudgeCriterion + from coder_eval.models import LLMJudgeCriterion criterion = LLMJudgeCriterion(description="x", prompt="grade this code") assert criterion.type == "llm_judge" - assert criterion.model == DEFAULT_JUDGE_MODEL + # Unset by default (not a materialized DEFAULT_JUDGE_MODEL default) so an + # unset per-criterion model survives a JSON round trip as None — see + # LLMJudgeChecker's precedence: criterion.model or route.model or + # DEFAULT_JUDGE_MODEL, applied at check time. + assert criterion.model is None assert criterion.temperature == 0.0 # Bumped from 1000 → 2000 when verbose verdict (findings) was added, # so output budgets fit the bullet evidence a typical judge emits. diff --git a/tests/test_route_seam_exhaustiveness.py b/tests/test_route_seam_exhaustiveness.py index 6621be96..449f4bdf 100644 --- a/tests/test_route_seam_exhaustiveness.py +++ b/tests/test_route_seam_exhaustiveness.py @@ -69,8 +69,12 @@ async def _stub_bedrock(**_: object) -> dict[str, object]: async def _stub_anthropic(**_: object) -> dict[str, object]: return {} + async def _stub_litellm(**_: object) -> dict[str, object]: + return {} + monkeypatch.setattr(llm_judge, "invoke_bedrock_judge_async", _stub_bedrock) monkeypatch.setattr(llm_judge, "invoke_anthropic_judge_async", _stub_anthropic) + monkeypatch.setattr(llm_judge, "invoke_litellm_judge_async", _stub_litellm) monkeypatch.setattr(llm_judge, "extract_verdict_from_anthropic_response", lambda _resp: (None, "stub")) monkeypatch.setattr(llm_judge, "token_usage_from_anthropic_dict", lambda _resp, **_kwargs: None) criterion = MagicMock() diff --git a/tests/test_routing.py b/tests/test_routing.py index 35b7b94e..025d2442 100644 --- a/tests/test_routing.py +++ b/tests/test_routing.py @@ -32,6 +32,20 @@ def test_direct_omits_path_when_unset(self, monkeypatch): assert "PATH" not in env assert model is None + def test_direct_neutralizes_inherited_bedrock_creds(self): + """An explicit route: direct (e.g. via checker_context.api_route.route on a + run whose agent is on Bedrock) must not let the CLI's own + AWS_BEARER_TOKEN_BEDROCK auto-selection silently spend the operator's + Bedrock token instead of ANTHROPIC_API_KEY (PR #137 review).""" + env, _ = ClaudeCodeAgent._build_sdk_env(DirectRoute()) + assert env["AWS_BEARER_TOKEN_BEDROCK"] == "" + assert env["CLAUDE_CODE_USE_BEDROCK"] == "" + + def test_direct_model_override_sets_anthropic_model(self): + env, model = ClaudeCodeAgent._build_sdk_env(DirectRoute(model="claude-haiku-4-5")) + assert env["ANTHROPIC_MODEL"] == "claude-haiku-4-5" + assert model == "claude-haiku-4-5" + def test_bedrock_basic_env(self, monkeypatch): """BedrockRoute produces CLAUDE_CODE_USE_BEDROCK, token, region, and forwards PATH.""" custom_path = f"/bedrock/bin{os.pathsep}/usr/bin" diff --git a/uv.lock b/uv.lock index 688a1b3c..004e5618 100644 --- a/uv.lock +++ b/uv.lock @@ -278,6 +278,34 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e5/ca/78d423b324b8d77900030fa59c4aa9054261ef0925631cd2501dd015b7b7/boolean_py-5.0-py3-none-any.whl", hash = "sha256:ef28a70bd43115208441b53a045d1549e2f0ec6e3d08a9d142cbc41c1938e8d9", size = 26577, upload-time = "2025-04-03T10:39:48.449Z" }, ] +[[package]] +name = "boto3" +version = "1.43.78" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "botocore" }, + { name = "jmespath" }, + { name = "s3transfer" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/be/55/e026c943f7f1ed6d2f5e6035713f21233bbe9ee975008662dc64ca0d4ced/boto3-1.43.78.tar.gz", hash = "sha256:2fa59116e298171ef59e7600a8be6c01177faef8af4b9a4314b7a57a04009ada", size = 112679, upload-time = "2026-08-21T19:37:36.84Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e3/55/288b1f0d987e15db0417f5ea43da564bfa042063f508c259eec893134628/boto3-1.43.78-py3-none-any.whl", hash = "sha256:893f06a171469618e17de78dc927aca6e74fcf45a70d2c5918e9ac9919e96cc9", size = 140028, upload-time = "2026-08-21T19:37:35.358Z" }, +] + +[[package]] +name = "botocore" +version = "1.43.78" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "jmespath" }, + { name = "python-dateutil" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/67/71/490aaa384855bf3b69405ded52ae77a0e5f4eeb2165044fa65466c4d3a73/botocore-1.43.78.tar.gz", hash = "sha256:e8238d22c1e1342025d75d2e33d154a375e7caad0fc67f77d77faa2d82668b94", size = 15982895, upload-time = "2026-08-21T19:37:32.402Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c5/5b/b7c5c22767c0e8eb4bb2bd0a972d77bd0be3dd9908e15d4487094a31fa4b/botocore-1.43.78-py3-none-any.whl", hash = "sha256:ddd020493235e264b3bd12606f239a3d3b2dd7cfb1d25a0691061183c290c228", size = 15677214, upload-time = "2026-08-21T19:37:28.894Z" }, +] + [[package]] name = "cachecontrol" version = "0.14.4" @@ -492,6 +520,9 @@ dev = [ { name = "pytest-xdist", extra = ["psutil"] }, { name = "ruff" }, ] +litellm = [ + { name = "litellm" }, +] uipath = [ { name = "uipath" }, ] @@ -509,6 +540,7 @@ requires-dist = [ { name = "httpx2", specifier = ">=2.12.0,<3.0.0" }, { name = "jmespath", specifier = ">=1.1.0" }, { name = "jsonschema", specifier = ">=4.26.0" }, + { name = "litellm", marker = "extra == 'litellm'", specifier = ">=1.95.0,<2.0.0" }, { name = "mcp", marker = "extra == 'dev'", specifier = ">=1.28.1" }, { name = "openai-codex", marker = "extra == 'codex'", specifier = ">=0.144.4" }, { name = "opentelemetry-sdk", specifier = ">=1.30.0,<2.0.0" }, @@ -532,7 +564,7 @@ requires-dist = [ { name = "typer", specifier = ">=0.24.1" }, { name = "uipath", marker = "extra == 'uipath'", specifier = ">=2.10.31" }, ] -provides-extras = ["dev", "uipath", "codex", "antigravity"] +provides-extras = ["dev", "uipath", "litellm", "codex", "antigravity"] [[package]] name = "colorama" @@ -722,6 +754,36 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ab/84/02fc1827e8cdded4aa65baef11296a9bbe595c474f0d6d758af082d849fd/execnet-2.1.2-py3-none-any.whl", hash = "sha256:67fba928dd5a544b783f6056f449e5e3931a5c378b128bc18501f7ea79e296ec", size = 40708, upload-time = "2025-11-12T09:56:36.333Z" }, ] +[[package]] +name = "fastuuid" +version = "0.14.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c3/7d/d9daedf0f2ebcacd20d599928f8913e9d2aea1d56d2d355a93bfa2b611d7/fastuuid-0.14.0.tar.gz", hash = "sha256:178947fc2f995b38497a74172adee64fdeb8b7ec18f2a5934d037641ba265d26", size = 18232, upload-time = "2025-10-19T22:19:22.402Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a5/83/ae12dd39b9a39b55d7f90abb8971f1a5f3c321fd72d5aa83f90dc67fe9ed/fastuuid-0.14.0-cp313-cp313-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:77a09cb7427e7af74c594e409f7731a0cf887221de2f698e1ca0ebf0f3139021", size = 510720, upload-time = "2025-10-19T22:42:34.633Z" }, + { url = "https://files.pythonhosted.org/packages/53/b0/a4b03ff5d00f563cc7546b933c28cb3f2a07344b2aec5834e874f7d44143/fastuuid-0.14.0-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:9bd57289daf7b153bfa3e8013446aa144ce5e8c825e9e366d455155ede5ea2dc", size = 262024, upload-time = "2025-10-19T22:30:25.482Z" }, + { url = "https://files.pythonhosted.org/packages/9c/6d/64aee0a0f6a58eeabadd582e55d0d7d70258ffdd01d093b30c53d668303b/fastuuid-0.14.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:ac60fc860cdf3c3f327374db87ab8e064c86566ca8c49d2e30df15eda1b0c2d5", size = 251679, upload-time = "2025-10-19T22:36:14.096Z" }, + { url = "https://files.pythonhosted.org/packages/60/f5/a7e9cda8369e4f7919d36552db9b2ae21db7915083bc6336f1b0082c8b2e/fastuuid-0.14.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ab32f74bd56565b186f036e33129da77db8be09178cd2f5206a5d4035fb2a23f", size = 277862, upload-time = "2025-10-19T22:36:23.302Z" }, + { url = "https://files.pythonhosted.org/packages/f0/d3/8ce11827c783affffd5bd4d6378b28eb6cc6d2ddf41474006b8d62e7448e/fastuuid-0.14.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:33e678459cf4addaedd9936bbb038e35b3f6b2061330fd8f2f6a1d80414c0f87", size = 278278, upload-time = "2025-10-19T22:29:43.809Z" }, + { url = "https://files.pythonhosted.org/packages/a2/51/680fb6352d0bbade04036da46264a8001f74b7484e2fd1f4da9e3db1c666/fastuuid-0.14.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1e3cc56742f76cd25ecb98e4b82a25f978ccffba02e4bdce8aba857b6d85d87b", size = 301788, upload-time = "2025-10-19T22:36:06.825Z" }, + { url = "https://files.pythonhosted.org/packages/fa/7c/2014b5785bd8ebdab04ec857635ebd84d5ee4950186a577db9eff0fb8ff6/fastuuid-0.14.0-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:cb9a030f609194b679e1660f7e32733b7a0f332d519c5d5a6a0a580991290022", size = 459819, upload-time = "2025-10-19T22:35:31.623Z" }, + { url = "https://files.pythonhosted.org/packages/01/d2/524d4ceeba9160e7a9bc2ea3e8f4ccf1ad78f3bde34090ca0c51f09a5e91/fastuuid-0.14.0-cp313-cp313-musllinux_1_1_i686.whl", hash = "sha256:09098762aad4f8da3a888eb9ae01c84430c907a297b97166b8abc07b640f2995", size = 478546, upload-time = "2025-10-19T22:26:03.023Z" }, + { url = "https://files.pythonhosted.org/packages/bc/17/354d04951ce114bf4afc78e27a18cfbd6ee319ab1829c2d5fb5e94063ac6/fastuuid-0.14.0-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:1383fff584fa249b16329a059c68ad45d030d5a4b70fb7c73a08d98fd53bcdab", size = 450921, upload-time = "2025-10-19T22:31:02.151Z" }, + { url = "https://files.pythonhosted.org/packages/fb/be/d7be8670151d16d88f15bb121c5b66cdb5ea6a0c2a362d0dcf30276ade53/fastuuid-0.14.0-cp313-cp313-win32.whl", hash = "sha256:a0809f8cc5731c066c909047f9a314d5f536c871a7a22e815cc4967c110ac9ad", size = 154559, upload-time = "2025-10-19T22:36:36.011Z" }, + { url = "https://files.pythonhosted.org/packages/22/1d/5573ef3624ceb7abf4a46073d3554e37191c868abc3aecd5289a72f9810a/fastuuid-0.14.0-cp313-cp313-win_amd64.whl", hash = "sha256:0df14e92e7ad3276327631c9e7cec09e32572ce82089c55cb1bb8df71cf394ed", size = 156539, upload-time = "2025-10-19T22:33:35.898Z" }, + { url = "https://files.pythonhosted.org/packages/16/c9/8c7660d1fe3862e3f8acabd9be7fc9ad71eb270f1c65cce9a2b7a31329ab/fastuuid-0.14.0-cp314-cp314-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:b852a870a61cfc26c884af205d502881a2e59cc07076b60ab4a951cc0c94d1ad", size = 510600, upload-time = "2025-10-19T22:43:44.17Z" }, + { url = "https://files.pythonhosted.org/packages/4c/f4/a989c82f9a90d0ad995aa957b3e572ebef163c5299823b4027986f133dfb/fastuuid-0.14.0-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:c7502d6f54cd08024c3ea9b3514e2d6f190feb2f46e6dbcd3747882264bb5f7b", size = 262069, upload-time = "2025-10-19T22:43:38.38Z" }, + { url = "https://files.pythonhosted.org/packages/da/6c/a1a24f73574ac995482b1326cf7ab41301af0fabaa3e37eeb6b3df00e6e2/fastuuid-0.14.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:1ca61b592120cf314cfd66e662a5b54a578c5a15b26305e1b8b618a6f22df714", size = 251543, upload-time = "2025-10-19T22:32:22.537Z" }, + { url = "https://files.pythonhosted.org/packages/1a/20/2a9b59185ba7a6c7b37808431477c2d739fcbdabbf63e00243e37bd6bf49/fastuuid-0.14.0-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:aa75b6657ec129d0abded3bec745e6f7ab642e6dba3a5272a68247e85f5f316f", size = 277798, upload-time = "2025-10-19T22:33:53.821Z" }, + { url = "https://files.pythonhosted.org/packages/ef/33/4105ca574f6ded0af6a797d39add041bcfb468a1255fbbe82fcb6f592da2/fastuuid-0.14.0-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a8a0dfea3972200f72d4c7df02c8ac70bad1bb4c58d7e0ec1e6f341679073a7f", size = 278283, upload-time = "2025-10-19T22:29:02.812Z" }, + { url = "https://files.pythonhosted.org/packages/fe/8c/fca59f8e21c4deb013f574eae05723737ddb1d2937ce87cb2a5d20992dc3/fastuuid-0.14.0-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1bf539a7a95f35b419f9ad105d5a8a35036df35fdafae48fb2fd2e5f318f0d75", size = 301627, upload-time = "2025-10-19T22:35:54.985Z" }, + { url = "https://files.pythonhosted.org/packages/cb/e2/f78c271b909c034d429218f2798ca4e89eeda7983f4257d7865976ddbb6c/fastuuid-0.14.0-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:9a133bf9cc78fdbd1179cb58a59ad0100aa32d8675508150f3658814aeefeaa4", size = 459778, upload-time = "2025-10-19T22:28:00.999Z" }, + { url = "https://files.pythonhosted.org/packages/1e/f0/5ff209d865897667a2ff3e7a572267a9ced8f7313919f6d6043aed8b1caa/fastuuid-0.14.0-cp314-cp314-musllinux_1_1_i686.whl", hash = "sha256:f54d5b36c56a2d5e1a31e73b950b28a0d83eb0c37b91d10408875a5a29494bad", size = 478605, upload-time = "2025-10-19T22:36:21.764Z" }, + { url = "https://files.pythonhosted.org/packages/e0/c8/2ce1c78f983a2c4987ea865d9516dbdfb141a120fd3abb977ae6f02ba7ca/fastuuid-0.14.0-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:ec27778c6ca3393ef662e2762dba8af13f4ec1aaa32d08d77f71f2a70ae9feb8", size = 450837, upload-time = "2025-10-19T22:34:37.178Z" }, + { url = "https://files.pythonhosted.org/packages/df/60/dad662ec9a33b4a5fe44f60699258da64172c39bd041da2994422cdc40fe/fastuuid-0.14.0-cp314-cp314-win32.whl", hash = "sha256:e23fc6a83f112de4be0cc1990e5b127c27663ae43f866353166f87df58e73d06", size = 154532, upload-time = "2025-10-19T22:35:18.217Z" }, + { url = "https://files.pythonhosted.org/packages/1f/f6/da4db31001e854025ffd26bc9ba0740a9cbba2c3259695f7c5834908b336/fastuuid-0.14.0-cp314-cp314-win_amd64.whl", hash = "sha256:df61342889d0f5e7a32f7284e55ef95103f2110fee433c2ae7c2c0956d76ac8a", size = 156457, upload-time = "2025-10-19T22:33:44.579Z" }, +] + [[package]] name = "filelock" version = "3.25.2" @@ -804,6 +866,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/9a/9a/e35b4a917281c0b8419d4207f4334c8e8c5dbf4f3f5f9ada73958d937dcc/frozenlist-1.8.0-py3-none-any.whl", hash = "sha256:0c18a16eab41e82c295618a77502e17b195883241c563b00f0aa5106fc4eaa0d", size = 13409, upload-time = "2025-10-06T05:38:16.721Z" }, ] +[[package]] +name = "fsspec" +version = "2026.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/00/78/f34251dadb8f3921264a1d9b8946f5e542014ee2614b285261b4e40e6775/fsspec-2026.7.0.tar.gz", hash = "sha256:c803c40f4cf860b49dea58ee3e1c33cb9c790520e233537e1340049f89b82a88", size = 317040, upload-time = "2026-07-28T16:34:51.052Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fd/3c/6a2bf344106328fd04963664a60b9bb6496fc25df8e962fcdc1367285fb9/fsspec-2026.7.0-py3-none-any.whl", hash = "sha256:b57ddbafedfaef7018c1ecab32aa200a9d7ca26b77965f64e48b70061249d279", size = 206583, upload-time = "2026-07-28T16:34:49.538Z" }, +] + [[package]] name = "google-antigravity" version = "0.1.7" @@ -882,6 +953,30 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", size = 37515, upload-time = "2025-04-24T03:35:24.344Z" }, ] +[[package]] +name = "hf-xet" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1b/ab/522a2ab67f27971a9d48ca666d4fca85ef7d5282d142e31fd087e27b1bbe/hf_xet-1.6.0.tar.gz", hash = "sha256:2e58454a340b3556dfa4972d5451aff4fba8dd42a236600ba1a1d2b1514f0fef", size = 920527, upload-time = "2026-08-03T22:33:13.243Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/41/62/3c062f593bd92ef4e77a0ef39541e3d82a0a1d3947c8a777a02a13a27828/hf_xet-1.6.0-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:70cbb9c896901600128cb9b6f06e132954fbede1db30f31f7c6c63f84cb7c31d", size = 4074584, upload-time = "2026-08-03T22:32:47.364Z" }, + { url = "https://files.pythonhosted.org/packages/bb/1e/c0ad437dd267a8e435bef594acf781bbc3874ff0b6435b4962d03ecf7cc4/hf_xet-1.6.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:23379c2f9ec8696d952b16414a2bae72cad86a52df869b050698ba60f538c675", size = 3867381, upload-time = "2026-08-03T22:32:49.049Z" }, + { url = "https://files.pythonhosted.org/packages/d5/ee/7c0d7b6ab336167531b1c30af2af003f054af4c749becbd7209ae33a77c3/hf_xet-1.6.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f2f7278c05c22fd60cb436cda1269649b3e81db65ecdc8496e5e164aa4143e7b", size = 4453982, upload-time = "2026-08-03T22:32:50.568Z" }, + { url = "https://files.pythonhosted.org/packages/63/06/ad8eab1c9525246650cbaa821caa3cdbaca734ab1a5b8c91bea09cbd8d69/hf_xet-1.6.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:948f15d3a9545cfe5932f6bd8b440f6ae630aee108f14b7bd6c561f7c2dcc522", size = 4249445, upload-time = "2026-08-03T22:32:52.391Z" }, + { url = "https://files.pythonhosted.org/packages/d8/26/1eee8aedb0dafc1ab9717dc9ac602cde33361b232dc06803f1f6ed18b58c/hf_xet-1.6.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:5153e6bb103ad49d6ea9f1b2e230db5a2ea32551ad09a706d2f61d7c7c80d80e", size = 4451099, upload-time = "2026-08-03T22:32:54.114Z" }, + { url = "https://files.pythonhosted.org/packages/67/57/0b88af1f194ab6c9c650547d9cc06bfeaab836ae4dcdb331676bfb8be95a/hf_xet-1.6.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:35cec30d75c6f9eb9c16a77cef68e85a103b72e24d4b473714ec9ff06428bab9", size = 4664712, upload-time = "2026-08-03T22:32:55.547Z" }, + { url = "https://files.pythonhosted.org/packages/53/a0/26b717a9d1840e8abf48dcec64b5ed8fbe472671d38ad28d30e147132b33/hf_xet-1.6.0-cp314-cp314t-win_amd64.whl", hash = "sha256:5789835d7c6bc9436962853192082374297fb72d7eff7e7762ec25ceb7e25338", size = 4025906, upload-time = "2026-08-03T22:32:57.391Z" }, + { url = "https://files.pythonhosted.org/packages/49/f6/4a9966633c6fef83af997e2cff68ec1963676d412bdfd096df2a93b8e185/hf_xet-1.6.0-cp314-cp314t-win_arm64.whl", hash = "sha256:75765820ce4700db3750c94acc8fe27c5fae4c9ec000a0dbac3ca082acf97765", size = 3849221, upload-time = "2026-08-03T22:32:59.123Z" }, + { url = "https://files.pythonhosted.org/packages/a2/50/7afa2c9c787405864fc47a0d1bbc02c62e9101947ed43c1f43899fc7d91d/hf_xet-1.6.0-cp38-abi3-macosx_10_12_x86_64.whl", hash = "sha256:633dc0cd71d32da58ab8c03ad38e2fac452c15c2b0a2866ebf6ededfe0a5061d", size = 4071729, upload-time = "2026-08-03T22:33:00.721Z" }, + { url = "https://files.pythonhosted.org/packages/4b/69/55b8dcf636142ae660fec1869fcac14c4da2e8412e14d6eee1523be77e9f/hf_xet-1.6.0-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:f0906082d9932ae0c0057fa194041c22b4e2cdb46b2592ef3b91f020d62a081a", size = 3876287, upload-time = "2026-08-03T22:33:02.251Z" }, + { url = "https://files.pythonhosted.org/packages/67/4e/a28359bf1c1ecf11eba22123168c138698f7cb576ac678f5a2e16cd5da08/hf_xet-1.6.0-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:d62671bb130879cef0ee4c9ebe47a14af6c66ec53e6d84dc15936e5ffdfac82f", size = 4464663, upload-time = "2026-08-03T22:33:03.802Z" }, + { url = "https://files.pythonhosted.org/packages/9a/69/1f0cbc2fb22ae6082d094f743d1b8945a3f36f6089cb95f42b7ee348cda7/hf_xet-1.6.0-cp38-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:0e6e21fa3cdfcdcd76748564bf593870a5e013f47d97cf10aed63aa222cff5b7", size = 4262538, upload-time = "2026-08-03T22:33:05.287Z" }, + { url = "https://files.pythonhosted.org/packages/d1/3a/4f4f2301ade26e404462d3336fa11f7958d914cabbabdd6e03c3c5d5658c/hf_xet-1.6.0-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:4fc74352a17015bd0ee90038bc9efe38db894cde45f268b6712b04fce8cd0acb", size = 4460520, upload-time = "2026-08-03T22:33:06.81Z" }, + { url = "https://files.pythonhosted.org/packages/ab/5f/311725e2a905534dfee2dcb5b08414f249147f1f12252bfc2bd24caa075c/hf_xet-1.6.0-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:8fb4f71cba6129110c3374a33f919001ff130488fc23553698e34cc1c2a1198c", size = 4675937, upload-time = "2026-08-03T22:33:08.616Z" }, + { url = "https://files.pythonhosted.org/packages/98/b7/8c59a66d15205024662f1d66968136f13893f96df1ddc5087e2e281fc95f/hf_xet-1.6.0-cp38-abi3-win_amd64.whl", hash = "sha256:fb4fadde1b2b70bf4c0c14a6dccbe7194b1c28947fefd5bbe3fed9d940676c3b", size = 4033128, upload-time = "2026-08-03T22:33:10.171Z" }, + { url = "https://files.pythonhosted.org/packages/73/63/ca511b6f802f28cf3489b280fe77475bcca8de85e81a6299d7916b5b5555/hf_xet-1.6.0-cp38-abi3-win_arm64.whl", hash = "sha256:3dc3e35441ba395006af5aaacc40ef2e603c51ef46c3530b9156185f00935ea3", size = 3859359, upload-time = "2026-08-03T22:33:11.725Z" }, +] + [[package]] name = "httpcore" version = "1.0.9" @@ -957,6 +1052,26 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/9b/43/832f631d32e4f1211caa2ba368317739fe71f0b8530e4c9d15dc454bac2a/httpx2_jsfetch-1.0-py3-none-any.whl", hash = "sha256:cb916b707601e69a07721aabc8f3f6659be3a6893bc1ff5c6f9e02241df2da32", size = 6382, upload-time = "2026-08-07T00:13:06.567Z" }, ] +[[package]] +name = "huggingface-hub" +version = "1.28.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "filelock" }, + { name = "fsspec" }, + { name = "hf-xet", marker = "platform_machine == 'AMD64' or platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'arm64' or platform_machine == 'x86_64'" }, + { name = "httpx" }, + { name = "packaging" }, + { name = "pyyaml" }, + { name = "tqdm" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c6/ae/222a91937ebee7f62c0ca8f5ee0afd97577caf24c0abb927d1f5c7e9f6d2/huggingface_hub-1.28.0.tar.gz", hash = "sha256:46a2e950c09234de54093d587d1675382f0d08dbd600d9fb599b5932f5b2c6cb", size = 959609, upload-time = "2026-08-18T12:27:15.101Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/51/0e/eafef18f1a75e125e68395db21131db0cf868a128ecd2fce69b4df6c584b/huggingface_hub-1.28.0-py3-none-any.whl", hash = "sha256:58a8bacb03072edfc38067065e9dc24bbb34805410fcd36a1632de0b329660bb", size = 793202, upload-time = "2026-08-18T12:27:12.719Z" }, +] + [[package]] name = "identify" version = "2.6.18" @@ -1005,6 +1120,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/15/aa/0aca39a37d3c7eb941ba736ede56d689e7be91cab5d9ca846bde3999eba6/isodate-0.7.2-py3-none-any.whl", hash = "sha256:28009937d8031054830160fce6d409ed342816b543597cece116d966c6d99e15", size = 22320, upload-time = "2024-10-08T23:04:09.501Z" }, ] +[[package]] +name = "jinja2" +version = "3.1.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markupsafe" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/df/bf/f7da0350254c0ed7c72f3e33cef02e048281fec7ecec5f032d4aac52226b/jinja2-3.1.6.tar.gz", hash = "sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d", size = 245115, upload-time = "2025-03-05T20:05:02.478Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67", size = 134899, upload-time = "2025-03-05T20:05:00.369Z" }, +] + [[package]] name = "jiter" version = "0.13.0" @@ -1104,6 +1231,37 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/af/40/791891d4c0c4dab4c5e187c17261cedc26285fd41541577f900470a45a4d/license_expression-30.4.4-py3-none-any.whl", hash = "sha256:421788fdcadb41f049d2dc934ce666626265aeccefddd25e162a26f23bcbf8a4", size = 120615, upload-time = "2025-07-22T11:13:31.217Z" }, ] +[[package]] +name = "litellm" +version = "1.98.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "aiohttp" }, + { name = "boto3" }, + { name = "click" }, + { name = "fastuuid" }, + { name = "httpx" }, + { name = "importlib-metadata" }, + { name = "jinja2" }, + { name = "jsonschema" }, + { name = "openai" }, + { name = "pydantic" }, + { name = "pydantic-settings" }, + { name = "python-dotenv" }, + { name = "tiktoken" }, + { name = "tokenizers" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9c/97/c9da198af273d700bf44d7d82eb21c5b8078c82574b31856b71b1298234b/litellm-1.98.0.tar.gz", hash = "sha256:0e6ba5d645a73ca6d0ffb4e8ec539d94b6e8fad691f2a54c6819011e6d0de8bf", size = 17577139, upload-time = "2026-08-22T22:19:21.931Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c8/90/2ef5e33b0a67b309be124a77e5098a261beffe28439221dbbc28e5f02e2e/litellm-1.98.0-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:9fda3497f1ec4686c943ce2aeab5767f8fb4a5989305d4b28d6d5d7e488b850c", size = 24026097, upload-time = "2026-08-22T22:19:03.567Z" }, + { url = "https://files.pythonhosted.org/packages/b0/56/b4569b4ef3640732d5770e0d3f46a395fd20f35594db1f65629595daed00/litellm-1.98.0-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:b89b6a0fc179d881191579f5309e622173dca802ee991b92e382acb918eea437", size = 23683944, upload-time = "2026-08-22T22:19:06.952Z" }, + { url = "https://files.pythonhosted.org/packages/d9/a7/9f03de0e8d767ff27964ea99bbf07e4c37a12f9d3c168c09f40e068535d4/litellm-1.98.0-cp310-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:3a95260f087a4cf763da85bbeeb2efa82caec243ad2394c9e6362ff747963738", size = 23824066, upload-time = "2026-08-22T22:19:09.714Z" }, + { url = "https://files.pythonhosted.org/packages/69/de/ab46b521e2a6e6a94a5cb91ba3debd6c4e922feaeb47158a389400b0a0fe/litellm-1.98.0-cp310-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:150993180bf049feafa3e20cf46ca0978c69cc66e2ef1639a47e852c682a4721", size = 24190393, upload-time = "2026-08-22T22:19:12.156Z" }, + { url = "https://files.pythonhosted.org/packages/c5/0a/d2f549d906b9b267b38b406dde721eb93db21b46ec907ae0a7a2a89a50f6/litellm-1.98.0-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:54d0bc2aba84644de5e84a265f24e5d436d91592ca5b7cc61cee478db297b0c3", size = 23901211, upload-time = "2026-08-22T22:19:14.708Z" }, + { url = "https://files.pythonhosted.org/packages/85/08/1bd1653297d9c92eaf04425f8a21da817fcde12e1d9469394c543df19d72/litellm-1.98.0-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:5e546d4af197c257d320299af11f4619f8e5a29f9bb7ce2d9974dd4b1e045499", size = 24290140, upload-time = "2026-08-22T22:19:17.145Z" }, + { url = "https://files.pythonhosted.org/packages/de/91/14d11ad7e290137400e5b30bcf23de86f811085f4a46756f60684ed0f064/litellm-1.98.0-cp310-abi3-win_amd64.whl", hash = "sha256:1daac9a9a9d052fdbe58ee711c9924dc81d349d4621286cbd96d77baa12158c4", size = 24079638, upload-time = "2026-08-22T22:19:19.558Z" }, +] + [[package]] name = "mando" version = "0.7.1" @@ -1128,6 +1286,58 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/94/54/e7d793b573f298e1c9013b8c4dade17d481164aa517d1d7148619c2cedbf/markdown_it_py-4.0.0-py3-none-any.whl", hash = "sha256:87327c59b172c5011896038353a81343b6754500a08cd7a4973bb48c6d578147", size = 87321, upload-time = "2025-08-11T12:57:51.923Z" }, ] +[[package]] +name = "markupsafe" +version = "3.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7e/99/7690b6d4034fffd95959cbe0c02de8deb3098cc577c67bb6a24fe5d7caa7/markupsafe-3.0.3.tar.gz", hash = "sha256:722695808f4b6457b320fdc131280796bdceb04ab50fe1795cd540799ebe1698", size = 80313, upload-time = "2025-09-27T18:37:40.426Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/38/2f/907b9c7bbba283e68f20259574b13d005c121a0fa4c175f9bed27c4597ff/markupsafe-3.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e1cf1972137e83c5d4c136c43ced9ac51d0e124706ee1c8aa8532c1287fa8795", size = 11622, upload-time = "2025-09-27T18:36:41.777Z" }, + { url = "https://files.pythonhosted.org/packages/9c/d9/5f7756922cdd676869eca1c4e3c0cd0df60ed30199ffd775e319089cb3ed/markupsafe-3.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:116bb52f642a37c115f517494ea5feb03889e04df47eeff5b130b1808ce7c219", size = 12029, upload-time = "2025-09-27T18:36:43.257Z" }, + { url = "https://files.pythonhosted.org/packages/00/07/575a68c754943058c78f30db02ee03a64b3c638586fba6a6dd56830b30a3/markupsafe-3.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:133a43e73a802c5562be9bbcd03d090aa5a1fe899db609c29e8c8d815c5f6de6", size = 24374, upload-time = "2025-09-27T18:36:44.508Z" }, + { url = "https://files.pythonhosted.org/packages/a9/21/9b05698b46f218fc0e118e1f8168395c65c8a2c750ae2bab54fc4bd4e0e8/markupsafe-3.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ccfcd093f13f0f0b7fdd0f198b90053bf7b2f02a3927a30e63f3ccc9df56b676", size = 22980, upload-time = "2025-09-27T18:36:45.385Z" }, + { url = "https://files.pythonhosted.org/packages/7f/71/544260864f893f18b6827315b988c146b559391e6e7e8f7252839b1b846a/markupsafe-3.0.3-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:509fa21c6deb7a7a273d629cf5ec029bc209d1a51178615ddf718f5918992ab9", size = 21990, upload-time = "2025-09-27T18:36:46.916Z" }, + { url = "https://files.pythonhosted.org/packages/c2/28/b50fc2f74d1ad761af2f5dcce7492648b983d00a65b8c0e0cb457c82ebbe/markupsafe-3.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a4afe79fb3de0b7097d81da19090f4df4f8d3a2b3adaa8764138aac2e44f3af1", size = 23784, upload-time = "2025-09-27T18:36:47.884Z" }, + { url = "https://files.pythonhosted.org/packages/ed/76/104b2aa106a208da8b17a2fb72e033a5a9d7073c68f7e508b94916ed47a9/markupsafe-3.0.3-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:795e7751525cae078558e679d646ae45574b47ed6e7771863fcc079a6171a0fc", size = 21588, upload-time = "2025-09-27T18:36:48.82Z" }, + { url = "https://files.pythonhosted.org/packages/b5/99/16a5eb2d140087ebd97180d95249b00a03aa87e29cc224056274f2e45fd6/markupsafe-3.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:8485f406a96febb5140bfeca44a73e3ce5116b2501ac54fe953e488fb1d03b12", size = 23041, upload-time = "2025-09-27T18:36:49.797Z" }, + { url = "https://files.pythonhosted.org/packages/19/bc/e7140ed90c5d61d77cea142eed9f9c303f4c4806f60a1044c13e3f1471d0/markupsafe-3.0.3-cp313-cp313-win32.whl", hash = "sha256:bdd37121970bfd8be76c5fb069c7751683bdf373db1ed6c010162b2a130248ed", size = 14543, upload-time = "2025-09-27T18:36:51.584Z" }, + { url = "https://files.pythonhosted.org/packages/05/73/c4abe620b841b6b791f2edc248f556900667a5a1cf023a6646967ae98335/markupsafe-3.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:9a1abfdc021a164803f4d485104931fb8f8c1efd55bc6b748d2f5774e78b62c5", size = 15113, upload-time = "2025-09-27T18:36:52.537Z" }, + { url = "https://files.pythonhosted.org/packages/f0/3a/fa34a0f7cfef23cf9500d68cb7c32dd64ffd58a12b09225fb03dd37d5b80/markupsafe-3.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:7e68f88e5b8799aa49c85cd116c932a1ac15caaa3f5db09087854d218359e485", size = 13911, upload-time = "2025-09-27T18:36:53.513Z" }, + { url = "https://files.pythonhosted.org/packages/e4/d7/e05cd7efe43a88a17a37b3ae96e79a19e846f3f456fe79c57ca61356ef01/markupsafe-3.0.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:218551f6df4868a8d527e3062d0fb968682fe92054e89978594c28e642c43a73", size = 11658, upload-time = "2025-09-27T18:36:54.819Z" }, + { url = "https://files.pythonhosted.org/packages/99/9e/e412117548182ce2148bdeacdda3bb494260c0b0184360fe0d56389b523b/markupsafe-3.0.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:3524b778fe5cfb3452a09d31e7b5adefeea8c5be1d43c4f810ba09f2ceb29d37", size = 12066, upload-time = "2025-09-27T18:36:55.714Z" }, + { url = "https://files.pythonhosted.org/packages/bc/e6/fa0ffcda717ef64a5108eaa7b4f5ed28d56122c9a6d70ab8b72f9f715c80/markupsafe-3.0.3-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4e885a3d1efa2eadc93c894a21770e4bc67899e3543680313b09f139e149ab19", size = 25639, upload-time = "2025-09-27T18:36:56.908Z" }, + { url = "https://files.pythonhosted.org/packages/96/ec/2102e881fe9d25fc16cb4b25d5f5cde50970967ffa5dddafdb771237062d/markupsafe-3.0.3-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8709b08f4a89aa7586de0aadc8da56180242ee0ada3999749b183aa23df95025", size = 23569, upload-time = "2025-09-27T18:36:57.913Z" }, + { url = "https://files.pythonhosted.org/packages/4b/30/6f2fce1f1f205fc9323255b216ca8a235b15860c34b6798f810f05828e32/markupsafe-3.0.3-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b8512a91625c9b3da6f127803b166b629725e68af71f8184ae7e7d54686a56d6", size = 23284, upload-time = "2025-09-27T18:36:58.833Z" }, + { url = "https://files.pythonhosted.org/packages/58/47/4a0ccea4ab9f5dcb6f79c0236d954acb382202721e704223a8aafa38b5c8/markupsafe-3.0.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:9b79b7a16f7fedff2495d684f2b59b0457c3b493778c9eed31111be64d58279f", size = 24801, upload-time = "2025-09-27T18:36:59.739Z" }, + { url = "https://files.pythonhosted.org/packages/6a/70/3780e9b72180b6fecb83a4814d84c3bf4b4ae4bf0b19c27196104149734c/markupsafe-3.0.3-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:12c63dfb4a98206f045aa9563db46507995f7ef6d83b2f68eda65c307c6829eb", size = 22769, upload-time = "2025-09-27T18:37:00.719Z" }, + { url = "https://files.pythonhosted.org/packages/98/c5/c03c7f4125180fc215220c035beac6b9cb684bc7a067c84fc69414d315f5/markupsafe-3.0.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:8f71bc33915be5186016f675cd83a1e08523649b0e33efdb898db577ef5bb009", size = 23642, upload-time = "2025-09-27T18:37:01.673Z" }, + { url = "https://files.pythonhosted.org/packages/80/d6/2d1b89f6ca4bff1036499b1e29a1d02d282259f3681540e16563f27ebc23/markupsafe-3.0.3-cp313-cp313t-win32.whl", hash = "sha256:69c0b73548bc525c8cb9a251cddf1931d1db4d2258e9599c28c07ef3580ef354", size = 14612, upload-time = "2025-09-27T18:37:02.639Z" }, + { url = "https://files.pythonhosted.org/packages/2b/98/e48a4bfba0a0ffcf9925fe2d69240bfaa19c6f7507b8cd09c70684a53c1e/markupsafe-3.0.3-cp313-cp313t-win_amd64.whl", hash = "sha256:1b4b79e8ebf6b55351f0d91fe80f893b4743f104bff22e90697db1590e47a218", size = 15200, upload-time = "2025-09-27T18:37:03.582Z" }, + { url = "https://files.pythonhosted.org/packages/0e/72/e3cc540f351f316e9ed0f092757459afbc595824ca724cbc5a5d4263713f/markupsafe-3.0.3-cp313-cp313t-win_arm64.whl", hash = "sha256:ad2cf8aa28b8c020ab2fc8287b0f823d0a7d8630784c31e9ee5edea20f406287", size = 13973, upload-time = "2025-09-27T18:37:04.929Z" }, + { url = "https://files.pythonhosted.org/packages/33/8a/8e42d4838cd89b7dde187011e97fe6c3af66d8c044997d2183fbd6d31352/markupsafe-3.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:eaa9599de571d72e2daf60164784109f19978b327a3910d3e9de8c97b5b70cfe", size = 11619, upload-time = "2025-09-27T18:37:06.342Z" }, + { url = "https://files.pythonhosted.org/packages/b5/64/7660f8a4a8e53c924d0fa05dc3a55c9cee10bbd82b11c5afb27d44b096ce/markupsafe-3.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:c47a551199eb8eb2121d4f0f15ae0f923d31350ab9280078d1e5f12b249e0026", size = 12029, upload-time = "2025-09-27T18:37:07.213Z" }, + { url = "https://files.pythonhosted.org/packages/da/ef/e648bfd021127bef5fa12e1720ffed0c6cbb8310c8d9bea7266337ff06de/markupsafe-3.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f34c41761022dd093b4b6896d4810782ffbabe30f2d443ff5f083e0cbbb8c737", size = 24408, upload-time = "2025-09-27T18:37:09.572Z" }, + { url = "https://files.pythonhosted.org/packages/41/3c/a36c2450754618e62008bf7435ccb0f88053e07592e6028a34776213d877/markupsafe-3.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:457a69a9577064c05a97c41f4e65148652db078a3a509039e64d3467b9e7ef97", size = 23005, upload-time = "2025-09-27T18:37:10.58Z" }, + { url = "https://files.pythonhosted.org/packages/bc/20/b7fdf89a8456b099837cd1dc21974632a02a999ec9bf7ca3e490aacd98e7/markupsafe-3.0.3-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e8afc3f2ccfa24215f8cb28dcf43f0113ac3c37c2f0f0806d8c70e4228c5cf4d", size = 22048, upload-time = "2025-09-27T18:37:11.547Z" }, + { url = "https://files.pythonhosted.org/packages/9a/a7/591f592afdc734f47db08a75793a55d7fbcc6902a723ae4cfbab61010cc5/markupsafe-3.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:ec15a59cf5af7be74194f7ab02d0f59a62bdcf1a537677ce67a2537c9b87fcda", size = 23821, upload-time = "2025-09-27T18:37:12.48Z" }, + { url = "https://files.pythonhosted.org/packages/7d/33/45b24e4f44195b26521bc6f1a82197118f74df348556594bd2262bda1038/markupsafe-3.0.3-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:0eb9ff8191e8498cca014656ae6b8d61f39da5f95b488805da4bb029cccbfbaf", size = 21606, upload-time = "2025-09-27T18:37:13.485Z" }, + { url = "https://files.pythonhosted.org/packages/ff/0e/53dfaca23a69fbfbbf17a4b64072090e70717344c52eaaaa9c5ddff1e5f0/markupsafe-3.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:2713baf880df847f2bece4230d4d094280f4e67b1e813eec43b4c0e144a34ffe", size = 23043, upload-time = "2025-09-27T18:37:14.408Z" }, + { url = "https://files.pythonhosted.org/packages/46/11/f333a06fc16236d5238bfe74daccbca41459dcd8d1fa952e8fbd5dccfb70/markupsafe-3.0.3-cp314-cp314-win32.whl", hash = "sha256:729586769a26dbceff69f7a7dbbf59ab6572b99d94576a5592625d5b411576b9", size = 14747, upload-time = "2025-09-27T18:37:15.36Z" }, + { url = "https://files.pythonhosted.org/packages/28/52/182836104b33b444e400b14f797212f720cbc9ed6ba34c800639d154e821/markupsafe-3.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:bdc919ead48f234740ad807933cdf545180bfbe9342c2bb451556db2ed958581", size = 15341, upload-time = "2025-09-27T18:37:16.496Z" }, + { url = "https://files.pythonhosted.org/packages/6f/18/acf23e91bd94fd7b3031558b1f013adfa21a8e407a3fdb32745538730382/markupsafe-3.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:5a7d5dc5140555cf21a6fefbdbf8723f06fcd2f63ef108f2854de715e4422cb4", size = 14073, upload-time = "2025-09-27T18:37:17.476Z" }, + { url = "https://files.pythonhosted.org/packages/3c/f0/57689aa4076e1b43b15fdfa646b04653969d50cf30c32a102762be2485da/markupsafe-3.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:1353ef0c1b138e1907ae78e2f6c63ff67501122006b0f9abad68fda5f4ffc6ab", size = 11661, upload-time = "2025-09-27T18:37:18.453Z" }, + { url = "https://files.pythonhosted.org/packages/89/c3/2e67a7ca217c6912985ec766c6393b636fb0c2344443ff9d91404dc4c79f/markupsafe-3.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1085e7fbddd3be5f89cc898938f42c0b3c711fdcb37d75221de2666af647c175", size = 12069, upload-time = "2025-09-27T18:37:19.332Z" }, + { url = "https://files.pythonhosted.org/packages/f0/00/be561dce4e6ca66b15276e184ce4b8aec61fe83662cce2f7d72bd3249d28/markupsafe-3.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1b52b4fb9df4eb9ae465f8d0c228a00624de2334f216f178a995ccdcf82c4634", size = 25670, upload-time = "2025-09-27T18:37:20.245Z" }, + { url = "https://files.pythonhosted.org/packages/50/09/c419f6f5a92e5fadde27efd190eca90f05e1261b10dbd8cbcb39cd8ea1dc/markupsafe-3.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fed51ac40f757d41b7c48425901843666a6677e3e8eb0abcff09e4ba6e664f50", size = 23598, upload-time = "2025-09-27T18:37:21.177Z" }, + { url = "https://files.pythonhosted.org/packages/22/44/a0681611106e0b2921b3033fc19bc53323e0b50bc70cffdd19f7d679bb66/markupsafe-3.0.3-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f190daf01f13c72eac4efd5c430a8de82489d9cff23c364c3ea822545032993e", size = 23261, upload-time = "2025-09-27T18:37:22.167Z" }, + { url = "https://files.pythonhosted.org/packages/5f/57/1b0b3f100259dc9fffe780cfb60d4be71375510e435efec3d116b6436d43/markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e56b7d45a839a697b5eb268c82a71bd8c7f6c94d6fd50c3d577fa39a9f1409f5", size = 24835, upload-time = "2025-09-27T18:37:23.296Z" }, + { url = "https://files.pythonhosted.org/packages/26/6a/4bf6d0c97c4920f1597cc14dd720705eca0bf7c787aebc6bb4d1bead5388/markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:f3e98bb3798ead92273dc0e5fd0f31ade220f59a266ffd8a4f6065e0a3ce0523", size = 22733, upload-time = "2025-09-27T18:37:24.237Z" }, + { url = "https://files.pythonhosted.org/packages/14/c7/ca723101509b518797fedc2fdf79ba57f886b4aca8a7d31857ba3ee8281f/markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5678211cb9333a6468fb8d8be0305520aa073f50d17f089b5b4b477ea6e67fdc", size = 23672, upload-time = "2025-09-27T18:37:25.271Z" }, + { url = "https://files.pythonhosted.org/packages/fb/df/5bd7a48c256faecd1d36edc13133e51397e41b73bb77e1a69deab746ebac/markupsafe-3.0.3-cp314-cp314t-win32.whl", hash = "sha256:915c04ba3851909ce68ccc2b8e2cd691618c4dc4c4232fb7982bca3f41fd8c3d", size = 14819, upload-time = "2025-09-27T18:37:26.285Z" }, + { url = "https://files.pythonhosted.org/packages/1a/8a/0402ba61a2f16038b48b39bccca271134be00c5c9f0f623208399333c448/markupsafe-3.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4faffd047e07c38848ce017e8725090413cd80cbc23d86e55c587bf979e579c9", size = 15426, upload-time = "2025-09-27T18:37:27.316Z" }, + { url = "https://files.pythonhosted.org/packages/70/bc/6f1c2f612465f5fa89b95bead1f44dcb607670fd42891d8fdcd5d039f4f4/markupsafe-3.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:32001d6a8fc98c8cb5c947787c5d08b0a50663d139f1305bac5885d98d9b40fa", size = 14146, upload-time = "2025-09-27T18:37:28.327Z" }, +] + [[package]] name = "mcp" version = "1.28.1" @@ -1362,6 +1572,25 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/be/9c/92789c596b8df838baa98fa71844d84283302f7604ed565dafe5a6b5041a/oauthlib-3.3.1-py3-none-any.whl", hash = "sha256:88119c938d2b8fb88561af5f6ee0eec8cc8d552b7bb1f712743136eb7523b7a1", size = 160065, upload-time = "2025-06-19T22:48:06.508Z" }, ] +[[package]] +name = "openai" +version = "2.54.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "distro" }, + { name = "httpx" }, + { name = "jiter" }, + { name = "pydantic" }, + { name = "sniffio" }, + { name = "tqdm" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/50/9a/8c75e8c8a5b407a0586faeb2afac91674ff955c191ecc1d6d3b6669f6788/openai-2.54.0.tar.gz", hash = "sha256:e3e6f8bc1ba30ddf381ace1a14340eed381cb984a1a59bd0f34b5be3b5d49cfa", size = 1100285, upload-time = "2026-08-11T18:46:59.035Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/64/a8/bb76c7356de8ad57f59d5ff993d434df0607f07f08bcc9c9a5c275e399c0/openai-2.54.0-py3-none-any.whl", hash = "sha256:89089789197ccdb87f173a03145ed1598d00795220c93e96cf712b1cbf5e5f2b", size = 1660351, upload-time = "2026-08-11T18:46:56.684Z" }, +] + [[package]] name = "openai-codex" version = "0.144.4" @@ -1979,6 +2208,18 @@ psutil = [ { name = "psutil" }, ] +[[package]] +name = "python-dateutil" +version = "2.9.0.post0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "six" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/66/c0/0c8b6ad9f17a802ee498c46e004a0eb49bc148f2fd230864601a86dcf6db/python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", size = 342432, upload-time = "2024-03-01T18:36:20.211Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", size = 229892, upload-time = "2024-03-01T18:36:18.57Z" }, +] + [[package]] name = "python-discovery" version = "1.2.0" @@ -2110,6 +2351,78 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2c/58/ca301544e1fa93ed4f80d724bf5b194f6e4b945841c5bfd555878eea9fcb/referencing-0.37.0-py3-none-any.whl", hash = "sha256:381329a9f99628c9069361716891d34ad94af76e461dcb0335825aecc7692231", size = 26766, upload-time = "2025-10-13T15:30:47.625Z" }, ] +[[package]] +name = "regex" +version = "2026.7.19" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/20/98/04b13f1ddfb63158025291c02e03eb42fbb7acb51d091d541050eb4e35e8/regex-2026.7.19.tar.gz", hash = "sha256:7e77b324909c1617cbb4c668677e2c6ae13f44d7c1de0d4f15f2e3c10f3315b5", size = 416440, upload-time = "2026-07-19T00:19:48.923Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5d/3d/84165e4299ff76f3a40fe1f2abf939e976f693383a08d2beea6af62bd2c1/regex-2026.7.19-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:f035d9dc1d25eff9d361456572231c7d27b5ccd473ca7dc0adfce732bd006d40", size = 496552, upload-time = "2026-07-19T00:17:36.808Z" }, + { url = "https://files.pythonhosted.org/packages/02/a2/a65293e6e4cf28eb7ee1be5335a5386c40d6742e9f47fafc8fec785e16c7/regex-2026.7.19-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:c42572142ed0b9d5d261ba727157c426510da78e20828b66bbb855098b8a4e38", size = 296983, upload-time = "2026-07-19T00:17:38.816Z" }, + { url = "https://files.pythonhosted.org/packages/95/47/2d0564e93d87bc48618360ddca232a2ca612bbdf53ce8465d45ca5ce14ee/regex-2026.7.19-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:40b34dd88658e4fedd2fddbf0275ac970d00614b731357f425722a3ed1983d11", size = 291832, upload-time = "2026-07-19T00:17:40.726Z" }, + { url = "https://files.pythonhosted.org/packages/07/cd/42dfbabff3dfc9603c501c0e2e2c5adbb09d127b267bf5348de0af338c15/regex-2026.7.19-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0c41c63992bf1874cebb6e7f56fd7d3c007924659a604ae3d90e427d40d4fd13", size = 796775, upload-time = "2026-07-19T00:17:42.382Z" }, + { url = "https://files.pythonhosted.org/packages/df/5d/f6a4839f2b934e3eed5973fd07f5929ee97d4c98939fb275ea23c274ee16/regex-2026.7.19-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1d3372064506b94dd2c67c845f2db8062e9e9ba84d04e33cb96d7d33c11fe1ae", size = 865687, upload-time = "2026-07-19T00:17:44.185Z" }, + { url = "https://files.pythonhosted.org/packages/14/b0/b47d6c36049bc59806a50bd4c86ced70bbe058d787f80281b1d7a9b0e024/regex-2026.7.19-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fce7760bf283405b2c7999cab3da4e72f7deca6396013115e3f7a955db9760da", size = 911962, upload-time = "2026-07-19T00:17:46.442Z" }, + { url = "https://files.pythonhosted.org/packages/2a/be/ff61f28f9273658cfe23acbbac5217221f6519960ed401e61dfdab12bc35/regex-2026.7.19-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c0d702548d89d572b2929879bc883bb7a4c4709efafe4512cadee56c55c9bd15", size = 801817, upload-time = "2026-07-19T00:17:48.25Z" }, + { url = "https://files.pythonhosted.org/packages/c3/bb/8b4f7f26b333f9f79e1b453613c39bb4776f51d38ae66dd0ba31d6b354ca/regex-2026.7.19-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d446c6ac40bb6e05025ccee55b84d80fe9bf8e93010ffc4bb9484f13d498835f", size = 776908, upload-time = "2026-07-19T00:17:50.183Z" }, + { url = "https://files.pythonhosted.org/packages/09/13/610110fc5921d380516d03c26b652555f08aa0d23ea78a771231873c3638/regex-2026.7.19-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4c3501bfa814ab07b5580741f9bf78dfdfe146a04057f82df9e2402d2a975939", size = 784426, upload-time = "2026-07-19T00:17:52.454Z" }, + { url = "https://files.pythonhosted.org/packages/ca/f5/1ef9e2a83a5947c57ebff0b377cb5727c3d5ec1992317a320d035cd0dbb6/regex-2026.7.19-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:c4585c3e64b4f9e583b4d2683f18f5d5d872b3d71dcf24594b74ecc23602fa96", size = 860600, upload-time = "2026-07-19T00:17:54.229Z" }, + { url = "https://files.pythonhosted.org/packages/a0/02/073af33a3ec149241d11c80acea91e722aa0adbf05addd50f251c4fe89c3/regex-2026.7.19-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:571fde9741eb0ccde23dd4e0c1d50fbae910e901fa7e629faf39b2dda740d220", size = 765950, upload-time = "2026-07-19T00:17:56.041Z" }, + { url = "https://files.pythonhosted.org/packages/81/a9/d1e9f819dc394a568ef370cd56cf25394e957a2235f8370f23b576e5a475/regex-2026.7.19-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:15b364b9b98d6d2fe1a85034c23a3180ff913f46caddc3895f6fd65186255ccc", size = 851794, upload-time = "2026-07-19T00:17:57.897Z" }, + { url = "https://files.pythonhosted.org/packages/03/3a/8ae83eda7579feacdf984e71fb9e70635fb6f832eeddca58427ec4fca926/regex-2026.7.19-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ffd8893ccc1c2fce6e0d6ca402d716fe1b29db70c7132609a05955e31b2aa8f2", size = 789845, upload-time = "2026-07-19T00:17:59.97Z" }, + { url = "https://files.pythonhosted.org/packages/4b/23/c195cbfe5a75fdec64d8f6554fd15237b837919d2c61bdc141d7c807b08b/regex-2026.7.19-cp313-cp313-win32.whl", hash = "sha256:f0fa4fa9c3632d708742baf2282f2055c11d888a790362670a403cbf48a2c404", size = 267135, upload-time = "2026-07-19T00:18:01.958Z" }, + { url = "https://files.pythonhosted.org/packages/b2/80/a11de8404b7272b70acb45c1c05987cce60b45d5693da2e176f0e390d564/regex-2026.7.19-cp313-cp313-win_amd64.whl", hash = "sha256:d51ffd3427640fa2da6ade574ceba932f210ad095f65fcc450a2b0a0d454868e", size = 277747, upload-time = "2026-07-19T00:18:04.121Z" }, + { url = "https://files.pythonhosted.org/packages/d1/29/0f5c8eff1b4f1f3d83276d365fccecf666afcc7d947420943bf394d07adb/regex-2026.7.19-cp313-cp313-win_arm64.whl", hash = "sha256:c670fe7be5b6020b76bc6e8d2196074657e1327595bca93a389e1a76ab130ad8", size = 277129, upload-time = "2026-07-19T00:18:05.821Z" }, + { url = "https://files.pythonhosted.org/packages/dc/4c/44b74742052cedda40f9ae469532a037112f7311a36669a891fba8984bb0/regex-2026.7.19-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:db47b561c9afd884baa1f96f797c9ca369872c4b65912bc691cfa99e68340af2", size = 501134, upload-time = "2026-07-19T00:18:07.567Z" }, + { url = "https://files.pythonhosted.org/packages/f0/45/bbd038b5e39ee5613a5a689290145b40058cc152c41de9cc23639d2b9734/regex-2026.7.19-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:65dcd28d3eba2ab7c2fd906485cc301392b47cc2234790d27d4e4814e02cdfda", size = 299418, upload-time = "2026-07-19T00:18:09.38Z" }, + { url = "https://files.pythonhosted.org/packages/65/38/c5bde94b4cedfd5850d64c3f08222d8e1600e84f6ee71d9b44b4b8163f74/regex-2026.7.19-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:f2e7f8e2ab6c2922be02c7ec45185aa5bd771e2e57b95455ee343a44d8130dff", size = 294486, upload-time = "2026-07-19T00:18:11.188Z" }, + { url = "https://files.pythonhosted.org/packages/d7/6a/2f5e107cb26c960b781967178899daf2787a7ab151844ed3c01d6fc95474/regex-2026.7.19-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fe31f28c94402043161876a258a9c6f757cb485905c7614ce8d6cd40e6b7bdc1", size = 811643, upload-time = "2026-07-19T00:18:12.975Z" }, + { url = "https://files.pythonhosted.org/packages/37/d4/a2f963406d7d73a62eed84ba05a258afb6cad1b21aa4517443ce40506b78/regex-2026.7.19-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f8f6fa298bb4f7f58a33334406218ba74716e68feddf5e4e54cd5d8082705abf", size = 871081, upload-time = "2026-07-19T00:18:14.733Z" }, + { url = "https://files.pythonhosted.org/packages/45/a3/44be546340bedb15f13063f5e7fe16793ea4d9ea2e805d09bd174ac27724/regex-2026.7.19-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:cc1b2440423a851fad781309dd87843868f4f66a6bcd1ddb9225cf4ec2c84732", size = 917372, upload-time = "2026-07-19T00:18:16.724Z" }, + { url = "https://files.pythonhosted.org/packages/f8/f6/e0870b0fd2a40dba0074e4b76e514b21313d37946c9248453e34ec43923e/regex-2026.7.19-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8ac59a0900474a52b7c04af8196affc22bd9842acb0950df12f7b813e983609a", size = 816089, upload-time = "2026-07-19T00:18:18.617Z" }, + { url = "https://files.pythonhosted.org/packages/ae/27/957e8e22690ad6634572b39b71f130a6105f4d0718bb16849eac00fff147/regex-2026.7.19-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4896db1f4ce0576765b8272aa922df324e0f5b9bb2c3d03044ff32a7234a9aba", size = 785206, upload-time = "2026-07-19T00:18:20.464Z" }, + { url = "https://files.pythonhosted.org/packages/76/a4/186e410941e731037c01166069ab86da9f65e8f8110c18009ccf4bd623ee/regex-2026.7.19-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:4e6883a021db30511d9fb8cfb0f222ce1f2c369f7d4d8b0448f449a93ba0bdfc", size = 800431, upload-time = "2026-07-19T00:18:22.716Z" }, + { url = "https://files.pythonhosted.org/packages/73/9f/e4e10e023d291d64a33e246610b724493bf1ce98e0e59c9b7c837e5acfb7/regex-2026.7.19-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:09523a592938aa9f587fb74467c63ff0cf88fc3df14c82ab0f0517dcf76aaa62", size = 864906, upload-time = "2026-07-19T00:18:24.772Z" }, + { url = "https://files.pythonhosted.org/packages/24/57/ccb20b6be5f1f52a053d1ba2a8f7a077edb9d918248b8490d7506c6832b3/regex-2026.7.19-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:1ebac3474b8589fce2f9b225b650afd61448f7c73a5d0255a10cc6366471aed1", size = 773559, upload-time = "2026-07-19T00:18:27.008Z" }, + { url = "https://files.pythonhosted.org/packages/a3/82/f3b263cf8fad927dc102891da8502e718b7ff9d19af7a2a07c03865d7188/regex-2026.7.19-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:4a0530bb1b8c1c985e7e2122e2b4d3aedd8a3c21c6bfddae6767c4405668b56e", size = 857739, upload-time = "2026-07-19T00:18:29.107Z" }, + { url = "https://files.pythonhosted.org/packages/47/2e/1687bd1b6c2aed5e672ccf845fc11557821fe7366d921b50889ea5ce57bf/regex-2026.7.19-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:2ef7eeb108c47ce7bcc9513e51bcb1bf57e8f483d52fce68a8642e3527141ae0", size = 804522, upload-time = "2026-07-19T00:18:31.362Z" }, + { url = "https://files.pythonhosted.org/packages/76/7c/cc4e7655181b2d9235b704f2c5e19d8eff002bbc437bae59baee0e381aca/regex-2026.7.19-cp313-cp313t-win32.whl", hash = "sha256:64b6ca7391a1395c2638dd5c7456d67bea44fc6c5e8e92c5dc8aa6a8f23292b4", size = 269141, upload-time = "2026-07-19T00:18:33.479Z" }, + { url = "https://files.pythonhosted.org/packages/bb/14/961b4c7b05a2391c32dbc85e27773076671ef8f97f36cec70fe414734c02/regex-2026.7.19-cp313-cp313t-win_amd64.whl", hash = "sha256:f04b9f56b0e0614c0126be12c2c2d9f8850c1e57af302bd0a63bed379d4af974", size = 280036, upload-time = "2026-07-19T00:18:35.419Z" }, + { url = "https://files.pythonhosted.org/packages/ce/67/795644550d788ddbb6dc458c95895f8009978ea6d6ea76b005eb3f45e8c9/regex-2026.7.19-cp313-cp313t-win_arm64.whl", hash = "sha256:fcee38cd8e5089d6d4f048ba1233b3ad76e5954f545382180889112ff5cb712d", size = 279394, upload-time = "2026-07-19T00:18:37.454Z" }, + { url = "https://files.pythonhosted.org/packages/d2/25/0c4c452f8ef3efe456745b2f33195f5904b573fb4c2ff3f0cb9ec188461e/regex-2026.7.19-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:a81758ed242b861b72e778ba34d41366441a2e10b16b472784c88da2dea7e2dd", size = 496750, upload-time = "2026-07-19T00:18:39.633Z" }, + { url = "https://files.pythonhosted.org/packages/24/9e/b70ca6c1704f6c7cd32a9e143c86cc5968d10981eca284bad670c245ea7d/regex-2026.7.19-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:4aa5435cdb3eb6f55fe98a171b05e3fbcd95fadaa4aa32acf62afd9b0cfdbcac", size = 297093, upload-time = "2026-07-19T00:18:41.583Z" }, + { url = "https://files.pythonhosted.org/packages/87/74/0b692da2520d51fbff19c88b83d97e4c702909dd02386c585998b7e2dbed/regex-2026.7.19-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:60be8693a1dadc210bbcbc0db3e26da5f7d01d1d5a3da594e99b4fa42df404f5", size = 292043, upload-time = "2026-07-19T00:18:43.347Z" }, + { url = "https://files.pythonhosted.org/packages/e3/a7/1d478e614016045a33feae57446215f9fd65b665a5ceb2f891fb3183bc52/regex-2026.7.19-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d19662dbedbe783d323196312d38f5ba53cf56296378252171985da6899887d3", size = 797214, upload-time = "2026-07-19T00:18:45.362Z" }, + { url = "https://files.pythonhosted.org/packages/aa/ae/11b9c9411d92c30e3d2db32df5a31133e4a99a8fc397a604fd08f6c4bffb/regex-2026.7.19-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d15df07081d91b76ff20d43f94592ee110330152d617b730fdbe5ef9fb680053", size = 866433, upload-time = "2026-07-19T00:18:47.315Z" }, + { url = "https://files.pythonhosted.org/packages/b1/62/2b2efc4992f91d6d204b24c647c9f9412e85379d92b7c0ab9fdae622327e/regex-2026.7.19-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:56ad4d9f77df871a99e25c37091052a02528ec0eb059de928ee33956b854b45b", size = 911360, upload-time = "2026-07-19T00:18:49.588Z" }, + { url = "https://files.pythonhosted.org/packages/14/71/986ceea9aa3da548bf1357cad89b63915ec6d21ec957c8113b29ece567df/regex-2026.7.19-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7322ec6cc9fba9d49ab888bb82d67ac5625627aa168f0165139b17018df3fb8a", size = 801275, upload-time = "2026-07-19T00:18:51.767Z" }, + { url = "https://files.pythonhosted.org/packages/15/be/ce9d9534b2cda96eab32c548261224b9b4e220a4126f098f60f42ae7b4cd/regex-2026.7.19-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9c7472192ebfad53a6be7c4a8bfb2d64b81c0e93a1fc8c57e1dd0b638297b5d1", size = 777131, upload-time = "2026-07-19T00:18:54.053Z" }, + { url = "https://files.pythonhosted.org/packages/61/2b/58b5c710f2c3929515a25f3a1ca0dad0dcd4518d4fff3cf23bc7adb8dcd2/regex-2026.7.19-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c10b82c2634df08dfb13b1f04e38fe310d086ee092f4f69c0c8da234251e556e", size = 785020, upload-time = "2026-07-19T00:18:56.579Z" }, + { url = "https://files.pythonhosted.org/packages/84/03/5fe091935b74f15fe0f97998c215cae418d1c0413f6258c7d4d2e83aa37f/regex-2026.7.19-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:17ed5692f6acc4183e98331101a5f9e4f64d72fe58b753da4d444a2c77d05b12", size = 861263, upload-time = "2026-07-19T00:18:58.64Z" }, + { url = "https://files.pythonhosted.org/packages/d8/fa/d60bf82e10841eef62a9e32aac401468f05fddfbcb2942e342b1ba3d2433/regex-2026.7.19-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:22a992de9a0d91bda927bf02b94351d737a0302905432c88a53de7c4b9ce62e2", size = 766199, upload-time = "2026-07-19T00:19:00.705Z" }, + { url = "https://files.pythonhosted.org/packages/bf/5d/11e64d151b0662b81d6bf644c74dc118d461df85bdf2577fadbbf751788a/regex-2026.7.19-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:618a0aed532be87294c4477b0481f3aa0f1520f4014a4374dd4cf789b4cd2c97", size = 851317, upload-time = "2026-07-19T00:19:03.015Z" }, + { url = "https://files.pythonhosted.org/packages/7c/34/532efb87488d90807bae6a443d357ee5e2728a478c597619c8aaa17cc0bd/regex-2026.7.19-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:2ce9e679f776649746729b6c86382da519ef649c8e34cc41df0d2e5e0f6c36d4", size = 789557, upload-time = "2026-07-19T00:19:05.338Z" }, + { url = "https://files.pythonhosted.org/packages/d6/90/3a8d5ca977171ec3ae21a71207d2228b2663bde14d7f7ef0e6363ecf9290/regex-2026.7.19-cp314-cp314-win32.whl", hash = "sha256:73f272fba87b8ccfe70a137d02a54af386f6d27aa509fbffdd978f5947aae1aa", size = 272531, upload-time = "2026-07-19T00:19:07.487Z" }, + { url = "https://files.pythonhosted.org/packages/96/e1/8862885e70409de70e8c005f57fb2e7be8d9ef0317250d60f4c9660a300d/regex-2026.7.19-cp314-cp314-win_amd64.whl", hash = "sha256:d721e53758b2cca74990185eb0671dd466d7a388a1a45d0c6f4c13cef41a68ac", size = 280831, upload-time = "2026-07-19T00:19:09.46Z" }, + { url = "https://files.pythonhosted.org/packages/08/82/2693e53e29f9104d9de95d37ce4dd826bd32d5f9c0085d3aa6ac042675c4/regex-2026.7.19-cp314-cp314-win_arm64.whl", hash = "sha256:65fa6cb38ed5e9c3637e68e544f598b39c3b86b808ed0627a67b68320384b459", size = 281099, upload-time = "2026-07-19T00:19:11.398Z" }, + { url = "https://files.pythonhosted.org/packages/92/b7/9a01aa16461a18cde9d7b9c3ab21e501db2ce33725f53014342b91df2b0a/regex-2026.7.19-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:5a2721c8720e2cb3c209925dfb9200199b4b07361c9e01d321719404b21458b3", size = 501121, upload-time = "2026-07-19T00:19:13.425Z" }, + { url = "https://files.pythonhosted.org/packages/f3/5e/bbaeca815dc9191c424c94a4fdc5c87c75748a64a6271821212ebdd4e1a3/regex-2026.7.19-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:199535629f25caf89698039af3d1ad5fcae7f933e2112c73f1cdf49165c99518", size = 299415, upload-time = "2026-07-19T00:19:15.43Z" }, + { url = "https://files.pythonhosted.org/packages/cd/d6/0dd1a321afaab95eb7ff44aa0f637301786f1dc71c6b797b9ed236ed8890/regex-2026.7.19-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:9b60d7814174f059e5de4ab98271cc5ba9259cfea55273a81544dceea32dc8d9", size = 294483, upload-time = "2026-07-19T00:19:17.879Z" }, + { url = "https://files.pythonhosted.org/packages/92/5f/40bacf91d0904f812e13bbbab3864604c463eced8afdc54aeaa50492ea95/regex-2026.7.19-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dbece16025afda5e3031af0c4059207e61dcf73ef13af844964f57f387d1c435", size = 811833, upload-time = "2026-07-19T00:19:20.102Z" }, + { url = "https://files.pythonhosted.org/packages/94/7c/4902744261f775aeede8b5627314b38482da29cf49a57b66a6fb753246c5/regex-2026.7.19-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d24ecb4f5e009ea0bd275ee37ad9953b32005e2e5e60f8bbae16da0dbbf0d3a0", size = 871270, upload-time = "2026-07-19T00:19:22.365Z" }, + { url = "https://files.pythonhosted.org/packages/16/70/6980c9be6bf21c0a60ed3e0aea39cf419ecf3b08d1d9947bc56e196ef186/regex-2026.7.19-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:8cae6fd77a5b72dae505084b1a2ee0360139faf72fedbab667cd7cc65aae7a6a", size = 917534, upload-time = "2026-07-19T00:19:24.529Z" }, + { url = "https://files.pythonhosted.org/packages/52/92/8b2bd872782ce8c42691e39acb38eb8efe014e5ddb78ad7d943d6f197ce9/regex-2026.7.19-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9724e6cb5e478cd7d8cabf027826178739cb18cf0e117d0e32814d479fa02276", size = 816135, upload-time = "2026-07-19T00:19:26.919Z" }, + { url = "https://files.pythonhosted.org/packages/de/2d/33a602f657bdc4041f17d79f92ab18261d255d91a06117a6e29df023e5e2/regex-2026.7.19-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:572fc57b0009c735ee56c175ea021b637a15551a312f56734277f923d6fd0f6c", size = 785492, upload-time = "2026-07-19T00:19:29.192Z" }, + { url = "https://files.pythonhosted.org/packages/9e/36/0987cf4cb271680064a70d24a475873775a151d0b7058698a006cb0cae4a/regex-2026.7.19-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:20568e182eb82d39a6bf7cff3fd58566f14c75c6f74b2c8c96537eecf9010e3a", size = 800658, upload-time = "2026-07-19T00:19:31.392Z" }, + { url = "https://files.pythonhosted.org/packages/a8/24/c14f31c135e1ba55fa4f9a58ca98d0842512bf6188230763c31c8f449e3b/regex-2026.7.19-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:1d58561843f0ff7dc78b4c28b5e2dc388f3eff94ebc8a232a3adba961fc00009", size = 865073, upload-time = "2026-07-19T00:19:33.485Z" }, + { url = "https://files.pythonhosted.org/packages/14/85/181a12211f22469f24d2de1ebddfe397d2396e2c29013b9a58134a91069a/regex-2026.7.19-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:61bb1bd45520aacd56dd80943bd34991fb5350afdd1f36f2282230fd5154a218", size = 773684, upload-time = "2026-07-19T00:19:35.599Z" }, + { url = "https://files.pythonhosted.org/packages/23/58/bd1a0c1a62251366f8d21f41b1ea3c76994962071b8b6ea42f72d505c0f0/regex-2026.7.19-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:cd3584591ea4429026cdb931b054342c2bcf189b44ff367f8d5c15bc092a2966", size = 857769, upload-time = "2026-07-19T00:19:37.738Z" }, + { url = "https://files.pythonhosted.org/packages/e4/4f/f7e2dad6756b2fe1fe75dd90a628c3b45f249d39f948dd90cd2476325417/regex-2026.7.19-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5cc26a66e212fa5d6c6170c3a40d99d888db3020c6fdab1523250d4341382e44", size = 804546, upload-time = "2026-07-19T00:19:40.229Z" }, + { url = "https://files.pythonhosted.org/packages/2b/d7/01d31d5bdb09bc026fab77f59a371fdf8f9b292e4810546c56182ca70498/regex-2026.7.19-cp314-cp314t-win32.whl", hash = "sha256:2c4e61e2e1be56f63ec3cc618aa9e0de81ef6f43d177205451840022e24f5b78", size = 274526, upload-time = "2026-07-19T00:19:42.398Z" }, + { url = "https://files.pythonhosted.org/packages/52/0e/cea4ce73bc0a8247a0748228ae6669984c7e1f8134b6fa66e59c0572e0ea/regex-2026.7.19-cp314-cp314t-win_amd64.whl", hash = "sha256:c639ea314df70a7b2811e8020448c75af8c9445f5a60f8a4ced81c306a9380c2", size = 283763, upload-time = "2026-07-19T00:19:44.644Z" }, + { url = "https://files.pythonhosted.org/packages/6f/b6/26e41975febae63b7a6e3e02f32cff6cff2e4f10d19c929082f56aebf7c6/regex-2026.7.19-cp314-cp314t-win_arm64.whl", hash = "sha256:9a15e785f244f3e07847b984ce8773fc3da10a9f3c131cc49a4c5b4d672b4547", size = 283451, upload-time = "2026-07-19T00:19:46.639Z" }, +] + [[package]] name = "requests" version = "2.33.0" @@ -2242,6 +2555,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8f/e8/726643a3ea68c727da31570bde48c7a10f1aa60eddd628d94078fec586ff/ruff-0.15.7-py3-none-win_arm64.whl", hash = "sha256:18e8d73f1c3fdf27931497972250340f92e8c861722161a9caeb89a58ead6ed2", size = 11023304, upload-time = "2026-03-19T16:26:51.669Z" }, ] +[[package]] +name = "s3transfer" +version = "0.19.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "botocore" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/76/43/35e4d8aa320bffe8287fe8f65f578fa2d2db0a64212f0e710dce58267854/s3transfer-0.19.2.tar.gz", hash = "sha256:ba0309fd86be3c27dbf78cdd813c13c5e1df16e5874b99d2535ebbdfb9892993", size = 165592, upload-time = "2026-07-22T19:30:44.432Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/bc/e7/5c595c75e9f41a44f30e526eda465ea0b4eec93470e074e4a111b253f13a/s3transfer-0.19.2-py3-none-any.whl", hash = "sha256:d8168eccca828cbb2cd573675333f3bddd254313a9c42494b84c76b539e8ba25", size = 90216, upload-time = "2026-07-22T19:30:43.251Z" }, +] + [[package]] name = "shellingham" version = "1.5.4" @@ -2342,6 +2667,80 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d7/c1/eb8f9debc45d3b7918a32ab756658a0904732f75e555402972246b0b8e71/tenacity-9.1.4-py3-none-any.whl", hash = "sha256:6095a360c919085f28c6527de529e76a06ad89b23659fa881ae0649b867a9d55", size = 28926, upload-time = "2026-02-07T10:45:32.24Z" }, ] +[[package]] +name = "tiktoken" +version = "0.14.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "regex" }, + { name = "requests" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/66/62/167a842aa0429d45f5e797354fd4343a96f6043d67d0513c675c7b8d36e6/tiktoken-0.14.0.tar.gz", hash = "sha256:231dec90efcdccf1b565a1416107736f1e09b1a08fe736ef9d6363e626d03874", size = 38898, upload-time = "2026-08-17T19:49:49.514Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/50/53/ee1453623bf65f019328721ccb6587846d2c5b7b82f34e73ca09101f072e/tiktoken-0.14.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e9c5fe393aab56469f04e432ff851216d3def3436cf5f07e442a240164bf500f", size = 1094198, upload-time = "2026-08-17T19:48:57.955Z" }, + { url = "https://files.pythonhosted.org/packages/ad/5f/6448cfe278c3664ba9ec5b5ac08344341f7dc3d42888476e215a14eda2be/tiktoken-0.14.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:cbe2cc3bba939bcdaf103e03df9d5039d33887080b315624be28ec69059e5f94", size = 1038820, upload-time = "2026-08-17T19:48:59.015Z" }, + { url = "https://files.pythonhosted.org/packages/69/3b/d67eac1bcce9dee3abe23aff5e3ded3116bbebaf67b80a0811c06d3806fc/tiktoken-0.14.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:2157f52e4b4d7ac5ecc7457b3716834706e7ef9a46f5144029bfeb7cf71f4e06", size = 1186175, upload-time = "2026-08-17T19:49:00.068Z" }, + { url = "https://files.pythonhosted.org/packages/37/62/cae690d9783146b0f81f564ada0f8f611de68178c0c9c7e1e969f0516b48/tiktoken-0.14.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:26e60f6a956ee171ab728b37b8439905d7ea1db435c30f9822f291e9861c861d", size = 1203884, upload-time = "2026-08-17T19:49:01.163Z" }, + { url = "https://files.pythonhosted.org/packages/b9/1e/633e30237b94e383cf814145499079f3bb9cdd4aeafc1bc42e01b0f810a6/tiktoken-0.14.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:380873f330b741c4435574f37edb20813d04603ace2d53e0a63560e1fec83010", size = 1250980, upload-time = "2026-08-17T19:49:02.274Z" }, + { url = "https://files.pythonhosted.org/packages/cb/56/4c12f07b812f84206f38d723eb1ebfdd34bad9309b5dbc0bee6bbcff4cbf/tiktoken-0.14.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:3fd7c14b1cb45b486c39fc9b3443bb341f3e2fc7e6f31247f3435a5836651632", size = 1315434, upload-time = "2026-08-17T19:49:03.434Z" }, + { url = "https://files.pythonhosted.org/packages/c9/e0/c65603f0c44811def666d3fbf611bf2af3b5e1ef613e06c19411419830b3/tiktoken-0.14.0-cp313-cp313-win_amd64.whl", hash = "sha256:90a762670c7f968184723769a06ed51f5cf5ce5dcd1e30164f25c72d85c2d1f1", size = 940883, upload-time = "2026-08-17T19:49:04.583Z" }, + { url = "https://files.pythonhosted.org/packages/59/b0/1cf129f4af8fc513931f931023def596b7c4bfc77026513cd9d851da9e88/tiktoken-0.14.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:e067f4cbcc5d036e8aff7fe7a6b530a8f4de2e4616ad9005a24a1879e24e6450", size = 1096273, upload-time = "2026-08-17T19:49:05.807Z" }, + { url = "https://files.pythonhosted.org/packages/62/85/2ae74575e321148484147e10b53c3b1717c59ebaa9edb4fe18b1f5c055f8/tiktoken-0.14.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:f2af4a336ea56d6c14f27741a0e1d8294a35dd0b038bcf990d232ebb54eb994b", size = 1040269, upload-time = "2026-08-17T19:49:06.943Z" }, + { url = "https://files.pythonhosted.org/packages/89/29/92a1120a12e4bcf2d5464350d1a91b68a433d63ce656bb7f806c27aec09c/tiktoken-0.14.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:f702e0aeeb6506e57687e881c59e844ebe8f0a6a097ddafe20e3ab25f387be4e", size = 1186101, upload-time = "2026-08-17T19:49:08.102Z" }, + { url = "https://files.pythonhosted.org/packages/5b/7d/144af98dc5ad68108451a82e2f5a17f80e2663f5115058b8dfd215c1ad02/tiktoken-0.14.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:e3442bbb2f0c588cec876061e37ae67b455b9df9978b003c8fe30e45f2ef5b42", size = 1204457, upload-time = "2026-08-17T19:49:09.28Z" }, + { url = "https://files.pythonhosted.org/packages/e6/1f/be7cb06ab2108f612f3e92e7b76cf391e192db0db37a984616f0cc32aafc/tiktoken-0.14.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:979c1524f753b662b0f3cd261b135afe6659cce33caaa7a5ea00dd1756b3055c", size = 1251716, upload-time = "2026-08-17T19:49:10.509Z" }, + { url = "https://files.pythonhosted.org/packages/ab/6b/81f158d0f90adb826cd704069c2129a046cb784a2a09861009519fc41cf4/tiktoken-0.14.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:2cc19ac87b41c9493c9778ff5847f0c8bbcf5bd0ec6b87ce06c1c802adc8a771", size = 1315432, upload-time = "2026-08-17T19:49:11.844Z" }, + { url = "https://files.pythonhosted.org/packages/fc/ec/f5fa35ec13f07279fdcaf3cc9c04bbb154ea591d23978651f2b672593e8a/tiktoken-0.14.0-cp314-cp314-win_amd64.whl", hash = "sha256:eceeff0c62419bc78d4b6e70a4762a4d25df3ae8f2d5946e3853ce93e7a57098", size = 988046, upload-time = "2026-08-17T19:49:13.282Z" }, + { url = "https://files.pythonhosted.org/packages/68/c9/7756717408d3d0dfea3f046c9466144b28afde39ff69d5808f2475dcd7f5/tiktoken-0.14.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:6eb94895c45f26bb8f5546e5fd8a069efcf6e3f108ea9d5cbe3bf6f7f3983438", size = 1096261, upload-time = "2026-08-17T19:49:14.351Z" }, + { url = "https://files.pythonhosted.org/packages/79/29/46ad8061f57bd9f8b2ea0aa82bf574e0f2aa040b0857a1582adba9957899/tiktoken-0.14.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:86951a971c53979ec857bd8c4a32dc227ab0fd33f6c12a3bd62d3fbf5f0bfcaa", size = 1040183, upload-time = "2026-08-17T19:49:15.707Z" }, + { url = "https://files.pythonhosted.org/packages/5a/7c/3184d17b868456f17b60b1a75f5ec0405618a43aa753336df341d8f11781/tiktoken-0.14.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:e2eca764c53490f8930dbce329e0769f11108d87d908282a80c5c130e26e7037", size = 1186719, upload-time = "2026-08-17T19:49:16.84Z" }, + { url = "https://files.pythonhosted.org/packages/0b/e8/46de4400d5bf859f640feee85bd7e32235f68ddf25db53c63be78e581e3a/tiktoken-0.14.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:26cc4b4840fa0e9f4b72ed489883e12f57e00d1021ca794720e3c29a12f0edef", size = 1204660, upload-time = "2026-08-17T19:49:17.987Z" }, + { url = "https://files.pythonhosted.org/packages/29/ce/af8964c38bc8226dd8950305b7a255fa33345d5572f78af7275a313d28e0/tiktoken-0.14.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:2fc834fbe3f6a0736905c36ab709537e6840dbd63b982dc9e0216ae7d305ba1a", size = 1250932, upload-time = "2026-08-17T19:49:19.28Z" }, + { url = "https://files.pythonhosted.org/packages/1d/4b/323631116fc986d9cc5bbeb2b8223c7c85e61a8bb94ea5ab4951023b149b/tiktoken-0.14.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:ca4db6ff5c5bf600f9b7761a0070ed44dfe5797a76bd432fb978bc480ef40c58", size = 1315190, upload-time = "2026-08-17T19:49:20.467Z" }, + { url = "https://files.pythonhosted.org/packages/18/8b/ba48a73729c9270989b36f37ab2ed5525e52690d715097c9fa791aaa5d05/tiktoken-0.14.0-cp314-cp314t-win_amd64.whl", hash = "sha256:7aab286a020660a039097912a088236b985d18a3090d73f136c4413d29d37ca0", size = 987717, upload-time = "2026-08-17T19:49:21.704Z" }, + { url = "https://files.pythonhosted.org/packages/1d/10/b73b7e319179e0f60b32475f783b044f9cece872c53b6662664e9084b0d0/tiktoken-0.14.0-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:14b47e3674f2624803a8acc8fb367b7e24fc53055f9df3296482fe9a3a34a232", size = 1096280, upload-time = "2026-08-17T19:49:22.779Z" }, + { url = "https://files.pythonhosted.org/packages/c2/6b/09999a9bf1d559670d1680e8f8e419ac0e2c5f6aac82e9bfdf70f260b30a/tiktoken-0.14.0-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:19d643d701fdaa70e5b9c7f8f96abcaffe77ca5e482a3a1a7dde46feb4284695", size = 1040433, upload-time = "2026-08-17T19:49:23.998Z" }, + { url = "https://files.pythonhosted.org/packages/cd/7b/8537be0836f3df99b2a636b44399bfa43cd757f2b8b4097dacb794cf24a7/tiktoken-0.14.0-cp315-cp315-manylinux_2_28_aarch64.whl", hash = "sha256:e4ddf863b59347deaa92302dcd90e5eb003cdc9be06ec2b692c38d1bdd9efd49", size = 1186989, upload-time = "2026-08-17T19:49:25.021Z" }, + { url = "https://files.pythonhosted.org/packages/7c/9d/f9c56d7a943a4468abf9ef37661bb9b8e0cd3aa8aa87368c7146cc3f3222/tiktoken-0.14.0-cp315-cp315-manylinux_2_28_x86_64.whl", hash = "sha256:60c47ca69ddda0dea8256fffd12e1b86f4b59734a20e4a70c61f63cc5f021df4", size = 1204615, upload-time = "2026-08-17T19:49:26.37Z" }, + { url = "https://files.pythonhosted.org/packages/4b/d2/98a38579db25c4a8a84e31dd95d9072ec5f21f7e70de591da0412e29b25b/tiktoken-0.14.0-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:728303a072163130c5b477b1f20d6211895569c1d5302c24ffc93a3009160871", size = 1251828, upload-time = "2026-08-17T19:49:27.423Z" }, + { url = "https://files.pythonhosted.org/packages/0c/83/467be424746c039c5493c0f4102feab16b9b48eb6f5c089b2a2438e3cde2/tiktoken-0.14.0-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:3c5349c9f916283bba32bec8af69b763e4faa304dc004d0eaaea66a3cf004c1f", size = 1316260, upload-time = "2026-08-17T19:49:29.101Z" }, + { url = "https://files.pythonhosted.org/packages/02/ee/ddf46ca78e371f5890e96b6e7d089a85b3536432be219851eb0481786ca8/tiktoken-0.14.0-cp315-cp315-win_amd64.whl", hash = "sha256:1b6e4adcfd285c44502aed51df98aaaca4f0fea028165dbf8a9e857b9f98d8ea", size = 988230, upload-time = "2026-08-17T19:49:30.246Z" }, + { url = "https://files.pythonhosted.org/packages/2a/00/5162e90c851a28da18ed382d34898b79a8022548e5619a64e14c03ce7c3d/tiktoken-0.14.0-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:11d8211b290855d2721334ff17dd9b3a17bfb26872be01f25d73612ef7ece890", size = 1096186, upload-time = "2026-08-17T19:49:31.656Z" }, + { url = "https://files.pythonhosted.org/packages/65/97/a5a7bfccf25b1bb65e82bae8edff11ac3c9c041c374b7b4a823d60c38133/tiktoken-0.14.0-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:d0781223705199b289faa59601bb9c2441712d4c600dd13c43d8fd6a33d22cd5", size = 1039947, upload-time = "2026-08-17T19:49:32.848Z" }, + { url = "https://files.pythonhosted.org/packages/fb/ba/ef427fc638f1439181c5e12dd26b70e881861f89c007aa7e5b36300f8342/tiktoken-0.14.0-cp315-cp315t-manylinux_2_28_aarch64.whl", hash = "sha256:2ea70afba6b9eddbf22c165142e5f0a2ad7aa36a452873c48b57bb2aeb8492ae", size = 1186997, upload-time = "2026-08-17T19:49:34.121Z" }, + { url = "https://files.pythonhosted.org/packages/3e/88/2f3f85a968cdc514152129af0a060ebcccb067005a2f29b0d5ef3c838514/tiktoken-0.14.0-cp315-cp315t-manylinux_2_28_x86_64.whl", hash = "sha256:78571efc311c30b73f31eb949a921d6dac39a5d9dc42d1cfa8f8db157b3447b1", size = 1205211, upload-time = "2026-08-17T19:49:35.284Z" }, + { url = "https://files.pythonhosted.org/packages/4e/f6/80760e98a08e6649d2d68afb6035af713121dfb615acce8c4f73810ec438/tiktoken-0.14.0-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:86f66c85e796f5d05d5c4a60ec1d40cbfebc47a32464053528c797163fa9ab89", size = 1251479, upload-time = "2026-08-17T19:49:36.419Z" }, + { url = "https://files.pythonhosted.org/packages/c5/84/50966fb6918a0fb9b32721277e5342bf729a2d74350074d662fbedf9772e/tiktoken-0.14.0-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:149d97453c4c98c04b081d64a85e635921269b532710d6faf81e9e82b790e7d3", size = 1316673, upload-time = "2026-08-17T19:49:37.756Z" }, + { url = "https://files.pythonhosted.org/packages/35/5e/9b01afd037bfa22a0033963fa091e0f75b6fb15cd85bffb42ff86e697323/tiktoken-0.14.0-cp315-cp315t-win_amd64.whl", hash = "sha256:561e7580f84a79859af1ef6f676968e9030fcc3fe195700b15235bca64f009c9", size = 987929, upload-time = "2026-08-17T19:49:38.947Z" }, +] + +[[package]] +name = "tokenizers" +version = "0.23.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "huggingface-hub" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c1/60/21f715d9faba5f5407ff759472ade058ec4a507ad62bcea47cb847239a73/tokenizers-0.23.1.tar.gz", hash = "sha256:1feeeadf865a7915adc25445dea30e9933e593c31bb96c277cee36de227c8bfa", size = 365748, upload-time = "2026-04-27T14:43:25.606Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/87/39/b87a87d5bb9470610b80a2d31df42fcffeaf35118b8b97952b2aff598cc7/tokenizers-0.23.1-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:e03d6ffcbe0d56ee9c1ccd070e70a13fa750727c0277e138152acbc0252c2224", size = 3146732, upload-time = "2026-04-27T14:43:15.427Z" }, + { url = "https://files.pythonhosted.org/packages/e2/6a/068ed9f6e444c9d7e9d55ce134181325700f3d7f30410721bdc8f848d727/tokenizers-0.23.1-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:e0948bbb1ac1d7cdfc9fb6d62c596e3b7550036ad60ecd654a66ad273326324e", size = 3054954, upload-time = "2026-04-27T14:43:13.745Z" }, + { url = "https://files.pythonhosted.org/packages/6c/36/e006edf031154cba92b8416057d92c3abe3635e4c4b0aa0b5b9bb39dde70/tokenizers-0.23.1-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1bf13402aff9bc533c89cb849ec3b412dc3fbeacc9744840e423d7bf3f7dc0e3", size = 3374081, upload-time = "2026-04-27T14:43:01.241Z" }, + { url = "https://files.pythonhosted.org/packages/a2/ef/7735d226f9c7f874a6bee5e3f27fb25ecabdf207d37b8cf45286d0795893/tokenizers-0.23.1-cp310-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f836ca703b89ae07919a309f9651f7a88fd5a33d5f718ba5ad0870ec0256bad6", size = 3247641, upload-time = "2026-04-27T14:43:03.856Z" }, + { url = "https://files.pythonhosted.org/packages/b9/d9/24827036f6e21297bfffda0768e58eb6096a4f411e932964a01707857931/tokenizers-0.23.1-cp310-abi3-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ae848657742035523fdf261773630cb819a26995fcd3d9ecae0c1daf6e5a4959", size = 3585624, upload-time = "2026-04-27T14:43:10.664Z" }, + { url = "https://files.pythonhosted.org/packages/0c/9a/22f3582b3a4f49358293a5206e25317621ee4526bfe9cdaa0f07a12e770e/tokenizers-0.23.1-cp310-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:53b09e85775d5187941e7bab30e941b4134ab4a7dd8c68e783d231fb7ca27c51", size = 3844062, upload-time = "2026-04-27T14:43:05.643Z" }, + { url = "https://files.pythonhosted.org/packages/7e/65/b8f8814eef95800f20721384136d9a1d22241d50b2874357cb70542c392f/tokenizers-0.23.1-cp310-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ea5a0ce170074329faaa8ea3f6400ecde604b6678192688533af80980daae71a", size = 3460098, upload-time = "2026-04-27T14:43:08.854Z" }, + { url = "https://files.pythonhosted.org/packages/0d/d5/1353e5f677ec27c2494fb6a6725e82d56c985f53e90ec511369e7e4f02c6/tokenizers-0.23.1-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5075b405006415ea148a992d093699c66eb01952bf59f4d5727089a98bda45a4", size = 3346235, upload-time = "2026-04-27T14:43:12.377Z" }, + { url = "https://files.pythonhosted.org/packages/71/89/39b6b8fc073fb6d413d0147aa333dc7eff7be65639ac9d19930a0b21bf33/tokenizers-0.23.1-cp310-abi3-manylinux_2_31_riscv64.whl", hash = "sha256:56f3a77de629917652f876294dc9fe6bad4a0c43bc229dc72e59bb23a0f4729a", size = 3426398, upload-time = "2026-04-27T14:43:07.264Z" }, + { url = "https://files.pythonhosted.org/packages/0f/80/127c854da64827e5b79264ce524993a90dddcb320e5cd42412c5c02f9e8a/tokenizers-0.23.1-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:9d10a6d957ef01896dc274e890eee27d41bd0e74ef31e60616f0fc311345184e", size = 9823279, upload-time = "2026-04-27T14:43:17.222Z" }, + { url = "https://files.pythonhosted.org/packages/fe/ba/44c2502feb1a058f096ddfb4e0996ef3225a01a388e1a9b094e91689fe93/tokenizers-0.23.1-cp310-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:1974288a609c343774f1b897c8b482c791ab17b75ab5c8c2b1737565c1d82288", size = 9644986, upload-time = "2026-04-27T14:43:19.45Z" }, + { url = "https://files.pythonhosted.org/packages/9e/c1/464019a9fb059870bfe4eebb4ba12208f3042035e258bf5e782906bd3847/tokenizers-0.23.1-cp310-abi3-musllinux_1_2_i686.whl", hash = "sha256:120468fb4c24faf0543c835a4fabafa4deb3f20a035c9b6e83d0b553a97615d4", size = 9976181, upload-time = "2026-04-27T14:43:21.463Z" }, + { url = "https://files.pythonhosted.org/packages/79/94/3ac1432bda31626071e9b6a12709b97ae05131c804b94c8f3ac622c5da32/tokenizers-0.23.1-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:e3d8f40ea6268047de7046906326abed5134f27d4e8447b23763afe5808c8a96", size = 10113853, upload-time = "2026-04-27T14:43:23.617Z" }, + { url = "https://files.pythonhosted.org/packages/6a/dd/631b21433c771b1382535326f0eca80b9c9cee2e64961dd993bc9ac4669e/tokenizers-0.23.1-cp310-abi3-win32.whl", hash = "sha256:93120a930b919416da7cd10a2f606ac9919cc69cacae7980fa2140e277660948", size = 2536263, upload-time = "2026-04-27T14:43:29.888Z" }, + { url = "https://files.pythonhosted.org/packages/97/c9/2553f72aaf65a2797d4229e37fa7fbe38ffbf3e32912d31bdd78b3323e59/tokenizers-0.23.1-cp310-abi3-win_amd64.whl", hash = "sha256:e7bfaf995c1bdbbd21d13539decb6650967013759318627d85daeb7881af16b7", size = 2798223, upload-time = "2026-04-27T14:43:28.51Z" }, + { url = "https://files.pythonhosted.org/packages/cd/2b/2be299bab55fc595e3d38567edb1a87f86e594842968fa9515a07bdcf422/tokenizers-0.23.1-cp310-abi3-win_arm64.whl", hash = "sha256:a26197957d8e4425dfba746315f3c425ea00cfa8367c5fbc4ec73447893dcea9", size = 2664127, upload-time = "2026-04-27T14:43:26.949Z" }, +] + [[package]] name = "tomli" version = "2.4.0" From de4c476f371cef4786faeff5b4ddb48aef4aa15d Mon Sep 17 00:00:00 2001 From: Akshaya Shanbhogue Date: Wed, 26 Aug 2026 12:38:05 -0700 Subject: [PATCH 4/9] feat(litellm-judge): support arbitrary litellm kwargs via params/auth MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit litellm.acompletion() takes dozens of provider-specific kwargs (aws_access_key_id, vertex_project, api_version, ...) that LiteLLMRoute had no way to express, and secrets couldn't be put in task YAML anyway. Extend checker_context.api_route with two new keys, litellm-route only: - `params`: arbitrary passthrough dict merged straight into the litellm.acompletion() call — no allowlist to maintain, litellm validates param names itself. - `auth`: maps a kwarg name to the ENV VAR NAME (never the secret value) to resolve it from right before the call — so an arbitrary provider's auth shape (IAM keys, an Azure AD token, ...) is representable without a dedicated field per provider and without a secret ever landing in YAML. The plain LITELLM_AUTH_TOKEN requirement is relaxed to "LITELLM_AUTH_TOKEN OR a non-empty `auth` override", since some providers (e.g. Bedrock via IAM) have no `api_key` concept at all. `validate_checker_context_shape` rejects `params`/`auth` on any route other than `litellm` at task-load time, and type-checks `auth`'s values are env-var-name strings. Co-Authored-By: Claude Sonnet 5 --- docs/TASK_DEFINITION_GUIDE.md | 13 +++ src/coder_eval/evaluation/judge_litellm.py | 42 +++++++++- src/coder_eval/models/routing.py | 59 ++++++++++--- src/coder_eval/models/tasks.py | 28 ++++++- src/coder_eval/orchestrator.py | 17 +++- tests/test_judge_litellm.py | 75 ++++++++++++++++- tests/test_litellm_route.py | 98 +++++++++++++++++++++- 7 files changed, 309 insertions(+), 23 deletions(-) diff --git a/docs/TASK_DEFINITION_GUIDE.md b/docs/TASK_DEFINITION_GUIDE.md index db09401f..c7b7b4fa 100644 --- a/docs/TASK_DEFINITION_GUIDE.md +++ b/docs/TASK_DEFINITION_GUIDE.md @@ -1314,6 +1314,19 @@ checker_context: - `route` selects which backend the WHOLE evaluation side calls (`llm_judge`, `agent_judge`, and the simulator all share one resolved eval route per run — this isn't per-criterion), **decoupled from the agent's own route** (a Claude agent can be graded by a differently-backed judge, or vice versa). This is a backend *name* (`direct` / `bedrock` / `litellm`), not a route object: credentials are never read from the task, always from the matching environment variables (`ANTHROPIC_API_KEY` for `direct`, `AWS_BEARER_TOKEN_BEDROCK`/`AWS_REGION` for `bedrock`, `LITELLM_BASE_URL`/`LITELLM_AUTH_TOKEN` for `litellm`). An unconfigured or unknown backend name raises at dispatch rather than silently falling back. **`route: litellm` dispatches `llm_judge` through the `litellm` library** (the `coder-eval[litellm]` extra, `litellm.acompletion`) rather than assuming one wire protocol — a gateway-routed judge model (e.g. an Azure AI `/openai/v1` deployment) rarely speaks Anthropic Messages, so this lets `model` carry its own provider hint (e.g. `azure_ai/gpt-5.6-luna`) and get that provider's actual request/response shape handled by the library. - `model` overrides the model that resolved route uses for **`llm_judge` only** — when the criterion itself leaves `model:` unset (precedence: an explicit per-criterion `model:` always wins; below that, `checker_context.api_route.model`; below that, the built-in `DEFAULT_JUDGE_MODEL`). This floor is deliberate and never the agent's own model — an unpinned judge must grade identically regardless of which model the agent under test is using, so `resolve_evaluation_route` never lets the agent's env-configured model (e.g. `BEDROCK_MODEL`) leak into `route.model` on its own; `route.model` is set only when this override was actually given. This works because every `ApiRoute` (`DirectRoute`/`BedrockRoute`/`LiteLLMRoute`) carries its own `model` field; the orchestrator bakes the override into the resolved route's `model` before any criterion runs, so `llm_judge` just reads `context.route.model` — it never reads `checker_context` directly. **`agent_judge` and the simulator do not honor this override** — `agent_judge`'s sub-agent model comes from the criterion's own `agent:` block (defaulted to a fixed judge model), and the simulator's model is pinned by `SimulationConfig.model` (see [Simulation](#simulation) below) — both independent of `checker_context.api_route.model` by design, for the same "measuring instrument stays fixed" reason. +- `params`/`auth` (**`route: litellm` only**) extend the judge call to any of the dozens of provider-specific kwargs `litellm.acompletion` accepts (`aws_access_key_id`, `vertex_project`, `api_version`, ...), which have no dedicated field on `LiteLLMRoute`: + ```yaml + checker_context: + api_route: + route: litellm + model: azure_ai/gpt-5.6-luna + params: # arbitrary passthrough kwargs to litellm.acompletion + api_version: "2024-05-01" + auth: # param name -> ENV VAR NAME (never the secret itself) + api_key: LITELLM_AUTH_TOKEN + aws_access_key_id: AWS_ACCESS_KEY_ID + ``` + `params` is merged straight into the `litellm.acompletion(**kwargs)` call — litellm validates the param names itself, so there's no allowlist to keep in sync here. `auth` maps a kwarg name to the *name* of an environment variable; the value is resolved right before the call, so no secret is ever written into task/experiment YAML — this is how an arbitrary provider's auth shape (IAM keys, an Azure AD token, a service-account path, ...) is representable without a dedicated field per provider. At least one of `LITELLM_AUTH_TOKEN` or an `auth` entry must be configured. Rejected at task-load time if given without `route: litellm`. `checker_context` merges shallow-per-namespace across `default_experiment.defaults.checker_context` → `experiment.defaults.checker_context` → `task.checker_context` → `variant.checker_context` (same 4-layer precedence as `agent`/`simulation`). So a judge-model A/B, or a judge-backend A/B, is a variant-level config change, not an edit to every task YAML. diff --git a/src/coder_eval/evaluation/judge_litellm.py b/src/coder_eval/evaluation/judge_litellm.py index 7c42db0b..a9f9e39d 100644 --- a/src/coder_eval/evaluation/judge_litellm.py +++ b/src/coder_eval/evaluation/judge_litellm.py @@ -27,6 +27,7 @@ from __future__ import annotations import logging +import os from typing import Any from coder_eval.errors import JudgeInfrastructureError @@ -66,8 +67,20 @@ async def invoke_litellm_judge_async( # (including AssertionError) and downgrades it to a scored 0.0 — the # opposite of the intended "internal-contract violation escalates to # FinalStatus.ERROR" behavior. - if not auth_token: - raise JudgeInfrastructureError("checker_context route 'litellm' requires LITELLM_AUTH_TOKEN to be set") + # + # `route.auth` can supply ITS OWN auth entirely — a provider that doesn't + # use `api_key` at all (e.g. Bedrock's aws_access_key_id/ + # aws_secret_access_key) has no reason to also need LITELLM_AUTH_TOKEN — + # so the plain LITELLM_AUTH_TOKEN requirement only applies when `auth` is + # empty too. `_resolve_backend_route` already enforces the same relaxed + # check at resolution time; this is the runtime backstop for a route + # built any other way. + if not auth_token and not route.auth: + msg = ( + "checker_context route 'litellm' requires LITELLM_AUTH_TOKEN to be set, " + "or an explicit `auth: {api_key: ENV_VAR}` override" + ) + raise JudgeInfrastructureError(msg) try: from litellm.exceptions import APIError, BadRequestError @@ -88,11 +101,26 @@ async def invoke_litellm_judge_async( }, } + def _resolve_auth() -> dict[str, str]: + """Resolve ``route.auth`` (kwarg name -> ENV VAR NAME) into kwarg name -> + secret value, right before the call so no resolved secret is ever stored + on the route object itself (only the env var *name* is).""" + if not route.auth: + return {} + resolved: dict[str, str] = {} + for param_name, env_var in route.auth.items(): + value = os.environ.get(env_var) + if not value: + raise JudgeInfrastructureError( + f"checker_context.api_route.auth[{param_name!r}] references env var {env_var!r}, which is not set" + ) + resolved[param_name] = value + return resolved + def _call_kwargs(*, include_temperature: bool) -> dict[str, Any]: kwargs: dict[str, Any] = { "model": model, "api_base": route.base_url, - "api_key": auth_token, "messages": [ {"role": "system", "content": system}, {"role": "user", "content": user}, @@ -108,8 +136,16 @@ def _call_kwargs(*, include_temperature: bool) -> dict[str, Any]: # handled below by retrying once without `temperature`. "drop_params": True, } + if auth_token: + kwargs["api_key"] = auth_token if include_temperature: kwargs["temperature"] = temperature + # `params` is arbitrary passthrough (e.g. aws_region_name, api_version, ...); + # `auth` (resolved secrets) applies LAST so it always wins over both the + # LITELLM_AUTH_TOKEN default above and anything in `params`. + if route.params: + kwargs.update(route.params) + kwargs.update(_resolve_auth()) return kwargs def _rejects_temperature(e: BadRequestError) -> bool: diff --git a/src/coder_eval/models/routing.py b/src/coder_eval/models/routing.py index c8c4ad3a..2b8f8bc9 100644 --- a/src/coder_eval/models/routing.py +++ b/src/coder_eval/models/routing.py @@ -3,7 +3,7 @@ from __future__ import annotations from dataclasses import dataclass, replace -from typing import TYPE_CHECKING, Literal +from typing import TYPE_CHECKING, Any, Literal from coder_eval.models.enums import ApiBackend from coder_eval.models.judge_defaults import DEFAULT_JUDGE_MODEL @@ -136,12 +136,25 @@ class LiteLLMRoute: a live rejection before the judge retries without it. Defaulting to omitted skips that wasted round trip for the common case; set ``True`` for a gateway/model known to accept it. + + ``params``/``auth`` (checker side only, from ``checker_context.api_route.{params,auth}``): + ``litellm.acompletion`` takes dozens of provider-specific kwargs (``aws_access_key_id``, + ``vertex_project``, ``api_version``, ...) that this route has no dedicated field for. + ``params`` is passed through verbatim as extra kwargs. ``auth`` maps a kwarg name to + the ENV VAR NAME to resolve it from at call time — e.g. ``{aws_access_key_id: + AWS_ACCESS_KEY_ID}`` — so an arbitrary provider's auth shape is representable + without a secret ever landing in the task YAML. Both are ``None`` unless a task + author set them; ``auth``'s values are env var *names*, never secrets, so it is + safe to record verbatim in ``environment_info`` (unlike ``params``, which a task + author could — but shouldn't — put a raw secret into). """ base_url: str model: str | None = None small_model: str | None = None include_temperature: bool = False + params: dict[str, Any] | None = None + auth: dict[str, str] | None = None ApiRoute = DirectRoute | BedrockRoute | LiteLLMRoute @@ -214,7 +227,14 @@ def resolve_route(settings: Settings) -> ApiRoute: ) -def _resolve_backend_route(settings: Settings, backend: ApiBackend, *, model_override: str | None = None) -> ApiRoute: +def _resolve_backend_route( + settings: Settings, + backend: ApiBackend, + *, + model_override: str | None = None, + params_override: dict[str, Any] | None = None, + auth_override: dict[str, str] | None = None, +) -> ApiRoute: """Build the ``ApiRoute`` for an EXPLICITLY-requested backend, from the same env-sourced ``Settings`` fields ``resolve_route`` reads for the agent — credentials always come from the environment, never from a task/variant. @@ -226,7 +246,10 @@ def _resolve_backend_route(settings: Settings, backend: ApiBackend, *, model_ove fail loudly, not degrade to a backend the task author didn't ask for. ``model_override`` (``checker_context.api_route.model``) wins over the - backend's own env-configured default model when set. + backend's own env-configured default model when set. ``params_override``/ + ``auth_override`` (``checker_context.api_route.{params,auth}``) only ever + land on a ``LiteLLMRoute`` — ``validate_checker_context_shape`` rejects them + on any other backend at load time, so they're ignored here otherwise. """ match backend: case ApiBackend.BEDROCK: @@ -242,16 +265,22 @@ def _resolve_backend_route(settings: Settings, backend: ApiBackend, *, model_ove raise ValueError("checker_context route 'direct' requires ANTHROPIC_API_KEY to be set") return DirectRoute(judge_transport="anthropic", model=model_override) case ApiBackend.LITELLM: - if not settings.litellm_base_url or not settings.litellm_auth_token: - raise ValueError( - "checker_context route 'litellm' requires LITELLM_BASE_URL and LITELLM_AUTH_TOKEN to be set" + if not settings.litellm_base_url: + raise ValueError("checker_context route 'litellm' requires LITELLM_BASE_URL to be set") + if not settings.litellm_auth_token and not auth_override: + msg = ( + "checker_context route 'litellm' requires LITELLM_AUTH_TOKEN to be set, " + "or an explicit `checker_context.api_route.auth` override" ) + raise ValueError(msg) judge_model = model_override or settings.litellm_model small_model = settings.litellm_small_model or judge_model return LiteLLMRoute( base_url=settings.litellm_base_url, model=judge_model, small_model=small_model, + params=params_override, + auth=auth_override, ) case _: # ApiBackend covers exactly BEDROCK/DIRECT/LITELLM above; this arm is @@ -266,17 +295,21 @@ def resolve_evaluation_route( *, backend_override: str | None = None, model_override: str | None = None, + params_override: dict[str, Any] | None = None, + auth_override: dict[str, str] | None = None, ) -> ApiRoute: """Resolve the route used by the *evaluation* side — the ``llm_judge`` / ``agent_judge`` criteria and the simulated user — which must stay on a constant Claude backend regardless of the agent under test, so grading and simulation stay comparable across models. - Both overrides come from the reserved ``checker_context.api_route`` namespace + All overrides come from the reserved ``checker_context.api_route`` namespace (see ``TaskDefinition.checker_context``) — ``route`` (``backend_override``) picks the backend, ``model`` (``model_override``) picks the model on - whichever route is resolved. Criteria never read either directly; they only - ever see the resulting ``CheckContext.route.model``. + whichever route is resolved, and ``params``/``auth`` (``params_override``/ + ``auth_override``) only ever apply when ``backend_override`` resolves to + ``litellm`` (see ``_resolve_backend_route``). Criteria never read any of + these directly; they only ever see the resulting ``CheckContext.route``. - ``backend_override`` set: build that backend's route from env, regardless of ``agent_route`` — an explicit task/variant choice always wins. Raises @@ -305,7 +338,13 @@ def resolve_evaluation_route( except ValueError as e: valid = ", ".join(b.value for b in ApiBackend) raise ValueError(f"checker_context route {backend_override!r} is not a known backend ({valid})") from e - return _resolve_backend_route(settings, backend, model_override=model_override) + return _resolve_backend_route( + settings, + backend, + model_override=model_override, + params_override=params_override, + auth_override=auth_override, + ) if isinstance(agent_route, BedrockRoute | DirectRoute): if isinstance(agent_route, BedrockRoute) and model_override: # Bedrock model ids must be region-qualified — reusing the agent's route diff --git a/src/coder_eval/models/tasks.py b/src/coder_eval/models/tasks.py index 50b32b2d..dd0b5361 100644 --- a/src/coder_eval/models/tasks.py +++ b/src/coder_eval/models/tasks.py @@ -101,7 +101,7 @@ def validate_checker_context_shape(value: dict[str, dict[str, Any]]) -> None: raise ValueError(msg) api_route = value.get("api_route") if api_route is not None: - known_keys = {"route", "model"} + known_keys = {"route", "model", "params", "auth"} unknown_keys = set(api_route) - known_keys if unknown_keys: msg = ( @@ -116,6 +116,23 @@ def validate_checker_context_shape(value: dict[str, dict[str, Any]]) -> None: valid = sorted(b.value for b in ApiBackend) msg = f"checker_context.api_route.route {route!r} is not a known backend ({valid})" raise ValueError(msg) from e + # `params`/`auth` only ever reach the litellm judge transport (see + # invoke_litellm_judge_async) — on any other backend they'd be silently + # dropped, which is worse than a load-time error. + params = api_route.get("params") + auth = api_route.get("auth") + if (params is not None or auth is not None) and route != "litellm": + msg = "checker_context.api_route.params/auth require route: litellm" + raise ValueError(msg) + if params is not None and not isinstance(params, dict): + raise ValueError(f"checker_context.api_route.params must be a mapping, got {type(params).__name__}") + if auth is not None: + if not isinstance(auth, dict): + raise ValueError(f"checker_context.api_route.auth must be a mapping, got {type(auth).__name__}") + bad = {k: v for k, v in auth.items() if not isinstance(k, str) or not isinstance(v, str)} + if bad: + msg = f"checker_context.api_route.auth must map param name -> ENV VAR NAME (both strings); got {bad!r}" + raise ValueError(msg) class SimulationConfig(BaseModel): @@ -488,8 +505,13 @@ class TaskDefinition(BaseModel): # noqa: CE009 -- soft-launch: see _warn_on_unk "route uses. Both are consumed by the orchestrator (`resolve_evaluation_route`) BEFORE " "`CheckContext` is built and baked into the resolved route's own `model` field — no criterion " "ever reads `checker_context` directly, only `CheckContext.route.model`. Credentials are " - "always resolved from environment variables, never from this field. Merged shallow-per-" - "namespace across default -> experiment-defaults -> task -> variant." + "always resolved from environment variables, never from this field. `params`/`auth` (only " + "with `route: litellm`) extend this to the judge's underlying `litellm.acompletion` call: " + "`params` is an arbitrary passthrough dict of extra kwargs (e.g. `{aws_region_name: " + "eu-north-1}`), and `auth` maps a kwarg name to the ENV VAR NAME (not the value!) to resolve " + "it from at call time (e.g. `{aws_access_key_id: AWS_ACCESS_KEY_ID}`) — so an arbitrary " + "provider's auth shape is representable without ever putting a secret in the task YAML. " + "Merged shallow-per-namespace across default -> experiment-defaults -> task -> variant." ), ) run_limits: RunLimits | None = Field( diff --git a/src/coder_eval/orchestrator.py b/src/coder_eval/orchestrator.py index 8caee878..a5fa75c3 100644 --- a/src/coder_eval/orchestrator.py +++ b/src/coder_eval/orchestrator.py @@ -151,12 +151,14 @@ async def _pump_stream( class EvalRouteOverrides(NamedTuple): - """``checker_context.api_route``'s ``(backend, model)`` pair. Named fields - (rather than a bare tuple) so a future transposition at a call site is a - typo'd attribute, not a silent positional swap of backend vs. model.""" + """``checker_context.api_route``'s override fields. Named fields (rather than + a bare tuple) so a future transposition at a call site is a typo'd attribute, + not a silent positional swap.""" backend: str | None model: str | None + params: dict[str, Any] | None + auth: dict[str, str] | None def _format_routing(route: ApiRoute, effective_model: str | None = None) -> str: @@ -1477,6 +1479,8 @@ def _eval_route_overrides(self) -> EvalRouteOverrides: return EvalRouteOverrides( backend=str(backend) if backend is not None else None, model=str(model) if model is not None else None, + params=api_route.get("params"), + auth=api_route.get("auth"), ) def _resolve_routes(self) -> None: @@ -1489,7 +1493,12 @@ def _resolve_routes(self) -> None: self.route = resolve_route(settings) overrides = self._eval_route_overrides() self.eval_route = resolve_evaluation_route( - settings, self.route, backend_override=overrides.backend, model_override=overrides.model + settings, + self.route, + backend_override=overrides.backend, + model_override=overrides.model, + params_override=overrides.params, + auth_override=overrides.auth, ) logger.info("API routing: %s", _format_routing(self.route, self.task.agent.model if self.task.agent else None)) self.success_checker = SuccessChecker(self.sandbox, route=self.eval_route) diff --git a/tests/test_judge_litellm.py b/tests/test_judge_litellm.py index 5be0bd1a..b0311e89 100644 --- a/tests/test_judge_litellm.py +++ b/tests/test_judge_litellm.py @@ -51,8 +51,19 @@ def _make_response(*, score: float = 0.5, rationale: str = "ok") -> MagicMock: return response -def _route(*, include_temperature: bool = False) -> LiteLLMRoute: - return LiteLLMRoute(base_url="http://gateway:4000", model="gpt-5.6-luna", include_temperature=include_temperature) +def _route( + *, + include_temperature: bool = False, + params: dict[str, Any] | None = None, + auth: dict[str, str] | None = None, +) -> LiteLLMRoute: + return LiteLLMRoute( + base_url="http://gateway:4000", + model="gpt-5.6-luna", + include_temperature=include_temperature, + params=params, + auth=auth, + ) async def _invoke(**overrides): @@ -205,3 +216,63 @@ async def test_invoke_litellm_judge_escalates_on_signature_break() -> None: pytest.raises(JudgeInfrastructureError, match="LiteLLM judge call failed"), ): await _invoke() + + +async def test_invoke_litellm_judge_passes_through_params() -> None: + """`route.params` is arbitrary passthrough merged straight into the + litellm.acompletion() kwargs — e.g. aws_region_name, api_version, ...""" + acompletion = AsyncMock(return_value=_make_response()) + with patch("litellm.acompletion", new=acompletion): + await _invoke(route=_route(params={"aws_region_name": "eu-north-1", "api_version": "2024-05-01"})) + kwargs = acompletion.call_args.kwargs + assert kwargs["aws_region_name"] == "eu-north-1" + assert kwargs["api_version"] == "2024-05-01" + + +async def test_invoke_litellm_judge_resolves_auth_from_env(monkeypatch: pytest.MonkeyPatch) -> None: + """`route.auth` maps a kwarg name -> ENV VAR NAME; the secret VALUE is only + ever resolved at call time, never stored on the route.""" + monkeypatch.setenv("MY_AWS_ACCESS_KEY_ID", "AKIA-fake") + monkeypatch.setenv("MY_AWS_SECRET_ACCESS_KEY", "secret-fake") + acompletion = AsyncMock(return_value=_make_response()) + route = _route( + auth={"aws_access_key_id": "MY_AWS_ACCESS_KEY_ID", "aws_secret_access_key": "MY_AWS_SECRET_ACCESS_KEY"} + ) + with patch("litellm.acompletion", new=acompletion): + await _invoke(route=route, auth_token=None) + kwargs = acompletion.call_args.kwargs + assert kwargs["aws_access_key_id"] == "AKIA-fake" + assert kwargs["aws_secret_access_key"] == "secret-fake" + + +async def test_invoke_litellm_judge_auth_api_key_overrides_auth_token(monkeypatch: pytest.MonkeyPatch) -> None: + """An explicit `auth: {api_key: ENV_VAR}` wins over the LITELLM_AUTH_TOKEN- + sourced `auth_token` default, and satisfies the "some api_key is configured" + requirement even when `auth_token` itself is None.""" + monkeypatch.setenv("OTHER_KEY", "sk-other") + acompletion = AsyncMock(return_value=_make_response()) + route = _route(auth={"api_key": "OTHER_KEY"}) + with patch("litellm.acompletion", new=acompletion): + await _invoke(route=route, auth_token=None) + assert acompletion.call_args.kwargs["api_key"] == "sk-other" + + +async def test_invoke_litellm_judge_raises_on_missing_env_var_for_auth() -> None: + route = _route(auth={"aws_access_key_id": "TOTALLY_UNSET_ENV_VAR_XYZ"}) + with pytest.raises(JudgeInfrastructureError, match="TOTALLY_UNSET_ENV_VAR_XYZ"): + await _invoke(route=route) + + +async def test_invoke_litellm_judge_params_do_not_shadow_required_kwargs() -> None: + """auth resolves AFTER params, so an auth-mapped key always wins over the + same key set via params (belt-and-braces; auth is the documented secret + channel).""" + acompletion = AsyncMock(return_value=_make_response()) + route = _route(params={"api_key": "leaked-from-params"}) + with patch("litellm.acompletion", new=acompletion): + await _invoke(route=route, auth_token="sk-master") + # No `auth` override -> the LITELLM_AUTH_TOKEN default is applied first, + # then params overwrites it (params has no special protection over the + # base kwargs) -- documents the actual precedence rather than asserting a + # stronger guarantee than the implementation provides. + assert acompletion.call_args.kwargs["api_key"] == "leaked-from-params" diff --git a/tests/test_litellm_route.py b/tests/test_litellm_route.py index 6e323625..22e8ee66 100644 --- a/tests/test_litellm_route.py +++ b/tests/test_litellm_route.py @@ -227,9 +227,45 @@ def test_override_to_litellm_builds_route_from_env(self, monkeypatch): def test_override_to_litellm_without_creds_raises(self, monkeypatch): agent_route = BedrockRoute(region="eu-north-1") settings = self._isolated_settings(monkeypatch, api_backend=ApiBackend.BEDROCK) - with pytest.raises(ValueError, match="requires LITELLM_BASE_URL and LITELLM_AUTH_TOKEN"): + with pytest.raises(ValueError, match="requires LITELLM_BASE_URL"): resolve_evaluation_route(settings, agent_route, backend_override="litellm") + def test_override_to_litellm_without_auth_token_but_with_auth_override(self, monkeypatch): + """An `auth` override (e.g. AWS creds for a provider with no `api_key` + concept) satisfies the credential requirement on its own — no + LITELLM_AUTH_TOKEN needed.""" + agent_route = BedrockRoute(region="eu-north-1") + settings = self._isolated_settings( + monkeypatch, api_backend=ApiBackend.BEDROCK, litellm_base_url="http://gateway:4000" + ) + ev = resolve_evaluation_route( + settings, + agent_route, + backend_override="litellm", + auth_override={"aws_access_key_id": "MY_KEY_ID"}, + ) + assert isinstance(ev, LiteLLMRoute) + assert ev.auth == {"aws_access_key_id": "MY_KEY_ID"} + + def test_override_to_litellm_threads_params_and_auth(self, monkeypatch): + agent_route = BedrockRoute(region="eu-north-1") + settings = self._isolated_settings( + monkeypatch, + api_backend=ApiBackend.BEDROCK, + litellm_base_url="http://gateway:4000", + litellm_auth_token="sk-master", + ) + ev = resolve_evaluation_route( + settings, + agent_route, + backend_override="litellm", + params_override={"aws_region_name": "eu-north-1"}, + auth_override={"api_key": "OTHER_ENV_VAR"}, + ) + assert isinstance(ev, LiteLLMRoute) + assert ev.params == {"aws_region_name": "eu-north-1"} + assert ev.auth == {"api_key": "OTHER_ENV_VAR"} + def test_unknown_backend_raises(self, monkeypatch): agent_route = DirectRoute() settings = self._isolated_settings(monkeypatch, api_backend=ApiBackend.DIRECT) @@ -237,6 +273,66 @@ def test_unknown_backend_raises(self, monkeypatch): resolve_evaluation_route(settings, agent_route, backend_override="not-a-backend") +class TestValidateCheckerContextShape: + """validate_checker_context_shape() — the load-time guard for + checker_context (previously 0% covered, per the PR #137 review).""" + + @staticmethod + def _validate(value): + from coder_eval.models.tasks import validate_checker_context_shape + + return validate_checker_context_shape(value) + + def test_accepts_empty(self): + self._validate({}) + + def test_accepts_route_and_model(self): + self._validate({"api_route": {"route": "bedrock", "model": "claude-haiku-4-5"}}) + + def test_rejects_unknown_namespace(self): + with pytest.raises(ValueError, match="unknown namespace"): + self._validate({"api_rotue": {"route": "bedrock"}}) + + def test_rejects_unknown_api_route_key(self): + with pytest.raises(ValueError, match="unknown key"): + self._validate({"api_route": {"rotue": "bedrock"}}) + + def test_rejects_unknown_backend_name(self): + with pytest.raises(ValueError, match="not a known backend"): + self._validate({"api_route": {"route": "not-a-backend"}}) + + def test_accepts_params_and_auth_with_litellm_route(self): + self._validate( + { + "api_route": { + "route": "litellm", + "params": {"aws_region_name": "eu-north-1"}, + "auth": {"api_key": "MY_ENV_VAR"}, + } + } + ) + + def test_rejects_params_without_litellm_route(self): + with pytest.raises(ValueError, match="require route: litellm"): + self._validate({"api_route": {"route": "bedrock", "params": {"x": 1}}}) + + def test_rejects_auth_without_litellm_route(self): + with pytest.raises(ValueError, match="require route: litellm"): + self._validate({"api_route": {"auth": {"api_key": "MY_ENV_VAR"}}}) + + def test_rejects_non_dict_params(self): + with pytest.raises(ValueError, match="params must be a mapping"): + self._validate({"api_route": {"route": "litellm", "params": "not-a-dict"}}) + + def test_rejects_non_dict_auth(self): + with pytest.raises(ValueError, match="auth must be a mapping"): + self._validate({"api_route": {"route": "litellm", "auth": "not-a-dict"}}) + + def test_rejects_non_string_auth_values(self): + with pytest.raises(ValueError, match="must map param name -> ENV VAR NAME"): + self._validate({"api_route": {"route": "litellm", "auth": {"api_key": 123}}}) + + class TestEvalRouteWiring: """The orchestrator must hand the simulated user the eval_route (constant Claude), never the agent's (possibly open-weight) route — guards the From f7583c01a65ded8dccbe8f065a42c67787b1bae3 Mon Sep 17 00:00:00 2001 From: Akshaya Shanbhogue Date: Wed, 26 Aug 2026 13:28:58 -0700 Subject: [PATCH 5/9] refactor(litellm-judge): drop settings coupling, rename auth to env_params Per review: the litellm judge transport was silently reading from coder_eval.config.settings (litellm_base_url/litellm_auth_token) even though checker_context.api_route.route: litellm is meant to be fully task-author-owned and independent of the agent's own LiteLLM backend. - LiteLLMRoute no longer carries base_url at all -- the agent's own LiteLLM backend (_build_sdk_env, environment_info recording) now reads settings.litellm_base_url directly instead of storing it on the route, mirroring how the bearer token is already handled. - The checker's litellm route is built ENTIRELY from checker_context.api_route.{params,env_params} -- no implicit fallback to the agent's LITELLM_BASE_URL/LITELLM_AUTH_TOKEN. `model` is now required when route: litellm (no default open-weight/gateway model). - Renamed `auth` -> `env_params` for clarity (it's not auth-specific -- api_base, aws_region_name, etc. can all be env-sourced too). - Removed LiteLLMRoute.include_temperature and the BadRequestError retry-without-temperature logic: invoke_litellm_judge_async no longer takes a `temperature` kwarg at all -- a gateway-routed model may reject it outright (observed live against an Azure AI deployment) with no reliable way to detect that in advance, so the task author opts in via `params: {temperature: ...}` if their model accepts it. Verified end-to-end against a real Azure AI gateway (checker_context: {api_route: {route: litellm, model: azure/gpt-5.6-luna, env_params: {api_base: LITELLM_BASE_URL, api_key: LITELLM_AUTH_TOKEN}}}) -- SUCCESS, score 1.000. Co-Authored-By: Claude Sonnet 5 --- docs/TASK_DEFINITION_GUIDE.md | 14 +- src/coder_eval/agents/claude_code_agent.py | 2 +- src/coder_eval/criteria/llm_judge.py | 3 - src/coder_eval/evaluation/judge_litellm.py | 159 +++++++--------- src/coder_eval/models/routing.py | 119 ++++++------ src/coder_eval/models/tasks.py | 39 ++-- src/coder_eval/orchestrator.py | 10 +- tests/test_judge_litellm.py | 209 +++++++-------------- tests/test_litellm_cost.py | 6 +- tests/test_litellm_route.py | 115 +++++------- tests/test_llm_judge_criterion.py | 5 +- tests/test_orchestrator.py | 11 +- tests/test_route_seam_exhaustiveness.py | 2 +- 13 files changed, 289 insertions(+), 405 deletions(-) diff --git a/docs/TASK_DEFINITION_GUIDE.md b/docs/TASK_DEFINITION_GUIDE.md index c7b7b4fa..dc985dcb 100644 --- a/docs/TASK_DEFINITION_GUIDE.md +++ b/docs/TASK_DEFINITION_GUIDE.md @@ -1312,21 +1312,21 @@ checker_context: model: claude-haiku-4-5 # model override for that route ``` -- `route` selects which backend the WHOLE evaluation side calls (`llm_judge`, `agent_judge`, and the simulator all share one resolved eval route per run — this isn't per-criterion), **decoupled from the agent's own route** (a Claude agent can be graded by a differently-backed judge, or vice versa). This is a backend *name* (`direct` / `bedrock` / `litellm`), not a route object: credentials are never read from the task, always from the matching environment variables (`ANTHROPIC_API_KEY` for `direct`, `AWS_BEARER_TOKEN_BEDROCK`/`AWS_REGION` for `bedrock`, `LITELLM_BASE_URL`/`LITELLM_AUTH_TOKEN` for `litellm`). An unconfigured or unknown backend name raises at dispatch rather than silently falling back. **`route: litellm` dispatches `llm_judge` through the `litellm` library** (the `coder-eval[litellm]` extra, `litellm.acompletion`) rather than assuming one wire protocol — a gateway-routed judge model (e.g. an Azure AI `/openai/v1` deployment) rarely speaks Anthropic Messages, so this lets `model` carry its own provider hint (e.g. `azure_ai/gpt-5.6-luna`) and get that provider's actual request/response shape handled by the library. +- `route` selects which backend the WHOLE evaluation side calls (`llm_judge`, `agent_judge`, and the simulator all share one resolved eval route per run — this isn't per-criterion), **decoupled from the agent's own route** (a Claude agent can be graded by a differently-backed judge, or vice versa). This is a backend *name* (`direct` / `bedrock` / `litellm`), not a route object. For `direct`/`bedrock` credentials are never read from the task, always from the matching environment variables (`ANTHROPIC_API_KEY` for `direct`, `AWS_BEARER_TOKEN_BEDROCK`/`AWS_REGION` for `bedrock`). An unconfigured or unknown backend name raises at dispatch rather than silently falling back. **`route: litellm` dispatches `llm_judge` through the `litellm` library** (the `coder-eval[litellm]` extra, `litellm.acompletion`) rather than assuming one wire protocol — a gateway-routed judge model (e.g. an Azure AI `/openai/v1` deployment) rarely speaks Anthropic Messages, so this lets `model` carry its own provider hint (e.g. `azure_ai/gpt-5.6-luna`) and get that provider's actual request/response shape handled by the library. Unlike the other two backends, `route: litellm` has NO implicit env-var fallback — see `params`/`env_params` below, which is how it's configured. `model` is required for `route: litellm` (there is no default open-weight/gateway model). - `model` overrides the model that resolved route uses for **`llm_judge` only** — when the criterion itself leaves `model:` unset (precedence: an explicit per-criterion `model:` always wins; below that, `checker_context.api_route.model`; below that, the built-in `DEFAULT_JUDGE_MODEL`). This floor is deliberate and never the agent's own model — an unpinned judge must grade identically regardless of which model the agent under test is using, so `resolve_evaluation_route` never lets the agent's env-configured model (e.g. `BEDROCK_MODEL`) leak into `route.model` on its own; `route.model` is set only when this override was actually given. This works because every `ApiRoute` (`DirectRoute`/`BedrockRoute`/`LiteLLMRoute`) carries its own `model` field; the orchestrator bakes the override into the resolved route's `model` before any criterion runs, so `llm_judge` just reads `context.route.model` — it never reads `checker_context` directly. **`agent_judge` and the simulator do not honor this override** — `agent_judge`'s sub-agent model comes from the criterion's own `agent:` block (defaulted to a fixed judge model), and the simulator's model is pinned by `SimulationConfig.model` (see [Simulation](#simulation) below) — both independent of `checker_context.api_route.model` by design, for the same "measuring instrument stays fixed" reason. -- `params`/`auth` (**`route: litellm` only**) extend the judge call to any of the dozens of provider-specific kwargs `litellm.acompletion` accepts (`aws_access_key_id`, `vertex_project`, `api_version`, ...), which have no dedicated field on `LiteLLMRoute`: +- `params`/`env_params` (**`route: litellm` only**) are how the call is actually configured — there is no fallback to the agent's own `LITELLM_BASE_URL`/`LITELLM_AUTH_TOKEN` env vars, since a gateway-routed judge model rarely reuses the agent's own LiteLLM proxy/credential. They also cover any of the dozens of other provider-specific kwargs `litellm.acompletion` accepts (`aws_access_key_id`, `vertex_project`, `api_version`, ...), which have no dedicated field on `LiteLLMRoute`: ```yaml checker_context: api_route: route: litellm - model: azure_ai/gpt-5.6-luna - params: # arbitrary passthrough kwargs to litellm.acompletion + model: azure/gpt-5.6-luna + params: # arbitrary literal passthrough kwargs to litellm.acompletion api_version: "2024-05-01" - auth: # param name -> ENV VAR NAME (never the secret itself) + env_params: # param name -> ENV VAR NAME (never the secret itself) + api_base: LITELLM_BASE_URL api_key: LITELLM_AUTH_TOKEN - aws_access_key_id: AWS_ACCESS_KEY_ID ``` - `params` is merged straight into the `litellm.acompletion(**kwargs)` call — litellm validates the param names itself, so there's no allowlist to keep in sync here. `auth` maps a kwarg name to the *name* of an environment variable; the value is resolved right before the call, so no secret is ever written into task/experiment YAML — this is how an arbitrary provider's auth shape (IAM keys, an Azure AD token, a service-account path, ...) is representable without a dedicated field per provider. At least one of `LITELLM_AUTH_TOKEN` or an `auth` entry must be configured. Rejected at task-load time if given without `route: litellm`. + `params` is merged straight into the `litellm.acompletion(**kwargs)` call — litellm validates the param names itself, so there's no allowlist to keep in sync here. `env_params` maps a kwarg name to the *name* of an environment variable; the value is resolved right before the call, so no secret is ever written into task/experiment YAML — this is how an arbitrary provider's config, including secrets (IAM keys, an Azure AD token, a service-account path, ...), is representable without a dedicated field per provider. `env_params` is resolved after `params`, so it always wins for the same key. Rejected at task-load time if given without `route: litellm`. `checker_context` merges shallow-per-namespace across `default_experiment.defaults.checker_context` → `experiment.defaults.checker_context` → `task.checker_context` → `variant.checker_context` (same 4-layer precedence as `agent`/`simulation`). So a judge-model A/B, or a judge-backend A/B, is a variant-level config change, not an edit to every task YAML. diff --git a/src/coder_eval/agents/claude_code_agent.py b/src/coder_eval/agents/claude_code_agent.py index 57b8b382..a7690ed5 100644 --- a/src/coder_eval/agents/claude_code_agent.py +++ b/src/coder_eval/agents/claude_code_agent.py @@ -836,7 +836,7 @@ def _build_sdk_env( # merges {**os.environ, ..., **options.env} at spawn, so setting # them here wins over the parent environment. env = { - "ANTHROPIC_BASE_URL": cr.base_url, + "ANTHROPIC_BASE_URL": settings.litellm_base_url or "", "ANTHROPIC_AUTH_TOKEN": settings.litellm_auth_token or "", # Neutralize any inherited ANTHROPIC_API_KEY: auth on this # route is the bearer ANTHROPIC_AUTH_TOKEN, and a stray diff --git a/src/coder_eval/criteria/llm_judge.py b/src/coder_eval/criteria/llm_judge.py index 3e0661de..42edad25 100644 --- a/src/coder_eval/criteria/llm_judge.py +++ b/src/coder_eval/criteria/llm_judge.py @@ -4,7 +4,6 @@ import logging from typing import TYPE_CHECKING -from coder_eval.config import settings from coder_eval.criteria.base import BaseCriterion, CheckContext, register_criterion from coder_eval.evaluation.judge_anthropic import invoke_anthropic_judge_async from coder_eval.evaluation.judge_bedrock import invoke_bedrock_judge_async @@ -252,11 +251,9 @@ async def _invoke_tool_channel( # whatever gateway their judge model actually lives behind. litellm_response = await invoke_litellm_judge_async( route=route, - auth_token=settings.litellm_auth_token, model=model, system=system_msg, user=user_msg, - temperature=criterion.temperature, max_tokens=criterion.max_tokens, tool_spec=SUBMIT_VERDICT_ANTHROPIC_TOOL, ) diff --git a/src/coder_eval/evaluation/judge_litellm.py b/src/coder_eval/evaluation/judge_litellm.py index a9f9e39d..705c1e04 100644 --- a/src/coder_eval/evaluation/judge_litellm.py +++ b/src/coder_eval/evaluation/judge_litellm.py @@ -1,19 +1,22 @@ """Single-completion invoker for the LiteLLM judge backend, via the ``litellm`` library (the ``coder-eval[litellm]`` extra) rather than a hand-rolled HTTP call. -``LiteLLMRoute``'s docstring frames it as an Anthropic-compatible proxy — true -for the AGENT side (``ClaudeCodeAgent`` points ``ANTHROPIC_BASE_URL`` at the -local ``litellm/start-litellm.sh`` proxy). The checker side reuses the same -route/env vars (``LITELLM_BASE_URL``/``LITELLM_AUTH_TOKEN``) for a different -purpose: task authors point this at whatever gateway their judge model lives -behind (an Azure AI ``/openai/v1`` deployment, a multi-model marketplace, ...), -which is rarely that same Anthropic-passthrough proxy. Calling through -``litellm.acompletion`` — rather than assuming one specific wire protocol — -lets ``model`` carry its own provider hint (e.g. ``azure_ai/gpt-5.6-luna``) -and get that provider's actual request/response shape handled by the library, -including per-provider quirks (Azure AI's ``api_base``/``api_key`` shape, -``max_tokens`` vs ``max_completion_tokens`` naming, unsupported-parameter -drops via ``drop_params``) instead of this module hand-coding them. +Unlike the AGENT's own LiteLLM backend (which points the Claude Code SDK at +``settings.litellm_base_url``/``settings.litellm_auth_token``), this module +reads NOTHING from ``coder_eval.config.settings`` — the task author fully owns +the call shape via ``LiteLLMRoute.params``/``LiteLLMRoute.env_params`` (see +that class's docstring). A gateway-routed judge model rarely reuses the same +proxy/credential the agent's own LiteLLM backend points at, so there is no +implicit fallback here; if the provider needs ``api_base``/``api_key``/ +whatever else, the task author names it via ``params``/``env_params`` like any +other kwarg. + +Calling through ``litellm.acompletion`` — rather than assuming one specific +wire protocol — lets ``model`` carry its own provider hint (e.g. +``azure_ai/gpt-5.6-luna``) and get that provider's actual request/response +shape handled by the library, including per-provider quirks (``max_tokens`` +vs ``max_completion_tokens`` naming, unsupported-parameter drops via +``drop_params``) instead of this module hand-coding them. ``litellm.acompletion`` always returns an OpenAI-shaped ``ModelResponse`` regardless of the underlying provider, so the caller reuses @@ -40,50 +43,41 @@ async def invoke_litellm_judge_async( *, route: LiteLLMRoute, - auth_token: str | None, model: str, system: str, user: str, - temperature: float, max_tokens: int, tool_spec: dict[str, Any], timeout_seconds: float = 120.0, ) -> dict[str, Any]: """One completion call via ``litellm.acompletion`` with a forced tool call. + Unlike ``invoke_anthropic_judge_async``/``invoke_bedrock_judge_async``, this + does NOT take a ``temperature`` — a gateway-routed model may reject it + outright (observed live against an Azure AI deployment), and there is no + uniform way to know in advance. Set it via ``route.params`` (e.g. + ``{temperature: 0.0}``) if the target model accepts it. + + Every other provider-specific kwarg (``api_base``, ``api_key``, + ``aws_access_key_id``, ...) comes from ``route.params``/``route.env_params`` + — see ``LiteLLMRoute``'s docstring. This function has no opinion on what a + valid call needs; an incomplete configuration surfaces as whatever error + ``litellm`` itself raises, wrapped below. + Returns the OpenAI-shaped response converted to a dict via ``model_dump`` so the caller can reuse ``extract_verdict_from_openai_response``. Raises: ValueError: ``model`` empty. - JudgeInfrastructureError: the ``litellm`` extra isn't installed; no - auth token configured; or the call fails (an eval-infra fault, - not the agent's fault — CE039). + JudgeInfrastructureError: the ``litellm`` extra isn't installed; an + ``env_params`` entry names an unset env var; or the call fails (an + eval-infra fault, not the agent's fault — CE039). """ if not model: raise ValueError("invoke_litellm_judge_async: model must not be empty") - # Raise (not assert): this call runs inside LLMJudgeChecker's - # handle_criterion_errors(_async) wrapper, which catches plain Exception - # (including AssertionError) and downgrades it to a scored 0.0 — the - # opposite of the intended "internal-contract violation escalates to - # FinalStatus.ERROR" behavior. - # - # `route.auth` can supply ITS OWN auth entirely — a provider that doesn't - # use `api_key` at all (e.g. Bedrock's aws_access_key_id/ - # aws_secret_access_key) has no reason to also need LITELLM_AUTH_TOKEN — - # so the plain LITELLM_AUTH_TOKEN requirement only applies when `auth` is - # empty too. `_resolve_backend_route` already enforces the same relaxed - # check at resolution time; this is the runtime backstop for a route - # built any other way. - if not auth_token and not route.auth: - msg = ( - "checker_context route 'litellm' requires LITELLM_AUTH_TOKEN to be set, " - "or an explicit `auth: {api_key: ENV_VAR}` override" - ) - raise JudgeInfrastructureError(msg) try: - from litellm.exceptions import APIError, BadRequestError + from litellm.exceptions import APIError from litellm.types.utils import ModelResponse import litellm @@ -101,75 +95,48 @@ async def invoke_litellm_judge_async( }, } - def _resolve_auth() -> dict[str, str]: - """Resolve ``route.auth`` (kwarg name -> ENV VAR NAME) into kwarg name -> - secret value, right before the call so no resolved secret is ever stored + def _resolve_env_params() -> dict[str, str]: + """Resolve ``route.env_params`` (kwarg name -> ENV VAR NAME) into kwarg + name -> value, right before the call so no resolved value is ever stored on the route object itself (only the env var *name* is).""" - if not route.auth: + if not route.env_params: return {} resolved: dict[str, str] = {} - for param_name, env_var in route.auth.items(): + for param_name, env_var in route.env_params.items(): value = os.environ.get(env_var) if not value: - raise JudgeInfrastructureError( - f"checker_context.api_route.auth[{param_name!r}] references env var {env_var!r}, which is not set" + msg = ( + f"checker_context.api_route.env_params[{param_name!r}] references env var " + f"{env_var!r}, which is not set" ) + raise JudgeInfrastructureError(msg) resolved[param_name] = value return resolved - def _call_kwargs(*, include_temperature: bool) -> dict[str, Any]: - kwargs: dict[str, Any] = { - "model": model, - "api_base": route.base_url, - "messages": [ - {"role": "system", "content": system}, - {"role": "user", "content": user}, - ], - "tools": [openai_tool], - "tool_choice": {"type": "function", "function": {"name": tool_spec["name"]}}, - "max_completion_tokens": max_tokens, - "timeout": timeout_seconds, - # `drop_params` only covers params litellm's own static model-cost map - # KNOWS a model rejects; a custom/gateway-routed model id (e.g. one - # behind an Azure AI deployment) isn't in that map, so an actual - # unsupported-parameter rejection still round-trips to the provider — - # handled below by retrying once without `temperature`. - "drop_params": True, - } - if auth_token: - kwargs["api_key"] = auth_token - if include_temperature: - kwargs["temperature"] = temperature - # `params` is arbitrary passthrough (e.g. aws_region_name, api_version, ...); - # `auth` (resolved secrets) applies LAST so it always wins over both the - # LITELLM_AUTH_TOKEN default above and anything in `params`. - if route.params: - kwargs.update(route.params) - kwargs.update(_resolve_auth()) - return kwargs - - def _rejects_temperature(e: BadRequestError) -> bool: - body = e.body if isinstance(e.body, dict) else {} - # The OpenAI SDK (which litellm's Azure path calls under the hood) - # unwraps the provider's `{"error": {...}}` envelope before attaching - # `.body` to the exception it raises — so `body` here is normally - # already the inner object (`{"param": "temperature", ...}`). Handle a - # still-wrapped shape too (a different provider path, or a future - # litellm/openai version) rather than assuming one or the other. - wrapped = body.get("error") - inner = wrapped if isinstance(wrapped, dict) else body - if inner.get("param") == "temperature": - return True - return "temperature" in str(e) and "not supported" in str(e).lower() + kwargs: dict[str, Any] = { + "model": model, + "messages": [ + {"role": "system", "content": system}, + {"role": "user", "content": user}, + ], + "tools": [openai_tool], + "tool_choice": {"type": "function", "function": {"name": tool_spec["name"]}}, + "max_completion_tokens": max_tokens, + "timeout": timeout_seconds, + # `drop_params` covers params litellm's own static model-cost map KNOWS a + # model rejects; a custom/gateway-routed model id (e.g. one behind an + # Azure AI deployment) usually isn't in that map, so this alone doesn't + # protect a `params`-supplied kwarg the target model live-rejects. + "drop_params": True, + } + # `params` (literal passthrough) applies first; `env_params` (resolved from + # env) applies LAST so it always wins over `params` for the same key. + if route.params: + kwargs.update(route.params) + kwargs.update(_resolve_env_params()) try: - try: - response = await litellm.acompletion(**_call_kwargs(include_temperature=route.include_temperature)) - except BadRequestError as e: - if not (route.include_temperature and _rejects_temperature(e)): - raise - logger.info("LiteLLM judge model %r rejects temperature; retrying without it", model) - response = await litellm.acompletion(**_call_kwargs(include_temperature=False)) + response = await litellm.acompletion(**kwargs) except APIError as e: raise JudgeInfrastructureError(f"LiteLLM judge API error: {e}") from e except Exception as e: diff --git a/src/coder_eval/models/routing.py b/src/coder_eval/models/routing.py index 2b8f8bc9..dd87b880 100644 --- a/src/coder_eval/models/routing.py +++ b/src/coder_eval/models/routing.py @@ -117,44 +117,49 @@ class BedrockRoute: @dataclass(frozen=True) class LiteLLMRoute: - """Route through a custom Anthropic-compatible endpoint (e.g. a LiteLLM - gateway fronting Bedrock open-weight models). - - The Claude Code SDK is pointed at ``base_url`` via ``ANTHROPIC_BASE_URL`` and - authenticates via ``ANTHROPIC_AUTH_TOKEN`` (bearer). The ``model``/``small_model`` - ids are passed **verbatim** (no Bedrock inference-profile qualification) — the - gateway maps them to its backend. - - Deliberately carries NO credential field — see ``BedrockRoute``'s docstring for - why. ``ClaudeCodeAgent._build_sdk_env`` reads ``settings.litellm_auth_token`` itself. - - ``include_temperature`` (checker side only — ``invoke_litellm_judge_async``): - whether the judge call sends ``temperature`` at all. Defaults to ``False`` - because a gateway-routed model id (e.g. an Azure AI deployment) isn't in - ``litellm``'s static param-support table, so an unsupported ``temperature`` - isn't caught by ``drop_params`` — it round-trips to the provider and back as - a live rejection before the judge retries without it. Defaulting to omitted - skips that wasted round trip for the common case; set ``True`` for a - gateway/model known to accept it. - - ``params``/``auth`` (checker side only, from ``checker_context.api_route.{params,auth}``): - ``litellm.acompletion`` takes dozens of provider-specific kwargs (``aws_access_key_id``, - ``vertex_project``, ``api_version``, ...) that this route has no dedicated field for. - ``params`` is passed through verbatim as extra kwargs. ``auth`` maps a kwarg name to - the ENV VAR NAME to resolve it from at call time — e.g. ``{aws_access_key_id: - AWS_ACCESS_KEY_ID}`` — so an arbitrary provider's auth shape is representable - without a secret ever landing in the task YAML. Both are ``None`` unless a task - author set them; ``auth``'s values are env var *names*, never secrets, so it is - safe to record verbatim in ``environment_info`` (unlike ``params``, which a task - author could — but shouldn't — put a raw secret into). + """Route through a custom endpoint — either the AGENT's own LiteLLM proxy + (an Anthropic-compatible gateway fronting Bedrock open-weight models), or, + on the CHECKER side (``checker_context.api_route.route: litellm``), an + arbitrary provider reached through the ``litellm`` library directly. + + AGENT side: the Claude Code SDK is pointed at the gateway via + ``ANTHROPIC_BASE_URL``/``ANTHROPIC_AUTH_TOKEN``. Deliberately carries NO + ``base_url``/credential field for this — same reasoning as ``BedrockRoute``'s + docstring: this route object flows through orchestrator state + (``environment_info`` recording, logging) that has no business handling + config that should always be read live from the environment. + ``ClaudeCodeAgent._build_sdk_env`` reads ``settings.litellm_base_url``/ + ``settings.litellm_auth_token`` itself, the same source ``resolve_route`` + validated before constructing this route. + + CHECKER side (``invoke_litellm_judge_async``): unlike the agent path, this + is NOT sourced from ``coder_eval.config.settings`` at all — the task author + fully owns it via ``params``/``env_params`` below (a gateway-routed judge + model rarely reuses the same proxy/credential the AGENT's own LiteLLM + backend points at). There is no implicit fallback to + ``settings.litellm_base_url``/``settings.litellm_auth_token``; if the + provider needs ``api_base``/``api_key``, the task author sets them via + ``params``/``env_params`` like any other kwarg. + + ``params``/``env_params`` (checker side only, from + ``checker_context.api_route.{params,env_params}``): ``litellm.acompletion`` + takes dozens of provider-specific kwargs (``api_base``, ``api_key``, + ``aws_access_key_id``, ``vertex_project``, ``api_version``, ...) that this + route has no dedicated field for. ``params`` is passed through verbatim as + extra kwargs. ``env_params`` maps a kwarg name to the ENV VAR NAME to + resolve it from at call time — e.g. ``{api_key: LITELLM_AUTH_TOKEN, + aws_access_key_id: AWS_ACCESS_KEY_ID}`` — so an arbitrary provider's config + (including secrets) is representable without a secret ever landing in the + task YAML. Both are ``None`` unless a task author set them; ``env_params``'s + values are env var *names*, never secrets, so it is safe to record verbatim + in ``environment_info`` (unlike ``params``, which a task author could — but + shouldn't — put a raw secret into). """ - base_url: str model: str | None = None small_model: str | None = None - include_temperature: bool = False params: dict[str, Any] | None = None - auth: dict[str, str] | None = None + env_params: dict[str, str] | None = None ApiRoute = DirectRoute | BedrockRoute | LiteLLMRoute @@ -221,7 +226,6 @@ def resolve_route(settings: Settings) -> ApiRoute: # No inference-profile qualification: the id is passed verbatim to the gateway. small_model = settings.litellm_small_model or settings.litellm_model return LiteLLMRoute( - base_url=settings.litellm_base_url, model=settings.litellm_model, small_model=small_model, ) @@ -233,11 +237,9 @@ def _resolve_backend_route( *, model_override: str | None = None, params_override: dict[str, Any] | None = None, - auth_override: dict[str, str] | None = None, + env_params_override: dict[str, str] | None = None, ) -> ApiRoute: - """Build the ``ApiRoute`` for an EXPLICITLY-requested backend, from the same - env-sourced ``Settings`` fields ``resolve_route`` reads for the agent — - credentials always come from the environment, never from a task/variant. + """Build the ``ApiRoute`` for an EXPLICITLY-requested backend. Used only by the ``checker_context.api_route`` override path (see ``resolve_evaluation_route``): raises ``ValueError`` naming the missing env @@ -246,10 +248,19 @@ def _resolve_backend_route( fail loudly, not degrade to a backend the task author didn't ask for. ``model_override`` (``checker_context.api_route.model``) wins over the - backend's own env-configured default model when set. ``params_override``/ - ``auth_override`` (``checker_context.api_route.{params,auth}``) only ever - land on a ``LiteLLMRoute`` — ``validate_checker_context_shape`` rejects them - on any other backend at load time, so they're ignored here otherwise. + backend's own env-configured default model when set. + + ``ApiBackend.LITELLM`` is the one exception to "env-sourced ``Settings`` + fields, credentials always come from the environment": unlike + BEDROCK/DIRECT (which reuse the agent's own env-configured credentials, since + grading still needs to reach the SAME Claude backend), a checker-side litellm + route is not assumed to share the agent's LiteLLM proxy/gateway at all — it + is built ENTIRELY from ``params_override``/``env_params_override`` + (``checker_context.api_route.{params,env_params}``), never from + ``settings.litellm_base_url``/``settings.litellm_auth_token``. Those two + settings fields are the AGENT's own LiteLLM-backend config (see + ``resolve_route``) — reusing them here would silently point the judge at + infrastructure the task author never named. """ match backend: case ApiBackend.BEDROCK: @@ -265,22 +276,16 @@ def _resolve_backend_route( raise ValueError("checker_context route 'direct' requires ANTHROPIC_API_KEY to be set") return DirectRoute(judge_transport="anthropic", model=model_override) case ApiBackend.LITELLM: - if not settings.litellm_base_url: - raise ValueError("checker_context route 'litellm' requires LITELLM_BASE_URL to be set") - if not settings.litellm_auth_token and not auth_override: + if not model_override: msg = ( - "checker_context route 'litellm' requires LITELLM_AUTH_TOKEN to be set, " - "or an explicit `checker_context.api_route.auth` override" + "checker_context route 'litellm' requires an explicit `checker_context.api_route.model` " + "— there is no default open-weight/gateway model to fall back to" ) raise ValueError(msg) - judge_model = model_override or settings.litellm_model - small_model = settings.litellm_small_model or judge_model return LiteLLMRoute( - base_url=settings.litellm_base_url, - model=judge_model, - small_model=small_model, + model=model_override, params=params_override, - auth=auth_override, + env_params=env_params_override, ) case _: # ApiBackend covers exactly BEDROCK/DIRECT/LITELLM above; this arm is @@ -296,7 +301,7 @@ def resolve_evaluation_route( backend_override: str | None = None, model_override: str | None = None, params_override: dict[str, Any] | None = None, - auth_override: dict[str, str] | None = None, + env_params_override: dict[str, str] | None = None, ) -> ApiRoute: """Resolve the route used by the *evaluation* side — the ``llm_judge`` / ``agent_judge`` criteria and the simulated user — which must stay on a @@ -306,8 +311,8 @@ def resolve_evaluation_route( All overrides come from the reserved ``checker_context.api_route`` namespace (see ``TaskDefinition.checker_context``) — ``route`` (``backend_override``) picks the backend, ``model`` (``model_override``) picks the model on - whichever route is resolved, and ``params``/``auth`` (``params_override``/ - ``auth_override``) only ever apply when ``backend_override`` resolves to + whichever route is resolved, and ``params``/``env_params`` (``params_override``/ + ``env_params_override``) only ever apply when ``backend_override`` resolves to ``litellm`` (see ``_resolve_backend_route``). Criteria never read any of these directly; they only ever see the resulting ``CheckContext.route``. @@ -343,7 +348,7 @@ def resolve_evaluation_route( backend, model_override=model_override, params_override=params_override, - auth_override=auth_override, + env_params_override=env_params_override, ) if isinstance(agent_route, BedrockRoute | DirectRoute): if isinstance(agent_route, BedrockRoute) and model_override: diff --git a/src/coder_eval/models/tasks.py b/src/coder_eval/models/tasks.py index dd0b5361..18960f12 100644 --- a/src/coder_eval/models/tasks.py +++ b/src/coder_eval/models/tasks.py @@ -101,7 +101,7 @@ def validate_checker_context_shape(value: dict[str, dict[str, Any]]) -> None: raise ValueError(msg) api_route = value.get("api_route") if api_route is not None: - known_keys = {"route", "model", "params", "auth"} + known_keys = {"route", "model", "params", "env_params"} unknown_keys = set(api_route) - known_keys if unknown_keys: msg = ( @@ -116,22 +116,27 @@ def validate_checker_context_shape(value: dict[str, dict[str, Any]]) -> None: valid = sorted(b.value for b in ApiBackend) msg = f"checker_context.api_route.route {route!r} is not a known backend ({valid})" raise ValueError(msg) from e - # `params`/`auth` only ever reach the litellm judge transport (see + # `params`/`env_params` only ever reach the litellm judge transport (see # invoke_litellm_judge_async) — on any other backend they'd be silently # dropped, which is worse than a load-time error. params = api_route.get("params") - auth = api_route.get("auth") - if (params is not None or auth is not None) and route != "litellm": - msg = "checker_context.api_route.params/auth require route: litellm" + env_params = api_route.get("env_params") + if (params is not None or env_params is not None) and route != "litellm": + msg = "checker_context.api_route.params/env_params require route: litellm" raise ValueError(msg) if params is not None and not isinstance(params, dict): raise ValueError(f"checker_context.api_route.params must be a mapping, got {type(params).__name__}") - if auth is not None: - if not isinstance(auth, dict): - raise ValueError(f"checker_context.api_route.auth must be a mapping, got {type(auth).__name__}") - bad = {k: v for k, v in auth.items() if not isinstance(k, str) or not isinstance(v, str)} + if env_params is not None: + if not isinstance(env_params, dict): + raise ValueError( + f"checker_context.api_route.env_params must be a mapping, got {type(env_params).__name__}" + ) + bad = {k: v for k, v in env_params.items() if not isinstance(k, str) or not isinstance(v, str)} if bad: - msg = f"checker_context.api_route.auth must map param name -> ENV VAR NAME (both strings); got {bad!r}" + msg = ( + f"checker_context.api_route.env_params must map param name -> ENV VAR NAME " + f"(both strings); got {bad!r}" + ) raise ValueError(msg) @@ -505,12 +510,14 @@ class TaskDefinition(BaseModel): # noqa: CE009 -- soft-launch: see _warn_on_unk "route uses. Both are consumed by the orchestrator (`resolve_evaluation_route`) BEFORE " "`CheckContext` is built and baked into the resolved route's own `model` field — no criterion " "ever reads `checker_context` directly, only `CheckContext.route.model`. Credentials are " - "always resolved from environment variables, never from this field. `params`/`auth` (only " - "with `route: litellm`) extend this to the judge's underlying `litellm.acompletion` call: " - "`params` is an arbitrary passthrough dict of extra kwargs (e.g. `{aws_region_name: " - "eu-north-1}`), and `auth` maps a kwarg name to the ENV VAR NAME (not the value!) to resolve " - "it from at call time (e.g. `{aws_access_key_id: AWS_ACCESS_KEY_ID}`) — so an arbitrary " - "provider's auth shape is representable without ever putting a secret in the task YAML. " + "always resolved from environment variables, never from this field — EXCEPT `route: litellm`, " + "whose call is built entirely from `params`/`env_params` (no implicit fallback to the agent's " + "own LITELLM_BASE_URL/LITELLM_AUTH_TOKEN): `params` is an arbitrary passthrough dict of extra " + "kwargs to the underlying `litellm.acompletion` call (e.g. `{api_base: https://my-gateway/v1}`), " + "and `env_params` maps a kwarg name to the ENV VAR NAME (not the value!) to resolve it from at " + "call time (e.g. `{api_key: MY_GATEWAY_TOKEN, aws_access_key_id: AWS_ACCESS_KEY_ID}`) — so an " + "arbitrary provider's config, including secrets, is representable without ever putting one in " + "the task YAML. `model` is required when `route: litellm` (no default open-weight/gateway model). " "Merged shallow-per-namespace across default -> experiment-defaults -> task -> variant." ), ) diff --git a/src/coder_eval/orchestrator.py b/src/coder_eval/orchestrator.py index a5fa75c3..4c80a849 100644 --- a/src/coder_eval/orchestrator.py +++ b/src/coder_eval/orchestrator.py @@ -158,7 +158,7 @@ class EvalRouteOverrides(NamedTuple): backend: str | None model: str | None params: dict[str, Any] | None - auth: dict[str, str] | None + env_params: dict[str, str] | None def _format_routing(route: ApiRoute, effective_model: str | None = None) -> str: @@ -1480,7 +1480,7 @@ def _eval_route_overrides(self) -> EvalRouteOverrides: backend=str(backend) if backend is not None else None, model=str(model) if model is not None else None, params=api_route.get("params"), - auth=api_route.get("auth"), + env_params=api_route.get("env_params"), ) def _resolve_routes(self) -> None: @@ -1498,7 +1498,7 @@ def _resolve_routes(self) -> None: backend_override=overrides.backend, model_override=overrides.model, params_override=overrides.params, - auth_override=overrides.auth, + env_params_override=overrides.env_params, ) logger.info("API routing: %s", _format_routing(self.route, self.task.agent.model if self.task.agent else None)) self.success_checker = SuccessChecker(self.sandbox, route=self.eval_route) @@ -1537,7 +1537,9 @@ def _record_route_environment_info(self) -> None: elif isinstance(self.route, LiteLLMRoute): # Host only (never the base_url or auth token) — mirrors the Codex # agent's host-only recording so secrets stay out of run artifacts. - self.result.environment_info["litellm_base_url_host"] = urlparse(self.route.base_url).hostname or "" + self.result.environment_info["litellm_base_url_host"] = ( + urlparse(settings.litellm_base_url or "").hostname or "" + ) if self.route.model: self.result.environment_info["litellm_model"] = self.route.model # Agent-specific routing (e.g. Codex custom-endpoint / Azure). No-op for diff --git a/tests/test_judge_litellm.py b/tests/test_judge_litellm.py index b0311e89..2c52706f 100644 --- a/tests/test_judge_litellm.py +++ b/tests/test_judge_litellm.py @@ -1,7 +1,9 @@ """Tests for the LiteLLM judge invoker, which calls through the ``litellm`` library (``litellm.acompletion``) rather than a hand-rolled HTTP client — ``litellm`` normalizes provider-specific request/response shapes so the judge -transport doesn't have to. +transport doesn't have to. Unlike the agent's own LiteLLM backend, this path +reads NOTHING from ``coder_eval.config.settings`` — everything comes from +``route.params``/``route.env_params``. """ from __future__ import annotations @@ -53,27 +55,18 @@ def _make_response(*, score: float = 0.5, rationale: str = "ok") -> MagicMock: def _route( *, - include_temperature: bool = False, params: dict[str, Any] | None = None, - auth: dict[str, str] | None = None, + env_params: dict[str, str] | None = None, ) -> LiteLLMRoute: - return LiteLLMRoute( - base_url="http://gateway:4000", - model="gpt-5.6-luna", - include_temperature=include_temperature, - params=params, - auth=auth, - ) + return LiteLLMRoute(model="gpt-5.6-luna", params=params, env_params=env_params) async def _invoke(**overrides): defaults = { - "route": _route(), - "auth_token": "sk-master", + "route": _route(params={"api_base": "http://gateway:4000", "api_key": "sk-master"}), "model": "azure_ai/gpt-5.6-luna", "system": "s", "user": "u", - "temperature": 0.0, "max_tokens": 10, "tool_spec": SUBMIT_VERDICT_ANTHROPIC_TOOL, } @@ -98,13 +91,13 @@ async def test_invoke_litellm_judge_calls_acompletion() -> None: async def test_invoke_litellm_judge_omits_temperature_by_default() -> None: - """LiteLLMRoute.include_temperature defaults to False: a gateway-routed model - id isn't in litellm's static param table, so an unsupported `temperature` - isn't caught by `drop_params` -- it round-trips to the provider and back as - a live rejection. Skip sending it at all unless the route opts in.""" + """Unlike invoke_anthropic_judge_async/invoke_bedrock_judge_async, there is no + `temperature` parameter at all here — a gateway-routed model may reject it + outright (observed live against an Azure AI deployment), so the task author + opts in via `params: {temperature: ...}` if their model accepts it.""" acompletion = AsyncMock(return_value=_make_response()) with patch("litellm.acompletion", new=acompletion): - await _invoke(temperature=0.7, max_tokens=321, system="sys", user="usr") + await _invoke(max_tokens=321, system="sys", user="usr") kwargs: dict[str, Any] = dict(acompletion.call_args.kwargs) assert "temperature" not in kwargs assert kwargs["max_completion_tokens"] == 321 @@ -114,84 +107,64 @@ async def test_invoke_litellm_judge_omits_temperature_by_default() -> None: ] -async def test_invoke_litellm_judge_sends_temperature_when_route_opts_in() -> None: +async def test_invoke_litellm_judge_sends_temperature_via_params() -> None: acompletion = AsyncMock(return_value=_make_response()) with patch("litellm.acompletion", new=acompletion): - await _invoke(route=_route(include_temperature=True), temperature=0.7) - kwargs: dict[str, Any] = dict(acompletion.call_args.kwargs) - assert kwargs["temperature"] == 0.7 + await _invoke(route=_route(params={"temperature": 0.7})) + assert acompletion.call_args.kwargs["temperature"] == 0.7 -async def test_invoke_litellm_judge_retries_without_temperature_when_rejected() -> None: - """A gateway-routed model litellm has no static param metadata for (so - `drop_params` can't preflight it) can still reject `temperature` live even - when the route opted in — observed against a real Azure AI deployment. - Must retry once without it rather than failing the whole judge call.""" - from litellm.exceptions import BadRequestError - - rejection = BadRequestError( - message="Unsupported parameter: 'temperature' is not supported with this model.", - model="azure_ai/gpt-5.6-luna", - llm_provider="azure_ai", - body={"error": {"message": "...", "param": "temperature", "code": None}}, - ) - acompletion = AsyncMock(side_effect=[rejection, _make_response(score=0.9)]) +async def test_invoke_litellm_judge_passes_through_params() -> None: + """`route.params` is arbitrary passthrough merged straight into the + litellm.acompletion() kwargs — e.g. api_base, aws_region_name, ...""" + acompletion = AsyncMock(return_value=_make_response()) with patch("litellm.acompletion", new=acompletion): - result = await _invoke(route=_route(include_temperature=True), temperature=0.3) - assert acompletion.call_count == 2 - first_kwargs, second_kwargs = (c.kwargs for c in acompletion.call_args_list) - assert first_kwargs["temperature"] == 0.3 - assert "temperature" not in second_kwargs - assert result["choices"][0]["message"]["tool_calls"][0]["function"]["name"] == "submit_verdict" - + await _invoke(route=_route(params={"aws_region_name": "eu-north-1", "api_version": "2024-05-01"})) + kwargs = acompletion.call_args.kwargs + assert kwargs["aws_region_name"] == "eu-north-1" + assert kwargs["api_version"] == "2024-05-01" -async def test_invoke_litellm_judge_reraises_unrelated_bad_request() -> None: - from litellm.exceptions import BadRequestError - rejection = BadRequestError( - message="Unsupported parameter: 'foo'.", - model="m", - llm_provider="azure_ai", - body={"error": {"message": "...", "param": "foo", "code": None}}, +async def test_invoke_litellm_judge_resolves_env_params_from_env(monkeypatch: pytest.MonkeyPatch) -> None: + """`route.env_params` maps a kwarg name -> ENV VAR NAME; the value is only + ever resolved at call time, never stored on the route.""" + monkeypatch.setenv("MY_AWS_ACCESS_KEY_ID", "AKIA-fake") + monkeypatch.setenv("MY_AWS_SECRET_ACCESS_KEY", "secret-fake") + acompletion = AsyncMock(return_value=_make_response()) + route = _route( + env_params={ + "aws_access_key_id": "MY_AWS_ACCESS_KEY_ID", + "aws_secret_access_key": "MY_AWS_SECRET_ACCESS_KEY", + } ) - acompletion = AsyncMock(side_effect=rejection) - with ( - patch("litellm.acompletion", new=acompletion), - pytest.raises(JudgeInfrastructureError, match="LiteLLM judge call failed"), - ): - await _invoke(route=_route(include_temperature=True)) - acompletion.assert_called_once() + with patch("litellm.acompletion", new=acompletion): + await _invoke(route=route) + kwargs = acompletion.call_args.kwargs + assert kwargs["aws_access_key_id"] == "AKIA-fake" + assert kwargs["aws_secret_access_key"] == "secret-fake" -async def test_invoke_litellm_judge_reraises_bad_request_when_route_did_not_opt_in() -> None: - """No point retrying-without-temperature when temperature was never sent.""" - from litellm.exceptions import BadRequestError +async def test_invoke_litellm_judge_env_params_override_params(monkeypatch: pytest.MonkeyPatch) -> None: + """`env_params` is resolved AFTER `params`, so it always wins for the same key.""" + monkeypatch.setenv("REAL_KEY", "sk-real") + acompletion = AsyncMock(return_value=_make_response()) + route = _route(params={"api_key": "sk-literal-in-yaml"}, env_params={"api_key": "REAL_KEY"}) + with patch("litellm.acompletion", new=acompletion): + await _invoke(route=route) + assert acompletion.call_args.kwargs["api_key"] == "sk-real" - rejection = BadRequestError( - message="Unsupported parameter: 'temperature' is not supported with this model.", - model="m", - llm_provider="azure_ai", - body={"error": {"message": "...", "param": "temperature", "code": None}}, - ) - acompletion = AsyncMock(side_effect=rejection) - with ( - patch("litellm.acompletion", new=acompletion), - pytest.raises(JudgeInfrastructureError, match="LiteLLM judge call failed"), - ): - await _invoke() - acompletion.assert_called_once() + +async def test_invoke_litellm_judge_raises_on_missing_env_var() -> None: + route = _route(env_params={"aws_access_key_id": "TOTALLY_UNSET_ENV_VAR_XYZ"}) + with pytest.raises(JudgeInfrastructureError, match="TOTALLY_UNSET_ENV_VAR_XYZ"): + await _invoke(route=route) async def test_invoke_litellm_judge_raises_on_empty_model() -> None: - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="model must not be empty"): await _invoke(model="") -async def test_invoke_litellm_judge_raises_on_missing_auth_token() -> None: - with pytest.raises(JudgeInfrastructureError, match="LITELLM_AUTH_TOKEN"): - await _invoke(auth_token=None) - - async def test_invoke_litellm_judge_raises_when_library_not_installed(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setitem(sys.modules, "litellm", None) with pytest.raises(JudgeInfrastructureError, match=r"pip install 'coder-eval\[litellm\]'"): @@ -209,70 +182,28 @@ async def test_invoke_litellm_judge_wraps_api_error() -> None: await _invoke() -async def test_invoke_litellm_judge_escalates_on_signature_break() -> None: - acompletion = AsyncMock(side_effect=TypeError("acompletion() got an unexpected keyword argument 'drop_params'")) +async def test_invoke_litellm_judge_wraps_bad_request_error() -> None: + from litellm.exceptions import BadRequestError + + rejection = BadRequestError( + message="Unsupported parameter: 'foo'.", + model="m", + llm_provider="azure_ai", + body={"error": {"message": "...", "param": "foo", "code": None}}, + ) + acompletion = AsyncMock(side_effect=rejection) with ( patch("litellm.acompletion", new=acompletion), pytest.raises(JudgeInfrastructureError, match="LiteLLM judge call failed"), ): await _invoke() + acompletion.assert_called_once() -async def test_invoke_litellm_judge_passes_through_params() -> None: - """`route.params` is arbitrary passthrough merged straight into the - litellm.acompletion() kwargs — e.g. aws_region_name, api_version, ...""" - acompletion = AsyncMock(return_value=_make_response()) - with patch("litellm.acompletion", new=acompletion): - await _invoke(route=_route(params={"aws_region_name": "eu-north-1", "api_version": "2024-05-01"})) - kwargs = acompletion.call_args.kwargs - assert kwargs["aws_region_name"] == "eu-north-1" - assert kwargs["api_version"] == "2024-05-01" - - -async def test_invoke_litellm_judge_resolves_auth_from_env(monkeypatch: pytest.MonkeyPatch) -> None: - """`route.auth` maps a kwarg name -> ENV VAR NAME; the secret VALUE is only - ever resolved at call time, never stored on the route.""" - monkeypatch.setenv("MY_AWS_ACCESS_KEY_ID", "AKIA-fake") - monkeypatch.setenv("MY_AWS_SECRET_ACCESS_KEY", "secret-fake") - acompletion = AsyncMock(return_value=_make_response()) - route = _route( - auth={"aws_access_key_id": "MY_AWS_ACCESS_KEY_ID", "aws_secret_access_key": "MY_AWS_SECRET_ACCESS_KEY"} - ) - with patch("litellm.acompletion", new=acompletion): - await _invoke(route=route, auth_token=None) - kwargs = acompletion.call_args.kwargs - assert kwargs["aws_access_key_id"] == "AKIA-fake" - assert kwargs["aws_secret_access_key"] == "secret-fake" - - -async def test_invoke_litellm_judge_auth_api_key_overrides_auth_token(monkeypatch: pytest.MonkeyPatch) -> None: - """An explicit `auth: {api_key: ENV_VAR}` wins over the LITELLM_AUTH_TOKEN- - sourced `auth_token` default, and satisfies the "some api_key is configured" - requirement even when `auth_token` itself is None.""" - monkeypatch.setenv("OTHER_KEY", "sk-other") - acompletion = AsyncMock(return_value=_make_response()) - route = _route(auth={"api_key": "OTHER_KEY"}) - with patch("litellm.acompletion", new=acompletion): - await _invoke(route=route, auth_token=None) - assert acompletion.call_args.kwargs["api_key"] == "sk-other" - - -async def test_invoke_litellm_judge_raises_on_missing_env_var_for_auth() -> None: - route = _route(auth={"aws_access_key_id": "TOTALLY_UNSET_ENV_VAR_XYZ"}) - with pytest.raises(JudgeInfrastructureError, match="TOTALLY_UNSET_ENV_VAR_XYZ"): - await _invoke(route=route) - - -async def test_invoke_litellm_judge_params_do_not_shadow_required_kwargs() -> None: - """auth resolves AFTER params, so an auth-mapped key always wins over the - same key set via params (belt-and-braces; auth is the documented secret - channel).""" - acompletion = AsyncMock(return_value=_make_response()) - route = _route(params={"api_key": "leaked-from-params"}) - with patch("litellm.acompletion", new=acompletion): - await _invoke(route=route, auth_token="sk-master") - # No `auth` override -> the LITELLM_AUTH_TOKEN default is applied first, - # then params overwrites it (params has no special protection over the - # base kwargs) -- documents the actual precedence rather than asserting a - # stronger guarantee than the implementation provides. - assert acompletion.call_args.kwargs["api_key"] == "leaked-from-params" +async def test_invoke_litellm_judge_escalates_on_signature_break() -> None: + acompletion = AsyncMock(side_effect=TypeError("acompletion() got an unexpected keyword argument 'drop_params'")) + with ( + patch("litellm.acompletion", new=acompletion), + pytest.raises(JudgeInfrastructureError, match="LiteLLM judge call failed"), + ): + await _invoke() diff --git a/tests/test_litellm_cost.py b/tests/test_litellm_cost.py index 7f4c220c..ea23c8c7 100644 --- a/tests/test_litellm_cost.py +++ b/tests/test_litellm_cost.py @@ -252,7 +252,7 @@ def test_joins_on_litellm_route(self, tmp_path, monkeypatch): ) monkeypatch.setattr(orch_mod.settings, "litellm_cost_log", str(log)) fake = SimpleNamespace( - route=LiteLLMRoute(base_url="http://x:4000", model="deepseek/deepseek-v4-pro"), + route=LiteLLMRoute(model="deepseek/deepseek-v4-pro"), result=_result([_turn(0, static_cost=0.5)]), _cost_correlation_run_id=run_id, _cost_attempt_nonce="att1", @@ -265,7 +265,7 @@ def test_joins_on_litellm_route(self, tmp_path, monkeypatch): def test_join_never_raises_on_bad_log(self, tmp_path, monkeypatch): monkeypatch.setattr(orch_mod.settings, "litellm_cost_log", str(tmp_path / "does-not-exist.jsonl")) fake = SimpleNamespace( - route=LiteLLMRoute(base_url="http://x:4000"), + route=LiteLLMRoute(), result=_result([_turn(0, static_cost=0.5)]), _cost_correlation_run_id="R", _cost_attempt_nonce="att1", @@ -289,7 +289,7 @@ def test_run_total_rederives_from_actual_after_join(self, tmp_path, monkeypatch) ) monkeypatch.setattr(orch_mod.settings, "litellm_cost_log", str(log)) fake = SimpleNamespace( - route=LiteLLMRoute(base_url="http://x:4000", model="deepseek/deepseek-v4-pro"), + route=LiteLLMRoute(model="deepseek/deepseek-v4-pro"), result=_result([_turn(0, static_cost=0.5), _turn(1, static_cost=0.5)]), _cost_correlation_run_id=run_id, _cost_attempt_nonce="att1", diff --git a/tests/test_litellm_route.py b/tests/test_litellm_route.py index 22e8ee66..83ea40c2 100644 --- a/tests/test_litellm_route.py +++ b/tests/test_litellm_route.py @@ -63,7 +63,7 @@ def test_direct_agent_route_is_reused_with_model_reset(self, monkeypatch): assert ev == DirectRoute(judge_transport="anthropic", model=None) def test_litellm_agent_pins_evaluation_to_bedrock_when_aws_creds_present(self, monkeypatch): - agent = LiteLLMRoute(base_url="http://x:4000", model="zai.glm-5") + agent = LiteLLMRoute(model="zai.glm-5") settings = self._isolated_settings( monkeypatch, api_backend=ApiBackend.LITELLM, @@ -78,7 +78,7 @@ def test_litellm_agent_pins_evaluation_to_bedrock_when_aws_creds_present(self, m assert ev.model is None def test_litellm_agent_falls_back_to_direct_when_only_anthropic_key(self, monkeypatch): - agent = LiteLLMRoute(base_url="http://x:4000", model="zai.glm-5") + agent = LiteLLMRoute(model="zai.glm-5") settings = self._isolated_settings(monkeypatch, api_backend=ApiBackend.LITELLM, anthropic_api_key="sk-ant") ev = resolve_evaluation_route(settings, agent) assert isinstance(ev, DirectRoute) @@ -87,7 +87,7 @@ def test_litellm_agent_falls_back_to_direct_when_only_anthropic_key(self, monkey def test_litellm_agent_unconfigured_yields_direct_with_no_transport(self, monkeypatch): # No Bedrock creds and no ANTHROPIC_API_KEY → DirectRoute(None), which makes # llm_judge fail with its clean "unconfigured" error rather than scoring 0.0. - agent = LiteLLMRoute(base_url="http://x:4000", model="zai.glm-5") + agent = LiteLLMRoute(model="zai.glm-5") settings = self._isolated_settings(monkeypatch, api_backend=ApiBackend.LITELLM) ev = resolve_evaluation_route(settings, agent) assert isinstance(ev, DirectRoute) @@ -138,7 +138,7 @@ def test_litellm_agent_pin_to_bedrock_no_override_strips_bedrock_model(self, mon # Agent on LiteLLM (open-weight); AWS creds present with a real # BEDROCK_MODEL configured for some unrelated purpose. No override -> # the pinned eval route must NOT inherit BEDROCK_MODEL. - agent_route = LiteLLMRoute(base_url="http://x:4000", model="zai.glm-5") + agent_route = LiteLLMRoute(model="zai.glm-5") settings = self._isolated_settings( monkeypatch, api_backend=ApiBackend.LITELLM, @@ -151,7 +151,7 @@ def test_litellm_agent_pin_to_bedrock_no_override_strips_bedrock_model(self, mon assert ev.model is None def test_litellm_agent_pin_to_bedrock_with_override(self, monkeypatch): - agent_route = LiteLLMRoute(base_url="http://x:4000", model="zai.glm-5") + agent_route = LiteLLMRoute(model="zai.glm-5") settings = self._isolated_settings( monkeypatch, api_backend=ApiBackend.LITELLM, @@ -211,60 +211,30 @@ def test_override_to_direct_without_key_raises(self, monkeypatch): with pytest.raises(ValueError, match="requires ANTHROPIC_API_KEY"): resolve_evaluation_route(settings, agent_route, backend_override="direct") - def test_override_to_litellm_builds_route_from_env(self, monkeypatch): - agent_route = BedrockRoute(region="eu-north-1") - settings = self._isolated_settings( - monkeypatch, - api_backend=ApiBackend.BEDROCK, - litellm_base_url="http://gateway:4000", - litellm_auth_token="sk-master", - ) - ev = resolve_evaluation_route(settings, agent_route, backend_override="litellm", model_override="gpt-5.6-luna") - assert isinstance(ev, LiteLLMRoute) - assert ev.base_url == "http://gateway:4000" - assert ev.model == "gpt-5.6-luna" - - def test_override_to_litellm_without_creds_raises(self, monkeypatch): + def test_override_to_litellm_builds_route_from_params_and_env_params(self, monkeypatch): + """No dependency on settings.litellm_base_url/litellm_auth_token at all — + the checker's litellm route is built entirely from params/env_params.""" agent_route = BedrockRoute(region="eu-north-1") settings = self._isolated_settings(monkeypatch, api_backend=ApiBackend.BEDROCK) - with pytest.raises(ValueError, match="requires LITELLM_BASE_URL"): - resolve_evaluation_route(settings, agent_route, backend_override="litellm") - - def test_override_to_litellm_without_auth_token_but_with_auth_override(self, monkeypatch): - """An `auth` override (e.g. AWS creds for a provider with no `api_key` - concept) satisfies the credential requirement on its own — no - LITELLM_AUTH_TOKEN needed.""" - agent_route = BedrockRoute(region="eu-north-1") - settings = self._isolated_settings( - monkeypatch, api_backend=ApiBackend.BEDROCK, litellm_base_url="http://gateway:4000" - ) ev = resolve_evaluation_route( settings, agent_route, backend_override="litellm", - auth_override={"aws_access_key_id": "MY_KEY_ID"}, + model_override="gpt-5.6-luna", + params_override={"api_base": "http://gateway:4000"}, + env_params_override={"api_key": "MY_ENV_VAR"}, ) assert isinstance(ev, LiteLLMRoute) - assert ev.auth == {"aws_access_key_id": "MY_KEY_ID"} + assert ev.model == "gpt-5.6-luna" + assert ev.params == {"api_base": "http://gateway:4000"} + assert ev.env_params == {"api_key": "MY_ENV_VAR"} - def test_override_to_litellm_threads_params_and_auth(self, monkeypatch): + def test_override_to_litellm_without_model_raises(self, monkeypatch): + """There is no default open-weight/gateway model to fall back to.""" agent_route = BedrockRoute(region="eu-north-1") - settings = self._isolated_settings( - monkeypatch, - api_backend=ApiBackend.BEDROCK, - litellm_base_url="http://gateway:4000", - litellm_auth_token="sk-master", - ) - ev = resolve_evaluation_route( - settings, - agent_route, - backend_override="litellm", - params_override={"aws_region_name": "eu-north-1"}, - auth_override={"api_key": "OTHER_ENV_VAR"}, - ) - assert isinstance(ev, LiteLLMRoute) - assert ev.params == {"aws_region_name": "eu-north-1"} - assert ev.auth == {"api_key": "OTHER_ENV_VAR"} + settings = self._isolated_settings(monkeypatch, api_backend=ApiBackend.BEDROCK) + with pytest.raises(ValueError, match=r"requires an explicit `checker_context\.api_route\.model`"): + resolve_evaluation_route(settings, agent_route, backend_override="litellm") def test_unknown_backend_raises(self, monkeypatch): agent_route = DirectRoute() @@ -301,13 +271,13 @@ def test_rejects_unknown_backend_name(self): with pytest.raises(ValueError, match="not a known backend"): self._validate({"api_route": {"route": "not-a-backend"}}) - def test_accepts_params_and_auth_with_litellm_route(self): + def test_accepts_params_and_env_params_with_litellm_route(self): self._validate( { "api_route": { "route": "litellm", "params": {"aws_region_name": "eu-north-1"}, - "auth": {"api_key": "MY_ENV_VAR"}, + "env_params": {"api_key": "MY_ENV_VAR"}, } } ) @@ -316,21 +286,21 @@ def test_rejects_params_without_litellm_route(self): with pytest.raises(ValueError, match="require route: litellm"): self._validate({"api_route": {"route": "bedrock", "params": {"x": 1}}}) - def test_rejects_auth_without_litellm_route(self): + def test_rejects_env_params_without_litellm_route(self): with pytest.raises(ValueError, match="require route: litellm"): - self._validate({"api_route": {"auth": {"api_key": "MY_ENV_VAR"}}}) + self._validate({"api_route": {"env_params": {"api_key": "MY_ENV_VAR"}}}) def test_rejects_non_dict_params(self): with pytest.raises(ValueError, match="params must be a mapping"): self._validate({"api_route": {"route": "litellm", "params": "not-a-dict"}}) - def test_rejects_non_dict_auth(self): - with pytest.raises(ValueError, match="auth must be a mapping"): - self._validate({"api_route": {"route": "litellm", "auth": "not-a-dict"}}) + def test_rejects_non_dict_env_params(self): + with pytest.raises(ValueError, match="env_params must be a mapping"): + self._validate({"api_route": {"route": "litellm", "env_params": "not-a-dict"}}) - def test_rejects_non_string_auth_values(self): + def test_rejects_non_string_env_params_values(self): with pytest.raises(ValueError, match="must map param name -> ENV VAR NAME"): - self._validate({"api_route": {"route": "litellm", "auth": {"api_key": 123}}}) + self._validate({"api_route": {"route": "litellm", "env_params": {"api_key": 123}}}) class TestEvalRouteWiring: @@ -346,7 +316,7 @@ async def test_simulator_receives_eval_route_not_agent_route(self, monkeypatch): from coder_eval.orchestrator import Orchestrator eval_route = BedrockRoute(region="eu-north-1", model="eu.anthropic.claude-sonnet-4-6") - agent_route = LiteLLMRoute(base_url="http://x:4000", model="zai.glm-5") + agent_route = LiteLLMRoute(model="zai.glm-5") captured: dict = {} class _SpySimulator: @@ -385,7 +355,8 @@ def test_resolves_custom_route_with_all_fields(self): ) route = resolve_route(settings) assert isinstance(route, LiteLLMRoute) - assert route.base_url == "http://localhost:4000" + # base_url/auth_token are NOT stored on the route (read live from + # settings by _build_sdk_env) -- only model/small_model live here. assert route.model == "deepseek.v3.2" def test_rejects_scheme_less_base_url(self): @@ -501,8 +472,8 @@ def test_custom_route_env_has_anthropic_vars_only(self, monkeypatch): from coder_eval.agents import claude_code_agent as claude_code_agent_mod monkeypatch.setattr(claude_code_agent_mod.settings, "litellm_auth_token", "sk-1") + monkeypatch.setattr(claude_code_agent_mod.settings, "litellm_base_url", "http://x:4000") route = LiteLLMRoute( - base_url="http://x:4000", model="deepseek.v3.2", small_model="deepseek.v3.2", ) @@ -519,7 +490,7 @@ def test_custom_route_env_has_anthropic_vars_only(self, monkeypatch): assert "AWS_REGION" not in env def test_custom_route_no_model_omits_model_vars(self): - route = LiteLLMRoute(base_url="http://x:4000") + route = LiteLLMRoute() env, model = ClaudeCodeAgent._build_sdk_env(route) assert model is None assert "ANTHROPIC_MODEL" not in env @@ -530,7 +501,7 @@ def test_custom_route_forwards_path(self, monkeypatch): custom_path = f"/custom/bin{os.pathsep}/usr/bin" monkeypatch.setenv("PATH", custom_path) - env, _ = ClaudeCodeAgent._build_sdk_env(LiteLLMRoute(base_url="http://x:4000")) + env, _ = ClaudeCodeAgent._build_sdk_env(LiteLLMRoute()) assert env["PATH"] == custom_path def test_custom_route_neutralizes_inherited_anthropic_api_key(self, monkeypatch): @@ -538,20 +509,20 @@ def test_custom_route_neutralizes_inherited_anthropic_api_key(self, monkeypatch) empty in options.env (not merely omitted) — else it would fight the bearer auth_token against the gateway.""" monkeypatch.setenv("ANTHROPIC_API_KEY", "leaked-key") - env, _ = ClaudeCodeAgent._build_sdk_env(LiteLLMRoute(base_url="http://x:4000")) + env, _ = ClaudeCodeAgent._build_sdk_env(LiteLLMRoute()) assert env["ANTHROPIC_API_KEY"] == "" def test_cost_log_tags_become_custom_headers(self): """cost_log_tags → ANTHROPIC_CUSTOM_HEADERS as newline-separated `Name: Value` pairs (the format Claude Code forwards verbatim), so the proxy-side cost log can join each call back to the run/task/turn.""" - route = LiteLLMRoute(base_url="http://x:4000", model="deepseek/deepseek-v4-pro") + route = LiteLLMRoute(model="deepseek/deepseek-v4-pro") tags = {"x-ce-run-id": "abc123", "x-ce-task-id": "calc/v1", "x-ce-iteration": "2"} env, _ = ClaudeCodeAgent._build_sdk_env(route, cost_log_tags=tags) assert env["ANTHROPIC_CUSTOM_HEADERS"] == "x-ce-run-id: abc123\nx-ce-task-id: calc/v1\nx-ce-iteration: 2" def test_no_cost_log_tags_omits_custom_headers(self): - route = LiteLLMRoute(base_url="http://x:4000") + route = LiteLLMRoute() env, _ = ClaudeCodeAgent._build_sdk_env(route) assert "ANTHROPIC_CUSTOM_HEADERS" not in env env2, _ = ClaudeCodeAgent._build_sdk_env(route, cost_log_tags={}) @@ -581,7 +552,7 @@ def test_cost_log_tags_gated_on_agent_capability_not_route(self): assert AgentRegistry.get(AgentKind.CLAUDE_CODE).agent_class.supports_cost_log_tags is True assert AgentRegistry.get(AgentKind.NONE).agent_class.supports_cost_log_tags is False - route = LiteLLMRoute(base_url="http://x:4000", model="deepseek/deepseek-v4-pro") + route = LiteLLMRoute(model="deepseek/deepseek-v4-pro") # A none-agent constructs fine on a LiteLLM route (the gate omits the kwarg)... assert create_agent(AgentKind.NONE, NoneAgentConfig(type=AgentKind.NONE), route=route) is not None # ...and it WOULD crash if the kwarg were forwarded — exactly what the gate prevents. @@ -593,7 +564,7 @@ def test_cost_log_tags_gated_on_agent_capability_not_route(self): def test_cost_log_tags_reject_header_injection(self): # A task_id/variant_id carrying a CR/LF would inject extra headers into every # SDK->proxy request; the seam must reject it (single-line ASCII only). - route = LiteLLMRoute(base_url="http://x:4000") + route = LiteLLMRoute() with pytest.raises(ValueError, match="single-line ASCII"): ClaudeCodeAgent._build_sdk_env(route, cost_log_tags={"x-ce-task-id": "ok\nAuthorization: Bearer forged"}) @@ -602,7 +573,7 @@ class TestResolveEffectiveModelCustom: """_resolve_effective_model() on the LiteLLM route — no prefixing.""" def test_config_model_synced_verbatim(self): - route = LiteLLMRoute(base_url="http://x:4000", model="deepseek.v3.2") + route = LiteLLMRoute(model="deepseek.v3.2") env, route_model = ClaudeCodeAgent._build_sdk_env(route) agent = _make_agent(route, config_model="zai.glm-5") effective = agent._resolve_effective_model("zai.glm-5", env, route_model) @@ -610,14 +581,14 @@ def test_config_model_synced_verbatim(self): assert env["ANTHROPIC_MODEL"] == "zai.glm-5" # no eu./anthropic. prefix def test_route_model_used_when_config_none(self): - route = LiteLLMRoute(base_url="http://x:4000", model="deepseek.v3.2") + route = LiteLLMRoute(model="deepseek.v3.2") env, route_model = ClaudeCodeAgent._build_sdk_env(route) agent = _make_agent(route) effective = agent._resolve_effective_model(None, env, route_model) assert effective == "deepseek.v3.2" def test_both_none_returns_none(self): - route = LiteLLMRoute(base_url="http://x:4000") + route = LiteLLMRoute() env, route_model = ClaudeCodeAgent._build_sdk_env(route) agent = _make_agent(route) effective = agent._resolve_effective_model(None, env, route_model) @@ -690,7 +661,7 @@ def _usage_after_finalize(self, effective_model: str | None) -> TokenUsage: from coder_eval.agents.claude_code_agent import _ClaudeTurnState agent = _make_agent( - LiteLLMRoute(base_url="http://x:4000", model="zai.glm-5"), + LiteLLMRoute(model="zai.glm-5"), config_model="zai.glm-5", ) stub = SimpleNamespace( diff --git a/tests/test_llm_judge_criterion.py b/tests/test_llm_judge_criterion.py index 8083c369..8b9a0e50 100644 --- a/tests/test_llm_judge_criterion.py +++ b/tests/test_llm_judge_criterion.py @@ -656,7 +656,7 @@ def test_judge_bedrock_route_with_explicit_route_model_still_wins(sandbox: Sandb def test_judge_litellm_route_uses_litellm_invoker(sandbox: Sandbox) -> None: from coder_eval.models.routing import LiteLLMRoute - route = LiteLLMRoute(base_url="http://gateway:4000", model="gpt-5-luna") + route = LiteLLMRoute(model="gpt-5-luna", params={"api_base": "http://gateway:4000"}) criterion = LLMJudgeCriterion(description="x", prompt="grade") with ( patch( @@ -673,7 +673,8 @@ def test_judge_litellm_route_uses_litellm_invoker(sandbox: Sandbox) -> None: assert kwargs["route"] is route # No explicit criterion.model set -> falls back to route.model (checker_context override). assert kwargs["model"] == "gpt-5-luna" - assert kwargs["temperature"] == criterion.temperature + # invoke_litellm_judge_async takes no `temperature` kwarg at all -- see its docstring. + assert "temperature" not in kwargs assert kwargs["max_tokens"] == criterion.max_tokens assert kwargs["tool_spec"]["name"] == "submit_verdict" assert m_bedrock.call_count == 0 diff --git a/tests/test_orchestrator.py b/tests/test_orchestrator.py index eadfb526..a7ff2d1f 100644 --- a/tests/test_orchestrator.py +++ b/tests/test_orchestrator.py @@ -49,7 +49,7 @@ def test_format_routing_non_direct_routes_unchanged(): def test_format_routing_litellm_shows_model(): - out = _format_routing(LiteLLMRoute(base_url="http://localhost:4000", model="zai.glm-5")) + out = _format_routing(LiteLLMRoute(model="zai.glm-5")) assert out.startswith("litellm") assert "zai.glm-5" in out @@ -57,7 +57,7 @@ def test_format_routing_litellm_shows_model(): def test_format_routing_litellm_effective_model_wins_over_route_default(): """The --model override (effective_model) must be logged, not the route's LITELLM_MODEL default.""" out = _format_routing( - LiteLLMRoute(base_url="http://localhost:4000", model="zai.glm-5"), + LiteLLMRoute(model="zai.glm-5"), effective_model="deepseek.v3.2", ) assert "deepseek.v3.2" in out @@ -117,11 +117,14 @@ def test_record_route_environment_info_bedrock(tmp_path): assert info["bedrock_model"] == "eu.anthropic.claude-sonnet-4-6" -def test_record_route_environment_info_litellm_records_host_only_no_secret(tmp_path): +def test_record_route_environment_info_litellm_records_host_only_no_secret(tmp_path, monkeypatch): """LiteLLM route records host + model, but NEVER the auth token or full base_url.""" + import coder_eval.orchestrator as orch_mod + + monkeypatch.setattr(orch_mod.settings, "litellm_base_url", "http://localhost:4000") orchestrator = _make_orchestrator_with_route( tmp_path, - LiteLLMRoute(base_url="http://localhost:4000", model="zai.glm-5"), + LiteLLMRoute(model="zai.glm-5"), ) orchestrator._record_route_environment_info() assert orchestrator.result is not None diff --git a/tests/test_route_seam_exhaustiveness.py b/tests/test_route_seam_exhaustiveness.py index 449f4bdf..c3fe38e7 100644 --- a/tests/test_route_seam_exhaustiveness.py +++ b/tests/test_route_seam_exhaustiveness.py @@ -30,7 +30,7 @@ _INSTANCES: list[object] = [ DirectRoute(), BedrockRoute(region="eu-north-1", model="x"), - LiteLLMRoute(base_url="http://localhost:4000", model="m"), + LiteLLMRoute(model="m"), ] From 0a2d47aa5aacd3055587d63551ab0ef6c288e118 Mon Sep 17 00:00:00 2001 From: Akshaya Shanbhogue Date: Wed, 26 Aug 2026 13:46:14 -0700 Subject: [PATCH 6/9] ci(fix): install litellm extra for pyright, address CodeQL findings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two CI gates (Quality Gate, Windows Smoke Test) were failing pyright: without `--extra litellm` in `uv sync`, `import litellm` in judge_litellm.py resolved to the repo's own top-level `litellm/` directory (the LiteLLM PROXY scripts, a namespace package with no `acompletion`/`exceptions`/`types.utils`) instead of the real PyPI package, since the real litellm distribution was never installed. Also fixes 4 CodeQL findings from the latest analysis: - routing.py: resolve_route()'s match (unlike its sibling _resolve_backend_route) had no `case _:`, so a 4th ApiBackend member would silently fall through and return None (mixed explicit/implicit returns) — added the same exhaustive-match guard. - test_llm_judge_criterion.py: two redundant local `import json` (already imported at module top). - test_orchestrator.py: `coder_eval.orchestrator` was imported both via `import ... as orch_mod` and `from ... import Orchestrator, ...` — patch `coder_eval.config.settings` directly instead (same singleton object orchestrator.py already imports). Co-Authored-By: Claude Sonnet 5 --- .github/workflows/pr-checks.yml | 4 ++-- src/coder_eval/models/routing.py | 5 +++++ tests/test_llm_judge_criterion.py | 4 ---- tests/test_orchestrator.py | 4 ++-- 4 files changed, 9 insertions(+), 8 deletions(-) diff --git a/.github/workflows/pr-checks.yml b/.github/workflows/pr-checks.yml index 533ee43b..c0667f23 100644 --- a/.github/workflows/pr-checks.yml +++ b/.github/workflows/pr-checks.yml @@ -80,7 +80,7 @@ jobs: pip install uv - name: Install project dependencies (hash-verified from uv.lock) - run: uv sync --frozen --extra dev --extra uipath --extra codex + run: uv sync --frozen --extra dev --extra uipath --extra codex --extra litellm # PHASE 1: Fast checks (fail early) - name: Check code formatting (ruff format) @@ -385,7 +385,7 @@ jobs: pip install uv - name: Install project dependencies (hash-verified from uv.lock) - run: uv sync --frozen --extra dev --extra uipath --extra codex + run: uv sync --frozen --extra dev --extra uipath --extra codex --extra litellm - name: Check code formatting (ruff format) run: .venv/Scripts/ruff format --check src/ tests/ diff --git a/src/coder_eval/models/routing.py b/src/coder_eval/models/routing.py index dd87b880..4dc5e0a0 100644 --- a/src/coder_eval/models/routing.py +++ b/src/coder_eval/models/routing.py @@ -229,6 +229,11 @@ def resolve_route(settings: Settings) -> ApiRoute: model=settings.litellm_model, small_model=small_model, ) + case _: + # ApiBackend covers exactly BEDROCK/DIRECT/LITELLM above; this arm is + # unreachable but makes the match exhaustive so every path returns + # explicitly (CodeQL: mixed explicit/implicit returns). + raise AssertionError(f"unhandled ApiBackend: {settings.api_backend!r}") def _resolve_backend_route( diff --git a/tests/test_llm_judge_criterion.py b/tests/test_llm_judge_criterion.py index 8b9a0e50..ca0a1450 100644 --- a/tests/test_llm_judge_criterion.py +++ b/tests/test_llm_judge_criterion.py @@ -573,8 +573,6 @@ def _tool_use_block(score: float, rationale: str = "ok") -> dict: def _openai_tool_call_block(score: float, rationale: str = "ok") -> dict: - import json - return { "choices": [ { @@ -600,8 +598,6 @@ def test_llm_judge_criterion_model_survives_json_round_trip() -> None: (isolation/docker_runner.py -> cli/run_task_internal_command.py::load_task). A `model_fields_set` sentinel would NOT survive this (every field is materialized by model_dump), silently making the judge-model override inert under --driver docker.""" - import json - criterion = LLMJudgeCriterion(description="x", prompt="grade") assert criterion.model is None reloaded = LLMJudgeCriterion.model_validate(json.loads(json.dumps(criterion.model_dump(mode="json")))) diff --git a/tests/test_orchestrator.py b/tests/test_orchestrator.py index a7ff2d1f..3fbce30a 100644 --- a/tests/test_orchestrator.py +++ b/tests/test_orchestrator.py @@ -119,9 +119,9 @@ def test_record_route_environment_info_bedrock(tmp_path): def test_record_route_environment_info_litellm_records_host_only_no_secret(tmp_path, monkeypatch): """LiteLLM route records host + model, but NEVER the auth token or full base_url.""" - import coder_eval.orchestrator as orch_mod + from coder_eval.config import settings - monkeypatch.setattr(orch_mod.settings, "litellm_base_url", "http://localhost:4000") + monkeypatch.setattr(settings, "litellm_base_url", "http://localhost:4000") orchestrator = _make_orchestrator_with_route( tmp_path, LiteLLMRoute(model="zai.glm-5"), From 9af53674caf85e63c33486c9dc9bfdfac9d0f9b7 Mon Sep 17 00:00:00 2001 From: Akshaya Shanbhogue Date: Wed, 26 Aug 2026 15:37:53 -0700 Subject: [PATCH 7/9] fix(checker-context): typed model, reject litellm+agent_judge/sim, live test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses bai-uipath's PR #137 follow-up review: - Blocking: checker_context.api_route.route: litellm dispatches llm_judge through the litellm library in-process, but agent_judge and the simulator run as real Claude Code CLI subprocesses speaking Anthropic Messages only. Orchestrator._reject_litellm_eval_route_if_unsupported() now raises a clear error at route-resolution time when route: litellm is combined with an enabled agent_judge criterion or simulation.enabled, instead of silently misrouting onto the agent's own unrelated LiteLLM settings. - checker_context is now typed (CheckerContext/ApiRouteContext pydantic models, extra="forbid") instead of a hand-validated open dict — deletes validate_checker_context_shape and its two call sites. A YAML `model: 5` is now rejected at load time instead of being str()-ified into a model id. _resolve_checker_context merges through the shared merge_layers engine (mirroring _resolve_simulation) and records config lineage. - Added tests/test_litellm_judge_live.py: a live regression test hitting a real gateway via litellm.acompletion, reusing the existing CODEX_API_KEY/ CODEX_BASE_URL/CODEX_MODEL CI secrets, wired into the codex-live-tests CI job — closes "nothing in the repo exercises the feature". - Docs: azure_ai/ -> azure/ (matches what actually ran), api_version pinning guidance, documents the new agent_judge/simulator restriction. - Fixed a vacuous test assertion: the orchestrator secret-leak test now actually sets litellm_auth_token before asserting it's absent from environment_info. Co-Authored-By: Claude Sonnet 5 --- .github/workflows/pr-checks.yml | 15 +- docs/TASK_DEFINITION_GUIDE.md | 5 +- src/coder_eval/models/__init__.py | 6 +- src/coder_eval/models/tasks.py | 155 +++++++++------------ src/coder_eval/orchestration/experiment.py | 54 +++---- src/coder_eval/orchestrator.py | 46 +++++- tests/test_litellm_judge_live.py | 69 +++++++++ tests/test_litellm_route.py | 39 ++++-- tests/test_orchestrator.py | 72 ++++++++++ 9 files changed, 315 insertions(+), 146 deletions(-) create mode 100644 tests/test_litellm_judge_live.py diff --git a/.github/workflows/pr-checks.yml b/.github/workflows/pr-checks.yml index c0667f23..94cce6d3 100644 --- a/.github/workflows/pr-checks.yml +++ b/.github/workflows/pr-checks.yml @@ -843,8 +843,8 @@ jobs: python -m pip install --upgrade "pip>=26.2" # PYSEC-2026-3721 fix floor pip install uv - - name: Install project dependencies (with codex extra) - run: uv sync --frozen --extra dev --extra uipath --extra codex + - name: Install project dependencies (with codex + litellm extras) + run: uv sync --frozen --extra dev --extra uipath --extra codex --extra litellm - name: Verify required secrets are present run: | @@ -854,14 +854,17 @@ jobs: fi echo "CODEX_API_KEY present." - - name: Run Codex live tests + - name: Run Codex + litellm-judge live tests run: | mkdir -p tmp # Run serially: `-n0` overrides the global `-n auto` (addopts). # Parallel xdist workers share ~/.codex and race the Codex SQLite # state migration (`duplicate column name: thread_id`); serial init - # migrates the fresh DB exactly once. - .venv/bin/pytest tests/test_codex_agent_live.py \ + # migrates the fresh DB exactly once. test_litellm_judge_live.py + # reuses these same CODEX_* secrets to exercise + # checker_context.api_route.route: litellm end-to-end (PR #137 + # review: "nothing in the repo exercises the feature"). + .venv/bin/pytest tests/test_codex_agent_live.py tests/test_litellm_judge_live.py \ -m live -n0 -v --tb=short --strict-markers -ra \ --junit-xml=tmp/junit-codex-live.xml @@ -878,7 +881,7 @@ jobs: passed = total - skipped - errors - failures print(f"codex-live passed={passed} skipped={skipped} errors={errors} failures={failures}") if passed < 1: - sys.exit("test_codex_agent_live.py reported zero PASSED tests (missing API key / silent skip?)") + sys.exit("Live Codex/litellm-judge tests reported zero PASSED tests (missing API key / silent skip?)") PY - name: Upload Codex live-test artifacts on failure diff --git a/docs/TASK_DEFINITION_GUIDE.md b/docs/TASK_DEFINITION_GUIDE.md index dc985dcb..b08a9bf4 100644 --- a/docs/TASK_DEFINITION_GUIDE.md +++ b/docs/TASK_DEFINITION_GUIDE.md @@ -1312,7 +1312,7 @@ checker_context: model: claude-haiku-4-5 # model override for that route ``` -- `route` selects which backend the WHOLE evaluation side calls (`llm_judge`, `agent_judge`, and the simulator all share one resolved eval route per run — this isn't per-criterion), **decoupled from the agent's own route** (a Claude agent can be graded by a differently-backed judge, or vice versa). This is a backend *name* (`direct` / `bedrock` / `litellm`), not a route object. For `direct`/`bedrock` credentials are never read from the task, always from the matching environment variables (`ANTHROPIC_API_KEY` for `direct`, `AWS_BEARER_TOKEN_BEDROCK`/`AWS_REGION` for `bedrock`). An unconfigured or unknown backend name raises at dispatch rather than silently falling back. **`route: litellm` dispatches `llm_judge` through the `litellm` library** (the `coder-eval[litellm]` extra, `litellm.acompletion`) rather than assuming one wire protocol — a gateway-routed judge model (e.g. an Azure AI `/openai/v1` deployment) rarely speaks Anthropic Messages, so this lets `model` carry its own provider hint (e.g. `azure_ai/gpt-5.6-luna`) and get that provider's actual request/response shape handled by the library. Unlike the other two backends, `route: litellm` has NO implicit env-var fallback — see `params`/`env_params` below, which is how it's configured. `model` is required for `route: litellm` (there is no default open-weight/gateway model). +- `route` selects which backend the WHOLE evaluation side calls (`llm_judge`, `agent_judge`, and the simulator all share one resolved eval route per run — this isn't per-criterion), **decoupled from the agent's own route** (a Claude agent can be graded by a differently-backed judge, or vice versa). This is a backend *name* (`direct` / `bedrock` / `litellm`), not a route object. For `direct`/`bedrock` credentials are never read from the task, always from the matching environment variables (`ANTHROPIC_API_KEY` for `direct`, `AWS_BEARER_TOKEN_BEDROCK`/`AWS_REGION` for `bedrock`). An unconfigured or unknown backend name raises at dispatch rather than silently falling back. **`route: litellm` dispatches `llm_judge` through the `litellm` library** (the `coder-eval[litellm]` extra, `litellm.acompletion`) rather than assuming one wire protocol — a gateway-routed judge model (e.g. an Azure AI `/openai/v1` deployment) rarely speaks Anthropic Messages, so this lets `model` carry its own provider hint (e.g. `azure/gpt-5.6-luna`) and get that provider's actual request/response shape handled by the library. Unlike the other two backends, `route: litellm` has NO implicit env-var fallback — see `params`/`env_params` below, which is how it's configured. `model` is required for `route: litellm` (there is no default open-weight/gateway model). - `model` overrides the model that resolved route uses for **`llm_judge` only** — when the criterion itself leaves `model:` unset (precedence: an explicit per-criterion `model:` always wins; below that, `checker_context.api_route.model`; below that, the built-in `DEFAULT_JUDGE_MODEL`). This floor is deliberate and never the agent's own model — an unpinned judge must grade identically regardless of which model the agent under test is using, so `resolve_evaluation_route` never lets the agent's env-configured model (e.g. `BEDROCK_MODEL`) leak into `route.model` on its own; `route.model` is set only when this override was actually given. This works because every `ApiRoute` (`DirectRoute`/`BedrockRoute`/`LiteLLMRoute`) carries its own `model` field; the orchestrator bakes the override into the resolved route's `model` before any criterion runs, so `llm_judge` just reads `context.route.model` — it never reads `checker_context` directly. **`agent_judge` and the simulator do not honor this override** — `agent_judge`'s sub-agent model comes from the criterion's own `agent:` block (defaulted to a fixed judge model), and the simulator's model is pinned by `SimulationConfig.model` (see [Simulation](#simulation) below) — both independent of `checker_context.api_route.model` by design, for the same "measuring instrument stays fixed" reason. - `params`/`env_params` (**`route: litellm` only**) are how the call is actually configured — there is no fallback to the agent's own `LITELLM_BASE_URL`/`LITELLM_AUTH_TOKEN` env vars, since a gateway-routed judge model rarely reuses the agent's own LiteLLM proxy/credential. They also cover any of the dozens of other provider-specific kwargs `litellm.acompletion` accepts (`aws_access_key_id`, `vertex_project`, `api_version`, ...), which have no dedicated field on `LiteLLMRoute`: ```yaml @@ -1326,7 +1326,8 @@ checker_context: api_base: LITELLM_BASE_URL api_key: LITELLM_AUTH_TOKEN ``` - `params` is merged straight into the `litellm.acompletion(**kwargs)` call — litellm validates the param names itself, so there's no allowlist to keep in sync here. `env_params` maps a kwarg name to the *name* of an environment variable; the value is resolved right before the call, so no secret is ever written into task/experiment YAML — this is how an arbitrary provider's config, including secrets (IAM keys, an Azure AD token, a service-account path, ...), is representable without a dedicated field per provider. `env_params` is resolved after `params`, so it always wins for the same key. Rejected at task-load time if given without `route: litellm`. + `params` is merged straight into the `litellm.acompletion(**kwargs)` call — litellm validates the param names itself, so there's no allowlist to keep in sync here. `env_params` maps a kwarg name to the *name* of an environment variable; the value is resolved right before the call, so no secret is ever written into task/experiment YAML — this is how an arbitrary provider's config, including secrets (IAM keys, an Azure AD token, a service-account path, ...), is representable without a dedicated field per provider. `env_params` is resolved after `params`, so it always wins for the same key. Rejected at task-load time if given without `route: litellm`. For an Azure deployment, pin `api_version` via `params` to whatever API version the agent side is actually configured for (e.g. Codex's `CODEX_API_VERSION`) — the judge has no way to inherit it, and a mismatched version can hit a different shape of the same endpoint. + **`route: litellm` is `llm_judge`-only** — `agent_judge` and the simulator run as real Claude Code CLI subprocesses that speak the Anthropic Messages protocol, so pointing them at an arbitrary litellm-fronted gateway (which may speak an entirely different wire protocol) isn't representable. The orchestrator rejects the combination at resolution time (a clear error, not a silent misroute) if the task has an enabled `agent_judge` criterion or `simulation.enabled: true` alongside `route: litellm` — use `route: bedrock`/`direct` for those instead. `checker_context` merges shallow-per-namespace across `default_experiment.defaults.checker_context` → `experiment.defaults.checker_context` → `task.checker_context` → `variant.checker_context` (same 4-layer precedence as `agent`/`simulation`). So a judge-model A/B, or a judge-backend A/B, is a variant-level config change, not an edit to every task YAML. diff --git a/src/coder_eval/models/__init__.py b/src/coder_eval/models/__init__.py index f524b2bb..297f170c 100644 --- a/src/coder_eval/models/__init__.py +++ b/src/coder_eval/models/__init__.py @@ -180,6 +180,8 @@ DEFAULT_SIMULATOR_MODEL, NORMALIZED_CRITERION_ALIASES, REMOVED_CRITERION_TYPES, + ApiRouteContext, + CheckerContext, CriteriaCheckTiming, Dataset, PostRunCommand, @@ -187,7 +189,6 @@ ReferenceSource, SimulationConfig, TaskDefinition, - validate_checker_context_shape, ) # Telemetry @@ -359,6 +360,8 @@ "merge_strategy_of", # Tasks "TaskDefinition", + "ApiRouteContext", + "CheckerContext", "DEFAULT_SIMULATION_STOP_TOKEN", "DEFAULT_SIMULATOR_MODEL", "NORMALIZED_CRITERION_ALIASES", @@ -369,7 +372,6 @@ "PreRunCommand", "ReferenceSource", "SimulationConfig", - "validate_checker_context_shape", # Mutations "PromptPrefix", "PromptSuffix", diff --git a/src/coder_eval/models/tasks.py b/src/coder_eval/models/tasks.py index 18960f12..75c2d866 100644 --- a/src/coder_eval/models/tasks.py +++ b/src/coder_eval/models/tasks.py @@ -78,66 +78,52 @@ class UnknownTaskFieldWarning(DeprecationWarning): :data:`NORMALIZED_CRITERION_ALIASES`.""" -def validate_checker_context_shape(value: dict[str, dict[str, Any]]) -> None: - """Reject an unknown ``checker_context`` namespace/key rather than silently - no-op'ing a typo. ``checker_context`` has no Pydantic schema of its own (it's - an open, namespaced bag — see ``TaskDefinition.checker_context``'s field - description), so a misspelled ``api_rotue`` or ``rotue:`` would otherwise - pass through, get merged across every experiment layer, and simply never be - read by ``Orchestrator._eval_route_overrides`` — an override silently doing - nothing, with no error anywhere. Called both from - ``TaskDefinition.validate_checker_context`` (catches a typo on the task's own - YAML) and from ``orchestration/experiment.py::_resolve_checker_context`` - (catches one introduced only at the experiment-defaults/variant layer, which - bypasses the field validator since ``model_copy`` doesn't re-validate). +class ApiRouteContext(BaseModel): + """``checker_context.api_route`` — see ``TaskDefinition.checker_context`` for + the full picture. A typed replacement for what used to be a hand-validated + open dict: ``extra="forbid"`` catches an unknown key (e.g. a misspelled + ``rotue:``) at load time for free, and ``model``/``params``/``env_params`` + are now real types instead of ``Any`` — a YAML ``model: 5`` is rejected here + rather than silently ``str()``-ified into a model id downstream. """ - known_namespaces = {"api_route"} - unknown_namespaces = set(value) - known_namespaces - if unknown_namespaces: - msg = ( - f"checker_context has unknown namespace(s) {sorted(unknown_namespaces)}; " - f"known namespaces: {sorted(known_namespaces)}" - ) - raise ValueError(msg) - api_route = value.get("api_route") - if api_route is not None: - known_keys = {"route", "model", "params", "env_params"} - unknown_keys = set(api_route) - known_keys - if unknown_keys: - msg = ( - f"checker_context.api_route has unknown key(s) {sorted(unknown_keys)}; known keys: {sorted(known_keys)}" - ) - raise ValueError(msg) - route = api_route.get("route") - if route is not None: - try: - ApiBackend(route) - except ValueError as e: - valid = sorted(b.value for b in ApiBackend) - msg = f"checker_context.api_route.route {route!r} is not a known backend ({valid})" - raise ValueError(msg) from e - # `params`/`env_params` only ever reach the litellm judge transport (see - # invoke_litellm_judge_async) — on any other backend they'd be silently - # dropped, which is worse than a load-time error. - params = api_route.get("params") - env_params = api_route.get("env_params") - if (params is not None or env_params is not None) and route != "litellm": - msg = "checker_context.api_route.params/env_params require route: litellm" - raise ValueError(msg) - if params is not None and not isinstance(params, dict): - raise ValueError(f"checker_context.api_route.params must be a mapping, got {type(params).__name__}") - if env_params is not None: - if not isinstance(env_params, dict): - raise ValueError( - f"checker_context.api_route.env_params must be a mapping, got {type(env_params).__name__}" - ) - bad = {k: v for k, v in env_params.items() if not isinstance(k, str) or not isinstance(v, str)} - if bad: - msg = ( - f"checker_context.api_route.env_params must map param name -> ENV VAR NAME " - f"(both strings); got {bad!r}" - ) - raise ValueError(msg) + + model_config = ConfigDict(extra="forbid") + + route: ApiBackend | None = Field( + default=None, + description="Backend the WHOLE evaluation side (llm_judge/agent_judge/simulator) calls.", + ) + model: str | None = Field(default=None, description="Model override for the resolved route.") + params: dict[str, Any] | None = Field( + default=None, + description="`route: litellm` only — arbitrary passthrough kwargs to litellm.acompletion.", + ) + env_params: dict[str, str] | None = Field( + default=None, + description=( + "`route: litellm` only — maps a litellm.acompletion kwarg name to the ENV VAR NAME " + "(never the value) to resolve it from at call time." + ), + ) + + @model_validator(mode="after") + def _validate_litellm_only_fields(self) -> Self: + """``params``/``env_params`` only ever reach the litellm judge transport + (see ``invoke_litellm_judge_async``) — on any other backend they'd be + silently dropped, which is worse than a load-time error.""" + if (self.params is not None or self.env_params is not None) and self.route != ApiBackend.LITELLM: + raise ValueError("checker_context.api_route.params/env_params require route: litellm") + return self + + +class CheckerContext(BaseModel): + """Task-authored config for the success-checking side. See + ``TaskDefinition.checker_context``'s field description for the full picture. + """ + + model_config = ConfigDict(extra="forbid") + + api_route: ApiRouteContext | None = None class SimulationConfig(BaseModel): @@ -500,25 +486,26 @@ class TaskDefinition(BaseModel): # noqa: CE009 -- soft-launch: see _warn_on_unk strategy="replace", # not layer-merged today; replace = the engine default if it ever is description="List of criteria that must all pass for task success", ) - checker_context: dict[str, dict[str, Any]] = Field( - default_factory=dict, + checker_context: CheckerContext | None = Field( + default=None, description=( - "Task-authored config for the success-checking side, namespaced by reserved key. Currently " - "the only recognized namespace is `api_route`, e.g. `{api_route: {route: litellm, model: " - "gpt-5}}`: `route` selects the backend the WHOLE evaluation side (llm_judge, agent_judge, the " - "simulator) calls, decoupled from the agent's own route; `model` overrides the model that " - "route uses. Both are consumed by the orchestrator (`resolve_evaluation_route`) BEFORE " - "`CheckContext` is built and baked into the resolved route's own `model` field — no criterion " - "ever reads `checker_context` directly, only `CheckContext.route.model`. Credentials are " - "always resolved from environment variables, never from this field — EXCEPT `route: litellm`, " - "whose call is built entirely from `params`/`env_params` (no implicit fallback to the agent's " - "own LITELLM_BASE_URL/LITELLM_AUTH_TOKEN): `params` is an arbitrary passthrough dict of extra " - "kwargs to the underlying `litellm.acompletion` call (e.g. `{api_base: https://my-gateway/v1}`), " - "and `env_params` maps a kwarg name to the ENV VAR NAME (not the value!) to resolve it from at " - "call time (e.g. `{api_key: MY_GATEWAY_TOKEN, aws_access_key_id: AWS_ACCESS_KEY_ID}`) — so an " - "arbitrary provider's config, including secrets, is representable without ever putting one in " - "the task YAML. `model` is required when `route: litellm` (no default open-weight/gateway model). " - "Merged shallow-per-namespace across default -> experiment-defaults -> task -> variant." + "Task-authored config for the success-checking side. Currently carries only `api_route` " + "(see ApiRouteContext), e.g. `{api_route: {route: litellm, model: gpt-5}}`: `route` selects " + "the backend the WHOLE evaluation side (llm_judge, agent_judge, the simulator) calls, " + "decoupled from the agent's own route; `model` overrides the model that route uses. Both " + "are consumed by the orchestrator (`resolve_evaluation_route`) BEFORE `CheckContext` is " + "built and baked into the resolved route's own `model` field — no criterion ever reads " + "`checker_context` directly, only `CheckContext.route.model`. Credentials are always " + "resolved from environment variables, never from this field — EXCEPT `route: litellm`, " + "whose call is built entirely from `params`/`env_params` (no implicit fallback to the " + "agent's own LITELLM_BASE_URL/LITELLM_AUTH_TOKEN): `params` is an arbitrary passthrough " + "dict of extra kwargs to the underlying `litellm.acompletion` call (e.g. `{api_base: " + "https://my-gateway/v1}`), and `env_params` maps a kwarg name to the ENV VAR NAME (not " + "the value!) to resolve it from at call time (e.g. `{api_key: MY_GATEWAY_TOKEN, " + "aws_access_key_id: AWS_ACCESS_KEY_ID}`) — so an arbitrary provider's config, including " + "secrets, is representable without ever putting one in the task YAML. `model` is required " + "when `route: litellm` (no default open-weight/gateway model). Merged field-by-field " + "across default -> experiment-defaults -> task -> variant (same engine as `agent`/`simulation`)." ), ) run_limits: RunLimits | None = Field( @@ -798,17 +785,3 @@ def validate_success_criteria(cls, v: Any) -> Any: if not v: raise ValueError("At least one success criterion must be defined") return v - - @field_validator("checker_context") - @classmethod - def validate_checker_context(cls, v: dict[str, dict[str, Any]]) -> dict[str, dict[str, Any]]: - """Reject an unknown namespace/key rather than silently no-op'ing a typo. - - Only catches a typo already present on the TASK's own YAML — a typo - introduced solely at the experiment-defaults/variant layer bypasses this - (``model_copy`` doesn't re-validate), so ``_resolve_checker_context`` - (``orchestration/experiment.py``) calls the same shared checker after - merging, to catch those too. - """ - validate_checker_context_shape(v) - return v diff --git a/src/coder_eval/orchestration/experiment.py b/src/coder_eval/orchestration/experiment.py index d852315f..25d4ed58 100644 --- a/src/coder_eval/orchestration/experiment.py +++ b/src/coder_eval/orchestration/experiment.py @@ -19,6 +19,7 @@ from ..models import ( AgentConfig, BaseAgentConfig, + CheckerContext, ConfigLineageEntry, ExperimentDefinition, ExperimentResult, @@ -35,7 +36,6 @@ VariantAggregate, VariantResult, apply_prompt_mutations, - validate_checker_context_shape, ) from ..path_utils import build_task_run_dir from .config import BatchRunConfig @@ -178,7 +178,8 @@ def _resolve_checker_context( experiment: ExperimentDefinition, task: TaskDefinition, variant: ExperimentVariant, -) -> dict[str, dict[str, Any]]: + lineage: dict[str, ConfigLineageEntry], +) -> CheckerContext: """Merge ``checker_context`` across the 4-layer precedence chain. Precedence (lowest to highest): @@ -187,30 +188,33 @@ def _resolve_checker_context( 3. task.checker_context 4. variant.checker_context - Unlike ``agent``/``simulation``, ``checker_context`` is an open, namespaced bag - (not a fixed model) — a later layer's namespace merges shallowly onto the same - namespace from an earlier layer (per-key overwrite within the namespace), - rather than replacing the whole namespace. A namespace absent from a layer is - left untouched by that layer. Currently the only recognized namespace is - ``api_route`` (``route``/``model``). + Mirrors ``_resolve_simulation``: runs through the generic resolver + (``merge_layers`` over ``CheckerContext``) so a later layer's fields + overwrite earlier ones (nested ``api_route`` merges field-by-field, the + type-aware default for a nested ``BaseModel``), then the merged dict is + validated by constructing a ``CheckerContext`` — which raises a helpful, + typed error (unknown key, wrong value type, `params`/`env_params` without + `route: litellm`) rather than a hand-rolled shape check. Lineage stays + coarse: a single ``checker_context`` entry crediting the most-specific source. """ - layers: list[dict[str, dict[str, Any]] | None] = [ - default_experiment.defaults.checker_context if default_experiment.defaults else None, - experiment.defaults.checker_context if experiment.defaults else None, - task.checker_context or None, - variant.checker_context, + specs: list[tuple[ConfigSource, dict[str, Any] | None]] = [ + ("default", default_experiment.defaults.checker_context if default_experiment.defaults else None), + ("experiment-defaults", experiment.defaults.checker_context if experiment.defaults else None), + ("task", task.checker_context.model_dump(exclude_unset=True) if task.checker_context else None), + ("variant", variant.checker_context), ] - merged: dict[str, dict[str, Any]] = {} - for layer in layers: - if not layer: - continue - for namespace, patch in layer.items(): - merged[namespace] = {**merged.get(namespace, {}), **patch} - # Catches a typo introduced only at the experiment-defaults/variant layer — - # TaskDefinition's own field validator only ever sees the task's raw YAML, - # not this merged result (model_copy doesn't re-validate). - validate_checker_context_shape(merged) - return merged + layers = [Layer(source=src, patch=patch) for src, patch in specs if patch is not None] + if not layers: + return CheckerContext() + + merged = merge_layers((CheckerContext,), layers, lineage_root="checker_context") + resolved = CheckerContext(**merged) + + most_specific: ConfigSource | None = next((src for src, patch in reversed(specs) if patch is not None), None) + if most_specific is not None: + lineage["checker_context"] = ConfigLineageEntry(value=merged, source=most_specific) + + return resolved def _resolve_repeats( @@ -487,7 +491,7 @@ def _add_rl(rl: RunLimits | None, source: ConfigSource) -> None: # Mirrors agent merge semantics — a later layer's keys overwrite earlier ones, and # the final dict is validated by building a SimulationConfig from it. resolved_simulation = _resolve_simulation(default_experiment, experiment, task, variant, lineage) - resolved_checker_context = _resolve_checker_context(default_experiment, experiment, task, variant) + resolved_checker_context = _resolve_checker_context(default_experiment, experiment, task, variant, lineage) # Build resolved task (copy with overrides) resolved_task = task.model_copy( diff --git a/src/coder_eval/orchestrator.py b/src/coder_eval/orchestrator.py index 4c80a849..043df8e3 100644 --- a/src/coder_eval/orchestrator.py +++ b/src/coder_eval/orchestrator.py @@ -35,6 +35,7 @@ CONTAINER_REFERENCE_DIR, DEFAULT_STOP_EARLY_GATE_THRESHOLD, ROUTE_NAMES, + AgentJudgeCriterion, AgentKind, ApiRoute, BedrockRoute, @@ -1473,14 +1474,14 @@ def _eval_route_overrides(self) -> EvalRouteOverrides: env-configured-default behavior. NOT currently ``-D``-reachable — task/variant YAML only. """ - api_route = self.task.checker_context.get("api_route", {}) - backend = api_route.get("route") - model = api_route.get("model") + api_route = self.task.checker_context.api_route if self.task.checker_context else None + if api_route is None: + return EvalRouteOverrides(backend=None, model=None, params=None, env_params=None) return EvalRouteOverrides( - backend=str(backend) if backend is not None else None, - model=str(model) if model is not None else None, - params=api_route.get("params"), - env_params=api_route.get("env_params"), + backend=api_route.route.value if api_route.route is not None else None, + model=api_route.model, + params=api_route.params, + env_params=api_route.env_params, ) def _resolve_routes(self) -> None: @@ -1500,9 +1501,40 @@ def _resolve_routes(self) -> None: params_override=overrides.params, env_params_override=overrides.env_params, ) + self._reject_litellm_eval_route_if_unsupported() logger.info("API routing: %s", _format_routing(self.route, self.task.agent.model if self.task.agent else None)) self.success_checker = SuccessChecker(self.sandbox, route=self.eval_route) + def _reject_litellm_eval_route_if_unsupported(self) -> None: + """``checker_context.api_route.route: litellm`` dispatches ``llm_judge`` + through the ``litellm`` library (protocol-agnostic — see + ``invoke_litellm_judge_async``'s module docstring), but ``agent_judge`` + and the simulator run as real Claude Code CLI subprocesses that speak + the Anthropic Messages protocol only. Handing them a ``LiteLLMRoute`` + built from arbitrary ``params``/``env_params`` (which may front an + OpenAI-/Vertex-shaped gateway with no Anthropic-compatible endpoint at + all) would either misroute onto the AGENT's own unrelated LiteLLM + settings or fail with a confusing SDK-level error — the exact + misrouting ``LiteLLMRoute``'s own docstring says must never happen. + Reject the combination loudly at resolution time instead. + """ + if not isinstance(self.eval_route, LiteLLMRoute): + return + offenders: list[str] = [] + if any(isinstance(c, AgentJudgeCriterion) and c.enabled for c in self.task.success_criteria): + offenders.append("an enabled agent_judge criterion") + if self.task.simulation is not None and self.task.simulation.enabled: + offenders.append("simulation.enabled") + if offenders: + named = " and ".join(offenders) + msg = ( + f"checker_context.api_route.route: litellm is llm_judge-only (it dispatches through the " + f"litellm library in-process, not a real Claude Code subprocess), but this task also has " + f"{named}, which run as Claude Code sub-agents requiring an Anthropic-compatible endpoint. " + f"Use route: bedrock/direct instead, or remove/disable {named}." + ) + raise ValueError(msg) + def _record_route_environment_info(self) -> None: """Persist resolved route + judge transport into ``result.environment_info``. diff --git a/tests/test_litellm_judge_live.py b/tests/test_litellm_judge_live.py new file mode 100644 index 00000000..550fe579 --- /dev/null +++ b/tests/test_litellm_judge_live.py @@ -0,0 +1,69 @@ +"""Live integration test for the ``checker_context.api_route.route: litellm`` +judge transport — hits a real gateway via ``litellm.acompletion()``. + +Reuses the same ``CODEX_API_KEY``/``CODEX_BASE_URL``/``CODEX_MODEL`` secrets the +Codex live tests already exercise (a real OpenAI-protocol-compatible deployment) +instead of a dedicated new secret, so this regression check runs in CI for free. +Addresses the PR #137 review gap: "Nothing in the repo exercises the feature. +No task, no experiment variant." — this is the litellm-route equivalent of +``tests/test_codex_agent_live.py``, at the invoker level rather than a full +``coder-eval run`` (no sandbox / agent turn needed to exercise the transport). + +Requirements: + - The ``[litellm]`` extra installed (``uv sync --extra litellm``). + - ``CODEX_API_KEY``, ``CODEX_BASE_URL``, ``CODEX_MODEL`` in the environment. + +Run with: ``pytest -m live``. +""" + +from __future__ import annotations + +import os + +import pytest + + +pytest.importorskip("litellm") + +from coder_eval.evaluation.judge_litellm import invoke_litellm_judge_async +from coder_eval.evaluation.verdict_tool import SUBMIT_VERDICT_ANTHROPIC_TOOL, extract_verdict_from_openai_response +from coder_eval.models.routing import LiteLLMRoute + + +_live = pytest.mark.live + + +def _have_creds() -> bool: + return bool(os.getenv("CODEX_API_KEY") and os.getenv("CODEX_BASE_URL") and os.getenv("CODEX_MODEL")) + + +_skip_reason = "Live litellm judge test needs [litellm] extra + CODEX_API_KEY/CODEX_BASE_URL/CODEX_MODEL" +pytestmark = [_live, pytest.mark.skipif(not _have_creds(), reason=_skip_reason)] + + +@_live +async def test_litellm_judge_route_scores_real_gateway_call(): + """Round-trips a real forced ``submit_verdict`` tool call through + ``litellm.acompletion`` against the same gateway/model the Codex live tests + use, with ``api_base``/``api_key`` resolved purely from ``env_params`` (no + LITELLM_BASE_URL/LITELLM_AUTH_TOKEN involved) — the exact shape + ``checker_context.api_route.route: litellm`` produces.""" + model = os.environ["CODEX_MODEL"] + route = LiteLLMRoute( + model=model, + env_params={"api_base": "CODEX_BASE_URL", "api_key": "CODEX_API_KEY"}, + ) + response = await invoke_litellm_judge_async( + route=route, + model=model, + system="You are a strict grader. Call the submit_verdict tool exactly once.", + user=( + 'Score 1.0 if 2 + 2 = 4, else 0.0. Call submit_verdict with score=1.0, rationale="arithmetic is correct".' + ), + max_tokens=200, + tool_spec=SUBMIT_VERDICT_ANTHROPIC_TOOL, + ) + verdict, err = extract_verdict_from_openai_response(response) + assert err is None, f"judge did not call submit_verdict: {err}" + assert verdict is not None + assert verdict.score == 1.0 diff --git a/tests/test_litellm_route.py b/tests/test_litellm_route.py index 83ea40c2..48256e19 100644 --- a/tests/test_litellm_route.py +++ b/tests/test_litellm_route.py @@ -243,36 +243,41 @@ def test_unknown_backend_raises(self, monkeypatch): resolve_evaluation_route(settings, agent_route, backend_override="not-a-backend") -class TestValidateCheckerContextShape: - """validate_checker_context_shape() — the load-time guard for - checker_context (previously 0% covered, per the PR #137 review).""" +class TestCheckerContextModel: + """CheckerContext/ApiRouteContext — the typed replacement for the old + hand-validated open dict (previously 0% covered, per the PR #137 review). + ``extra="forbid"`` + real field types now do what the hand-rolled + ``validate_checker_context_shape`` used to.""" @staticmethod def _validate(value): - from coder_eval.models.tasks import validate_checker_context_shape + from coder_eval.models import CheckerContext - return validate_checker_context_shape(value) + return CheckerContext(**value) def test_accepts_empty(self): self._validate({}) def test_accepts_route_and_model(self): - self._validate({"api_route": {"route": "bedrock", "model": "claude-haiku-4-5"}}) + cc = self._validate({"api_route": {"route": "bedrock", "model": "claude-haiku-4-5"}}) + assert cc.api_route is not None + assert cc.api_route.route == ApiBackend.BEDROCK + assert cc.api_route.model == "claude-haiku-4-5" def test_rejects_unknown_namespace(self): - with pytest.raises(ValueError, match="unknown namespace"): + with pytest.raises(ValueError, match=r"[Ee]xtra"): self._validate({"api_rotue": {"route": "bedrock"}}) def test_rejects_unknown_api_route_key(self): - with pytest.raises(ValueError, match="unknown key"): + with pytest.raises(ValueError, match=r"[Ee]xtra"): self._validate({"api_route": {"rotue": "bedrock"}}) def test_rejects_unknown_backend_name(self): - with pytest.raises(ValueError, match="not a known backend"): + with pytest.raises(ValueError): self._validate({"api_route": {"route": "not-a-backend"}}) def test_accepts_params_and_env_params_with_litellm_route(self): - self._validate( + cc = self._validate( { "api_route": { "route": "litellm", @@ -281,6 +286,9 @@ def test_accepts_params_and_env_params_with_litellm_route(self): } } ) + assert cc.api_route is not None + assert cc.api_route.params == {"aws_region_name": "eu-north-1"} + assert cc.api_route.env_params == {"api_key": "MY_ENV_VAR"} def test_rejects_params_without_litellm_route(self): with pytest.raises(ValueError, match="require route: litellm"): @@ -291,17 +299,22 @@ def test_rejects_env_params_without_litellm_route(self): self._validate({"api_route": {"env_params": {"api_key": "MY_ENV_VAR"}}}) def test_rejects_non_dict_params(self): - with pytest.raises(ValueError, match="params must be a mapping"): + with pytest.raises(ValueError): self._validate({"api_route": {"route": "litellm", "params": "not-a-dict"}}) def test_rejects_non_dict_env_params(self): - with pytest.raises(ValueError, match="env_params must be a mapping"): + with pytest.raises(ValueError): self._validate({"api_route": {"route": "litellm", "env_params": "not-a-dict"}}) def test_rejects_non_string_env_params_values(self): - with pytest.raises(ValueError, match="must map param name -> ENV VAR NAME"): + with pytest.raises(ValueError): self._validate({"api_route": {"route": "litellm", "env_params": {"api_key": 123}}}) + def test_rejects_non_string_model(self): + """A YAML `model: 5` must be rejected here, not str()-ified downstream.""" + with pytest.raises(ValueError): + self._validate({"api_route": {"route": "bedrock", "model": 5}}) + class TestEvalRouteWiring: """The orchestrator must hand the simulated user the eval_route (constant diff --git a/tests/test_orchestrator.py b/tests/test_orchestrator.py index 3fbce30a..511464d1 100644 --- a/tests/test_orchestrator.py +++ b/tests/test_orchestrator.py @@ -105,6 +105,77 @@ def test_record_route_environment_info_direct_none_serialized_as_string(tmp_path assert orchestrator.result.environment_info["judge_transport"] == "none" +class TestRejectLitellmEvalRouteIfUnsupported: + """checker_context.api_route.route: litellm dispatches llm_judge through the + litellm library in-process, but agent_judge and the simulator are real + Claude Code CLI subprocesses that speak Anthropic Messages only — handing + them an arbitrary litellm-fronted route would misroute or fail silently. + PR #137 review: 'route: litellm breaks the simulator and agent_judge.'""" + + @staticmethod + def _orchestrator_with_criteria(tmp_path: Path, success_criteria, simulation=None) -> Orchestrator: + task_file = Path("tasks/hello_date.yaml") + task, _ = load_task(task_file) + task = task.model_copy(update={"success_criteria": success_criteria, "simulation": simulation}) + orchestrator = Orchestrator(task=task, run_dir=tmp_path / "run", variant_id="t") + orchestrator.eval_route = LiteLLMRoute(model="gpt-5.6-luna") + return orchestrator + + def test_llm_judge_only_is_fine(self, tmp_path): + from coder_eval.models import LLMJudgeCriterion + + orchestrator = self._orchestrator_with_criteria(tmp_path, [LLMJudgeCriterion(description="x", prompt="grade")]) + orchestrator._reject_litellm_eval_route_if_unsupported() # no raise + + def test_rejects_enabled_agent_judge(self, tmp_path): + from coder_eval.models import AgentJudgeCriterion + + orchestrator = self._orchestrator_with_criteria( + tmp_path, [AgentJudgeCriterion(description="x", prompt="grade")] + ) + with pytest.raises(ValueError, match="agent_judge"): + orchestrator._reject_litellm_eval_route_if_unsupported() + + def test_allows_disabled_agent_judge(self, tmp_path): + from coder_eval.models import AgentJudgeCriterion + + orchestrator = self._orchestrator_with_criteria( + tmp_path, [AgentJudgeCriterion(description="x", prompt="grade", enabled=False)] + ) + orchestrator._reject_litellm_eval_route_if_unsupported() # no raise + + def test_rejects_enabled_simulation(self, tmp_path): + from coder_eval.models import LLMJudgeCriterion, SimulationConfig + + orchestrator = self._orchestrator_with_criteria( + tmp_path, + [LLMJudgeCriterion(description="x", prompt="grade")], + simulation=SimulationConfig(enabled=True, persona="p", goal="g"), + ) + with pytest.raises(ValueError, match=r"simulation\.enabled"): + orchestrator._reject_litellm_eval_route_if_unsupported() + + def test_allows_disabled_simulation(self, tmp_path): + from coder_eval.models import LLMJudgeCriterion, SimulationConfig + + orchestrator = self._orchestrator_with_criteria( + tmp_path, + [LLMJudgeCriterion(description="x", prompt="grade")], + simulation=SimulationConfig(enabled=False, persona="p", goal="g"), + ) + orchestrator._reject_litellm_eval_route_if_unsupported() # no raise + + def test_non_litellm_eval_route_is_never_checked(self, tmp_path): + """Bedrock/Direct eval routes are always fine with agent_judge/simulation.""" + from coder_eval.models import AgentJudgeCriterion + + orchestrator = self._orchestrator_with_criteria( + tmp_path, [AgentJudgeCriterion(description="x", prompt="grade")] + ) + orchestrator.eval_route = BedrockRoute(region="eu-north-1") + orchestrator._reject_litellm_eval_route_if_unsupported() # no raise + + def test_record_route_environment_info_bedrock(tmp_path): orchestrator = _make_orchestrator_with_route( tmp_path, BedrockRoute(region="eu-north-1", model="eu.anthropic.claude-sonnet-4-6") @@ -122,6 +193,7 @@ def test_record_route_environment_info_litellm_records_host_only_no_secret(tmp_p from coder_eval.config import settings monkeypatch.setattr(settings, "litellm_base_url", "http://localhost:4000") + monkeypatch.setattr(settings, "litellm_auth_token", "sk-super-secret") orchestrator = _make_orchestrator_with_route( tmp_path, LiteLLMRoute(model="zai.glm-5"), From a15970bbd9636d3eaa941b19ab441fc260587e7d Mon Sep 17 00:00:00 2001 From: Akshaya Shanbhogue Date: Wed, 26 Aug 2026 15:50:43 -0700 Subject: [PATCH 8/9] ci: retrigger checks (previous push did not trigger CI) From cfd821747c40fc16337ad3deb5777a27c2282398 Mon Sep 17 00:00:00 2001 From: Akshaya Shanbhogue Date: Wed, 26 Aug 2026 15:54:13 -0700 Subject: [PATCH 9/9] ci: retrigger checks (GH Actions appeared stalled repo-wide)