diff --git a/.github/workflows/pr-checks.yml b/.github/workflows/pr-checks.yml index 533ee43b..94cce6d3 100644 --- a/.github/workflows/pr-checks.yml +++ b/.github/workflows/pr-checks.yml @@ -80,7 +80,7 @@ jobs: pip install uv - name: Install project dependencies (hash-verified from uv.lock) - run: uv sync --frozen --extra dev --extra uipath --extra codex + run: uv sync --frozen --extra dev --extra uipath --extra codex --extra litellm # PHASE 1: Fast checks (fail early) - name: Check code formatting (ruff format) @@ -385,7 +385,7 @@ jobs: pip install uv - name: Install project dependencies (hash-verified from uv.lock) - run: uv sync --frozen --extra dev --extra uipath --extra codex + run: uv sync --frozen --extra dev --extra uipath --extra codex --extra litellm - name: Check code formatting (ruff format) run: .venv/Scripts/ruff format --check src/ tests/ @@ -843,8 +843,8 @@ jobs: python -m pip install --upgrade "pip>=26.2" # PYSEC-2026-3721 fix floor pip install uv - - name: Install project dependencies (with codex extra) - run: uv sync --frozen --extra dev --extra uipath --extra codex + - name: Install project dependencies (with codex + litellm extras) + run: uv sync --frozen --extra dev --extra uipath --extra codex --extra litellm - name: Verify required secrets are present run: | @@ -854,14 +854,17 @@ jobs: fi echo "CODEX_API_KEY present." - - name: Run Codex live tests + - name: Run Codex + litellm-judge live tests run: | mkdir -p tmp # Run serially: `-n0` overrides the global `-n auto` (addopts). # Parallel xdist workers share ~/.codex and race the Codex SQLite # state migration (`duplicate column name: thread_id`); serial init - # migrates the fresh DB exactly once. - .venv/bin/pytest tests/test_codex_agent_live.py \ + # migrates the fresh DB exactly once. test_litellm_judge_live.py + # reuses these same CODEX_* secrets to exercise + # checker_context.api_route.route: litellm end-to-end (PR #137 + # review: "nothing in the repo exercises the feature"). + .venv/bin/pytest tests/test_codex_agent_live.py tests/test_litellm_judge_live.py \ -m live -n0 -v --tb=short --strict-markers -ra \ --junit-xml=tmp/junit-codex-live.xml @@ -878,7 +881,7 @@ jobs: passed = total - skipped - errors - failures print(f"codex-live passed={passed} skipped={skipped} errors={errors} failures={failures}") if passed < 1: - sys.exit("test_codex_agent_live.py reported zero PASSED tests (missing API key / silent skip?)") + sys.exit("Live Codex/litellm-judge tests reported zero PASSED tests (missing API key / silent skip?)") PY - name: Upload Codex live-test artifacts on failure diff --git a/docs/AB_EXPERIMENTS.md b/docs/AB_EXPERIMENTS.md index 4392d7f6..0e73d6a3 100644 --- a/docs/AB_EXPERIMENTS.md +++ b/docs/AB_EXPERIMENTS.md @@ -130,6 +130,7 @@ From `ExperimentVariant` (`coder_eval/models/experiment.py`): | `initial_prompt_file` | str | Prompt replacement loaded from a file | | `run_limits` | block | Per-key cap overrides (`max_turns`, `task_timeout`, token/USD budgets) | | `driver` | `tempdir`/`docker` | Sandbox driver — enables tempdir-vs-docker arms | +| `checker_context` | dict | Backend/model override for the evaluation side (judge, simulator) — see [Checker Context](TASK_DEFINITION_GUIDE.md#checker-context); **not** currently `-D`-reachable | The `agent` dict is the lever for most A/B tests. Anything on `AgentConfig` is fair game: `model`, `permission_mode`, `allowed_tools`, `disallowed_tools`, diff --git a/docs/DIALOG_MODE.md b/docs/DIALOG_MODE.md index 0b4bad29..622c3c3b 100644 --- a/docs/DIALOG_MODE.md +++ b/docs/DIALOG_MODE.md @@ -60,9 +60,11 @@ The mechanics: - The simulator is a **tools-disabled Claude Code agent** with `allowed_tools: []`, an explicit deny-list, and no plugins or settings sources. It is pure text-in / text-out, and it **cannot see the sandbox** — no files, no terminal, no agent reasoning. Only what the agent writes in the chat. -- The simulator shares the coding agent's resolved `ApiRoute`, so backend and model come from the - run's routing (`--backend direct` / `--backend bedrock`) rather than from the `simulation:` block. - There is no model field here to set. +- The simulator runs on the run's resolved *evaluation* `ApiRoute` — the coding agent's own route + (`--backend direct` / `--backend bedrock`) unless `checker_context.api_route.route` overrides it + (see [Checker Context](TASK_DEFINITION_GUIDE.md#checker-context)). The **model** is separately + pinned by `simulation.model` (see [Simulation](TASK_DEFINITION_GUIDE.md#simulation)), not inherited + from the route — so an A/B varying the subject model doesn't silently vary the simulated user too. **Agent-kind constraint.** The *subject* agent can be any registered kind — the dialog driver only calls the agent's `communicate()`, so Codex and plugin agents work. The *simulator*, however, is diff --git a/docs/TASK_DEFINITION_GUIDE.md b/docs/TASK_DEFINITION_GUIDE.md index a16edb43..b08a9bf4 100644 --- a/docs/TASK_DEFINITION_GUIDE.md +++ b/docs/TASK_DEFINITION_GUIDE.md @@ -35,6 +35,7 @@ Complete reference for defining evaluation tasks in Coder Eval. - [llm_judge](#llm_judge) - [agent_judge](#agent_judge) - [skill_triggered](#skill_triggered) +- [Checker Context](#checker-context) - [Reference Solutions](#reference-solutions) - [Pre-Run Commands](#pre-run-commands) - [Post-Run Commands](#post-run-commands) @@ -63,6 +64,7 @@ reference: { ... } # Optional reference solution (a directory pre_run: [ ... ] # Optional pre-run commands (before agent starts) post_run: [ ... ] # Optional post-run commands dataset: { ... } # Optional dataset fan-out (one task -> N row-tasks) +checker_context: { ... } # Optional: backend/model for the evaluation side (judge, simulator) ``` ### `dataset` @@ -1295,6 +1297,40 @@ Observed label is `"yes"` when either signal is found, else `"no"`. Expected lab **Typical pattern.** Label each dataset row with its true skill (`expected_skill`, `""` for negatives) and stack one `skill_triggered` criterion per skill against the same dataset — each gets its own confusion matrix from the same agent traces. This is the natural companion to a skill A/B experiment (skill plugin on vs. off); see the [A/B Experiment Guide](AB_EXPERIMENTS.md#recipe-ab-a-skill). +## Checker Context + +`checker_context` carries task-authored config for the success-checking side, namespaced by reserved key. Currently the only recognized namespace is **`api_route`**: + +```yaml +success_criteria: + - type: llm_judge + prompt: "Grade the fix for correctness." + +checker_context: + api_route: + route: bedrock # which backend the whole eval side (llm_judge/agent_judge/simulator) uses + model: claude-haiku-4-5 # model override for that route +``` + +- `route` selects which backend the WHOLE evaluation side calls (`llm_judge`, `agent_judge`, and the simulator all share one resolved eval route per run — this isn't per-criterion), **decoupled from the agent's own route** (a Claude agent can be graded by a differently-backed judge, or vice versa). This is a backend *name* (`direct` / `bedrock` / `litellm`), not a route object. For `direct`/`bedrock` credentials are never read from the task, always from the matching environment variables (`ANTHROPIC_API_KEY` for `direct`, `AWS_BEARER_TOKEN_BEDROCK`/`AWS_REGION` for `bedrock`). An unconfigured or unknown backend name raises at dispatch rather than silently falling back. **`route: litellm` dispatches `llm_judge` through the `litellm` library** (the `coder-eval[litellm]` extra, `litellm.acompletion`) rather than assuming one wire protocol — a gateway-routed judge model (e.g. an Azure AI `/openai/v1` deployment) rarely speaks Anthropic Messages, so this lets `model` carry its own provider hint (e.g. `azure/gpt-5.6-luna`) and get that provider's actual request/response shape handled by the library. Unlike the other two backends, `route: litellm` has NO implicit env-var fallback — see `params`/`env_params` below, which is how it's configured. `model` is required for `route: litellm` (there is no default open-weight/gateway model). +- `model` overrides the model that resolved route uses for **`llm_judge` only** — when the criterion itself leaves `model:` unset (precedence: an explicit per-criterion `model:` always wins; below that, `checker_context.api_route.model`; below that, the built-in `DEFAULT_JUDGE_MODEL`). This floor is deliberate and never the agent's own model — an unpinned judge must grade identically regardless of which model the agent under test is using, so `resolve_evaluation_route` never lets the agent's env-configured model (e.g. `BEDROCK_MODEL`) leak into `route.model` on its own; `route.model` is set only when this override was actually given. This works because every `ApiRoute` (`DirectRoute`/`BedrockRoute`/`LiteLLMRoute`) carries its own `model` field; the orchestrator bakes the override into the resolved route's `model` before any criterion runs, so `llm_judge` just reads `context.route.model` — it never reads `checker_context` directly. **`agent_judge` and the simulator do not honor this override** — `agent_judge`'s sub-agent model comes from the criterion's own `agent:` block (defaulted to a fixed judge model), and the simulator's model is pinned by `SimulationConfig.model` (see [Simulation](#simulation) below) — both independent of `checker_context.api_route.model` by design, for the same "measuring instrument stays fixed" reason. +- `params`/`env_params` (**`route: litellm` only**) are how the call is actually configured — there is no fallback to the agent's own `LITELLM_BASE_URL`/`LITELLM_AUTH_TOKEN` env vars, since a gateway-routed judge model rarely reuses the agent's own LiteLLM proxy/credential. They also cover any of the dozens of other provider-specific kwargs `litellm.acompletion` accepts (`aws_access_key_id`, `vertex_project`, `api_version`, ...), which have no dedicated field on `LiteLLMRoute`: + ```yaml + checker_context: + api_route: + route: litellm + model: azure/gpt-5.6-luna + params: # arbitrary literal passthrough kwargs to litellm.acompletion + api_version: "2024-05-01" + env_params: # param name -> ENV VAR NAME (never the secret itself) + api_base: LITELLM_BASE_URL + api_key: LITELLM_AUTH_TOKEN + ``` + `params` is merged straight into the `litellm.acompletion(**kwargs)` call — litellm validates the param names itself, so there's no allowlist to keep in sync here. `env_params` maps a kwarg name to the *name* of an environment variable; the value is resolved right before the call, so no secret is ever written into task/experiment YAML — this is how an arbitrary provider's config, including secrets (IAM keys, an Azure AD token, a service-account path, ...), is representable without a dedicated field per provider. `env_params` is resolved after `params`, so it always wins for the same key. Rejected at task-load time if given without `route: litellm`. For an Azure deployment, pin `api_version` via `params` to whatever API version the agent side is actually configured for (e.g. Codex's `CODEX_API_VERSION`) — the judge has no way to inherit it, and a mismatched version can hit a different shape of the same endpoint. + **`route: litellm` is `llm_judge`-only** — `agent_judge` and the simulator run as real Claude Code CLI subprocesses that speak the Anthropic Messages protocol, so pointing them at an arbitrary litellm-fronted gateway (which may speak an entirely different wire protocol) isn't representable. The orchestrator rejects the combination at resolution time (a clear error, not a silent misroute) if the task has an enabled `agent_judge` criterion or `simulation.enabled: true` alongside `route: litellm` — use `route: bedrock`/`direct` for those instead. + +`checker_context` merges shallow-per-namespace across `default_experiment.defaults.checker_context` → `experiment.defaults.checker_context` → `task.checker_context` → `variant.checker_context` (same 4-layer precedence as `agent`/`simulation`). So a judge-model A/B, or a judge-backend A/B, is a variant-level config change, not an edit to every task YAML. + ## Reference Solutions A reference solution is always a **directory**, given relative to the task YAML's own directory: @@ -1577,7 +1613,7 @@ simulation: | `check_criteria` | `end_of_dialog` | `end_of_dialog`, `every_turn`, or `both`. | | `model` | `anthropic.claude-sonnet-4-6` | Model that plays the simulated user. Auto-translated to the run's backend (Bedrock inference profile / bare Anthropic alias), the same way [`llm_judge`](#llm_judge)'s `model` is. | -The simulator runs as a tools-disabled Claude Code agent sharing the coding agent's `ApiRoute`, so temperature and sampling are resolved at the route level (same `-b` flag as the coding agent) and are not configured on this block. The **model is not**: it is pinned by `model` above. Inheriting it from the route meant `BEDROCK_MODEL` decided who the simulated user was, so an A/B varying the subject model silently varied its interlocutor too. Hold `model` fixed across variants for the same reason you hold a judge model fixed — the simulator is part of the measuring instrument, not the thing being measured. +The simulator runs as a tools-disabled Claude Code agent on the run's resolved *evaluation* `ApiRoute` (the coding agent's own route unless [`checker_context.api_route.route`](#checker-context) overrides it), so temperature and sampling are resolved at the route level (same `-b` flag as the coding agent by default) and are not configured on this block. The **model is not**: it is pinned by `model` above. Inheriting it from the route meant `BEDROCK_MODEL` decided who the simulated user was, so an A/B varying the subject model silently varied its interlocutor too. Hold `model` fixed across variants for the same reason you hold a judge model fixed — the simulator is part of the measuring instrument, not the thing being measured. **Semantics:** diff --git a/plugins/coder-eval/reference/criteria.md b/plugins/coder-eval/reference/criteria.md index ba5441d4..a946bc07 100644 --- a/plugins/coder-eval/reference/criteria.md +++ b/plugins/coder-eval/reference/criteria.md @@ -209,7 +209,7 @@ Optional: | `include_tool_calls` | When true, include a summary of the latest agent turn's tool calls (via summarize_commands). No-op when turn_records is unavailable. | | `include_dialog` | When true, include the full user<->agent conversation across all turns in the judge prompt. In simulation mode the user side is generated by an LLM simulator and may invent premises — the judge should treat any claim made only by the simulated user as possibly fabricated, and not penalize the agent for going along with it unless the task description contradicts it. | | `max_dialog_chars` | Aggregate cap on dialog text rendered into the judge prompt. Prevents an N-turn simulation from blowing out the judge's context window. Per-message truncation uses max_file_chars; trailing turns are dropped when this aggregate budget is exceeded (a degraded note is recorded). | -| `model` | Judge model id (e.g. 'anthropic.claude-sonnet-4-6'). On a BedrockRoute / DirectRoute the value is auto-translated: trailing '-vN[:M]' suffixes and the 'anthropic.' prefix are stripped where the backend doesn't accept them; on Bedrock the cross-region inference-profile prefix is added based on AWS_REGION. | +| `model` | Judge model id (e.g. 'anthropic.claude-sonnet-4-6'). Leave unset to fall back to checker_context.api_route.model when set, else the built-in default ('anthropic.claude-sonnet-4-6') — the fallback is never the agent's own model, so an unpinned judge grades identically across agent-model A/Bs. On a BedrockRoute / DirectRoute the value is auto-translated: trailing '-vN[:M]' suffixes and the 'anthropic.' prefix are stripped where the backend doesn't accept them; on Bedrock the cross-region inference-profile prefix is added based on AWS_REGION. | | `temperature` | Sampling temperature for the judge model. 0.0 keeps grading deterministic. | | `max_tokens` | Output token cap. Defaults to 2000 — large enough for the verbose verdict (score + rationale + a handful of findings) without runaway. | | `max_file_chars` | Per-file content truncation applied before building the prompt. | diff --git a/pyproject.toml b/pyproject.toml index a438018d..ee3e280e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -89,6 +89,19 @@ dev = [ uipath = [ "uipath>=2.10.31", ] +# Optional extra that enables the `checker_context.api_route.route: litellm` +# judge backend (llm_judge only) via the `litellm` library's `acompletion` — +# it normalizes provider-specific quirks (Azure AI's api_base/api_key shape, +# max_tokens vs max_completion_tokens naming, unsupported-param drops, ...) +# so the judge transport doesn't hand-roll per-provider HTTP. NOT the same +# thing as the LiteLLM PROXY (litellm/start-litellm.sh) the AGENT's own +# `route: litellm` points at over HTTP — this extra calls the library +# in-process. Without this extra, the framework still installs and runs; +# the litellm-route judge path fails at dispatch with a clear hint pointing +# back here. +litellm = [ + "litellm>=1.95.0,<2.0.0", +] # Optional extra that enables Codex agent support: # - CodexAgent implementation using official openai-codex SDK # Without this extra, the framework still installs and runs; Codex-dependent diff --git a/src/coder_eval/agents/claude_code_agent.py b/src/coder_eval/agents/claude_code_agent.py index e23f69ea..a7690ed5 100644 --- a/src/coder_eval/agents/claude_code_agent.py +++ b/src/coder_eval/agents/claude_code_agent.py @@ -36,6 +36,7 @@ from coder_eval.agents._logging import PrefixedAdapter, log_raw_sdk_event from coder_eval.agents.registry import AgentRegistry from coder_eval.agents.watchdog import ThreadedWatchdog +from coder_eval.config import settings from coder_eval.errors import ( TurnTimeoutError, format_timeout_reason, @@ -792,9 +793,16 @@ def _build_sdk_env( match route: case BedrockRoute() as br: + # `or ""` rather than asserting non-None here (unlike judge_bedrock.py's + # invoke_bedrock_judge_async): reaching a BedrockRoute at all already implies + # validate_api_keys()/resolve_route() confirmed the token upstream, and this + # is a pure env-dict builder with no error-reporting seam of its own — an + # empty token still produces a clear downstream SDK auth failure rather than + # a crash here. Kept deliberately lenient; do not "fix" to assert without + # also deciding how the resulting AssertionError should surface to the caller. env: dict[str, str] = { "CLAUDE_CODE_USE_BEDROCK": "1", - "AWS_BEARER_TOKEN_BEDROCK": br.bearer_token, + "AWS_BEARER_TOKEN_BEDROCK": settings.aws_bearer_token_bedrock or "", "AWS_REGION": br.region, } if br.disable_attribution_header: @@ -806,8 +814,21 @@ def _build_sdk_env( env["ANTHROPIC_SMALL_FAST_MODEL"] = br.small_model return {**base_env, **env}, br.model - case DirectRoute(): - return base_env, None + case DirectRoute() as dr: + # Neutralize inherited Bedrock creds: the CLI auto-selects Bedrock + # DIRECT when AWS_BEARER_TOKEN_BEDROCK is present in the inherited + # environment (same auto-selection the LiteLLM arm above guards + # against), so an explicit `route: direct` (e.g. via + # checker_context.api_route.route on a run whose agent is on + # Bedrock) would otherwise silently spend the operator's Bedrock + # bearer token instead of ANTHROPIC_API_KEY (PR #137 review). + env = { + "AWS_BEARER_TOKEN_BEDROCK": "", + "CLAUDE_CODE_USE_BEDROCK": "", + } + if dr.model: + env["ANTHROPIC_MODEL"] = dr.model + return {**base_env, **env}, dr.model case LiteLLMRoute() as cr: # Point the SDK at the custom Anthropic-compatible endpoint (e.g. @@ -815,8 +836,8 @@ def _build_sdk_env( # merges {**os.environ, ..., **options.env} at spawn, so setting # them here wins over the parent environment. env = { - "ANTHROPIC_BASE_URL": cr.base_url, - "ANTHROPIC_AUTH_TOKEN": cr.auth_token, + "ANTHROPIC_BASE_URL": settings.litellm_base_url or "", + "ANTHROPIC_AUTH_TOKEN": settings.litellm_auth_token or "", # Neutralize any inherited ANTHROPIC_API_KEY: auth on this # route is the bearer ANTHROPIC_AUTH_TOKEN, and a stray # x-api-key (e.g. a real Anthropic key exported from .env) diff --git a/src/coder_eval/criteria/base.py b/src/coder_eval/criteria/base.py index c825ed2d..4f82ee47 100644 --- a/src/coder_eval/criteria/base.py +++ b/src/coder_eval/criteria/base.py @@ -63,6 +63,12 @@ class CheckContext: ``reference_comparison`` is entirely dependent on it and scores 0.0 without one. Checkers that consume neither field receive the context anyway (uniform ``_check_impl`` signature) and ignore it. + + A judge model override (``checker_context.api_route.model``) is deliberately + NOT a separate field here — it's baked into ``route.model`` by the orchestrator + before this object is built (see ``resolve_evaluation_route``), so criteria + read one thing (``route.model``) regardless of whether the value came from an + env-configured backend default or a task-authored override. """ route: "ApiRoute | None" = None diff --git a/src/coder_eval/criteria/llm_judge.py b/src/coder_eval/criteria/llm_judge.py index 390676ef..42edad25 100644 --- a/src/coder_eval/criteria/llm_judge.py +++ b/src/coder_eval/criteria/llm_judge.py @@ -15,14 +15,18 @@ format_details, scrub_reference, ) +from coder_eval.evaluation.judge_litellm import invoke_litellm_judge_async from coder_eval.evaluation.judge_usage import ( token_usage_from_anthropic_dict, + token_usage_from_openai_dict, ) from coder_eval.evaluation.verdict_tool import ( SUBMIT_VERDICT_ANTHROPIC_TOOL, extract_verdict_from_anthropic_response, + extract_verdict_from_openai_response, ) from coder_eval.models import ( + DEFAULT_JUDGE_MODEL, BedrockRoute, CriterionResult, DirectRoute, @@ -70,6 +74,14 @@ async def _check_impl_async( ctx = context or CheckContext() route = ctx.route reference_dir = ctx.reference_dir + # Precedence: an explicit per-criterion `model:` always wins; otherwise fall + # back to `checker_context.api_route.model` (baked into route.model by + # resolve_evaluation_route — set only when a real override was given, never + # the agent's own model); otherwise DEFAULT_JUDGE_MODEL. `criterion.model` is + # `None` (not a materialized default) when unset, so this precedence survives + # a `model_dump(mode="json")` / reload round trip (e.g. the docker driver's + # task-serialization step) unlike a `model_fields_set` check would. + judge_model = criterion.model or (route.model if route is not None else None) or DEFAULT_JUDGE_MODEL # Master enablement gate. Skipped criteria don't make an LLM call and don't # affect cost; weighted score includes them as 1.0 so they don't penalize. @@ -138,6 +150,7 @@ async def _check_impl_async( # from the usage the backend reported in its response. verdict, parse_error, raw_verdict_text, response_usage = await _invoke_tool_channel( criterion=criterion, + model=judge_model, route=route, system_msg=_SYSTEM_PROMPT, user_msg=user_msg, @@ -186,12 +199,18 @@ def _maybe_transcript() -> JudgeTranscript | None: async def _invoke_tool_channel( *, criterion: LLMJudgeCriterion, + model: str, route: "ApiRoute | None", system_msg: str, user_msg: str, ) -> tuple[JudgeVerdict | None, str | None, str, TokenUsage | None]: """Dispatch the tool-channel invocation by route, via non-blocking async clients. + ``model`` is the resolved judge model — ``criterion.model`` unless the task + left it unset and ``route.model`` (from ``checker_context.api_route.model``) + supplied a default (see ``_check_impl_async``); every backend call below + uses ``model``, never ``criterion.model`` directly. + Returns ``(verdict, parse_error, raw_verdict_text, response_usage)``. ``raw_verdict_text`` is the JSON-dumped verdict for the transcript when present, or a fallback marker when the model failed to call the tool — @@ -204,7 +223,7 @@ async def _invoke_tool_channel( case BedrockRoute(): response = await invoke_bedrock_judge_async( route=route, - model=criterion.model, + model=model, system=system_msg, user=user_msg, temperature=criterion.temperature, @@ -212,10 +231,10 @@ async def _invoke_tool_channel( tool_spec=SUBMIT_VERDICT_ANTHROPIC_TOOL, ) verdict, err = extract_verdict_from_anthropic_response(response) - response_usage = token_usage_from_anthropic_dict(response, model=criterion.model) + response_usage = token_usage_from_anthropic_dict(response, model=model) case DirectRoute(): anthropic_response = await invoke_anthropic_judge_async( - model=criterion.model, + model=model, system=system_msg, user=user_msg, temperature=criterion.temperature, @@ -223,13 +242,23 @@ async def _invoke_tool_channel( tool_spec=SUBMIT_VERDICT_ANTHROPIC_TOOL, ) verdict, err = extract_verdict_from_anthropic_response(anthropic_response) - response_usage = token_usage_from_anthropic_dict(anthropic_response, model=criterion.model) + response_usage = token_usage_from_anthropic_dict(anthropic_response, model=model) case LiteLLMRoute(): - # Defensive: the evaluation route is pinned to Bedrock/Direct by - # resolve_evaluation_route, so a LiteLLM route should never reach the - # judge. Fail loudly rather than silently scoring 0.0. (Explicit arm - # keeps the match exhaustive so pyright flags any future route member.) - return None, "llm_judge: evaluation route must be Bedrock/Direct, got LiteLLM", "(litellm route)", None + # Reachable via an explicit `checker_context.api_route.route: litellm` + # override (see resolve_evaluation_route). Dispatches through the + # `litellm` library (see invoke_litellm_judge_async's module docstring) + # rather than assuming one wire protocol — task authors point this at + # whatever gateway their judge model actually lives behind. + litellm_response = await invoke_litellm_judge_async( + route=route, + model=model, + system=system_msg, + user=user_msg, + max_tokens=criterion.max_tokens, + tool_spec=SUBMIT_VERDICT_ANTHROPIC_TOOL, + ) + verdict, err = extract_verdict_from_openai_response(litellm_response) + response_usage = token_usage_from_openai_dict(litellm_response, model=model) case None: # Handled by the unconfigured-arm guard in _check_impl_async before # dispatch; defensive only. diff --git a/src/coder_eval/evaluation/checker.py b/src/coder_eval/evaluation/checker.py index 52f94c0d..0e7c2c4d 100644 --- a/src/coder_eval/evaluation/checker.py +++ b/src/coder_eval/evaluation/checker.py @@ -81,6 +81,8 @@ def __init__( ``agent_judge``) can route through the same backend (Direct / Bedrock) as the main coding agent. ``None`` is acceptable for non-sub-agent criteria; ``agent_judge`` requires a route. + Any ``checker_context.api_route.model`` override is already + baked into ``route.model`` by ``resolve_evaluation_route``. """ self.sandbox = sandbox self._checker_instances: dict[str, BaseCriterion[Any]] = {} diff --git a/src/coder_eval/evaluation/judge_bedrock.py b/src/coder_eval/evaluation/judge_bedrock.py index ca201645..e9f5c474 100644 --- a/src/coder_eval/evaluation/judge_bedrock.py +++ b/src/coder_eval/evaluation/judge_bedrock.py @@ -29,6 +29,7 @@ import httpx2 +from coder_eval.config import settings from coder_eval.errors import JudgeInfrastructureError from coder_eval.errors.categories import RetryConfig from coder_eval.errors.retry import compute_backoff @@ -67,9 +68,17 @@ async def invoke_bedrock_judge_async( Raises: ValueError: ``model`` empty. - JudgeInfrastructureError: retries exhausted, non-retryable HTTP failure - (e.g. 400/401/403), or a non-dict JSON body. + JudgeInfrastructureError: no bearer token configured; retries exhausted; + non-retryable HTTP failure (e.g. 400/401/403); or a non-dict JSON body. """ + # Raise (not assert): this call runs inside LLMJudgeChecker's + # handle_criterion_errors(_async) wrapper, which catches plain Exception + # (including AssertionError) and downgrades it to a scored 0.0 — the + # opposite of the intended "internal-contract violation escalates to + # FinalStatus.ERROR" behavior. JudgeInfrastructureError is in + # _ESCALATING_EXCEPTIONS, so it propagates instead of being scored. + if settings.aws_bearer_token_bedrock is None: + raise JudgeInfrastructureError("Bedrock requires aws_bearer_token_bedrock") qualified = to_bedrock_model(model, route.region) url = f"https://bedrock-runtime.{route.region}.amazonaws.com/model/{qualified}/invoke" body = { @@ -82,7 +91,7 @@ async def invoke_bedrock_judge_async( "tool_choice": {"type": "tool", "name": tool_spec["name"]}, } headers = { - "Authorization": f"Bearer {route.bearer_token}", + "Authorization": f"Bearer {settings.aws_bearer_token_bedrock}", "Content-Type": "application/json", "Accept": "application/json", } diff --git a/src/coder_eval/evaluation/judge_litellm.py b/src/coder_eval/evaluation/judge_litellm.py new file mode 100644 index 00000000..705c1e04 --- /dev/null +++ b/src/coder_eval/evaluation/judge_litellm.py @@ -0,0 +1,148 @@ +"""Single-completion invoker for the LiteLLM judge backend, via the ``litellm`` +library (the ``coder-eval[litellm]`` extra) rather than a hand-rolled HTTP call. + +Unlike the AGENT's own LiteLLM backend (which points the Claude Code SDK at +``settings.litellm_base_url``/``settings.litellm_auth_token``), this module +reads NOTHING from ``coder_eval.config.settings`` — the task author fully owns +the call shape via ``LiteLLMRoute.params``/``LiteLLMRoute.env_params`` (see +that class's docstring). A gateway-routed judge model rarely reuses the same +proxy/credential the agent's own LiteLLM backend points at, so there is no +implicit fallback here; if the provider needs ``api_base``/``api_key``/ +whatever else, the task author names it via ``params``/``env_params`` like any +other kwarg. + +Calling through ``litellm.acompletion`` — rather than assuming one specific +wire protocol — lets ``model`` carry its own provider hint (e.g. +``azure_ai/gpt-5.6-luna``) and get that provider's actual request/response +shape handled by the library, including per-provider quirks (``max_tokens`` +vs ``max_completion_tokens`` naming, unsupported-parameter drops via +``drop_params``) instead of this module hand-coding them. + +``litellm.acompletion`` always returns an OpenAI-shaped ``ModelResponse`` +regardless of the underlying provider, so the caller reuses +``extract_verdict_from_openai_response``/``token_usage_from_openai_dict`` +unchanged. + +Async on purpose: mirrors ``invoke_anthropic_judge_async`` / +``invoke_bedrock_judge_async`` — the judge's only network call, no sync twin. +""" + +from __future__ import annotations + +import logging +import os +from typing import Any + +from coder_eval.errors import JudgeInfrastructureError +from coder_eval.models import LiteLLMRoute + + +logger = logging.getLogger(__name__) + + +async def invoke_litellm_judge_async( + *, + route: LiteLLMRoute, + model: str, + system: str, + user: str, + max_tokens: int, + tool_spec: dict[str, Any], + timeout_seconds: float = 120.0, +) -> dict[str, Any]: + """One completion call via ``litellm.acompletion`` with a forced tool call. + + Unlike ``invoke_anthropic_judge_async``/``invoke_bedrock_judge_async``, this + does NOT take a ``temperature`` — a gateway-routed model may reject it + outright (observed live against an Azure AI deployment), and there is no + uniform way to know in advance. Set it via ``route.params`` (e.g. + ``{temperature: 0.0}``) if the target model accepts it. + + Every other provider-specific kwarg (``api_base``, ``api_key``, + ``aws_access_key_id``, ...) comes from ``route.params``/``route.env_params`` + — see ``LiteLLMRoute``'s docstring. This function has no opinion on what a + valid call needs; an incomplete configuration surfaces as whatever error + ``litellm`` itself raises, wrapped below. + + Returns the OpenAI-shaped response converted to a dict via ``model_dump`` + so the caller can reuse ``extract_verdict_from_openai_response``. + + Raises: + ValueError: ``model`` empty. + JudgeInfrastructureError: the ``litellm`` extra isn't installed; an + ``env_params`` entry names an unset env var; or the call fails (an + eval-infra fault, not the agent's fault — CE039). + """ + if not model: + raise ValueError("invoke_litellm_judge_async: model must not be empty") + + try: + from litellm.exceptions import APIError + from litellm.types.utils import ModelResponse + + import litellm + except ImportError as e: + raise JudgeInfrastructureError( + "checker_context route 'litellm' needs the litellm library. Install with: pip install 'coder-eval[litellm]'" + ) from e + + openai_tool = { + "type": "function", + "function": { + "name": tool_spec["name"], + "description": tool_spec["description"], + "parameters": tool_spec["input_schema"], + }, + } + + def _resolve_env_params() -> dict[str, str]: + """Resolve ``route.env_params`` (kwarg name -> ENV VAR NAME) into kwarg + name -> value, right before the call so no resolved value is ever stored + on the route object itself (only the env var *name* is).""" + if not route.env_params: + return {} + resolved: dict[str, str] = {} + for param_name, env_var in route.env_params.items(): + value = os.environ.get(env_var) + if not value: + msg = ( + f"checker_context.api_route.env_params[{param_name!r}] references env var " + f"{env_var!r}, which is not set" + ) + raise JudgeInfrastructureError(msg) + resolved[param_name] = value + return resolved + + kwargs: dict[str, Any] = { + "model": model, + "messages": [ + {"role": "system", "content": system}, + {"role": "user", "content": user}, + ], + "tools": [openai_tool], + "tool_choice": {"type": "function", "function": {"name": tool_spec["name"]}}, + "max_completion_tokens": max_tokens, + "timeout": timeout_seconds, + # `drop_params` covers params litellm's own static model-cost map KNOWS a + # model rejects; a custom/gateway-routed model id (e.g. one behind an + # Azure AI deployment) usually isn't in that map, so this alone doesn't + # protect a `params`-supplied kwarg the target model live-rejects. + "drop_params": True, + } + # `params` (literal passthrough) applies first; `env_params` (resolved from + # env) applies LAST so it always wins over `params` for the same key. + if route.params: + kwargs.update(route.params) + kwargs.update(_resolve_env_params()) + + try: + response = await litellm.acompletion(**kwargs) + except APIError as e: + raise JudgeInfrastructureError(f"LiteLLM judge API error: {e}") from e + except Exception as e: + raise JudgeInfrastructureError(f"LiteLLM judge call failed: {e}") from e + if not isinstance(response, ModelResponse): + # Never actually streamed (no `stream=True` above) -- defensive only, + # keeps pyright's ModelResponse | CustomStreamWrapper union honest. + raise JudgeInfrastructureError(f"LiteLLM judge returned an unexpected response type: {type(response)}") + return response.model_dump() diff --git a/src/coder_eval/evaluation/judge_usage.py b/src/coder_eval/evaluation/judge_usage.py index dfe51dd0..e3ab0aab 100644 --- a/src/coder_eval/evaluation/judge_usage.py +++ b/src/coder_eval/evaluation/judge_usage.py @@ -64,3 +64,45 @@ def token_usage_from_anthropic_dict(resp: dict[str, Any], *, model: str | None = cache_read_tokens=tu.cache_read_input_tokens, ) return tu + + +def token_usage_from_openai_dict(resp: dict[str, Any], *, model: str | None = None) -> TokenUsage | None: + """Extract usage from an OpenAI Chat-Completions-shaped response dict. + + ``invoke_litellm_judge_async`` (parsed ``/chat/completions`` JSON) carries an + OpenAI-shaped ``usage`` block: ``prompt_tokens``/``completion_tokens``, with + the cached prefix (if any) nested under ``prompt_tokens_details.cached_tokens``. + Returns ``None`` when usage is missing or carries no tokens. + + Follows the OpenAI/Codex cache-bucket convention documented on + ``TokenUsage``: ``prompt_tokens`` is the FULL prompt inclusive of the cached + prefix, so the fresh (uncached) slice is ``prompt_tokens - cached_tokens``; + OpenAI bills no separate cache-write fee, so ``cache_creation_input_tokens`` + is always 0 (mirrors ``CodexAgent._token_usage_from_sdk``). + + ``model`` prices the call from the rate card, same as the Anthropic-shaped + extractor above. + """ + u = resp.get("usage") + if not isinstance(u, dict): + return None + prompt_tokens = _coerce_int(u.get("prompt_tokens")) + details = u.get("prompt_tokens_details") + cached = _coerce_int(details.get("cached_tokens")) if isinstance(details, dict) else 0 + tu = TokenUsage( + uncached_input_tokens=max(prompt_tokens - cached, 0), + output_tokens=_coerce_int(u.get("completion_tokens")), + cache_creation_input_tokens=0, + cache_read_input_tokens=cached, + ) + if tu.is_empty(): + return None + if model: + tu.total_cost_usd = calculate_cost( + model, + uncached_input_tokens=tu.uncached_input_tokens, + output_tokens=tu.output_tokens, + cache_creation_tokens=tu.cache_creation_input_tokens, + cache_read_tokens=tu.cache_read_input_tokens, + ) + return tu diff --git a/src/coder_eval/evaluation/verdict_tool.py b/src/coder_eval/evaluation/verdict_tool.py index 815cc4c6..16186a48 100644 --- a/src/coder_eval/evaluation/verdict_tool.py +++ b/src/coder_eval/evaluation/verdict_tool.py @@ -14,6 +14,7 @@ from __future__ import annotations +import json from dataclasses import dataclass from typing import Any @@ -163,6 +164,50 @@ def extract_verdict_from_anthropic_response( return None, _format_validation_error(e) +def extract_verdict_from_openai_response( + response: dict[str, Any], +) -> tuple[JudgeVerdict | None, str | None]: + """Read the LAST ``submit_verdict`` tool call from an OpenAI Chat-Completions-shaped response. + + Walks ``response["choices"][0]["message"]["tool_calls"]`` for entries named + ``submit_verdict``, JSON-decodes the last one's ``function.arguments`` + string (OpenAI, unlike Anthropic, delivers tool args as a JSON string, not + a dict), and validates it against ``JudgeVerdict``. Used by + ``invoke_litellm_judge_async``. + + LAST-call discipline + non-dict/undecodable guard, mirroring + ``extract_verdict_from_anthropic_response``: a final ``submit_verdict`` + call whose arguments are missing, not valid JSON, or not an object is + treated as an invalid-args failure, not "did not call". + """ + choices = response.get("choices") or [] + message = choices[0].get("message") if choices and isinstance(choices[0], dict) else None + tool_calls = message.get("tool_calls") if isinstance(message, dict) else None + saw_submit_verdict = False + last_input: dict[str, Any] | None = None + for call in tool_calls or []: + if not isinstance(call, dict): + continue + function = call.get("function") + if not isinstance(function, dict) or function.get("name") != SUBMIT_VERDICT_TOOL_NAME: + continue + saw_submit_verdict = True + arguments = function.get("arguments") + try: + decoded = json.loads(arguments) if isinstance(arguments, str) else None + except ValueError: + decoded = None + last_input = decoded if isinstance(decoded, dict) else None + if not saw_submit_verdict: + return None, "Judge did not call submit_verdict" + if last_input is None: + return None, "submit_verdict input must be an object" + try: + return JudgeVerdict.model_validate(last_input), None + except ValidationError as e: + return None, _format_validation_error(e) + + def _format_validation_error(e: ValidationError) -> str: """Normalize a Pydantic ``ValidationError`` into the legacy vocabulary. diff --git a/src/coder_eval/models/__init__.py b/src/coder_eval/models/__init__.py index d181241d..297f170c 100644 --- a/src/coder_eval/models/__init__.py +++ b/src/coder_eval/models/__init__.py @@ -180,6 +180,8 @@ DEFAULT_SIMULATOR_MODEL, NORMALIZED_CRITERION_ALIASES, REMOVED_CRITERION_TYPES, + ApiRouteContext, + CheckerContext, CriteriaCheckTiming, Dataset, PostRunCommand, @@ -358,6 +360,8 @@ "merge_strategy_of", # Tasks "TaskDefinition", + "ApiRouteContext", + "CheckerContext", "DEFAULT_SIMULATION_STOP_TOKEN", "DEFAULT_SIMULATOR_MODEL", "NORMALIZED_CRITERION_ALIASES", diff --git a/src/coder_eval/models/criteria.py b/src/coder_eval/models/criteria.py index 7e54637c..b0822638 100644 --- a/src/coder_eval/models/criteria.py +++ b/src/coder_eval/models/criteria.py @@ -1327,10 +1327,13 @@ class LLMJudgeCriterion(BaseSuccessCriterion): "aggregate budget is exceeded (a degraded note is recorded)." ), ) - model: str = Field( - default=DEFAULT_JUDGE_MODEL, + model: str | None = Field( + default=None, description=( - "Judge model id (e.g. 'anthropic.claude-sonnet-4-6'). " + "Judge model id (e.g. 'anthropic.claude-sonnet-4-6'). Leave unset to fall back to " + "checker_context.api_route.model when set, else the built-in default " + f"({DEFAULT_JUDGE_MODEL!r}) — the fallback is never the agent's own model, so an " + "unpinned judge grades identically across agent-model A/Bs. " "On a BedrockRoute / DirectRoute the value is auto-translated: " "trailing '-vN[:M]' suffixes and the 'anthropic.' prefix are stripped where " "the backend doesn't accept them; on Bedrock the cross-region inference-profile " diff --git a/src/coder_eval/models/experiment.py b/src/coder_eval/models/experiment.py index 847a8475..fe6ae6ff 100644 --- a/src/coder_eval/models/experiment.py +++ b/src/coder_eval/models/experiment.py @@ -38,6 +38,14 @@ class ExperimentVariant(BaseModel): ge=1, description="Number of replicates for each task under this variant. None = inherit.", ) + checker_context: dict[str, dict[str, Any]] | None = Field( + default=None, + description=( + "Partial TaskDefinition.checker_context overrides for this variant — currently the " + "reserved `api_route` namespace (`route`/`model`). Shallow-merged per-namespace onto the " + "task's (and/or experiment defaults') checker_context." + ), + ) template_sources: list[TemplateSource] | None = Field( default=None, description="Additional template sources appended after task's base templates" ) @@ -98,6 +106,14 @@ class ExperimentDefaults(BaseModel): description="Default number of replicates across all variants. None = 1 (no repetition).", ) agent: dict[str, Any] | None = Field(default=None, description="Partial agent config defaults") + checker_context: dict[str, dict[str, Any]] | None = Field( + default=None, + description=( + "Default TaskDefinition.checker_context applied to all variants — currently the reserved " + "`api_route` namespace (e.g. `{api_route: {model: gpt-5}}`). Shallow-merged per-namespace " + "with the task's and the variant's checker_context." + ), + ) simulation: dict[str, Any] | None = Field( default=None, description=( diff --git a/src/coder_eval/models/routing.py b/src/coder_eval/models/routing.py index dfdbd627..4dc5e0a0 100644 --- a/src/coder_eval/models/routing.py +++ b/src/coder_eval/models/routing.py @@ -2,8 +2,8 @@ from __future__ import annotations -from dataclasses import dataclass -from typing import TYPE_CHECKING, Literal +from dataclasses import dataclass, replace +from typing import TYPE_CHECKING, Any, Literal from coder_eval.models.enums import ApiBackend from coder_eval.models.judge_defaults import DEFAULT_JUDGE_MODEL @@ -87,13 +87,26 @@ class DirectRoute: """ judge_transport: JudgeTransport | None = "anthropic" + # Unlike BedrockRoute/LiteLLMRoute, the AGENT never reads this — the Claude Agent + # SDK picks its own default when unset. It exists so ``checker_context.api_route.model`` + # (see resolve_evaluation_route) has somewhere to land when the eval side is on Direct. + model: str | None = None @dataclass(frozen=True) class BedrockRoute: - """Route through AWS Bedrock with bearer token authentication.""" + """Route through AWS Bedrock with bearer token authentication. + + Deliberately carries NO credential field: the bearer token is a secret, and + a route object flows through orchestrator state (``CheckContext``, + ``environment_info`` recording, logging) that has no business handling one. + Every consumer that actually needs the token (``ClaudeCodeAgent._build_sdk_env`` + for the agent subprocess, ``judge_bedrock.invoke_bedrock_judge_async`` for the + judge's HTTP call) reads ``settings.aws_bearer_token_bedrock`` itself, via the + shared ``coder_eval.config.settings`` singleton — the same source ``resolve_route`` + validated before constructing this route in the first place. + """ - bearer_token: str region: str model: str | None = None # Cross-region model ID, e.g. "eu.anthropic.claude-sonnet-4-6" small_model: str | None = None # Cross-region small model ID @@ -104,19 +117,49 @@ class BedrockRoute: @dataclass(frozen=True) class LiteLLMRoute: - """Route through a custom Anthropic-compatible endpoint (e.g. a LiteLLM - gateway fronting Bedrock open-weight models). - - The Claude Code SDK is pointed at ``base_url`` via ``ANTHROPIC_BASE_URL`` and - authenticates with ``auth_token`` via ``ANTHROPIC_AUTH_TOKEN`` (bearer). The - ``model``/``small_model`` ids are passed **verbatim** (no Bedrock - inference-profile qualification) — the gateway maps them to its backend. + """Route through a custom endpoint — either the AGENT's own LiteLLM proxy + (an Anthropic-compatible gateway fronting Bedrock open-weight models), or, + on the CHECKER side (``checker_context.api_route.route: litellm``), an + arbitrary provider reached through the ``litellm`` library directly. + + AGENT side: the Claude Code SDK is pointed at the gateway via + ``ANTHROPIC_BASE_URL``/``ANTHROPIC_AUTH_TOKEN``. Deliberately carries NO + ``base_url``/credential field for this — same reasoning as ``BedrockRoute``'s + docstring: this route object flows through orchestrator state + (``environment_info`` recording, logging) that has no business handling + config that should always be read live from the environment. + ``ClaudeCodeAgent._build_sdk_env`` reads ``settings.litellm_base_url``/ + ``settings.litellm_auth_token`` itself, the same source ``resolve_route`` + validated before constructing this route. + + CHECKER side (``invoke_litellm_judge_async``): unlike the agent path, this + is NOT sourced from ``coder_eval.config.settings`` at all — the task author + fully owns it via ``params``/``env_params`` below (a gateway-routed judge + model rarely reuses the same proxy/credential the AGENT's own LiteLLM + backend points at). There is no implicit fallback to + ``settings.litellm_base_url``/``settings.litellm_auth_token``; if the + provider needs ``api_base``/``api_key``, the task author sets them via + ``params``/``env_params`` like any other kwarg. + + ``params``/``env_params`` (checker side only, from + ``checker_context.api_route.{params,env_params}``): ``litellm.acompletion`` + takes dozens of provider-specific kwargs (``api_base``, ``api_key``, + ``aws_access_key_id``, ``vertex_project``, ``api_version``, ...) that this + route has no dedicated field for. ``params`` is passed through verbatim as + extra kwargs. ``env_params`` maps a kwarg name to the ENV VAR NAME to + resolve it from at call time — e.g. ``{api_key: LITELLM_AUTH_TOKEN, + aws_access_key_id: AWS_ACCESS_KEY_ID}`` — so an arbitrary provider's config + (including secrets) is representable without a secret ever landing in the + task YAML. Both are ``None`` unless a task author set them; ``env_params``'s + values are env var *names*, never secrets, so it is safe to record verbatim + in ``environment_info`` (unlike ``params``, which a task author could — but + shouldn't — put a raw secret into). """ - base_url: str - auth_token: str model: str | None = None small_model: str | None = None + params: dict[str, Any] | None = None + env_params: dict[str, str] | None = None ApiRoute = DirectRoute | BedrockRoute | LiteLLMRoute @@ -130,6 +173,16 @@ class LiteLLMRoute: } +def _bedrock_model_pair(model: str | None, small_model: str | None, region: str) -> tuple[str | None, str | None]: + """Resolve ``(model, small_model)`` into Bedrock inference-profile ids, defaulting + ``small_model`` to ``model`` when unset. Shared by every ``BedrockRoute`` construction + site (``resolve_route``, ``_resolve_backend_route``, ``resolve_evaluation_route``'s + pin-to-Claude branch) so the qualification logic can't drift between them. + """ + resolved_small = small_model or model + return to_bedrock_inference_profile(model, region), to_bedrock_inference_profile(resolved_small, region) + + def resolve_route(settings: Settings) -> ApiRoute: """Resolve an ``ApiRoute`` from static settings. @@ -155,13 +208,10 @@ def resolve_route(settings: Settings) -> ApiRoute: # ClaudeCodeAgent._build_sdk_env). Leaving it unset made every # WebFetch fail with "model issues" under the Bedrock backend. The main # model is always a valid fallback, so default to it. - small_model = settings.bedrock_small_model or settings.bedrock_model - return BedrockRoute( - bearer_token=settings.aws_bearer_token_bedrock, - region=settings.aws_region, - model=to_bedrock_inference_profile(settings.bedrock_model, settings.aws_region), - small_model=to_bedrock_inference_profile(small_model, settings.aws_region), + model, small_model = _bedrock_model_pair( + settings.bedrock_model, settings.bedrock_small_model, settings.aws_region ) + return BedrockRoute(region=settings.aws_region, model=model, small_model=small_model) case ApiBackend.DIRECT: return DirectRoute(judge_transport=_resolve_direct_judge_transport(settings)) case ApiBackend.LITELLM: @@ -176,40 +226,150 @@ def resolve_route(settings: Settings) -> ApiRoute: # No inference-profile qualification: the id is passed verbatim to the gateway. small_model = settings.litellm_small_model or settings.litellm_model return LiteLLMRoute( - base_url=settings.litellm_base_url, - auth_token=settings.litellm_auth_token, model=settings.litellm_model, small_model=small_model, ) - - -def resolve_evaluation_route(settings: Settings, agent_route: ApiRoute) -> ApiRoute: + case _: + # ApiBackend covers exactly BEDROCK/DIRECT/LITELLM above; this arm is + # unreachable but makes the match exhaustive so every path returns + # explicitly (CodeQL: mixed explicit/implicit returns). + raise AssertionError(f"unhandled ApiBackend: {settings.api_backend!r}") + + +def _resolve_backend_route( + settings: Settings, + backend: ApiBackend, + *, + model_override: str | None = None, + params_override: dict[str, Any] | None = None, + env_params_override: dict[str, str] | None = None, +) -> ApiRoute: + """Build the ``ApiRoute`` for an EXPLICITLY-requested backend. + + Used only by the ``checker_context.api_route`` override path (see + ``resolve_evaluation_route``): raises ``ValueError`` naming the missing env + var when that backend isn't configured, rather than silently falling back + to a different backend — an explicit override that can't be honored must + fail loudly, not degrade to a backend the task author didn't ask for. + + ``model_override`` (``checker_context.api_route.model``) wins over the + backend's own env-configured default model when set. + + ``ApiBackend.LITELLM`` is the one exception to "env-sourced ``Settings`` + fields, credentials always come from the environment": unlike + BEDROCK/DIRECT (which reuse the agent's own env-configured credentials, since + grading still needs to reach the SAME Claude backend), a checker-side litellm + route is not assumed to share the agent's LiteLLM proxy/gateway at all — it + is built ENTIRELY from ``params_override``/``env_params_override`` + (``checker_context.api_route.{params,env_params}``), never from + ``settings.litellm_base_url``/``settings.litellm_auth_token``. Those two + settings fields are the AGENT's own LiteLLM-backend config (see + ``resolve_route``) — reusing them here would silently point the judge at + infrastructure the task author never named. + """ + match backend: + case ApiBackend.BEDROCK: + if not settings.aws_bearer_token_bedrock or not settings.aws_region: + raise ValueError( + "checker_context route 'bedrock' requires AWS_BEARER_TOKEN_BEDROCK and AWS_REGION to be set" + ) + judge_model = model_override or settings.bedrock_model or DEFAULT_JUDGE_MODEL + model, small_model = _bedrock_model_pair(judge_model, settings.bedrock_small_model, settings.aws_region) + return BedrockRoute(region=settings.aws_region, model=model, small_model=small_model) + case ApiBackend.DIRECT: + if not settings.anthropic_api_key: + raise ValueError("checker_context route 'direct' requires ANTHROPIC_API_KEY to be set") + return DirectRoute(judge_transport="anthropic", model=model_override) + case ApiBackend.LITELLM: + if not model_override: + msg = ( + "checker_context route 'litellm' requires an explicit `checker_context.api_route.model` " + "— there is no default open-weight/gateway model to fall back to" + ) + raise ValueError(msg) + return LiteLLMRoute( + model=model_override, + params=params_override, + env_params=env_params_override, + ) + case _: + # ApiBackend covers exactly BEDROCK/DIRECT/LITELLM above; this arm is + # unreachable but makes the match exhaustive so every path returns + # explicitly (CodeQL: mixed explicit/implicit returns, PR #137 review). + raise AssertionError(f"unhandled ApiBackend: {backend!r}") + + +def resolve_evaluation_route( + settings: Settings, + agent_route: ApiRoute, + *, + backend_override: str | None = None, + model_override: str | None = None, + params_override: dict[str, Any] | None = None, + env_params_override: dict[str, str] | None = None, +) -> ApiRoute: """Resolve the route used by the *evaluation* side — the ``llm_judge`` / ``agent_judge`` criteria and the simulated user — which must stay on a constant Claude backend regardless of the agent under test, so grading and simulation stay comparable across models. - - Agent on Bedrock/Direct: the judge already runs on Claude via that route, - so reuse it unchanged (no behavior change for existing runs). - - Agent on LiteLLM (open-weight): the agent route cannot serve a Claude - judge, so pin evaluation to Bedrock (preferred, from the AWS bearer token) - or Direct (``ANTHROPIC_API_KEY``). If neither is configured, fall back to a - ``DirectRoute`` with no judge transport so ``llm_judge`` fails with its - clean "unconfigured" error rather than silently scoring 0.0. + All overrides come from the reserved ``checker_context.api_route`` namespace + (see ``TaskDefinition.checker_context``) — ``route`` (``backend_override``) + picks the backend, ``model`` (``model_override``) picks the model on + whichever route is resolved, and ``params``/``env_params`` (``params_override``/ + ``env_params_override``) only ever apply when ``backend_override`` resolves to + ``litellm`` (see ``_resolve_backend_route``). Criteria never read any of + these directly; they only ever see the resulting ``CheckContext.route``. + + - ``backend_override`` set: build that backend's route from env, regardless + of ``agent_route`` — an explicit task/variant choice always wins. Raises + ``ValueError`` if the string isn't a known ``ApiBackend`` or that backend + isn't configured (see ``_resolve_backend_route``). + - Agent on Bedrock/Direct (no ``backend_override``): the judge already runs + on Claude via that route, so reuse it — except its ``model`` is always + reset to ``model_override`` (``None`` when unset). The agent's own + env-sourced model (e.g. ``BEDROCK_MODEL``) must NOT leak into the judge's + default: ``route.model`` must mean "an explicit override was given", not + "whatever the agent happens to be using" — otherwise an unpinned judge + silently starts grading with a different model whenever the agent's + model changes, breaking before/after comparability (PR #137 review: + "the judge loses DEFAULT_JUDGE_MODEL as its floor"). + - Agent on LiteLLM (open-weight, no ``backend_override``): the agent route + cannot serve a Claude judge, so pin evaluation to Bedrock (preferred, from + the AWS bearer token) or Direct (``ANTHROPIC_API_KEY``), honoring + ``model_override`` there too — same "no override, no baked-in model" rule + as above. If neither backend is configured, fall back to a ``DirectRoute`` + with no judge transport so ``llm_judge`` fails with its clean + "unconfigured" error rather than silently scoring 0.0. """ + if backend_override is not None: + try: + backend = ApiBackend(backend_override) + except ValueError as e: + valid = ", ".join(b.value for b in ApiBackend) + raise ValueError(f"checker_context route {backend_override!r} is not a known backend ({valid})") from e + return _resolve_backend_route( + settings, + backend, + model_override=model_override, + params_override=params_override, + env_params_override=env_params_override, + ) if isinstance(agent_route, BedrockRoute | DirectRoute): - return agent_route + if isinstance(agent_route, BedrockRoute) and model_override: + # Bedrock model ids must be region-qualified — reusing the agent's route + # verbatim would ship a bare alias straight to the Bedrock API (400). + qualified_model, _ = _bedrock_model_pair(model_override, None, agent_route.region) + return replace(agent_route, model=qualified_model) + return replace(agent_route, model=model_override) # agent_route is LiteLLMRoute → pin evaluation to a constant Claude backend. if settings.aws_bearer_token_bedrock and settings.aws_region: - judge_model = settings.bedrock_model or DEFAULT_JUDGE_MODEL - qualified = to_bedrock_inference_profile(judge_model, settings.aws_region) - return BedrockRoute( - bearer_token=settings.aws_bearer_token_bedrock, - region=settings.aws_region, - model=qualified, - small_model=qualified, - ) - return DirectRoute(judge_transport=_resolve_direct_judge_transport(settings)) + if model_override: + model, small_model = _bedrock_model_pair(model_override, None, settings.aws_region) + else: + model, small_model = None, None + return BedrockRoute(region=settings.aws_region, model=model, small_model=small_model) + return DirectRoute(judge_transport=_resolve_direct_judge_transport(settings), model=model_override) def _resolve_direct_judge_transport(settings: Settings) -> JudgeTransport | None: diff --git a/src/coder_eval/models/tasks.py b/src/coder_eval/models/tasks.py index b3034727..75c2d866 100644 --- a/src/coder_eval/models/tasks.py +++ b/src/coder_eval/models/tasks.py @@ -17,7 +17,7 @@ RunCommandCriterion, SuccessCriterion, ) -from coder_eval.models.enums import AgentKind +from coder_eval.models.enums import AgentKind, ApiBackend from coder_eval.models.judge_defaults import DEFAULT_JUDGE_MODEL from coder_eval.models.limits import RunLimits from coder_eval.models.merge_strategy import MergeField @@ -78,6 +78,54 @@ class UnknownTaskFieldWarning(DeprecationWarning): :data:`NORMALIZED_CRITERION_ALIASES`.""" +class ApiRouteContext(BaseModel): + """``checker_context.api_route`` — see ``TaskDefinition.checker_context`` for + the full picture. A typed replacement for what used to be a hand-validated + open dict: ``extra="forbid"`` catches an unknown key (e.g. a misspelled + ``rotue:``) at load time for free, and ``model``/``params``/``env_params`` + are now real types instead of ``Any`` — a YAML ``model: 5`` is rejected here + rather than silently ``str()``-ified into a model id downstream. + """ + + model_config = ConfigDict(extra="forbid") + + route: ApiBackend | None = Field( + default=None, + description="Backend the WHOLE evaluation side (llm_judge/agent_judge/simulator) calls.", + ) + model: str | None = Field(default=None, description="Model override for the resolved route.") + params: dict[str, Any] | None = Field( + default=None, + description="`route: litellm` only — arbitrary passthrough kwargs to litellm.acompletion.", + ) + env_params: dict[str, str] | None = Field( + default=None, + description=( + "`route: litellm` only — maps a litellm.acompletion kwarg name to the ENV VAR NAME " + "(never the value) to resolve it from at call time." + ), + ) + + @model_validator(mode="after") + def _validate_litellm_only_fields(self) -> Self: + """``params``/``env_params`` only ever reach the litellm judge transport + (see ``invoke_litellm_judge_async``) — on any other backend they'd be + silently dropped, which is worse than a load-time error.""" + if (self.params is not None or self.env_params is not None) and self.route != ApiBackend.LITELLM: + raise ValueError("checker_context.api_route.params/env_params require route: litellm") + return self + + +class CheckerContext(BaseModel): + """Task-authored config for the success-checking side. See + ``TaskDefinition.checker_context``'s field description for the full picture. + """ + + model_config = ConfigDict(extra="forbid") + + api_route: ApiRouteContext | None = None + + class SimulationConfig(BaseModel): """Configuration for multi-turn user simulation. @@ -438,6 +486,28 @@ class TaskDefinition(BaseModel): # noqa: CE009 -- soft-launch: see _warn_on_unk strategy="replace", # not layer-merged today; replace = the engine default if it ever is description="List of criteria that must all pass for task success", ) + checker_context: CheckerContext | None = Field( + default=None, + description=( + "Task-authored config for the success-checking side. Currently carries only `api_route` " + "(see ApiRouteContext), e.g. `{api_route: {route: litellm, model: gpt-5}}`: `route` selects " + "the backend the WHOLE evaluation side (llm_judge, agent_judge, the simulator) calls, " + "decoupled from the agent's own route; `model` overrides the model that route uses. Both " + "are consumed by the orchestrator (`resolve_evaluation_route`) BEFORE `CheckContext` is " + "built and baked into the resolved route's own `model` field — no criterion ever reads " + "`checker_context` directly, only `CheckContext.route.model`. Credentials are always " + "resolved from environment variables, never from this field — EXCEPT `route: litellm`, " + "whose call is built entirely from `params`/`env_params` (no implicit fallback to the " + "agent's own LITELLM_BASE_URL/LITELLM_AUTH_TOKEN): `params` is an arbitrary passthrough " + "dict of extra kwargs to the underlying `litellm.acompletion` call (e.g. `{api_base: " + "https://my-gateway/v1}`), and `env_params` maps a kwarg name to the ENV VAR NAME (not " + "the value!) to resolve it from at call time (e.g. `{api_key: MY_GATEWAY_TOKEN, " + "aws_access_key_id: AWS_ACCESS_KEY_ID}`) — so an arbitrary provider's config, including " + "secrets, is representable without ever putting one in the task YAML. `model` is required " + "when `route: litellm` (no default open-weight/gateway model). Merged field-by-field " + "across default -> experiment-defaults -> task -> variant (same engine as `agent`/`simulation`)." + ), + ) run_limits: RunLimits | None = Field( default=None, description=( diff --git a/src/coder_eval/orchestration/experiment.py b/src/coder_eval/orchestration/experiment.py index d103436b..25d4ed58 100644 --- a/src/coder_eval/orchestration/experiment.py +++ b/src/coder_eval/orchestration/experiment.py @@ -19,6 +19,7 @@ from ..models import ( AgentConfig, BaseAgentConfig, + CheckerContext, ConfigLineageEntry, ExperimentDefinition, ExperimentResult, @@ -172,6 +173,50 @@ def _resolve_simulation( return resolved +def _resolve_checker_context( + default_experiment: ExperimentDefinition, + experiment: ExperimentDefinition, + task: TaskDefinition, + variant: ExperimentVariant, + lineage: dict[str, ConfigLineageEntry], +) -> CheckerContext: + """Merge ``checker_context`` across the 4-layer precedence chain. + + Precedence (lowest to highest): + 1. default_experiment.defaults.checker_context + 2. experiment.defaults.checker_context + 3. task.checker_context + 4. variant.checker_context + + Mirrors ``_resolve_simulation``: runs through the generic resolver + (``merge_layers`` over ``CheckerContext``) so a later layer's fields + overwrite earlier ones (nested ``api_route`` merges field-by-field, the + type-aware default for a nested ``BaseModel``), then the merged dict is + validated by constructing a ``CheckerContext`` — which raises a helpful, + typed error (unknown key, wrong value type, `params`/`env_params` without + `route: litellm`) rather than a hand-rolled shape check. Lineage stays + coarse: a single ``checker_context`` entry crediting the most-specific source. + """ + specs: list[tuple[ConfigSource, dict[str, Any] | None]] = [ + ("default", default_experiment.defaults.checker_context if default_experiment.defaults else None), + ("experiment-defaults", experiment.defaults.checker_context if experiment.defaults else None), + ("task", task.checker_context.model_dump(exclude_unset=True) if task.checker_context else None), + ("variant", variant.checker_context), + ] + layers = [Layer(source=src, patch=patch) for src, patch in specs if patch is not None] + if not layers: + return CheckerContext() + + merged = merge_layers((CheckerContext,), layers, lineage_root="checker_context") + resolved = CheckerContext(**merged) + + most_specific: ConfigSource | None = next((src for src, patch in reversed(specs) if patch is not None), None) + if most_specific is not None: + lineage["checker_context"] = ConfigLineageEntry(value=merged, source=most_specific) + + return resolved + + def _resolve_repeats( default_experiment: ExperimentDefinition, experiment: ExperimentDefinition, @@ -446,6 +491,7 @@ def _add_rl(rl: RunLimits | None, source: ConfigSource) -> None: # Mirrors agent merge semantics — a later layer's keys overwrite earlier ones, and # the final dict is validated by building a SimulationConfig from it. resolved_simulation = _resolve_simulation(default_experiment, experiment, task, variant, lineage) + resolved_checker_context = _resolve_checker_context(default_experiment, experiment, task, variant, lineage) # Build resolved task (copy with overrides) resolved_task = task.model_copy( @@ -456,6 +502,7 @@ def _add_rl(rl: RunLimits | None, source: ConfigSource) -> None: "post_run": resolved_post_run, "pre_run": resolved_pre_run, "simulation": resolved_simulation, + "checker_context": resolved_checker_context, } ) diff --git a/src/coder_eval/orchestrator.py b/src/coder_eval/orchestrator.py index 63a643f1..043df8e3 100644 --- a/src/coder_eval/orchestrator.py +++ b/src/coder_eval/orchestrator.py @@ -12,7 +12,7 @@ from datetime import datetime from inspect import isawaitable from pathlib import Path -from typing import Any +from typing import Any, NamedTuple from urllib.parse import urlparse from .agent import Agent @@ -35,6 +35,7 @@ CONTAINER_REFERENCE_DIR, DEFAULT_STOP_EARLY_GATE_THRESHOLD, ROUTE_NAMES, + AgentJudgeCriterion, AgentKind, ApiRoute, BedrockRoute, @@ -150,6 +151,17 @@ async def _pump_stream( _UTTERANCE_TAG_RE = re.compile(r"^\[(ASSISTANT|RESULT - SUCCESS|RESULT - ERROR|TOOL USE)\](?: (.*))?$") +class EvalRouteOverrides(NamedTuple): + """``checker_context.api_route``'s override fields. Named fields (rather than + a bare tuple) so a future transposition at a call site is a typo'd attribute, + not a silent positional swap.""" + + backend: str | None + model: str | None + params: dict[str, Any] | None + env_params: dict[str, str] | None + + def _format_routing(route: ApiRoute, effective_model: str | None = None) -> str: """Format the route name for the ``API routing:`` log line. @@ -1276,12 +1288,7 @@ async def _setup(self) -> None: self.sandbox.reference_dir = self._reference_dir self.result.sandbox_path = str(self.sandbox.sandbox_dir) - self.route = resolve_route(settings) - self.eval_route = resolve_evaluation_route(settings, self.route) - logger.info( - "API routing: %s", _format_routing(self.route, self.task.agent.model if self.task.agent else None) - ) - self.success_checker = SuccessChecker(self.sandbox, route=self.eval_route) + self._resolve_routes() self._record_route_environment_info() return @@ -1347,10 +1354,7 @@ async def _setup_sandbox() -> Any: self.result.sandbox_path = str(sandbox_dir) # Determine API routing from settings.api_backend enum - self.route = resolve_route(settings) - self.eval_route = resolve_evaluation_route(settings, self.route) - logger.info("API routing: %s", _format_routing(self.route, self.task.agent.model if self.task.agent else None)) - self.success_checker = SuccessChecker(self.sandbox, route=self.eval_route) + self._resolve_routes() # Create and start the agent. For a no-op (type: none) task this dispatches # to NoOpAgent, whose start/communicate/stop are no-ops — the orchestrator @@ -1456,6 +1460,81 @@ def _sync_sandbox_command_path_with_agent(self) -> None: if isinstance(path, str) and path: self.sandbox.set_command_base_path(path) + def _eval_route_overrides(self) -> EvalRouteOverrides: + """The ``(backend, model)`` pair from ``task.checker_context.api_route``, if any. + + A task/variant-authored choice for the WHOLE evaluation side (``llm_judge``, + ``agent_judge``, the simulator all share one ``eval_route``), decoupled from + the agent's own route/model — resolved into a credentialed ``ApiRoute`` (from + env vars) by ``resolve_evaluation_route``, which bakes ``model`` into the + resolved route's own ``model`` field. Reserved under one ``api_route`` + namespace (not per-criterion-type): there is exactly one eval route per run, + not one per criterion. Either element is ``None`` when unset, in which case + ``resolve_evaluation_route`` falls back to its existing pin-to-Claude / + env-configured-default behavior. NOT currently ``-D``-reachable — task/variant + YAML only. + """ + api_route = self.task.checker_context.api_route if self.task.checker_context else None + if api_route is None: + return EvalRouteOverrides(backend=None, model=None, params=None, env_params=None) + return EvalRouteOverrides( + backend=api_route.route.value if api_route.route is not None else None, + model=api_route.model, + params=api_route.params, + env_params=api_route.env_params, + ) + + def _resolve_routes(self) -> None: + """Resolve ``self.route``/``self.eval_route``, log routing, and build + ``self.success_checker``. Shared by the evaluate-only and normal setup + paths in ``_setup`` — identical logic, previously duplicated at each call + site. Requires ``self.sandbox`` to already be set. + """ + assert self.sandbox is not None + self.route = resolve_route(settings) + overrides = self._eval_route_overrides() + self.eval_route = resolve_evaluation_route( + settings, + self.route, + backend_override=overrides.backend, + model_override=overrides.model, + params_override=overrides.params, + env_params_override=overrides.env_params, + ) + self._reject_litellm_eval_route_if_unsupported() + logger.info("API routing: %s", _format_routing(self.route, self.task.agent.model if self.task.agent else None)) + self.success_checker = SuccessChecker(self.sandbox, route=self.eval_route) + + def _reject_litellm_eval_route_if_unsupported(self) -> None: + """``checker_context.api_route.route: litellm`` dispatches ``llm_judge`` + through the ``litellm`` library (protocol-agnostic — see + ``invoke_litellm_judge_async``'s module docstring), but ``agent_judge`` + and the simulator run as real Claude Code CLI subprocesses that speak + the Anthropic Messages protocol only. Handing them a ``LiteLLMRoute`` + built from arbitrary ``params``/``env_params`` (which may front an + OpenAI-/Vertex-shaped gateway with no Anthropic-compatible endpoint at + all) would either misroute onto the AGENT's own unrelated LiteLLM + settings or fail with a confusing SDK-level error — the exact + misrouting ``LiteLLMRoute``'s own docstring says must never happen. + Reject the combination loudly at resolution time instead. + """ + if not isinstance(self.eval_route, LiteLLMRoute): + return + offenders: list[str] = [] + if any(isinstance(c, AgentJudgeCriterion) and c.enabled for c in self.task.success_criteria): + offenders.append("an enabled agent_judge criterion") + if self.task.simulation is not None and self.task.simulation.enabled: + offenders.append("simulation.enabled") + if offenders: + named = " and ".join(offenders) + msg = ( + f"checker_context.api_route.route: litellm is llm_judge-only (it dispatches through the " + f"litellm library in-process, not a real Claude Code subprocess), but this task also has " + f"{named}, which run as Claude Code sub-agents requiring an Anthropic-compatible endpoint. " + f"Use route: bedrock/direct instead, or remove/disable {named}." + ) + raise ValueError(msg) + def _record_route_environment_info(self) -> None: """Persist resolved route + judge transport into ``result.environment_info``. @@ -1472,6 +1551,13 @@ def _record_route_environment_info(self) -> None: # it, distinct from the agent's api_routing. if self.eval_route is not None: self.result.environment_info["eval_routing"] = ROUTE_NAMES[type(self.eval_route)] + # bedrock_model/litellm_model below are sourced from self.route (the + # AGENT's route) — record the judge/simulator's own model separately so + # a checker_context.api_route.model override (or the LiteLLM-agent + # pinned-to-Bedrock default) is visible in run artifacts, not just + # inferable from the agent's model. + if self.eval_route.model: + self.result.environment_info["eval_model"] = self.eval_route.model if isinstance(self.route, BedrockRoute): self.result.environment_info["aws_region"] = self.route.region if self.route.model: @@ -1483,7 +1569,9 @@ def _record_route_environment_info(self) -> None: elif isinstance(self.route, LiteLLMRoute): # Host only (never the base_url or auth token) — mirrors the Codex # agent's host-only recording so secrets stay out of run artifacts. - self.result.environment_info["litellm_base_url_host"] = urlparse(self.route.base_url).hostname or "" + self.result.environment_info["litellm_base_url_host"] = ( + urlparse(settings.litellm_base_url or "").hostname or "" + ) if self.route.model: self.result.environment_info["litellm_model"] = self.route.model # Agent-specific routing (e.g. Codex custom-endpoint / Azure). No-op for diff --git a/tests/test_config_precedence.py b/tests/test_config_precedence.py index b726786c..d968fda3 100644 --- a/tests/test_config_precedence.py +++ b/tests/test_config_precedence.py @@ -381,7 +381,6 @@ def test_resolve_route_bedrock(): ) route = resolve_route(s) assert isinstance(route, BedrockRoute) - assert route.bearer_token == "tok-123" assert route.region == "us-east-1" assert route.model == "eu.anthropic.claude-sonnet-4-6" diff --git a/tests/test_judge_bedrock.py b/tests/test_judge_bedrock.py index a3ea103a..d0a0aeee 100644 --- a/tests/test_judge_bedrock.py +++ b/tests/test_judge_bedrock.py @@ -27,6 +27,13 @@ def _make_response(*, status_code: int = 200, json_data: Any = None, text: str = return response +@pytest.fixture(autouse=True) +def _bearer_token(monkeypatch: pytest.MonkeyPatch) -> None: + """invoke_bedrock_judge_async reads the bearer token from settings, not the + route, so every test needs one set regardless of whether it inspects it.""" + monkeypatch.setattr(judge_bedrock.settings, "aws_bearer_token_bedrock", "test-token") + + @pytest.fixture def no_sleep(monkeypatch: pytest.MonkeyPatch) -> list[float]: """Capture backoff sleeps instead of actually sleeping.""" @@ -40,7 +47,7 @@ async def fake_sleep(s: float) -> None: def _route() -> BedrockRoute: - return BedrockRoute(bearer_token="test-token", region="eu-north-1") + return BedrockRoute(region="eu-north-1") def _tool_use_response(score: float = 0.5, rationale: str = "ok") -> dict[str, Any]: diff --git a/tests/test_judge_burn_in_live.py b/tests/test_judge_burn_in_live.py index 67cfc3f1..bf780fac 100644 --- a/tests/test_judge_burn_in_live.py +++ b/tests/test_judge_burn_in_live.py @@ -110,7 +110,7 @@ def test_llm_judge_bedrock_tool_channel(hello_sandbox: Sandbox) -> None: result = SuccessChecker( hello_sandbox, init_registry=False, - route=BedrockRoute(bearer_token=bearer, region=region), + route=BedrockRoute(region=region), ).check(criterion) assert result.error is None, f"Bedrock judge failed: {result.error}\n{result.details}" diff --git a/tests/test_judge_litellm.py b/tests/test_judge_litellm.py new file mode 100644 index 00000000..2c52706f --- /dev/null +++ b/tests/test_judge_litellm.py @@ -0,0 +1,209 @@ +"""Tests for the LiteLLM judge invoker, which calls through the ``litellm`` +library (``litellm.acompletion``) rather than a hand-rolled HTTP client — +``litellm`` normalizes provider-specific request/response shapes so the judge +transport doesn't have to. Unlike the agent's own LiteLLM backend, this path +reads NOTHING from ``coder_eval.config.settings`` — everything comes from +``route.params``/``route.env_params``. +""" + +from __future__ import annotations + +import sys +from typing import Any +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +from coder_eval.errors import JudgeInfrastructureError +from coder_eval.evaluation.judge_litellm import invoke_litellm_judge_async +from coder_eval.evaluation.verdict_tool import SUBMIT_VERDICT_ANTHROPIC_TOOL +from coder_eval.models.routing import LiteLLMRoute + + +def _make_response(*, score: float = 0.5, rationale: str = "ok") -> MagicMock: + """Mimic litellm's OpenAI-shaped ``ModelResponse``: ``model_dump()`` returns + an OpenAI Chat-Completions-native tool_calls dict, regardless of the + underlying provider litellm actually routed to. + + ``spec=ModelResponse`` so ``invoke_litellm_judge_async``'s defensive + ``isinstance(response, ModelResponse)`` guard (against the + ``ModelResponse | CustomStreamWrapper`` union ``acompletion`` is typed to + return) passes against the mock the same as it would the real object.""" + import json + + from litellm.types.utils import ModelResponse + + response = MagicMock(spec=ModelResponse) + response.model_dump.return_value = { + "choices": [ + { + "message": { + "tool_calls": [ + { + "function": { + "name": "submit_verdict", + "arguments": json.dumps({"score": score, "rationale": rationale}), + } + } + ] + } + } + ] + } + return response + + +def _route( + *, + params: dict[str, Any] | None = None, + env_params: dict[str, str] | None = None, +) -> LiteLLMRoute: + return LiteLLMRoute(model="gpt-5.6-luna", params=params, env_params=env_params) + + +async def _invoke(**overrides): + defaults = { + "route": _route(params={"api_base": "http://gateway:4000", "api_key": "sk-master"}), + "model": "azure_ai/gpt-5.6-luna", + "system": "s", + "user": "u", + "max_tokens": 10, + "tool_spec": SUBMIT_VERDICT_ANTHROPIC_TOOL, + } + defaults.update(overrides) + return await invoke_litellm_judge_async(**defaults) + + +async def test_invoke_litellm_judge_calls_acompletion() -> None: + acompletion = AsyncMock(return_value=_make_response(score=0.42)) + with patch("litellm.acompletion", new=acompletion): + result = await _invoke() + acompletion.assert_called_once() + kwargs = acompletion.call_args.kwargs + # Model id travels verbatim, provider prefix and all -- litellm routes on it. + assert kwargs["model"] == "azure_ai/gpt-5.6-luna" + assert kwargs["api_base"] == "http://gateway:4000" + assert kwargs["api_key"] == "sk-master" + assert kwargs["drop_params"] is True + assert kwargs["tools"][0]["function"]["name"] == "submit_verdict" + assert kwargs["tool_choice"] == {"type": "function", "function": {"name": "submit_verdict"}} + assert result["choices"][0]["message"]["tool_calls"][0]["function"]["name"] == "submit_verdict" + + +async def test_invoke_litellm_judge_omits_temperature_by_default() -> None: + """Unlike invoke_anthropic_judge_async/invoke_bedrock_judge_async, there is no + `temperature` parameter at all here — a gateway-routed model may reject it + outright (observed live against an Azure AI deployment), so the task author + opts in via `params: {temperature: ...}` if their model accepts it.""" + acompletion = AsyncMock(return_value=_make_response()) + with patch("litellm.acompletion", new=acompletion): + await _invoke(max_tokens=321, system="sys", user="usr") + kwargs: dict[str, Any] = dict(acompletion.call_args.kwargs) + assert "temperature" not in kwargs + assert kwargs["max_completion_tokens"] == 321 + assert kwargs["messages"] == [ + {"role": "system", "content": "sys"}, + {"role": "user", "content": "usr"}, + ] + + +async def test_invoke_litellm_judge_sends_temperature_via_params() -> None: + acompletion = AsyncMock(return_value=_make_response()) + with patch("litellm.acompletion", new=acompletion): + await _invoke(route=_route(params={"temperature": 0.7})) + assert acompletion.call_args.kwargs["temperature"] == 0.7 + + +async def test_invoke_litellm_judge_passes_through_params() -> None: + """`route.params` is arbitrary passthrough merged straight into the + litellm.acompletion() kwargs — e.g. api_base, aws_region_name, ...""" + acompletion = AsyncMock(return_value=_make_response()) + with patch("litellm.acompletion", new=acompletion): + await _invoke(route=_route(params={"aws_region_name": "eu-north-1", "api_version": "2024-05-01"})) + kwargs = acompletion.call_args.kwargs + assert kwargs["aws_region_name"] == "eu-north-1" + assert kwargs["api_version"] == "2024-05-01" + + +async def test_invoke_litellm_judge_resolves_env_params_from_env(monkeypatch: pytest.MonkeyPatch) -> None: + """`route.env_params` maps a kwarg name -> ENV VAR NAME; the value is only + ever resolved at call time, never stored on the route.""" + monkeypatch.setenv("MY_AWS_ACCESS_KEY_ID", "AKIA-fake") + monkeypatch.setenv("MY_AWS_SECRET_ACCESS_KEY", "secret-fake") + acompletion = AsyncMock(return_value=_make_response()) + route = _route( + env_params={ + "aws_access_key_id": "MY_AWS_ACCESS_KEY_ID", + "aws_secret_access_key": "MY_AWS_SECRET_ACCESS_KEY", + } + ) + with patch("litellm.acompletion", new=acompletion): + await _invoke(route=route) + kwargs = acompletion.call_args.kwargs + assert kwargs["aws_access_key_id"] == "AKIA-fake" + assert kwargs["aws_secret_access_key"] == "secret-fake" + + +async def test_invoke_litellm_judge_env_params_override_params(monkeypatch: pytest.MonkeyPatch) -> None: + """`env_params` is resolved AFTER `params`, so it always wins for the same key.""" + monkeypatch.setenv("REAL_KEY", "sk-real") + acompletion = AsyncMock(return_value=_make_response()) + route = _route(params={"api_key": "sk-literal-in-yaml"}, env_params={"api_key": "REAL_KEY"}) + with patch("litellm.acompletion", new=acompletion): + await _invoke(route=route) + assert acompletion.call_args.kwargs["api_key"] == "sk-real" + + +async def test_invoke_litellm_judge_raises_on_missing_env_var() -> None: + route = _route(env_params={"aws_access_key_id": "TOTALLY_UNSET_ENV_VAR_XYZ"}) + with pytest.raises(JudgeInfrastructureError, match="TOTALLY_UNSET_ENV_VAR_XYZ"): + await _invoke(route=route) + + +async def test_invoke_litellm_judge_raises_on_empty_model() -> None: + with pytest.raises(ValueError, match="model must not be empty"): + await _invoke(model="") + + +async def test_invoke_litellm_judge_raises_when_library_not_installed(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setitem(sys.modules, "litellm", None) + with pytest.raises(JudgeInfrastructureError, match=r"pip install 'coder-eval\[litellm\]'"): + await _invoke() + + +async def test_invoke_litellm_judge_wraps_api_error() -> None: + from litellm.exceptions import APIError + + acompletion = AsyncMock(side_effect=APIError(status_code=500, message="boom", llm_provider="azure_ai", model="m")) + with ( + patch("litellm.acompletion", new=acompletion), + pytest.raises(JudgeInfrastructureError, match="LiteLLM judge API error"), + ): + await _invoke() + + +async def test_invoke_litellm_judge_wraps_bad_request_error() -> None: + from litellm.exceptions import BadRequestError + + rejection = BadRequestError( + message="Unsupported parameter: 'foo'.", + model="m", + llm_provider="azure_ai", + body={"error": {"message": "...", "param": "foo", "code": None}}, + ) + acompletion = AsyncMock(side_effect=rejection) + with ( + patch("litellm.acompletion", new=acompletion), + pytest.raises(JudgeInfrastructureError, match="LiteLLM judge call failed"), + ): + await _invoke() + acompletion.assert_called_once() + + +async def test_invoke_litellm_judge_escalates_on_signature_break() -> None: + acompletion = AsyncMock(side_effect=TypeError("acompletion() got an unexpected keyword argument 'drop_params'")) + with ( + patch("litellm.acompletion", new=acompletion), + pytest.raises(JudgeInfrastructureError, match="LiteLLM judge call failed"), + ): + await _invoke() diff --git a/tests/test_litellm_cost.py b/tests/test_litellm_cost.py index 2e5c77b5..ea23c8c7 100644 --- a/tests/test_litellm_cost.py +++ b/tests/test_litellm_cost.py @@ -252,7 +252,7 @@ def test_joins_on_litellm_route(self, tmp_path, monkeypatch): ) monkeypatch.setattr(orch_mod.settings, "litellm_cost_log", str(log)) fake = SimpleNamespace( - route=LiteLLMRoute(base_url="http://x:4000", auth_token="k", model="deepseek/deepseek-v4-pro"), + route=LiteLLMRoute(model="deepseek/deepseek-v4-pro"), result=_result([_turn(0, static_cost=0.5)]), _cost_correlation_run_id=run_id, _cost_attempt_nonce="att1", @@ -265,7 +265,7 @@ def test_joins_on_litellm_route(self, tmp_path, monkeypatch): def test_join_never_raises_on_bad_log(self, tmp_path, monkeypatch): monkeypatch.setattr(orch_mod.settings, "litellm_cost_log", str(tmp_path / "does-not-exist.jsonl")) fake = SimpleNamespace( - route=LiteLLMRoute(base_url="http://x:4000", auth_token="k"), + route=LiteLLMRoute(), result=_result([_turn(0, static_cost=0.5)]), _cost_correlation_run_id="R", _cost_attempt_nonce="att1", @@ -289,7 +289,7 @@ def test_run_total_rederives_from_actual_after_join(self, tmp_path, monkeypatch) ) monkeypatch.setattr(orch_mod.settings, "litellm_cost_log", str(log)) fake = SimpleNamespace( - route=LiteLLMRoute(base_url="http://x:4000", auth_token="k", model="deepseek/deepseek-v4-pro"), + route=LiteLLMRoute(model="deepseek/deepseek-v4-pro"), result=_result([_turn(0, static_cost=0.5), _turn(1, static_cost=0.5)]), _cost_correlation_run_id=run_id, _cost_attempt_nonce="att1", diff --git a/tests/test_litellm_judge_live.py b/tests/test_litellm_judge_live.py new file mode 100644 index 00000000..550fe579 --- /dev/null +++ b/tests/test_litellm_judge_live.py @@ -0,0 +1,69 @@ +"""Live integration test for the ``checker_context.api_route.route: litellm`` +judge transport — hits a real gateway via ``litellm.acompletion()``. + +Reuses the same ``CODEX_API_KEY``/``CODEX_BASE_URL``/``CODEX_MODEL`` secrets the +Codex live tests already exercise (a real OpenAI-protocol-compatible deployment) +instead of a dedicated new secret, so this regression check runs in CI for free. +Addresses the PR #137 review gap: "Nothing in the repo exercises the feature. +No task, no experiment variant." — this is the litellm-route equivalent of +``tests/test_codex_agent_live.py``, at the invoker level rather than a full +``coder-eval run`` (no sandbox / agent turn needed to exercise the transport). + +Requirements: + - The ``[litellm]`` extra installed (``uv sync --extra litellm``). + - ``CODEX_API_KEY``, ``CODEX_BASE_URL``, ``CODEX_MODEL`` in the environment. + +Run with: ``pytest -m live``. +""" + +from __future__ import annotations + +import os + +import pytest + + +pytest.importorskip("litellm") + +from coder_eval.evaluation.judge_litellm import invoke_litellm_judge_async +from coder_eval.evaluation.verdict_tool import SUBMIT_VERDICT_ANTHROPIC_TOOL, extract_verdict_from_openai_response +from coder_eval.models.routing import LiteLLMRoute + + +_live = pytest.mark.live + + +def _have_creds() -> bool: + return bool(os.getenv("CODEX_API_KEY") and os.getenv("CODEX_BASE_URL") and os.getenv("CODEX_MODEL")) + + +_skip_reason = "Live litellm judge test needs [litellm] extra + CODEX_API_KEY/CODEX_BASE_URL/CODEX_MODEL" +pytestmark = [_live, pytest.mark.skipif(not _have_creds(), reason=_skip_reason)] + + +@_live +async def test_litellm_judge_route_scores_real_gateway_call(): + """Round-trips a real forced ``submit_verdict`` tool call through + ``litellm.acompletion`` against the same gateway/model the Codex live tests + use, with ``api_base``/``api_key`` resolved purely from ``env_params`` (no + LITELLM_BASE_URL/LITELLM_AUTH_TOKEN involved) — the exact shape + ``checker_context.api_route.route: litellm`` produces.""" + model = os.environ["CODEX_MODEL"] + route = LiteLLMRoute( + model=model, + env_params={"api_base": "CODEX_BASE_URL", "api_key": "CODEX_API_KEY"}, + ) + response = await invoke_litellm_judge_async( + route=route, + model=model, + system="You are a strict grader. Call the submit_verdict tool exactly once.", + user=( + 'Score 1.0 if 2 + 2 = 4, else 0.0. Call submit_verdict with score=1.0, rationale="arithmetic is correct".' + ), + max_tokens=200, + tool_spec=SUBMIT_VERDICT_ANTHROPIC_TOOL, + ) + verdict, err = extract_verdict_from_openai_response(response) + assert err is None, f"judge did not call submit_verdict: {err}" + assert verdict is not None + assert verdict.score == 1.0 diff --git a/tests/test_litellm_route.py b/tests/test_litellm_route.py index 3d7a7a8c..48256e19 100644 --- a/tests/test_litellm_route.py +++ b/tests/test_litellm_route.py @@ -46,18 +46,24 @@ def _isolated_settings(monkeypatch, **kwargs): monkeypatch.delenv(var, raising=False) return Settings(_env_file=None, **kwargs) - def test_bedrock_agent_route_is_reused_unchanged(self, monkeypatch): - route = BedrockRoute(bearer_token="tok", region="eu-north-1", model="eu.anthropic.claude-sonnet-4-6") + def test_bedrock_agent_route_is_reused_with_model_reset(self, monkeypatch): + # Same region/other fields reused verbatim, but `model` is reset to None + # (no checker_context override) rather than carrying over the agent's own + # env-sourced model — see TestResolveEvaluationRouteJudgeFloor. + route = BedrockRoute(region="eu-north-1", model="eu.anthropic.claude-sonnet-4-6") settings = self._isolated_settings(monkeypatch, api_backend=ApiBackend.BEDROCK) - assert resolve_evaluation_route(settings, route) is route + ev = resolve_evaluation_route(settings, route) + assert ev is not route + assert ev == BedrockRoute(region="eu-north-1", model=None) - def test_direct_agent_route_is_reused_unchanged(self, monkeypatch): - route = DirectRoute(judge_transport="anthropic") + def test_direct_agent_route_is_reused_with_model_reset(self, monkeypatch): + route = DirectRoute(judge_transport="anthropic", model="gpt-4") settings = self._isolated_settings(monkeypatch, api_backend=ApiBackend.DIRECT) - assert resolve_evaluation_route(settings, route) is route + ev = resolve_evaluation_route(settings, route) + assert ev == DirectRoute(judge_transport="anthropic", model=None) def test_litellm_agent_pins_evaluation_to_bedrock_when_aws_creds_present(self, monkeypatch): - agent = LiteLLMRoute(base_url="http://x:4000", auth_token="sk-1", model="zai.glm-5") + agent = LiteLLMRoute(model="zai.glm-5") settings = self._isolated_settings( monkeypatch, api_backend=ApiBackend.LITELLM, @@ -66,13 +72,13 @@ def test_litellm_agent_pins_evaluation_to_bedrock_when_aws_creds_present(self, m ) ev = resolve_evaluation_route(settings, agent) assert isinstance(ev, BedrockRoute) - assert ev.bearer_token == "aws-tok" assert ev.region == "eu-north-1" - # Judge + simulator run on a Claude model, region-qualified. - assert ev.model == "eu.anthropic.claude-sonnet-4-6" + # No checker_context override -> model is None, so llm_judge falls back + # to DEFAULT_JUDGE_MODEL rather than an env-sourced value. + assert ev.model is None def test_litellm_agent_falls_back_to_direct_when_only_anthropic_key(self, monkeypatch): - agent = LiteLLMRoute(base_url="http://x:4000", auth_token="sk-1", model="zai.glm-5") + agent = LiteLLMRoute(model="zai.glm-5") settings = self._isolated_settings(monkeypatch, api_backend=ApiBackend.LITELLM, anthropic_api_key="sk-ant") ev = resolve_evaluation_route(settings, agent) assert isinstance(ev, DirectRoute) @@ -81,13 +87,235 @@ def test_litellm_agent_falls_back_to_direct_when_only_anthropic_key(self, monkey def test_litellm_agent_unconfigured_yields_direct_with_no_transport(self, monkeypatch): # No Bedrock creds and no ANTHROPIC_API_KEY → DirectRoute(None), which makes # llm_judge fail with its clean "unconfigured" error rather than scoring 0.0. - agent = LiteLLMRoute(base_url="http://x:4000", auth_token="sk-1", model="zai.glm-5") + agent = LiteLLMRoute(model="zai.glm-5") settings = self._isolated_settings(monkeypatch, api_backend=ApiBackend.LITELLM) ev = resolve_evaluation_route(settings, agent) assert isinstance(ev, DirectRoute) assert ev.judge_transport is None +class TestResolveEvaluationRouteJudgeFloor: + """Regression coverage for PR #137 review Axis 8 ('the judge loses + DEFAULT_JUDGE_MODEL as its floor'): resolve_evaluation_route must never bake + the agent's own env-sourced model into the eval route's `model` unless a real + checker_context.api_route.model override was given — otherwise an unpinned + llm_judge silently starts grading with a different model whenever the + agent's model changes (BEDROCK_MODEL), breaking before/after comparability. + """ + + @staticmethod + def _isolated_settings(monkeypatch, **kwargs): + for var in ("AWS_BEARER_TOKEN_BEDROCK", "AWS_REGION", "ANTHROPIC_API_KEY", "BEDROCK_MODEL"): + monkeypatch.delenv(var, raising=False) + return Settings(_env_file=None, **kwargs) + + def test_bedrock_agent_route_no_override_strips_agent_model(self, monkeypatch): + # Simulates an agent route built with a real BEDROCK_MODEL (e.g. opus), + # reused for the eval side with no checker_context override -> the eval + # route's model must be None so llm_judge falls back to DEFAULT_JUDGE_MODEL, + # not the agent's model. + agent_route = BedrockRoute(region="eu-north-1", model="eu.anthropic.claude-opus-4-1") + settings = self._isolated_settings(monkeypatch, api_backend=ApiBackend.BEDROCK) + ev = resolve_evaluation_route(settings, agent_route) + assert isinstance(ev, BedrockRoute) + assert ev.model is None + + def test_bedrock_agent_route_with_override_qualifies_model(self, monkeypatch): + agent_route = BedrockRoute(region="eu-north-1", model="eu.anthropic.claude-opus-4-1") + settings = self._isolated_settings(monkeypatch, api_backend=ApiBackend.BEDROCK) + ev = resolve_evaluation_route(settings, agent_route, model_override="claude-haiku-4-5") + assert isinstance(ev, BedrockRoute) + assert ev.model == "eu.anthropic.claude-haiku-4-5" + + def test_direct_agent_route_no_override_strips_agent_model(self, monkeypatch): + agent_route = DirectRoute(judge_transport="anthropic", model="gpt-4") + settings = self._isolated_settings(monkeypatch, api_backend=ApiBackend.DIRECT) + ev = resolve_evaluation_route(settings, agent_route) + assert isinstance(ev, DirectRoute) + assert ev.model is None + + def test_litellm_agent_pin_to_bedrock_no_override_strips_bedrock_model(self, monkeypatch): + # Agent on LiteLLM (open-weight); AWS creds present with a real + # BEDROCK_MODEL configured for some unrelated purpose. No override -> + # the pinned eval route must NOT inherit BEDROCK_MODEL. + agent_route = LiteLLMRoute(model="zai.glm-5") + settings = self._isolated_settings( + monkeypatch, + api_backend=ApiBackend.LITELLM, + aws_bearer_token_bedrock="aws-tok", + aws_region="eu-north-1", + bedrock_model="claude-opus-4-1", + ) + ev = resolve_evaluation_route(settings, agent_route) + assert isinstance(ev, BedrockRoute) + assert ev.model is None + + def test_litellm_agent_pin_to_bedrock_with_override(self, monkeypatch): + agent_route = LiteLLMRoute(model="zai.glm-5") + settings = self._isolated_settings( + monkeypatch, + api_backend=ApiBackend.LITELLM, + aws_bearer_token_bedrock="aws-tok", + aws_region="eu-north-1", + ) + ev = resolve_evaluation_route(settings, agent_route, model_override="claude-haiku-4-5") + assert isinstance(ev, BedrockRoute) + assert ev.model == "eu.anthropic.claude-haiku-4-5" + + +class TestBackendOverride: + """resolve_evaluation_route(backend_override=...) — the checker_context.api_route.route + override path, dispatched through _resolve_backend_route. Zero coverage before this PR + review (Axis 3 blocker).""" + + @staticmethod + def _isolated_settings(monkeypatch, **kwargs): + for var in ( + "AWS_BEARER_TOKEN_BEDROCK", + "AWS_REGION", + "ANTHROPIC_API_KEY", + "LITELLM_BASE_URL", + "LITELLM_AUTH_TOKEN", + ): + monkeypatch.delenv(var, raising=False) + return Settings(_env_file=None, **kwargs) + + def test_override_to_bedrock_builds_route_from_env(self, monkeypatch): + agent_route = DirectRoute() + settings = self._isolated_settings( + monkeypatch, + api_backend=ApiBackend.DIRECT, + aws_bearer_token_bedrock="aws-tok", + aws_region="eu-north-1", + ) + ev = resolve_evaluation_route(settings, agent_route, backend_override="bedrock") + assert isinstance(ev, BedrockRoute) + assert ev.region == "eu-north-1" + + def test_override_to_bedrock_without_creds_raises(self, monkeypatch): + agent_route = DirectRoute() + settings = self._isolated_settings(monkeypatch, api_backend=ApiBackend.DIRECT) + with pytest.raises(ValueError, match="requires AWS_BEARER_TOKEN_BEDROCK and AWS_REGION"): + resolve_evaluation_route(settings, agent_route, backend_override="bedrock") + + def test_override_to_direct_builds_route_from_env(self, monkeypatch): + agent_route = BedrockRoute(region="eu-north-1") + settings = self._isolated_settings(monkeypatch, api_backend=ApiBackend.BEDROCK, anthropic_api_key="sk-ant") + ev = resolve_evaluation_route(settings, agent_route, backend_override="direct") + assert isinstance(ev, DirectRoute) + assert ev.judge_transport == "anthropic" + + def test_override_to_direct_without_key_raises(self, monkeypatch): + agent_route = BedrockRoute(region="eu-north-1") + settings = self._isolated_settings(monkeypatch, api_backend=ApiBackend.BEDROCK) + with pytest.raises(ValueError, match="requires ANTHROPIC_API_KEY"): + resolve_evaluation_route(settings, agent_route, backend_override="direct") + + def test_override_to_litellm_builds_route_from_params_and_env_params(self, monkeypatch): + """No dependency on settings.litellm_base_url/litellm_auth_token at all — + the checker's litellm route is built entirely from params/env_params.""" + agent_route = BedrockRoute(region="eu-north-1") + settings = self._isolated_settings(monkeypatch, api_backend=ApiBackend.BEDROCK) + ev = resolve_evaluation_route( + settings, + agent_route, + backend_override="litellm", + model_override="gpt-5.6-luna", + params_override={"api_base": "http://gateway:4000"}, + env_params_override={"api_key": "MY_ENV_VAR"}, + ) + assert isinstance(ev, LiteLLMRoute) + assert ev.model == "gpt-5.6-luna" + assert ev.params == {"api_base": "http://gateway:4000"} + assert ev.env_params == {"api_key": "MY_ENV_VAR"} + + def test_override_to_litellm_without_model_raises(self, monkeypatch): + """There is no default open-weight/gateway model to fall back to.""" + agent_route = BedrockRoute(region="eu-north-1") + settings = self._isolated_settings(monkeypatch, api_backend=ApiBackend.BEDROCK) + with pytest.raises(ValueError, match=r"requires an explicit `checker_context\.api_route\.model`"): + resolve_evaluation_route(settings, agent_route, backend_override="litellm") + + def test_unknown_backend_raises(self, monkeypatch): + agent_route = DirectRoute() + settings = self._isolated_settings(monkeypatch, api_backend=ApiBackend.DIRECT) + with pytest.raises(ValueError, match="is not a known backend"): + resolve_evaluation_route(settings, agent_route, backend_override="not-a-backend") + + +class TestCheckerContextModel: + """CheckerContext/ApiRouteContext — the typed replacement for the old + hand-validated open dict (previously 0% covered, per the PR #137 review). + ``extra="forbid"`` + real field types now do what the hand-rolled + ``validate_checker_context_shape`` used to.""" + + @staticmethod + def _validate(value): + from coder_eval.models import CheckerContext + + return CheckerContext(**value) + + def test_accepts_empty(self): + self._validate({}) + + def test_accepts_route_and_model(self): + cc = self._validate({"api_route": {"route": "bedrock", "model": "claude-haiku-4-5"}}) + assert cc.api_route is not None + assert cc.api_route.route == ApiBackend.BEDROCK + assert cc.api_route.model == "claude-haiku-4-5" + + def test_rejects_unknown_namespace(self): + with pytest.raises(ValueError, match=r"[Ee]xtra"): + self._validate({"api_rotue": {"route": "bedrock"}}) + + def test_rejects_unknown_api_route_key(self): + with pytest.raises(ValueError, match=r"[Ee]xtra"): + self._validate({"api_route": {"rotue": "bedrock"}}) + + def test_rejects_unknown_backend_name(self): + with pytest.raises(ValueError): + self._validate({"api_route": {"route": "not-a-backend"}}) + + def test_accepts_params_and_env_params_with_litellm_route(self): + cc = self._validate( + { + "api_route": { + "route": "litellm", + "params": {"aws_region_name": "eu-north-1"}, + "env_params": {"api_key": "MY_ENV_VAR"}, + } + } + ) + assert cc.api_route is not None + assert cc.api_route.params == {"aws_region_name": "eu-north-1"} + assert cc.api_route.env_params == {"api_key": "MY_ENV_VAR"} + + def test_rejects_params_without_litellm_route(self): + with pytest.raises(ValueError, match="require route: litellm"): + self._validate({"api_route": {"route": "bedrock", "params": {"x": 1}}}) + + def test_rejects_env_params_without_litellm_route(self): + with pytest.raises(ValueError, match="require route: litellm"): + self._validate({"api_route": {"env_params": {"api_key": "MY_ENV_VAR"}}}) + + def test_rejects_non_dict_params(self): + with pytest.raises(ValueError): + self._validate({"api_route": {"route": "litellm", "params": "not-a-dict"}}) + + def test_rejects_non_dict_env_params(self): + with pytest.raises(ValueError): + self._validate({"api_route": {"route": "litellm", "env_params": "not-a-dict"}}) + + def test_rejects_non_string_env_params_values(self): + with pytest.raises(ValueError): + self._validate({"api_route": {"route": "litellm", "env_params": {"api_key": 123}}}) + + def test_rejects_non_string_model(self): + """A YAML `model: 5` must be rejected here, not str()-ified downstream.""" + with pytest.raises(ValueError): + self._validate({"api_route": {"route": "bedrock", "model": 5}}) + + class TestEvalRouteWiring: """The orchestrator must hand the simulated user the eval_route (constant Claude), never the agent's (possibly open-weight) route — guards the @@ -100,8 +328,8 @@ async def test_simulator_receives_eval_route_not_agent_route(self, monkeypatch): from coder_eval import orchestrator as orch_mod from coder_eval.orchestrator import Orchestrator - eval_route = BedrockRoute(bearer_token="t", region="eu-north-1", model="eu.anthropic.claude-sonnet-4-6") - agent_route = LiteLLMRoute(base_url="http://x:4000", auth_token="k", model="zai.glm-5") + eval_route = BedrockRoute(region="eu-north-1", model="eu.anthropic.claude-sonnet-4-6") + agent_route = LiteLLMRoute(model="zai.glm-5") captured: dict = {} class _SpySimulator: @@ -140,8 +368,8 @@ def test_resolves_custom_route_with_all_fields(self): ) route = resolve_route(settings) assert isinstance(route, LiteLLMRoute) - assert route.base_url == "http://localhost:4000" - assert route.auth_token == "sk-master" + # base_url/auth_token are NOT stored on the route (read live from + # settings by _build_sdk_env) -- only model/small_model live here. assert route.model == "deepseek.v3.2" def test_rejects_scheme_less_base_url(self): @@ -253,10 +481,12 @@ def test_none_agent_skips_custom_validation(self): class TestBuildSdkEnvCustom: """_build_sdk_env() for the LiteLLM route.""" - def test_custom_route_env_has_anthropic_vars_only(self): + def test_custom_route_env_has_anthropic_vars_only(self, monkeypatch): + from coder_eval.agents import claude_code_agent as claude_code_agent_mod + + monkeypatch.setattr(claude_code_agent_mod.settings, "litellm_auth_token", "sk-1") + monkeypatch.setattr(claude_code_agent_mod.settings, "litellm_base_url", "http://x:4000") route = LiteLLMRoute( - base_url="http://x:4000", - auth_token="sk-1", model="deepseek.v3.2", small_model="deepseek.v3.2", ) @@ -273,7 +503,7 @@ def test_custom_route_env_has_anthropic_vars_only(self): assert "AWS_REGION" not in env def test_custom_route_no_model_omits_model_vars(self): - route = LiteLLMRoute(base_url="http://x:4000", auth_token="sk-1") + route = LiteLLMRoute() env, model = ClaudeCodeAgent._build_sdk_env(route) assert model is None assert "ANTHROPIC_MODEL" not in env @@ -284,7 +514,7 @@ def test_custom_route_forwards_path(self, monkeypatch): custom_path = f"/custom/bin{os.pathsep}/usr/bin" monkeypatch.setenv("PATH", custom_path) - env, _ = ClaudeCodeAgent._build_sdk_env(LiteLLMRoute(base_url="http://x:4000", auth_token="sk-1")) + env, _ = ClaudeCodeAgent._build_sdk_env(LiteLLMRoute()) assert env["PATH"] == custom_path def test_custom_route_neutralizes_inherited_anthropic_api_key(self, monkeypatch): @@ -292,20 +522,20 @@ def test_custom_route_neutralizes_inherited_anthropic_api_key(self, monkeypatch) empty in options.env (not merely omitted) — else it would fight the bearer auth_token against the gateway.""" monkeypatch.setenv("ANTHROPIC_API_KEY", "leaked-key") - env, _ = ClaudeCodeAgent._build_sdk_env(LiteLLMRoute(base_url="http://x:4000", auth_token="sk-1")) + env, _ = ClaudeCodeAgent._build_sdk_env(LiteLLMRoute()) assert env["ANTHROPIC_API_KEY"] == "" def test_cost_log_tags_become_custom_headers(self): """cost_log_tags → ANTHROPIC_CUSTOM_HEADERS as newline-separated `Name: Value` pairs (the format Claude Code forwards verbatim), so the proxy-side cost log can join each call back to the run/task/turn.""" - route = LiteLLMRoute(base_url="http://x:4000", auth_token="sk-1", model="deepseek/deepseek-v4-pro") + route = LiteLLMRoute(model="deepseek/deepseek-v4-pro") tags = {"x-ce-run-id": "abc123", "x-ce-task-id": "calc/v1", "x-ce-iteration": "2"} env, _ = ClaudeCodeAgent._build_sdk_env(route, cost_log_tags=tags) assert env["ANTHROPIC_CUSTOM_HEADERS"] == "x-ce-run-id: abc123\nx-ce-task-id: calc/v1\nx-ce-iteration: 2" def test_no_cost_log_tags_omits_custom_headers(self): - route = LiteLLMRoute(base_url="http://x:4000", auth_token="sk-1") + route = LiteLLMRoute() env, _ = ClaudeCodeAgent._build_sdk_env(route) assert "ANTHROPIC_CUSTOM_HEADERS" not in env env2, _ = ClaudeCodeAgent._build_sdk_env(route, cost_log_tags={}) @@ -314,7 +544,7 @@ def test_no_cost_log_tags_omits_custom_headers(self): def test_cost_log_tags_ignored_on_non_litellm_routes(self): """The tag is a LiteLLM-only concern; Bedrock/Direct must not emit it.""" tags = {"x-ce-run-id": "abc123"} - bedrock = BedrockRoute(bearer_token="t", region="eu-north-1", model="x") + bedrock = BedrockRoute(region="eu-north-1", model="x") env_b, _ = ClaudeCodeAgent._build_sdk_env(bedrock, cost_log_tags=tags) assert "ANTHROPIC_CUSTOM_HEADERS" not in env_b env_d, _ = ClaudeCodeAgent._build_sdk_env(DirectRoute(), cost_log_tags=tags) @@ -335,7 +565,7 @@ def test_cost_log_tags_gated_on_agent_capability_not_route(self): assert AgentRegistry.get(AgentKind.CLAUDE_CODE).agent_class.supports_cost_log_tags is True assert AgentRegistry.get(AgentKind.NONE).agent_class.supports_cost_log_tags is False - route = LiteLLMRoute(base_url="http://x:4000", auth_token="sk-1", model="deepseek/deepseek-v4-pro") + route = LiteLLMRoute(model="deepseek/deepseek-v4-pro") # A none-agent constructs fine on a LiteLLM route (the gate omits the kwarg)... assert create_agent(AgentKind.NONE, NoneAgentConfig(type=AgentKind.NONE), route=route) is not None # ...and it WOULD crash if the kwarg were forwarded — exactly what the gate prevents. @@ -347,7 +577,7 @@ def test_cost_log_tags_gated_on_agent_capability_not_route(self): def test_cost_log_tags_reject_header_injection(self): # A task_id/variant_id carrying a CR/LF would inject extra headers into every # SDK->proxy request; the seam must reject it (single-line ASCII only). - route = LiteLLMRoute(base_url="http://x:4000", auth_token="sk-1") + route = LiteLLMRoute() with pytest.raises(ValueError, match="single-line ASCII"): ClaudeCodeAgent._build_sdk_env(route, cost_log_tags={"x-ce-task-id": "ok\nAuthorization: Bearer forged"}) @@ -356,7 +586,7 @@ class TestResolveEffectiveModelCustom: """_resolve_effective_model() on the LiteLLM route — no prefixing.""" def test_config_model_synced_verbatim(self): - route = LiteLLMRoute(base_url="http://x:4000", auth_token="sk-1", model="deepseek.v3.2") + route = LiteLLMRoute(model="deepseek.v3.2") env, route_model = ClaudeCodeAgent._build_sdk_env(route) agent = _make_agent(route, config_model="zai.glm-5") effective = agent._resolve_effective_model("zai.glm-5", env, route_model) @@ -364,14 +594,14 @@ def test_config_model_synced_verbatim(self): assert env["ANTHROPIC_MODEL"] == "zai.glm-5" # no eu./anthropic. prefix def test_route_model_used_when_config_none(self): - route = LiteLLMRoute(base_url="http://x:4000", auth_token="sk-1", model="deepseek.v3.2") + route = LiteLLMRoute(model="deepseek.v3.2") env, route_model = ClaudeCodeAgent._build_sdk_env(route) agent = _make_agent(route) effective = agent._resolve_effective_model(None, env, route_model) assert effective == "deepseek.v3.2" def test_both_none_returns_none(self): - route = LiteLLMRoute(base_url="http://x:4000", auth_token="sk-1") + route = LiteLLMRoute() env, route_model = ClaudeCodeAgent._build_sdk_env(route) agent = _make_agent(route) effective = agent._resolve_effective_model(None, env, route_model) @@ -444,7 +674,7 @@ def _usage_after_finalize(self, effective_model: str | None) -> TokenUsage: from coder_eval.agents.claude_code_agent import _ClaudeTurnState agent = _make_agent( - LiteLLMRoute(base_url="http://x:4000", auth_token="k", model="zai.glm-5"), + LiteLLMRoute(model="zai.glm-5"), config_model="zai.glm-5", ) stub = SimpleNamespace( diff --git a/tests/test_llm_judge_criterion.py b/tests/test_llm_judge_criterion.py index cf9ac941..ca0a1450 100644 --- a/tests/test_llm_judge_criterion.py +++ b/tests/test_llm_judge_criterion.py @@ -20,6 +20,7 @@ LLMJudgeCriterion, TurnRecord, ) +from coder_eval.models.judge_defaults import DEFAULT_JUDGE_MODEL from coder_eval.sandbox import Sandbox @@ -571,10 +572,42 @@ def _tool_use_block(score: float, rationale: str = "ok") -> dict: } +def _openai_tool_call_block(score: float, rationale: str = "ok") -> dict: + return { + "choices": [ + { + "message": { + "tool_calls": [ + { + "function": { + "name": "submit_verdict", + "arguments": json.dumps({"score": score, "rationale": rationale}), + } + } + ] + } + } + ] + } + + +def test_llm_judge_criterion_model_survives_json_round_trip() -> None: + """Regression test for PR #137 review Axis 2: an unset LLMJudgeCriterion.model + must stay unset (None) after a model_dump(mode="json") / reload round trip — + the exact shape the docker driver's task serialization performs + (isolation/docker_runner.py -> cli/run_task_internal_command.py::load_task). + A `model_fields_set` sentinel would NOT survive this (every field is materialized + by model_dump), silently making the judge-model override inert under --driver docker.""" + criterion = LLMJudgeCriterion(description="x", prompt="grade") + assert criterion.model is None + reloaded = LLMJudgeCriterion.model_validate(json.loads(json.dumps(criterion.model_dump(mode="json")))) + assert reloaded.model is None + + def test_judge_bedrock_route_uses_bedrock_invoker(sandbox: Sandbox) -> None: from coder_eval.models.routing import BedrockRoute - route = BedrockRoute(bearer_token="t", region="eu-north-1") + route = BedrockRoute(region="eu-north-1") criterion = LLMJudgeCriterion(description="x", prompt="grade") with ( patch( @@ -587,13 +620,63 @@ def test_judge_bedrock_route_uses_bedrock_invoker(sandbox: Sandbox) -> None: m_bedrock.assert_called_once() kwargs = m_bedrock.call_args.kwargs assert kwargs["route"] is route - assert kwargs["model"] == criterion.model + # No explicit criterion.model and no checker_context override on the route -> + # falls back to DEFAULT_JUDGE_MODEL, never the agent's own model. + assert kwargs["model"] == DEFAULT_JUDGE_MODEL assert kwargs["temperature"] == criterion.temperature assert kwargs["max_tokens"] == criterion.max_tokens assert kwargs["tool_spec"]["name"] == "submit_verdict" assert m_anthropic.call_count == 0 +def test_judge_bedrock_route_with_explicit_route_model_still_wins(sandbox: Sandbox) -> None: + """A route.model that IS set (a real checker_context.api_route.model override, + as resolve_evaluation_route only ever produces) is honored — contrast with + the routing-level regression test in test_litellm_route.py, which asserts + resolve_evaluation_route itself never bakes the agent's own model into + route.model absent such an override.""" + from coder_eval.models.routing import BedrockRoute + + route = BedrockRoute(region="eu-north-1", model="eu.anthropic.claude-opus-4-1") + criterion = LLMJudgeCriterion(description="x", prompt="grade") + with ( + patch( + "coder_eval.criteria.llm_judge.invoke_bedrock_judge_async", new=AsyncMock(return_value=_tool_use_block(0.7)) + ) as m_bedrock, + patch("coder_eval.criteria.llm_judge.invoke_anthropic_judge_async", new=AsyncMock()), + ): + SuccessChecker(sandbox, init_registry=False, route=route).check(criterion) + assert m_bedrock.call_args.kwargs["model"] == "eu.anthropic.claude-opus-4-1" + + +def test_judge_litellm_route_uses_litellm_invoker(sandbox: Sandbox) -> None: + from coder_eval.models.routing import LiteLLMRoute + + route = LiteLLMRoute(model="gpt-5-luna", params={"api_base": "http://gateway:4000"}) + criterion = LLMJudgeCriterion(description="x", prompt="grade") + with ( + patch( + "coder_eval.criteria.llm_judge.invoke_litellm_judge_async", + new=AsyncMock(return_value=_openai_tool_call_block(0.9)), + ) as m_litellm, + patch("coder_eval.criteria.llm_judge.invoke_bedrock_judge_async", new=AsyncMock()) as m_bedrock, + patch("coder_eval.criteria.llm_judge.invoke_anthropic_judge_async", new=AsyncMock()) as m_anthropic, + ): + result = SuccessChecker(sandbox, init_registry=False, route=route).check(criterion) + assert result.score == 0.9 + m_litellm.assert_called_once() + kwargs = m_litellm.call_args.kwargs + assert kwargs["route"] is route + # No explicit criterion.model set -> falls back to route.model (checker_context override). + assert kwargs["model"] == "gpt-5-luna" + # invoke_litellm_judge_async takes no `temperature` kwarg at all -- see its docstring. + assert "temperature" not in kwargs + assert kwargs["max_tokens"] == criterion.max_tokens + assert kwargs["tool_spec"]["name"] == "submit_verdict" + assert m_bedrock.call_count == 0 + assert m_anthropic.call_count == 0 + + def test_judge_direct_route_uses_anthropic_invoker(sandbox: Sandbox) -> None: from coder_eval.models.routing import DirectRoute @@ -615,7 +698,7 @@ def test_judge_direct_route_uses_anthropic_invoker(sandbox: Sandbox) -> None: def test_judge_bedrock_invoke_runtime_error_maps_to_score_zero(sandbox: Sandbox) -> None: from coder_eval.models.routing import BedrockRoute - route = BedrockRoute(bearer_token="t", region="eu-north-1") + route = BedrockRoute(region="eu-north-1") criterion = LLMJudgeCriterion(description="x", prompt="grade") with patch( "coder_eval.criteria.llm_judge.invoke_bedrock_judge_async", @@ -706,7 +789,7 @@ def test_judge_bedrock_route_threads_model_unchanged(sandbox: Sandbox) -> None: """Translation happens INSIDE the helper, not at the dispatch site.""" from coder_eval.models.routing import BedrockRoute - route = BedrockRoute(bearer_token="t", region="eu-north-1") + route = BedrockRoute(region="eu-north-1") criterion = LLMJudgeCriterion(description="x", prompt="grade", model="anthropic.claude-opus-4-6-v1") with patch( "coder_eval.criteria.llm_judge.invoke_bedrock_judge_async", @@ -1021,9 +1104,7 @@ def test_llm_judge_tool_channel_bedrock(sandbox: Sandbox) -> None: with patch( "coder_eval.criteria.llm_judge.invoke_bedrock_judge_async", new=AsyncMock(return_value=bedrock_response) ) as mock_invoke: - result = SuccessChecker( - sandbox, init_registry=False, route=BedrockRoute(bearer_token="t", region="us-east-1") - ).check(criterion) + result = SuccessChecker(sandbox, init_registry=False, route=BedrockRoute(region="us-east-1")).check(criterion) assert result.score == 0.81 # Confirm we passed the Anthropic-native tool spec. kwargs = mock_invoke.call_args.kwargs @@ -1100,9 +1181,7 @@ def test_judge_usage_bedrock_from_response(sandbox: Sandbox) -> None: score=0.6, usage={"input_tokens": 900, "output_tokens": 40, "cache_read_input_tokens": 100} ) with patch("coder_eval.criteria.llm_judge.invoke_bedrock_judge_async", new=AsyncMock(return_value=resp)): - result = SuccessChecker( - sandbox, init_registry=False, route=BedrockRoute(bearer_token="t", region="us-east-1") - ).check(criterion) + result = SuccessChecker(sandbox, init_registry=False, route=BedrockRoute(region="us-east-1")).check(criterion) assert isinstance(result, JudgeCriterionResult) assert result.token_usage is not None assert result.token_usage.uncached_input_tokens == 900 diff --git a/tests/test_models.py b/tests/test_models.py index 3f533557..60fa21a2 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -187,11 +187,15 @@ class TestLLMJudgeCriterion: def test_llm_judge_criterion_defaults(self): """Constructing with the minimum required fields yields documented defaults.""" - from coder_eval.models import DEFAULT_JUDGE_MODEL, LLMJudgeCriterion + from coder_eval.models import LLMJudgeCriterion criterion = LLMJudgeCriterion(description="x", prompt="grade this code") assert criterion.type == "llm_judge" - assert criterion.model == DEFAULT_JUDGE_MODEL + # Unset by default (not a materialized DEFAULT_JUDGE_MODEL default) so an + # unset per-criterion model survives a JSON round trip as None — see + # LLMJudgeChecker's precedence: criterion.model or route.model or + # DEFAULT_JUDGE_MODEL, applied at check time. + assert criterion.model is None assert criterion.temperature == 0.0 # Bumped from 1000 → 2000 when verbose verdict (findings) was added, # so output budgets fit the bullet evidence a typical judge emits. diff --git a/tests/test_orchestrator.py b/tests/test_orchestrator.py index 2df31856..511464d1 100644 --- a/tests/test_orchestrator.py +++ b/tests/test_orchestrator.py @@ -45,11 +45,11 @@ def test_format_routing_direct_judge_transport_none_renders_as_none(): def test_format_routing_non_direct_routes_unchanged(): """BedrockRoute keeps the original bare-name format — judge transport is a Direct-only concern.""" - assert _format_routing(BedrockRoute(bearer_token="t", region="us-east-1")) == "aws_bedrock" + assert _format_routing(BedrockRoute(region="us-east-1")) == "aws_bedrock" def test_format_routing_litellm_shows_model(): - out = _format_routing(LiteLLMRoute(base_url="http://localhost:4000", auth_token="k", model="zai.glm-5")) + out = _format_routing(LiteLLMRoute(model="zai.glm-5")) assert out.startswith("litellm") assert "zai.glm-5" in out @@ -57,7 +57,7 @@ def test_format_routing_litellm_shows_model(): def test_format_routing_litellm_effective_model_wins_over_route_default(): """The --model override (effective_model) must be logged, not the route's LITELLM_MODEL default.""" out = _format_routing( - LiteLLMRoute(base_url="http://localhost:4000", auth_token="k", model="zai.glm-5"), + LiteLLMRoute(model="zai.glm-5"), effective_model="deepseek.v3.2", ) assert "deepseek.v3.2" in out @@ -105,9 +105,80 @@ def test_record_route_environment_info_direct_none_serialized_as_string(tmp_path assert orchestrator.result.environment_info["judge_transport"] == "none" +class TestRejectLitellmEvalRouteIfUnsupported: + """checker_context.api_route.route: litellm dispatches llm_judge through the + litellm library in-process, but agent_judge and the simulator are real + Claude Code CLI subprocesses that speak Anthropic Messages only — handing + them an arbitrary litellm-fronted route would misroute or fail silently. + PR #137 review: 'route: litellm breaks the simulator and agent_judge.'""" + + @staticmethod + def _orchestrator_with_criteria(tmp_path: Path, success_criteria, simulation=None) -> Orchestrator: + task_file = Path("tasks/hello_date.yaml") + task, _ = load_task(task_file) + task = task.model_copy(update={"success_criteria": success_criteria, "simulation": simulation}) + orchestrator = Orchestrator(task=task, run_dir=tmp_path / "run", variant_id="t") + orchestrator.eval_route = LiteLLMRoute(model="gpt-5.6-luna") + return orchestrator + + def test_llm_judge_only_is_fine(self, tmp_path): + from coder_eval.models import LLMJudgeCriterion + + orchestrator = self._orchestrator_with_criteria(tmp_path, [LLMJudgeCriterion(description="x", prompt="grade")]) + orchestrator._reject_litellm_eval_route_if_unsupported() # no raise + + def test_rejects_enabled_agent_judge(self, tmp_path): + from coder_eval.models import AgentJudgeCriterion + + orchestrator = self._orchestrator_with_criteria( + tmp_path, [AgentJudgeCriterion(description="x", prompt="grade")] + ) + with pytest.raises(ValueError, match="agent_judge"): + orchestrator._reject_litellm_eval_route_if_unsupported() + + def test_allows_disabled_agent_judge(self, tmp_path): + from coder_eval.models import AgentJudgeCriterion + + orchestrator = self._orchestrator_with_criteria( + tmp_path, [AgentJudgeCriterion(description="x", prompt="grade", enabled=False)] + ) + orchestrator._reject_litellm_eval_route_if_unsupported() # no raise + + def test_rejects_enabled_simulation(self, tmp_path): + from coder_eval.models import LLMJudgeCriterion, SimulationConfig + + orchestrator = self._orchestrator_with_criteria( + tmp_path, + [LLMJudgeCriterion(description="x", prompt="grade")], + simulation=SimulationConfig(enabled=True, persona="p", goal="g"), + ) + with pytest.raises(ValueError, match=r"simulation\.enabled"): + orchestrator._reject_litellm_eval_route_if_unsupported() + + def test_allows_disabled_simulation(self, tmp_path): + from coder_eval.models import LLMJudgeCriterion, SimulationConfig + + orchestrator = self._orchestrator_with_criteria( + tmp_path, + [LLMJudgeCriterion(description="x", prompt="grade")], + simulation=SimulationConfig(enabled=False, persona="p", goal="g"), + ) + orchestrator._reject_litellm_eval_route_if_unsupported() # no raise + + def test_non_litellm_eval_route_is_never_checked(self, tmp_path): + """Bedrock/Direct eval routes are always fine with agent_judge/simulation.""" + from coder_eval.models import AgentJudgeCriterion + + orchestrator = self._orchestrator_with_criteria( + tmp_path, [AgentJudgeCriterion(description="x", prompt="grade")] + ) + orchestrator.eval_route = BedrockRoute(region="eu-north-1") + orchestrator._reject_litellm_eval_route_if_unsupported() # no raise + + def test_record_route_environment_info_bedrock(tmp_path): orchestrator = _make_orchestrator_with_route( - tmp_path, BedrockRoute(bearer_token="t", region="eu-north-1", model="eu.anthropic.claude-sonnet-4-6") + tmp_path, BedrockRoute(region="eu-north-1", model="eu.anthropic.claude-sonnet-4-6") ) orchestrator._record_route_environment_info() assert orchestrator.result is not None @@ -117,11 +188,15 @@ def test_record_route_environment_info_bedrock(tmp_path): assert info["bedrock_model"] == "eu.anthropic.claude-sonnet-4-6" -def test_record_route_environment_info_litellm_records_host_only_no_secret(tmp_path): +def test_record_route_environment_info_litellm_records_host_only_no_secret(tmp_path, monkeypatch): """LiteLLM route records host + model, but NEVER the auth token or full base_url.""" + from coder_eval.config import settings + + monkeypatch.setattr(settings, "litellm_base_url", "http://localhost:4000") + monkeypatch.setattr(settings, "litellm_auth_token", "sk-super-secret") orchestrator = _make_orchestrator_with_route( tmp_path, - LiteLLMRoute(base_url="http://localhost:4000", auth_token="sk-super-secret", model="zai.glm-5"), + LiteLLMRoute(model="zai.glm-5"), ) orchestrator._record_route_environment_info() assert orchestrator.result is not None diff --git a/tests/test_route_seam_exhaustiveness.py b/tests/test_route_seam_exhaustiveness.py index b58a1307..c3fe38e7 100644 --- a/tests/test_route_seam_exhaustiveness.py +++ b/tests/test_route_seam_exhaustiveness.py @@ -29,8 +29,8 @@ # forces this dict to grow whenever the union does. _INSTANCES: list[object] = [ DirectRoute(), - BedrockRoute(bearer_token="t", region="eu-north-1", model="x"), - LiteLLMRoute(base_url="http://localhost:4000", auth_token="k", model="m"), + BedrockRoute(region="eu-north-1", model="x"), + LiteLLMRoute(model="m"), ] @@ -69,13 +69,17 @@ async def _stub_bedrock(**_: object) -> dict[str, object]: async def _stub_anthropic(**_: object) -> dict[str, object]: return {} + async def _stub_litellm(**_: object) -> dict[str, object]: + return {} + monkeypatch.setattr(llm_judge, "invoke_bedrock_judge_async", _stub_bedrock) monkeypatch.setattr(llm_judge, "invoke_anthropic_judge_async", _stub_anthropic) + monkeypatch.setattr(llm_judge, "invoke_litellm_judge_async", _stub_litellm) monkeypatch.setattr(llm_judge, "extract_verdict_from_anthropic_response", lambda _resp: (None, "stub")) monkeypatch.setattr(llm_judge, "token_usage_from_anthropic_dict", lambda _resp, **_kwargs: None) criterion = MagicMock() for r in _INSTANCES: - result = await _invoke_tool_channel(criterion=criterion, route=r, system_msg="s", user_msg="u") # type: ignore[arg-type] + result = await _invoke_tool_channel(criterion=criterion, model="m", route=r, system_msg="s", user_msg="u") # type: ignore[arg-type] # (verdict, parse_error, raw_text, response_usage) — a 4-tuple means the # route matched an explicit arm rather than falling through. assert isinstance(result, tuple) and len(result) == 4 diff --git a/tests/test_routing.py b/tests/test_routing.py index 35bf11eb..025d2442 100644 --- a/tests/test_routing.py +++ b/tests/test_routing.py @@ -5,6 +5,7 @@ from claude_agent_sdk import ClaudeAgentOptions +from coder_eval.agents import claude_code_agent as claude_code_agent_mod from coder_eval.agents.claude_code_agent import ClaudeCodeAgent from coder_eval.config import Settings from coder_eval.models import AgentKind, BedrockRoute, DirectRoute, parse_agent_config @@ -31,11 +32,26 @@ def test_direct_omits_path_when_unset(self, monkeypatch): assert "PATH" not in env assert model is None + def test_direct_neutralizes_inherited_bedrock_creds(self): + """An explicit route: direct (e.g. via checker_context.api_route.route on a + run whose agent is on Bedrock) must not let the CLI's own + AWS_BEARER_TOKEN_BEDROCK auto-selection silently spend the operator's + Bedrock token instead of ANTHROPIC_API_KEY (PR #137 review).""" + env, _ = ClaudeCodeAgent._build_sdk_env(DirectRoute()) + assert env["AWS_BEARER_TOKEN_BEDROCK"] == "" + assert env["CLAUDE_CODE_USE_BEDROCK"] == "" + + def test_direct_model_override_sets_anthropic_model(self): + env, model = ClaudeCodeAgent._build_sdk_env(DirectRoute(model="claude-haiku-4-5")) + assert env["ANTHROPIC_MODEL"] == "claude-haiku-4-5" + assert model == "claude-haiku-4-5" + def test_bedrock_basic_env(self, monkeypatch): """BedrockRoute produces CLAUDE_CODE_USE_BEDROCK, token, region, and forwards PATH.""" custom_path = f"/bedrock/bin{os.pathsep}/usr/bin" monkeypatch.setenv("PATH", custom_path) - route = BedrockRoute(bearer_token="tok-123", region="us-east-1") + monkeypatch.setattr(claude_code_agent_mod.settings, "aws_bearer_token_bedrock", "tok-123") + route = BedrockRoute(region="us-east-1") env, _ = ClaudeCodeAgent._build_sdk_env(route) assert env["CLAUDE_CODE_USE_BEDROCK"] == "1" assert env["AWS_BEARER_TOKEN_BEDROCK"] == "tok-123" @@ -44,26 +60,26 @@ def test_bedrock_basic_env(self, monkeypatch): def test_bedrock_attribution_header_disabled(self): """disable_attribution_header=True sets CLAUDE_CODE_ATTRIBUTION_HEADER=0.""" - route = BedrockRoute(bearer_token="t", region="r", disable_attribution_header=True) + route = BedrockRoute(region="r", disable_attribution_header=True) env, _ = ClaudeCodeAgent._build_sdk_env(route) assert env["CLAUDE_CODE_ATTRIBUTION_HEADER"] == "0" def test_bedrock_attribution_header_enabled(self): """disable_attribution_header=False omits the header key.""" - route = BedrockRoute(bearer_token="t", region="r", disable_attribution_header=False) + route = BedrockRoute(region="r", disable_attribution_header=False) env, _ = ClaudeCodeAgent._build_sdk_env(route) assert "CLAUDE_CODE_ATTRIBUTION_HEADER" not in env def test_bedrock_model_override(self): """BedrockRoute.model returned as effective_model.""" - route = BedrockRoute(bearer_token="t", region="r", model="eu.anthropic.claude-sonnet-4-6") + route = BedrockRoute(region="r", model="eu.anthropic.claude-sonnet-4-6") env, model = ClaudeCodeAgent._build_sdk_env(route) assert model == "eu.anthropic.claude-sonnet-4-6" assert env["ANTHROPIC_MODEL"] == "eu.anthropic.claude-sonnet-4-6" def test_bedrock_no_model_returns_none(self): """BedrockRoute without model returns None (use task config).""" - route = BedrockRoute(bearer_token="t", region="r") + route = BedrockRoute(region="r") env, model = ClaudeCodeAgent._build_sdk_env(route) assert model is None assert "ANTHROPIC_MODEL" not in env @@ -80,7 +96,7 @@ def test_propagates_plugin_tools_dir_when_set(self, monkeypatch): def test_propagates_plugin_tools_dir_on_bedrock_route(self, monkeypatch): """Pin must reach the SDK regardless of routing — bedrock edition.""" monkeypatch.setenv("PLUGIN_TOOLS_DIR", "/pinned/tools/@uipath") - env, _ = ClaudeCodeAgent._build_sdk_env(BedrockRoute(bearer_token="t", region="r")) + env, _ = ClaudeCodeAgent._build_sdk_env(BedrockRoute(region="r")) assert env["PLUGIN_TOOLS_DIR"] == "/pinned/tools/@uipath" def test_omits_plugin_tools_dir_when_unset(self, monkeypatch): @@ -103,13 +119,13 @@ def test_external_plugin_tools_dir_beats_fallback(self, monkeypatch): def test_bedrock_default_disables_attribution_header(self): """Default BedrockRoute (no explicit disable_attribution_header) sets CLAUDE_CODE_ATTRIBUTION_HEADER=0.""" - route = BedrockRoute(bearer_token="t", region="r") + route = BedrockRoute(region="r") env, _ = ClaudeCodeAgent._build_sdk_env(route) assert env["CLAUDE_CODE_ATTRIBUTION_HEADER"] == "0" def test_bedrock_small_model(self): """BedrockRoute.small_model appears in env as ANTHROPIC_SMALL_FAST_MODEL.""" - route = BedrockRoute(bearer_token="t", region="r", small_model="eu.anthropic.claude-haiku-4-5") + route = BedrockRoute(region="r", small_model="eu.anthropic.claude-haiku-4-5") env, _ = ClaudeCodeAgent._build_sdk_env(route) assert env["ANTHROPIC_SMALL_FAST_MODEL"] == "eu.anthropic.claude-haiku-4-5" @@ -279,7 +295,7 @@ class TestResolveEffectiveModel: def test_config_model_overrides_bedrock_route_model(self): """Task/CLI model wins over BEDROCK_MODEL and is synced into env.""" - route = BedrockRoute(bearer_token="t", region="us-east-2", model="us.anthropic.claude-sonnet-4-6") + route = BedrockRoute(region="us-east-2", model="us.anthropic.claude-sonnet-4-6") env, route_model = ClaudeCodeAgent._build_sdk_env(route) agent = _make_agent(route, config_model="global.anthropic.claude-sonnet-4-6") effective = agent._resolve_effective_model("global.anthropic.claude-sonnet-4-6", env, route_model) @@ -288,7 +304,7 @@ def test_config_model_overrides_bedrock_route_model(self): def test_route_model_used_when_config_none(self): """BEDROCK_MODEL is the fallback when no task/CLI model is set.""" - route = BedrockRoute(bearer_token="t", region="us-east-2", model="us.anthropic.claude-sonnet-4-6") + route = BedrockRoute(region="us-east-2", model="us.anthropic.claude-sonnet-4-6") env, route_model = ClaudeCodeAgent._build_sdk_env(route) agent = _make_agent(route) effective = agent._resolve_effective_model(None, env, route_model) @@ -297,7 +313,7 @@ def test_route_model_used_when_config_none(self): def test_bare_config_model_auto_prefixes_for_bedrock(self): """A bare --model on a Bedrock route gets the region's inference-profile prefix.""" - route = BedrockRoute(bearer_token="t", region="eu-north-1", model="eu.anthropic.claude-sonnet-4-6") + route = BedrockRoute(region="eu-north-1", model="eu.anthropic.claude-sonnet-4-6") env, route_model = ClaudeCodeAgent._build_sdk_env(route) agent = _make_agent(route, config_model="anthropic.claude-sonnet-4-6") effective = agent._resolve_effective_model("anthropic.claude-sonnet-4-6", env, route_model) @@ -306,7 +322,7 @@ def test_bare_config_model_auto_prefixes_for_bedrock(self): def test_both_none_returns_none(self): """No model anywhere → None, no env mutation.""" - route = BedrockRoute(bearer_token="t", region="us-east-2") + route = BedrockRoute(region="us-east-2") env, route_model = ClaudeCodeAgent._build_sdk_env(route) agent = _make_agent(route) effective = agent._resolve_effective_model(None, env, route_model) @@ -320,7 +336,7 @@ def test_config_model_injects_anthropic_model_when_route_has_none(self): by every Bedrock code path, so config_model must be injected into env on Bedrock even when _build_sdk_env left ANTHROPIC_MODEL absent. """ - route = BedrockRoute(bearer_token="t", region="eu-north-1") # no model + route = BedrockRoute(region="eu-north-1") # no model env, route_model = ClaudeCodeAgent._build_sdk_env(route) assert "ANTHROPIC_MODEL" not in env # precondition agent = _make_agent(route, config_model="claude-sonnet-4-6") @@ -340,9 +356,10 @@ def test_direct_route_does_not_inject_or_prefix(self): class TestSdkOptionsDumpRedaction: """Test that _dump_sdk_options redacts sensitive values.""" - def test_bedrock_token_redacted_in_dump(self): + def test_bedrock_token_redacted_in_dump(self, monkeypatch): """AWS_BEARER_TOKEN_BEDROCK must not appear in plain text in sdk_options dump.""" - route = BedrockRoute(bearer_token="SECRET_TOKEN_123", region="us-east-1") + monkeypatch.setattr(claude_code_agent_mod.settings, "aws_bearer_token_bedrock", "SECRET_TOKEN_123") + route = BedrockRoute(region="us-east-1") env, _ = ClaudeCodeAgent._build_sdk_env(route) opts = ClaudeAgentOptions(cwd=tempfile.gettempdir(), env=env) dump = dump_dataclass(opts) diff --git a/uv.lock b/uv.lock index 688a1b3c..004e5618 100644 --- a/uv.lock +++ b/uv.lock @@ -278,6 +278,34 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e5/ca/78d423b324b8d77900030fa59c4aa9054261ef0925631cd2501dd015b7b7/boolean_py-5.0-py3-none-any.whl", hash = "sha256:ef28a70bd43115208441b53a045d1549e2f0ec6e3d08a9d142cbc41c1938e8d9", size = 26577, upload-time = "2025-04-03T10:39:48.449Z" }, ] +[[package]] +name = "boto3" +version = "1.43.78" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "botocore" }, + { name = "jmespath" }, + { name = "s3transfer" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/be/55/e026c943f7f1ed6d2f5e6035713f21233bbe9ee975008662dc64ca0d4ced/boto3-1.43.78.tar.gz", hash = "sha256:2fa59116e298171ef59e7600a8be6c01177faef8af4b9a4314b7a57a04009ada", size = 112679, upload-time = "2026-08-21T19:37:36.84Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e3/55/288b1f0d987e15db0417f5ea43da564bfa042063f508c259eec893134628/boto3-1.43.78-py3-none-any.whl", hash = "sha256:893f06a171469618e17de78dc927aca6e74fcf45a70d2c5918e9ac9919e96cc9", size = 140028, upload-time = "2026-08-21T19:37:35.358Z" }, +] + +[[package]] +name = "botocore" +version = "1.43.78" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "jmespath" }, + { name = "python-dateutil" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/67/71/490aaa384855bf3b69405ded52ae77a0e5f4eeb2165044fa65466c4d3a73/botocore-1.43.78.tar.gz", hash = "sha256:e8238d22c1e1342025d75d2e33d154a375e7caad0fc67f77d77faa2d82668b94", size = 15982895, upload-time = "2026-08-21T19:37:32.402Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c5/5b/b7c5c22767c0e8eb4bb2bd0a972d77bd0be3dd9908e15d4487094a31fa4b/botocore-1.43.78-py3-none-any.whl", hash = "sha256:ddd020493235e264b3bd12606f239a3d3b2dd7cfb1d25a0691061183c290c228", size = 15677214, upload-time = "2026-08-21T19:37:28.894Z" }, +] + [[package]] name = "cachecontrol" version = "0.14.4" @@ -492,6 +520,9 @@ dev = [ { name = "pytest-xdist", extra = ["psutil"] }, { name = "ruff" }, ] +litellm = [ + { name = "litellm" }, +] uipath = [ { name = "uipath" }, ] @@ -509,6 +540,7 @@ requires-dist = [ { name = "httpx2", specifier = ">=2.12.0,<3.0.0" }, { name = "jmespath", specifier = ">=1.1.0" }, { name = "jsonschema", specifier = ">=4.26.0" }, + { name = "litellm", marker = "extra == 'litellm'", specifier = ">=1.95.0,<2.0.0" }, { name = "mcp", marker = "extra == 'dev'", specifier = ">=1.28.1" }, { name = "openai-codex", marker = "extra == 'codex'", specifier = ">=0.144.4" }, { name = "opentelemetry-sdk", specifier = ">=1.30.0,<2.0.0" }, @@ -532,7 +564,7 @@ requires-dist = [ { name = "typer", specifier = ">=0.24.1" }, { name = "uipath", marker = "extra == 'uipath'", specifier = ">=2.10.31" }, ] -provides-extras = ["dev", "uipath", "codex", "antigravity"] +provides-extras = ["dev", "uipath", "litellm", "codex", "antigravity"] [[package]] name = "colorama" @@ -722,6 +754,36 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ab/84/02fc1827e8cdded4aa65baef11296a9bbe595c474f0d6d758af082d849fd/execnet-2.1.2-py3-none-any.whl", hash = "sha256:67fba928dd5a544b783f6056f449e5e3931a5c378b128bc18501f7ea79e296ec", size = 40708, upload-time = "2025-11-12T09:56:36.333Z" }, ] +[[package]] +name = "fastuuid" +version = "0.14.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c3/7d/d9daedf0f2ebcacd20d599928f8913e9d2aea1d56d2d355a93bfa2b611d7/fastuuid-0.14.0.tar.gz", hash = "sha256:178947fc2f995b38497a74172adee64fdeb8b7ec18f2a5934d037641ba265d26", size = 18232, upload-time = "2025-10-19T22:19:22.402Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a5/83/ae12dd39b9a39b55d7f90abb8971f1a5f3c321fd72d5aa83f90dc67fe9ed/fastuuid-0.14.0-cp313-cp313-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:77a09cb7427e7af74c594e409f7731a0cf887221de2f698e1ca0ebf0f3139021", size = 510720, upload-time = "2025-10-19T22:42:34.633Z" }, + { url = "https://files.pythonhosted.org/packages/53/b0/a4b03ff5d00f563cc7546b933c28cb3f2a07344b2aec5834e874f7d44143/fastuuid-0.14.0-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:9bd57289daf7b153bfa3e8013446aa144ce5e8c825e9e366d455155ede5ea2dc", size = 262024, upload-time = "2025-10-19T22:30:25.482Z" }, + { url = "https://files.pythonhosted.org/packages/9c/6d/64aee0a0f6a58eeabadd582e55d0d7d70258ffdd01d093b30c53d668303b/fastuuid-0.14.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:ac60fc860cdf3c3f327374db87ab8e064c86566ca8c49d2e30df15eda1b0c2d5", size = 251679, upload-time = "2025-10-19T22:36:14.096Z" }, + { url = "https://files.pythonhosted.org/packages/60/f5/a7e9cda8369e4f7919d36552db9b2ae21db7915083bc6336f1b0082c8b2e/fastuuid-0.14.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ab32f74bd56565b186f036e33129da77db8be09178cd2f5206a5d4035fb2a23f", size = 277862, upload-time = "2025-10-19T22:36:23.302Z" }, + { url = "https://files.pythonhosted.org/packages/f0/d3/8ce11827c783affffd5bd4d6378b28eb6cc6d2ddf41474006b8d62e7448e/fastuuid-0.14.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:33e678459cf4addaedd9936bbb038e35b3f6b2061330fd8f2f6a1d80414c0f87", size = 278278, upload-time = "2025-10-19T22:29:43.809Z" }, + { url = "https://files.pythonhosted.org/packages/a2/51/680fb6352d0bbade04036da46264a8001f74b7484e2fd1f4da9e3db1c666/fastuuid-0.14.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1e3cc56742f76cd25ecb98e4b82a25f978ccffba02e4bdce8aba857b6d85d87b", size = 301788, upload-time = "2025-10-19T22:36:06.825Z" }, + { url = "https://files.pythonhosted.org/packages/fa/7c/2014b5785bd8ebdab04ec857635ebd84d5ee4950186a577db9eff0fb8ff6/fastuuid-0.14.0-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:cb9a030f609194b679e1660f7e32733b7a0f332d519c5d5a6a0a580991290022", size = 459819, upload-time = "2025-10-19T22:35:31.623Z" }, + { url = "https://files.pythonhosted.org/packages/01/d2/524d4ceeba9160e7a9bc2ea3e8f4ccf1ad78f3bde34090ca0c51f09a5e91/fastuuid-0.14.0-cp313-cp313-musllinux_1_1_i686.whl", hash = "sha256:09098762aad4f8da3a888eb9ae01c84430c907a297b97166b8abc07b640f2995", size = 478546, upload-time = "2025-10-19T22:26:03.023Z" }, + { url = "https://files.pythonhosted.org/packages/bc/17/354d04951ce114bf4afc78e27a18cfbd6ee319ab1829c2d5fb5e94063ac6/fastuuid-0.14.0-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:1383fff584fa249b16329a059c68ad45d030d5a4b70fb7c73a08d98fd53bcdab", size = 450921, upload-time = "2025-10-19T22:31:02.151Z" }, + { url = "https://files.pythonhosted.org/packages/fb/be/d7be8670151d16d88f15bb121c5b66cdb5ea6a0c2a362d0dcf30276ade53/fastuuid-0.14.0-cp313-cp313-win32.whl", hash = "sha256:a0809f8cc5731c066c909047f9a314d5f536c871a7a22e815cc4967c110ac9ad", size = 154559, upload-time = "2025-10-19T22:36:36.011Z" }, + { url = "https://files.pythonhosted.org/packages/22/1d/5573ef3624ceb7abf4a46073d3554e37191c868abc3aecd5289a72f9810a/fastuuid-0.14.0-cp313-cp313-win_amd64.whl", hash = "sha256:0df14e92e7ad3276327631c9e7cec09e32572ce82089c55cb1bb8df71cf394ed", size = 156539, upload-time = "2025-10-19T22:33:35.898Z" }, + { url = "https://files.pythonhosted.org/packages/16/c9/8c7660d1fe3862e3f8acabd9be7fc9ad71eb270f1c65cce9a2b7a31329ab/fastuuid-0.14.0-cp314-cp314-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:b852a870a61cfc26c884af205d502881a2e59cc07076b60ab4a951cc0c94d1ad", size = 510600, upload-time = "2025-10-19T22:43:44.17Z" }, + { url = "https://files.pythonhosted.org/packages/4c/f4/a989c82f9a90d0ad995aa957b3e572ebef163c5299823b4027986f133dfb/fastuuid-0.14.0-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:c7502d6f54cd08024c3ea9b3514e2d6f190feb2f46e6dbcd3747882264bb5f7b", size = 262069, upload-time = "2025-10-19T22:43:38.38Z" }, + { url = "https://files.pythonhosted.org/packages/da/6c/a1a24f73574ac995482b1326cf7ab41301af0fabaa3e37eeb6b3df00e6e2/fastuuid-0.14.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:1ca61b592120cf314cfd66e662a5b54a578c5a15b26305e1b8b618a6f22df714", size = 251543, upload-time = "2025-10-19T22:32:22.537Z" }, + { url = "https://files.pythonhosted.org/packages/1a/20/2a9b59185ba7a6c7b37808431477c2d739fcbdabbf63e00243e37bd6bf49/fastuuid-0.14.0-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:aa75b6657ec129d0abded3bec745e6f7ab642e6dba3a5272a68247e85f5f316f", size = 277798, upload-time = "2025-10-19T22:33:53.821Z" }, + { url = "https://files.pythonhosted.org/packages/ef/33/4105ca574f6ded0af6a797d39add041bcfb468a1255fbbe82fcb6f592da2/fastuuid-0.14.0-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a8a0dfea3972200f72d4c7df02c8ac70bad1bb4c58d7e0ec1e6f341679073a7f", size = 278283, upload-time = "2025-10-19T22:29:02.812Z" }, + { url = "https://files.pythonhosted.org/packages/fe/8c/fca59f8e21c4deb013f574eae05723737ddb1d2937ce87cb2a5d20992dc3/fastuuid-0.14.0-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1bf539a7a95f35b419f9ad105d5a8a35036df35fdafae48fb2fd2e5f318f0d75", size = 301627, upload-time = "2025-10-19T22:35:54.985Z" }, + { url = "https://files.pythonhosted.org/packages/cb/e2/f78c271b909c034d429218f2798ca4e89eeda7983f4257d7865976ddbb6c/fastuuid-0.14.0-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:9a133bf9cc78fdbd1179cb58a59ad0100aa32d8675508150f3658814aeefeaa4", size = 459778, upload-time = "2025-10-19T22:28:00.999Z" }, + { url = "https://files.pythonhosted.org/packages/1e/f0/5ff209d865897667a2ff3e7a572267a9ced8f7313919f6d6043aed8b1caa/fastuuid-0.14.0-cp314-cp314-musllinux_1_1_i686.whl", hash = "sha256:f54d5b36c56a2d5e1a31e73b950b28a0d83eb0c37b91d10408875a5a29494bad", size = 478605, upload-time = "2025-10-19T22:36:21.764Z" }, + { url = "https://files.pythonhosted.org/packages/e0/c8/2ce1c78f983a2c4987ea865d9516dbdfb141a120fd3abb977ae6f02ba7ca/fastuuid-0.14.0-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:ec27778c6ca3393ef662e2762dba8af13f4ec1aaa32d08d77f71f2a70ae9feb8", size = 450837, upload-time = "2025-10-19T22:34:37.178Z" }, + { url = "https://files.pythonhosted.org/packages/df/60/dad662ec9a33b4a5fe44f60699258da64172c39bd041da2994422cdc40fe/fastuuid-0.14.0-cp314-cp314-win32.whl", hash = "sha256:e23fc6a83f112de4be0cc1990e5b127c27663ae43f866353166f87df58e73d06", size = 154532, upload-time = "2025-10-19T22:35:18.217Z" }, + { url = "https://files.pythonhosted.org/packages/1f/f6/da4db31001e854025ffd26bc9ba0740a9cbba2c3259695f7c5834908b336/fastuuid-0.14.0-cp314-cp314-win_amd64.whl", hash = "sha256:df61342889d0f5e7a32f7284e55ef95103f2110fee433c2ae7c2c0956d76ac8a", size = 156457, upload-time = "2025-10-19T22:33:44.579Z" }, +] + [[package]] name = "filelock" version = "3.25.2" @@ -804,6 +866,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/9a/9a/e35b4a917281c0b8419d4207f4334c8e8c5dbf4f3f5f9ada73958d937dcc/frozenlist-1.8.0-py3-none-any.whl", hash = "sha256:0c18a16eab41e82c295618a77502e17b195883241c563b00f0aa5106fc4eaa0d", size = 13409, upload-time = "2025-10-06T05:38:16.721Z" }, ] +[[package]] +name = "fsspec" +version = "2026.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/00/78/f34251dadb8f3921264a1d9b8946f5e542014ee2614b285261b4e40e6775/fsspec-2026.7.0.tar.gz", hash = "sha256:c803c40f4cf860b49dea58ee3e1c33cb9c790520e233537e1340049f89b82a88", size = 317040, upload-time = "2026-07-28T16:34:51.052Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fd/3c/6a2bf344106328fd04963664a60b9bb6496fc25df8e962fcdc1367285fb9/fsspec-2026.7.0-py3-none-any.whl", hash = "sha256:b57ddbafedfaef7018c1ecab32aa200a9d7ca26b77965f64e48b70061249d279", size = 206583, upload-time = "2026-07-28T16:34:49.538Z" }, +] + [[package]] name = "google-antigravity" version = "0.1.7" @@ -882,6 +953,30 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", size = 37515, upload-time = "2025-04-24T03:35:24.344Z" }, ] +[[package]] +name = "hf-xet" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1b/ab/522a2ab67f27971a9d48ca666d4fca85ef7d5282d142e31fd087e27b1bbe/hf_xet-1.6.0.tar.gz", hash = "sha256:2e58454a340b3556dfa4972d5451aff4fba8dd42a236600ba1a1d2b1514f0fef", size = 920527, upload-time = "2026-08-03T22:33:13.243Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/41/62/3c062f593bd92ef4e77a0ef39541e3d82a0a1d3947c8a777a02a13a27828/hf_xet-1.6.0-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:70cbb9c896901600128cb9b6f06e132954fbede1db30f31f7c6c63f84cb7c31d", size = 4074584, upload-time = "2026-08-03T22:32:47.364Z" }, + { url = "https://files.pythonhosted.org/packages/bb/1e/c0ad437dd267a8e435bef594acf781bbc3874ff0b6435b4962d03ecf7cc4/hf_xet-1.6.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:23379c2f9ec8696d952b16414a2bae72cad86a52df869b050698ba60f538c675", size = 3867381, upload-time = "2026-08-03T22:32:49.049Z" }, + { url = "https://files.pythonhosted.org/packages/d5/ee/7c0d7b6ab336167531b1c30af2af003f054af4c749becbd7209ae33a77c3/hf_xet-1.6.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f2f7278c05c22fd60cb436cda1269649b3e81db65ecdc8496e5e164aa4143e7b", size = 4453982, upload-time = "2026-08-03T22:32:50.568Z" }, + { url = "https://files.pythonhosted.org/packages/63/06/ad8eab1c9525246650cbaa821caa3cdbaca734ab1a5b8c91bea09cbd8d69/hf_xet-1.6.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:948f15d3a9545cfe5932f6bd8b440f6ae630aee108f14b7bd6c561f7c2dcc522", size = 4249445, upload-time = "2026-08-03T22:32:52.391Z" }, + { url = "https://files.pythonhosted.org/packages/d8/26/1eee8aedb0dafc1ab9717dc9ac602cde33361b232dc06803f1f6ed18b58c/hf_xet-1.6.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:5153e6bb103ad49d6ea9f1b2e230db5a2ea32551ad09a706d2f61d7c7c80d80e", size = 4451099, upload-time = "2026-08-03T22:32:54.114Z" }, + { url = "https://files.pythonhosted.org/packages/67/57/0b88af1f194ab6c9c650547d9cc06bfeaab836ae4dcdb331676bfb8be95a/hf_xet-1.6.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:35cec30d75c6f9eb9c16a77cef68e85a103b72e24d4b473714ec9ff06428bab9", size = 4664712, upload-time = "2026-08-03T22:32:55.547Z" }, + { url = "https://files.pythonhosted.org/packages/53/a0/26b717a9d1840e8abf48dcec64b5ed8fbe472671d38ad28d30e147132b33/hf_xet-1.6.0-cp314-cp314t-win_amd64.whl", hash = "sha256:5789835d7c6bc9436962853192082374297fb72d7eff7e7762ec25ceb7e25338", size = 4025906, upload-time = "2026-08-03T22:32:57.391Z" }, + { url = "https://files.pythonhosted.org/packages/49/f6/4a9966633c6fef83af997e2cff68ec1963676d412bdfd096df2a93b8e185/hf_xet-1.6.0-cp314-cp314t-win_arm64.whl", hash = "sha256:75765820ce4700db3750c94acc8fe27c5fae4c9ec000a0dbac3ca082acf97765", size = 3849221, upload-time = "2026-08-03T22:32:59.123Z" }, + { url = "https://files.pythonhosted.org/packages/a2/50/7afa2c9c787405864fc47a0d1bbc02c62e9101947ed43c1f43899fc7d91d/hf_xet-1.6.0-cp38-abi3-macosx_10_12_x86_64.whl", hash = "sha256:633dc0cd71d32da58ab8c03ad38e2fac452c15c2b0a2866ebf6ededfe0a5061d", size = 4071729, upload-time = "2026-08-03T22:33:00.721Z" }, + { url = "https://files.pythonhosted.org/packages/4b/69/55b8dcf636142ae660fec1869fcac14c4da2e8412e14d6eee1523be77e9f/hf_xet-1.6.0-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:f0906082d9932ae0c0057fa194041c22b4e2cdb46b2592ef3b91f020d62a081a", size = 3876287, upload-time = "2026-08-03T22:33:02.251Z" }, + { url = "https://files.pythonhosted.org/packages/67/4e/a28359bf1c1ecf11eba22123168c138698f7cb576ac678f5a2e16cd5da08/hf_xet-1.6.0-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:d62671bb130879cef0ee4c9ebe47a14af6c66ec53e6d84dc15936e5ffdfac82f", size = 4464663, upload-time = "2026-08-03T22:33:03.802Z" }, + { url = "https://files.pythonhosted.org/packages/9a/69/1f0cbc2fb22ae6082d094f743d1b8945a3f36f6089cb95f42b7ee348cda7/hf_xet-1.6.0-cp38-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:0e6e21fa3cdfcdcd76748564bf593870a5e013f47d97cf10aed63aa222cff5b7", size = 4262538, upload-time = "2026-08-03T22:33:05.287Z" }, + { url = "https://files.pythonhosted.org/packages/d1/3a/4f4f2301ade26e404462d3336fa11f7958d914cabbabdd6e03c3c5d5658c/hf_xet-1.6.0-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:4fc74352a17015bd0ee90038bc9efe38db894cde45f268b6712b04fce8cd0acb", size = 4460520, upload-time = "2026-08-03T22:33:06.81Z" }, + { url = "https://files.pythonhosted.org/packages/ab/5f/311725e2a905534dfee2dcb5b08414f249147f1f12252bfc2bd24caa075c/hf_xet-1.6.0-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:8fb4f71cba6129110c3374a33f919001ff130488fc23553698e34cc1c2a1198c", size = 4675937, upload-time = "2026-08-03T22:33:08.616Z" }, + { url = "https://files.pythonhosted.org/packages/98/b7/8c59a66d15205024662f1d66968136f13893f96df1ddc5087e2e281fc95f/hf_xet-1.6.0-cp38-abi3-win_amd64.whl", hash = "sha256:fb4fadde1b2b70bf4c0c14a6dccbe7194b1c28947fefd5bbe3fed9d940676c3b", size = 4033128, upload-time = "2026-08-03T22:33:10.171Z" }, + { url = "https://files.pythonhosted.org/packages/73/63/ca511b6f802f28cf3489b280fe77475bcca8de85e81a6299d7916b5b5555/hf_xet-1.6.0-cp38-abi3-win_arm64.whl", hash = "sha256:3dc3e35441ba395006af5aaacc40ef2e603c51ef46c3530b9156185f00935ea3", size = 3859359, upload-time = "2026-08-03T22:33:11.725Z" }, +] + [[package]] name = "httpcore" version = "1.0.9" @@ -957,6 +1052,26 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/9b/43/832f631d32e4f1211caa2ba368317739fe71f0b8530e4c9d15dc454bac2a/httpx2_jsfetch-1.0-py3-none-any.whl", hash = "sha256:cb916b707601e69a07721aabc8f3f6659be3a6893bc1ff5c6f9e02241df2da32", size = 6382, upload-time = "2026-08-07T00:13:06.567Z" }, ] +[[package]] +name = "huggingface-hub" +version = "1.28.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "filelock" }, + { name = "fsspec" }, + { name = "hf-xet", marker = "platform_machine == 'AMD64' or platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'arm64' or platform_machine == 'x86_64'" }, + { name = "httpx" }, + { name = "packaging" }, + { name = "pyyaml" }, + { name = "tqdm" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c6/ae/222a91937ebee7f62c0ca8f5ee0afd97577caf24c0abb927d1f5c7e9f6d2/huggingface_hub-1.28.0.tar.gz", hash = "sha256:46a2e950c09234de54093d587d1675382f0d08dbd600d9fb599b5932f5b2c6cb", size = 959609, upload-time = "2026-08-18T12:27:15.101Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/51/0e/eafef18f1a75e125e68395db21131db0cf868a128ecd2fce69b4df6c584b/huggingface_hub-1.28.0-py3-none-any.whl", hash = "sha256:58a8bacb03072edfc38067065e9dc24bbb34805410fcd36a1632de0b329660bb", size = 793202, upload-time = "2026-08-18T12:27:12.719Z" }, +] + [[package]] name = "identify" version = "2.6.18" @@ -1005,6 +1120,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/15/aa/0aca39a37d3c7eb941ba736ede56d689e7be91cab5d9ca846bde3999eba6/isodate-0.7.2-py3-none-any.whl", hash = "sha256:28009937d8031054830160fce6d409ed342816b543597cece116d966c6d99e15", size = 22320, upload-time = "2024-10-08T23:04:09.501Z" }, ] +[[package]] +name = "jinja2" +version = "3.1.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markupsafe" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/df/bf/f7da0350254c0ed7c72f3e33cef02e048281fec7ecec5f032d4aac52226b/jinja2-3.1.6.tar.gz", hash = "sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d", size = 245115, upload-time = "2025-03-05T20:05:02.478Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67", size = 134899, upload-time = "2025-03-05T20:05:00.369Z" }, +] + [[package]] name = "jiter" version = "0.13.0" @@ -1104,6 +1231,37 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/af/40/791891d4c0c4dab4c5e187c17261cedc26285fd41541577f900470a45a4d/license_expression-30.4.4-py3-none-any.whl", hash = "sha256:421788fdcadb41f049d2dc934ce666626265aeccefddd25e162a26f23bcbf8a4", size = 120615, upload-time = "2025-07-22T11:13:31.217Z" }, ] +[[package]] +name = "litellm" +version = "1.98.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "aiohttp" }, + { name = "boto3" }, + { name = "click" }, + { name = "fastuuid" }, + { name = "httpx" }, + { name = "importlib-metadata" }, + { name = "jinja2" }, + { name = "jsonschema" }, + { name = "openai" }, + { name = "pydantic" }, + { name = "pydantic-settings" }, + { name = "python-dotenv" }, + { name = "tiktoken" }, + { name = "tokenizers" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9c/97/c9da198af273d700bf44d7d82eb21c5b8078c82574b31856b71b1298234b/litellm-1.98.0.tar.gz", hash = "sha256:0e6ba5d645a73ca6d0ffb4e8ec539d94b6e8fad691f2a54c6819011e6d0de8bf", size = 17577139, upload-time = "2026-08-22T22:19:21.931Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c8/90/2ef5e33b0a67b309be124a77e5098a261beffe28439221dbbc28e5f02e2e/litellm-1.98.0-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:9fda3497f1ec4686c943ce2aeab5767f8fb4a5989305d4b28d6d5d7e488b850c", size = 24026097, upload-time = "2026-08-22T22:19:03.567Z" }, + { url = "https://files.pythonhosted.org/packages/b0/56/b4569b4ef3640732d5770e0d3f46a395fd20f35594db1f65629595daed00/litellm-1.98.0-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:b89b6a0fc179d881191579f5309e622173dca802ee991b92e382acb918eea437", size = 23683944, upload-time = "2026-08-22T22:19:06.952Z" }, + { url = "https://files.pythonhosted.org/packages/d9/a7/9f03de0e8d767ff27964ea99bbf07e4c37a12f9d3c168c09f40e068535d4/litellm-1.98.0-cp310-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:3a95260f087a4cf763da85bbeeb2efa82caec243ad2394c9e6362ff747963738", size = 23824066, upload-time = "2026-08-22T22:19:09.714Z" }, + { url = "https://files.pythonhosted.org/packages/69/de/ab46b521e2a6e6a94a5cb91ba3debd6c4e922feaeb47158a389400b0a0fe/litellm-1.98.0-cp310-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:150993180bf049feafa3e20cf46ca0978c69cc66e2ef1639a47e852c682a4721", size = 24190393, upload-time = "2026-08-22T22:19:12.156Z" }, + { url = "https://files.pythonhosted.org/packages/c5/0a/d2f549d906b9b267b38b406dde721eb93db21b46ec907ae0a7a2a89a50f6/litellm-1.98.0-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:54d0bc2aba84644de5e84a265f24e5d436d91592ca5b7cc61cee478db297b0c3", size = 23901211, upload-time = "2026-08-22T22:19:14.708Z" }, + { url = "https://files.pythonhosted.org/packages/85/08/1bd1653297d9c92eaf04425f8a21da817fcde12e1d9469394c543df19d72/litellm-1.98.0-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:5e546d4af197c257d320299af11f4619f8e5a29f9bb7ce2d9974dd4b1e045499", size = 24290140, upload-time = "2026-08-22T22:19:17.145Z" }, + { url = "https://files.pythonhosted.org/packages/de/91/14d11ad7e290137400e5b30bcf23de86f811085f4a46756f60684ed0f064/litellm-1.98.0-cp310-abi3-win_amd64.whl", hash = "sha256:1daac9a9a9d052fdbe58ee711c9924dc81d349d4621286cbd96d77baa12158c4", size = 24079638, upload-time = "2026-08-22T22:19:19.558Z" }, +] + [[package]] name = "mando" version = "0.7.1" @@ -1128,6 +1286,58 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/94/54/e7d793b573f298e1c9013b8c4dade17d481164aa517d1d7148619c2cedbf/markdown_it_py-4.0.0-py3-none-any.whl", hash = "sha256:87327c59b172c5011896038353a81343b6754500a08cd7a4973bb48c6d578147", size = 87321, upload-time = "2025-08-11T12:57:51.923Z" }, ] +[[package]] +name = "markupsafe" +version = "3.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7e/99/7690b6d4034fffd95959cbe0c02de8deb3098cc577c67bb6a24fe5d7caa7/markupsafe-3.0.3.tar.gz", hash = "sha256:722695808f4b6457b320fdc131280796bdceb04ab50fe1795cd540799ebe1698", size = 80313, upload-time = "2025-09-27T18:37:40.426Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/38/2f/907b9c7bbba283e68f20259574b13d005c121a0fa4c175f9bed27c4597ff/markupsafe-3.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e1cf1972137e83c5d4c136c43ced9ac51d0e124706ee1c8aa8532c1287fa8795", size = 11622, upload-time = "2025-09-27T18:36:41.777Z" }, + { url = "https://files.pythonhosted.org/packages/9c/d9/5f7756922cdd676869eca1c4e3c0cd0df60ed30199ffd775e319089cb3ed/markupsafe-3.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:116bb52f642a37c115f517494ea5feb03889e04df47eeff5b130b1808ce7c219", size = 12029, upload-time = "2025-09-27T18:36:43.257Z" }, + { url = "https://files.pythonhosted.org/packages/00/07/575a68c754943058c78f30db02ee03a64b3c638586fba6a6dd56830b30a3/markupsafe-3.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:133a43e73a802c5562be9bbcd03d090aa5a1fe899db609c29e8c8d815c5f6de6", size = 24374, upload-time = "2025-09-27T18:36:44.508Z" }, + { url = "https://files.pythonhosted.org/packages/a9/21/9b05698b46f218fc0e118e1f8168395c65c8a2c750ae2bab54fc4bd4e0e8/markupsafe-3.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ccfcd093f13f0f0b7fdd0f198b90053bf7b2f02a3927a30e63f3ccc9df56b676", size = 22980, upload-time = "2025-09-27T18:36:45.385Z" }, + { url = "https://files.pythonhosted.org/packages/7f/71/544260864f893f18b6827315b988c146b559391e6e7e8f7252839b1b846a/markupsafe-3.0.3-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:509fa21c6deb7a7a273d629cf5ec029bc209d1a51178615ddf718f5918992ab9", size = 21990, upload-time = "2025-09-27T18:36:46.916Z" }, + { url = "https://files.pythonhosted.org/packages/c2/28/b50fc2f74d1ad761af2f5dcce7492648b983d00a65b8c0e0cb457c82ebbe/markupsafe-3.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a4afe79fb3de0b7097d81da19090f4df4f8d3a2b3adaa8764138aac2e44f3af1", size = 23784, upload-time = "2025-09-27T18:36:47.884Z" }, + { url = "https://files.pythonhosted.org/packages/ed/76/104b2aa106a208da8b17a2fb72e033a5a9d7073c68f7e508b94916ed47a9/markupsafe-3.0.3-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:795e7751525cae078558e679d646ae45574b47ed6e7771863fcc079a6171a0fc", size = 21588, upload-time = "2025-09-27T18:36:48.82Z" }, + { url = "https://files.pythonhosted.org/packages/b5/99/16a5eb2d140087ebd97180d95249b00a03aa87e29cc224056274f2e45fd6/markupsafe-3.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:8485f406a96febb5140bfeca44a73e3ce5116b2501ac54fe953e488fb1d03b12", size = 23041, upload-time = "2025-09-27T18:36:49.797Z" }, + { url = "https://files.pythonhosted.org/packages/19/bc/e7140ed90c5d61d77cea142eed9f9c303f4c4806f60a1044c13e3f1471d0/markupsafe-3.0.3-cp313-cp313-win32.whl", hash = "sha256:bdd37121970bfd8be76c5fb069c7751683bdf373db1ed6c010162b2a130248ed", size = 14543, upload-time = "2025-09-27T18:36:51.584Z" }, + { url = "https://files.pythonhosted.org/packages/05/73/c4abe620b841b6b791f2edc248f556900667a5a1cf023a6646967ae98335/markupsafe-3.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:9a1abfdc021a164803f4d485104931fb8f8c1efd55bc6b748d2f5774e78b62c5", size = 15113, upload-time = "2025-09-27T18:36:52.537Z" }, + { url = "https://files.pythonhosted.org/packages/f0/3a/fa34a0f7cfef23cf9500d68cb7c32dd64ffd58a12b09225fb03dd37d5b80/markupsafe-3.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:7e68f88e5b8799aa49c85cd116c932a1ac15caaa3f5db09087854d218359e485", size = 13911, upload-time = "2025-09-27T18:36:53.513Z" }, + { url = "https://files.pythonhosted.org/packages/e4/d7/e05cd7efe43a88a17a37b3ae96e79a19e846f3f456fe79c57ca61356ef01/markupsafe-3.0.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:218551f6df4868a8d527e3062d0fb968682fe92054e89978594c28e642c43a73", size = 11658, upload-time = "2025-09-27T18:36:54.819Z" }, + { url = "https://files.pythonhosted.org/packages/99/9e/e412117548182ce2148bdeacdda3bb494260c0b0184360fe0d56389b523b/markupsafe-3.0.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:3524b778fe5cfb3452a09d31e7b5adefeea8c5be1d43c4f810ba09f2ceb29d37", size = 12066, upload-time = "2025-09-27T18:36:55.714Z" }, + { url = "https://files.pythonhosted.org/packages/bc/e6/fa0ffcda717ef64a5108eaa7b4f5ed28d56122c9a6d70ab8b72f9f715c80/markupsafe-3.0.3-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4e885a3d1efa2eadc93c894a21770e4bc67899e3543680313b09f139e149ab19", size = 25639, upload-time = "2025-09-27T18:36:56.908Z" }, + { url = "https://files.pythonhosted.org/packages/96/ec/2102e881fe9d25fc16cb4b25d5f5cde50970967ffa5dddafdb771237062d/markupsafe-3.0.3-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8709b08f4a89aa7586de0aadc8da56180242ee0ada3999749b183aa23df95025", size = 23569, upload-time = "2025-09-27T18:36:57.913Z" }, + { url = "https://files.pythonhosted.org/packages/4b/30/6f2fce1f1f205fc9323255b216ca8a235b15860c34b6798f810f05828e32/markupsafe-3.0.3-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b8512a91625c9b3da6f127803b166b629725e68af71f8184ae7e7d54686a56d6", size = 23284, upload-time = "2025-09-27T18:36:58.833Z" }, + { url = "https://files.pythonhosted.org/packages/58/47/4a0ccea4ab9f5dcb6f79c0236d954acb382202721e704223a8aafa38b5c8/markupsafe-3.0.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:9b79b7a16f7fedff2495d684f2b59b0457c3b493778c9eed31111be64d58279f", size = 24801, upload-time = "2025-09-27T18:36:59.739Z" }, + { url = "https://files.pythonhosted.org/packages/6a/70/3780e9b72180b6fecb83a4814d84c3bf4b4ae4bf0b19c27196104149734c/markupsafe-3.0.3-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:12c63dfb4a98206f045aa9563db46507995f7ef6d83b2f68eda65c307c6829eb", size = 22769, upload-time = "2025-09-27T18:37:00.719Z" }, + { url = "https://files.pythonhosted.org/packages/98/c5/c03c7f4125180fc215220c035beac6b9cb684bc7a067c84fc69414d315f5/markupsafe-3.0.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:8f71bc33915be5186016f675cd83a1e08523649b0e33efdb898db577ef5bb009", size = 23642, upload-time = "2025-09-27T18:37:01.673Z" }, + { url = "https://files.pythonhosted.org/packages/80/d6/2d1b89f6ca4bff1036499b1e29a1d02d282259f3681540e16563f27ebc23/markupsafe-3.0.3-cp313-cp313t-win32.whl", hash = "sha256:69c0b73548bc525c8cb9a251cddf1931d1db4d2258e9599c28c07ef3580ef354", size = 14612, upload-time = "2025-09-27T18:37:02.639Z" }, + { url = "https://files.pythonhosted.org/packages/2b/98/e48a4bfba0a0ffcf9925fe2d69240bfaa19c6f7507b8cd09c70684a53c1e/markupsafe-3.0.3-cp313-cp313t-win_amd64.whl", hash = "sha256:1b4b79e8ebf6b55351f0d91fe80f893b4743f104bff22e90697db1590e47a218", size = 15200, upload-time = "2025-09-27T18:37:03.582Z" }, + { url = "https://files.pythonhosted.org/packages/0e/72/e3cc540f351f316e9ed0f092757459afbc595824ca724cbc5a5d4263713f/markupsafe-3.0.3-cp313-cp313t-win_arm64.whl", hash = "sha256:ad2cf8aa28b8c020ab2fc8287b0f823d0a7d8630784c31e9ee5edea20f406287", size = 13973, upload-time = "2025-09-27T18:37:04.929Z" }, + { url = "https://files.pythonhosted.org/packages/33/8a/8e42d4838cd89b7dde187011e97fe6c3af66d8c044997d2183fbd6d31352/markupsafe-3.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:eaa9599de571d72e2daf60164784109f19978b327a3910d3e9de8c97b5b70cfe", size = 11619, upload-time = "2025-09-27T18:37:06.342Z" }, + { url = "https://files.pythonhosted.org/packages/b5/64/7660f8a4a8e53c924d0fa05dc3a55c9cee10bbd82b11c5afb27d44b096ce/markupsafe-3.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:c47a551199eb8eb2121d4f0f15ae0f923d31350ab9280078d1e5f12b249e0026", size = 12029, upload-time = "2025-09-27T18:37:07.213Z" }, + { url = "https://files.pythonhosted.org/packages/da/ef/e648bfd021127bef5fa12e1720ffed0c6cbb8310c8d9bea7266337ff06de/markupsafe-3.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f34c41761022dd093b4b6896d4810782ffbabe30f2d443ff5f083e0cbbb8c737", size = 24408, upload-time = "2025-09-27T18:37:09.572Z" }, + { url = "https://files.pythonhosted.org/packages/41/3c/a36c2450754618e62008bf7435ccb0f88053e07592e6028a34776213d877/markupsafe-3.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:457a69a9577064c05a97c41f4e65148652db078a3a509039e64d3467b9e7ef97", size = 23005, upload-time = "2025-09-27T18:37:10.58Z" }, + { url = "https://files.pythonhosted.org/packages/bc/20/b7fdf89a8456b099837cd1dc21974632a02a999ec9bf7ca3e490aacd98e7/markupsafe-3.0.3-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e8afc3f2ccfa24215f8cb28dcf43f0113ac3c37c2f0f0806d8c70e4228c5cf4d", size = 22048, upload-time = "2025-09-27T18:37:11.547Z" }, + { url = "https://files.pythonhosted.org/packages/9a/a7/591f592afdc734f47db08a75793a55d7fbcc6902a723ae4cfbab61010cc5/markupsafe-3.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:ec15a59cf5af7be74194f7ab02d0f59a62bdcf1a537677ce67a2537c9b87fcda", size = 23821, upload-time = "2025-09-27T18:37:12.48Z" }, + { url = "https://files.pythonhosted.org/packages/7d/33/45b24e4f44195b26521bc6f1a82197118f74df348556594bd2262bda1038/markupsafe-3.0.3-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:0eb9ff8191e8498cca014656ae6b8d61f39da5f95b488805da4bb029cccbfbaf", size = 21606, upload-time = "2025-09-27T18:37:13.485Z" }, + { url = "https://files.pythonhosted.org/packages/ff/0e/53dfaca23a69fbfbbf17a4b64072090e70717344c52eaaaa9c5ddff1e5f0/markupsafe-3.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:2713baf880df847f2bece4230d4d094280f4e67b1e813eec43b4c0e144a34ffe", size = 23043, upload-time = "2025-09-27T18:37:14.408Z" }, + { url = "https://files.pythonhosted.org/packages/46/11/f333a06fc16236d5238bfe74daccbca41459dcd8d1fa952e8fbd5dccfb70/markupsafe-3.0.3-cp314-cp314-win32.whl", hash = "sha256:729586769a26dbceff69f7a7dbbf59ab6572b99d94576a5592625d5b411576b9", size = 14747, upload-time = "2025-09-27T18:37:15.36Z" }, + { url = "https://files.pythonhosted.org/packages/28/52/182836104b33b444e400b14f797212f720cbc9ed6ba34c800639d154e821/markupsafe-3.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:bdc919ead48f234740ad807933cdf545180bfbe9342c2bb451556db2ed958581", size = 15341, upload-time = "2025-09-27T18:37:16.496Z" }, + { url = "https://files.pythonhosted.org/packages/6f/18/acf23e91bd94fd7b3031558b1f013adfa21a8e407a3fdb32745538730382/markupsafe-3.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:5a7d5dc5140555cf21a6fefbdbf8723f06fcd2f63ef108f2854de715e4422cb4", size = 14073, upload-time = "2025-09-27T18:37:17.476Z" }, + { url = "https://files.pythonhosted.org/packages/3c/f0/57689aa4076e1b43b15fdfa646b04653969d50cf30c32a102762be2485da/markupsafe-3.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:1353ef0c1b138e1907ae78e2f6c63ff67501122006b0f9abad68fda5f4ffc6ab", size = 11661, upload-time = "2025-09-27T18:37:18.453Z" }, + { url = "https://files.pythonhosted.org/packages/89/c3/2e67a7ca217c6912985ec766c6393b636fb0c2344443ff9d91404dc4c79f/markupsafe-3.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1085e7fbddd3be5f89cc898938f42c0b3c711fdcb37d75221de2666af647c175", size = 12069, upload-time = "2025-09-27T18:37:19.332Z" }, + { url = "https://files.pythonhosted.org/packages/f0/00/be561dce4e6ca66b15276e184ce4b8aec61fe83662cce2f7d72bd3249d28/markupsafe-3.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1b52b4fb9df4eb9ae465f8d0c228a00624de2334f216f178a995ccdcf82c4634", size = 25670, upload-time = "2025-09-27T18:37:20.245Z" }, + { url = "https://files.pythonhosted.org/packages/50/09/c419f6f5a92e5fadde27efd190eca90f05e1261b10dbd8cbcb39cd8ea1dc/markupsafe-3.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fed51ac40f757d41b7c48425901843666a6677e3e8eb0abcff09e4ba6e664f50", size = 23598, upload-time = "2025-09-27T18:37:21.177Z" }, + { url = "https://files.pythonhosted.org/packages/22/44/a0681611106e0b2921b3033fc19bc53323e0b50bc70cffdd19f7d679bb66/markupsafe-3.0.3-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f190daf01f13c72eac4efd5c430a8de82489d9cff23c364c3ea822545032993e", size = 23261, upload-time = "2025-09-27T18:37:22.167Z" }, + { url = "https://files.pythonhosted.org/packages/5f/57/1b0b3f100259dc9fffe780cfb60d4be71375510e435efec3d116b6436d43/markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e56b7d45a839a697b5eb268c82a71bd8c7f6c94d6fd50c3d577fa39a9f1409f5", size = 24835, upload-time = "2025-09-27T18:37:23.296Z" }, + { url = "https://files.pythonhosted.org/packages/26/6a/4bf6d0c97c4920f1597cc14dd720705eca0bf7c787aebc6bb4d1bead5388/markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:f3e98bb3798ead92273dc0e5fd0f31ade220f59a266ffd8a4f6065e0a3ce0523", size = 22733, upload-time = "2025-09-27T18:37:24.237Z" }, + { url = "https://files.pythonhosted.org/packages/14/c7/ca723101509b518797fedc2fdf79ba57f886b4aca8a7d31857ba3ee8281f/markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5678211cb9333a6468fb8d8be0305520aa073f50d17f089b5b4b477ea6e67fdc", size = 23672, upload-time = "2025-09-27T18:37:25.271Z" }, + { url = "https://files.pythonhosted.org/packages/fb/df/5bd7a48c256faecd1d36edc13133e51397e41b73bb77e1a69deab746ebac/markupsafe-3.0.3-cp314-cp314t-win32.whl", hash = "sha256:915c04ba3851909ce68ccc2b8e2cd691618c4dc4c4232fb7982bca3f41fd8c3d", size = 14819, upload-time = "2025-09-27T18:37:26.285Z" }, + { url = "https://files.pythonhosted.org/packages/1a/8a/0402ba61a2f16038b48b39bccca271134be00c5c9f0f623208399333c448/markupsafe-3.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4faffd047e07c38848ce017e8725090413cd80cbc23d86e55c587bf979e579c9", size = 15426, upload-time = "2025-09-27T18:37:27.316Z" }, + { url = "https://files.pythonhosted.org/packages/70/bc/6f1c2f612465f5fa89b95bead1f44dcb607670fd42891d8fdcd5d039f4f4/markupsafe-3.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:32001d6a8fc98c8cb5c947787c5d08b0a50663d139f1305bac5885d98d9b40fa", size = 14146, upload-time = "2025-09-27T18:37:28.327Z" }, +] + [[package]] name = "mcp" version = "1.28.1" @@ -1362,6 +1572,25 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/be/9c/92789c596b8df838baa98fa71844d84283302f7604ed565dafe5a6b5041a/oauthlib-3.3.1-py3-none-any.whl", hash = "sha256:88119c938d2b8fb88561af5f6ee0eec8cc8d552b7bb1f712743136eb7523b7a1", size = 160065, upload-time = "2025-06-19T22:48:06.508Z" }, ] +[[package]] +name = "openai" +version = "2.54.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "distro" }, + { name = "httpx" }, + { name = "jiter" }, + { name = "pydantic" }, + { name = "sniffio" }, + { name = "tqdm" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/50/9a/8c75e8c8a5b407a0586faeb2afac91674ff955c191ecc1d6d3b6669f6788/openai-2.54.0.tar.gz", hash = "sha256:e3e6f8bc1ba30ddf381ace1a14340eed381cb984a1a59bd0f34b5be3b5d49cfa", size = 1100285, upload-time = "2026-08-11T18:46:59.035Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/64/a8/bb76c7356de8ad57f59d5ff993d434df0607f07f08bcc9c9a5c275e399c0/openai-2.54.0-py3-none-any.whl", hash = "sha256:89089789197ccdb87f173a03145ed1598d00795220c93e96cf712b1cbf5e5f2b", size = 1660351, upload-time = "2026-08-11T18:46:56.684Z" }, +] + [[package]] name = "openai-codex" version = "0.144.4" @@ -1979,6 +2208,18 @@ psutil = [ { name = "psutil" }, ] +[[package]] +name = "python-dateutil" +version = "2.9.0.post0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "six" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/66/c0/0c8b6ad9f17a802ee498c46e004a0eb49bc148f2fd230864601a86dcf6db/python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", size = 342432, upload-time = "2024-03-01T18:36:20.211Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", size = 229892, upload-time = "2024-03-01T18:36:18.57Z" }, +] + [[package]] name = "python-discovery" version = "1.2.0" @@ -2110,6 +2351,78 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2c/58/ca301544e1fa93ed4f80d724bf5b194f6e4b945841c5bfd555878eea9fcb/referencing-0.37.0-py3-none-any.whl", hash = "sha256:381329a9f99628c9069361716891d34ad94af76e461dcb0335825aecc7692231", size = 26766, upload-time = "2025-10-13T15:30:47.625Z" }, ] +[[package]] +name = "regex" +version = "2026.7.19" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/20/98/04b13f1ddfb63158025291c02e03eb42fbb7acb51d091d541050eb4e35e8/regex-2026.7.19.tar.gz", hash = "sha256:7e77b324909c1617cbb4c668677e2c6ae13f44d7c1de0d4f15f2e3c10f3315b5", size = 416440, upload-time = "2026-07-19T00:19:48.923Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5d/3d/84165e4299ff76f3a40fe1f2abf939e976f693383a08d2beea6af62bd2c1/regex-2026.7.19-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:f035d9dc1d25eff9d361456572231c7d27b5ccd473ca7dc0adfce732bd006d40", size = 496552, upload-time = "2026-07-19T00:17:36.808Z" }, + { url = "https://files.pythonhosted.org/packages/02/a2/a65293e6e4cf28eb7ee1be5335a5386c40d6742e9f47fafc8fec785e16c7/regex-2026.7.19-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:c42572142ed0b9d5d261ba727157c426510da78e20828b66bbb855098b8a4e38", size = 296983, upload-time = "2026-07-19T00:17:38.816Z" }, + { url = "https://files.pythonhosted.org/packages/95/47/2d0564e93d87bc48618360ddca232a2ca612bbdf53ce8465d45ca5ce14ee/regex-2026.7.19-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:40b34dd88658e4fedd2fddbf0275ac970d00614b731357f425722a3ed1983d11", size = 291832, upload-time = "2026-07-19T00:17:40.726Z" }, + { url = "https://files.pythonhosted.org/packages/07/cd/42dfbabff3dfc9603c501c0e2e2c5adbb09d127b267bf5348de0af338c15/regex-2026.7.19-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0c41c63992bf1874cebb6e7f56fd7d3c007924659a604ae3d90e427d40d4fd13", size = 796775, upload-time = "2026-07-19T00:17:42.382Z" }, + { url = "https://files.pythonhosted.org/packages/df/5d/f6a4839f2b934e3eed5973fd07f5929ee97d4c98939fb275ea23c274ee16/regex-2026.7.19-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1d3372064506b94dd2c67c845f2db8062e9e9ba84d04e33cb96d7d33c11fe1ae", size = 865687, upload-time = "2026-07-19T00:17:44.185Z" }, + { url = "https://files.pythonhosted.org/packages/14/b0/b47d6c36049bc59806a50bd4c86ced70bbe058d787f80281b1d7a9b0e024/regex-2026.7.19-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fce7760bf283405b2c7999cab3da4e72f7deca6396013115e3f7a955db9760da", size = 911962, upload-time = "2026-07-19T00:17:46.442Z" }, + { url = "https://files.pythonhosted.org/packages/2a/be/ff61f28f9273658cfe23acbbac5217221f6519960ed401e61dfdab12bc35/regex-2026.7.19-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c0d702548d89d572b2929879bc883bb7a4c4709efafe4512cadee56c55c9bd15", size = 801817, upload-time = "2026-07-19T00:17:48.25Z" }, + { url = "https://files.pythonhosted.org/packages/c3/bb/8b4f7f26b333f9f79e1b453613c39bb4776f51d38ae66dd0ba31d6b354ca/regex-2026.7.19-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d446c6ac40bb6e05025ccee55b84d80fe9bf8e93010ffc4bb9484f13d498835f", size = 776908, upload-time = "2026-07-19T00:17:50.183Z" }, + { url = "https://files.pythonhosted.org/packages/09/13/610110fc5921d380516d03c26b652555f08aa0d23ea78a771231873c3638/regex-2026.7.19-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4c3501bfa814ab07b5580741f9bf78dfdfe146a04057f82df9e2402d2a975939", size = 784426, upload-time = "2026-07-19T00:17:52.454Z" }, + { url = "https://files.pythonhosted.org/packages/ca/f5/1ef9e2a83a5947c57ebff0b377cb5727c3d5ec1992317a320d035cd0dbb6/regex-2026.7.19-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:c4585c3e64b4f9e583b4d2683f18f5d5d872b3d71dcf24594b74ecc23602fa96", size = 860600, upload-time = "2026-07-19T00:17:54.229Z" }, + { url = "https://files.pythonhosted.org/packages/a0/02/073af33a3ec149241d11c80acea91e722aa0adbf05addd50f251c4fe89c3/regex-2026.7.19-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:571fde9741eb0ccde23dd4e0c1d50fbae910e901fa7e629faf39b2dda740d220", size = 765950, upload-time = "2026-07-19T00:17:56.041Z" }, + { url = "https://files.pythonhosted.org/packages/81/a9/d1e9f819dc394a568ef370cd56cf25394e957a2235f8370f23b576e5a475/regex-2026.7.19-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:15b364b9b98d6d2fe1a85034c23a3180ff913f46caddc3895f6fd65186255ccc", size = 851794, upload-time = "2026-07-19T00:17:57.897Z" }, + { url = "https://files.pythonhosted.org/packages/03/3a/8ae83eda7579feacdf984e71fb9e70635fb6f832eeddca58427ec4fca926/regex-2026.7.19-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ffd8893ccc1c2fce6e0d6ca402d716fe1b29db70c7132609a05955e31b2aa8f2", size = 789845, upload-time = "2026-07-19T00:17:59.97Z" }, + { url = "https://files.pythonhosted.org/packages/4b/23/c195cbfe5a75fdec64d8f6554fd15237b837919d2c61bdc141d7c807b08b/regex-2026.7.19-cp313-cp313-win32.whl", hash = "sha256:f0fa4fa9c3632d708742baf2282f2055c11d888a790362670a403cbf48a2c404", size = 267135, upload-time = "2026-07-19T00:18:01.958Z" }, + { url = "https://files.pythonhosted.org/packages/b2/80/a11de8404b7272b70acb45c1c05987cce60b45d5693da2e176f0e390d564/regex-2026.7.19-cp313-cp313-win_amd64.whl", hash = "sha256:d51ffd3427640fa2da6ade574ceba932f210ad095f65fcc450a2b0a0d454868e", size = 277747, upload-time = "2026-07-19T00:18:04.121Z" }, + { url = "https://files.pythonhosted.org/packages/d1/29/0f5c8eff1b4f1f3d83276d365fccecf666afcc7d947420943bf394d07adb/regex-2026.7.19-cp313-cp313-win_arm64.whl", hash = "sha256:c670fe7be5b6020b76bc6e8d2196074657e1327595bca93a389e1a76ab130ad8", size = 277129, upload-time = "2026-07-19T00:18:05.821Z" }, + { url = "https://files.pythonhosted.org/packages/dc/4c/44b74742052cedda40f9ae469532a037112f7311a36669a891fba8984bb0/regex-2026.7.19-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:db47b561c9afd884baa1f96f797c9ca369872c4b65912bc691cfa99e68340af2", size = 501134, upload-time = "2026-07-19T00:18:07.567Z" }, + { url = "https://files.pythonhosted.org/packages/f0/45/bbd038b5e39ee5613a5a689290145b40058cc152c41de9cc23639d2b9734/regex-2026.7.19-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:65dcd28d3eba2ab7c2fd906485cc301392b47cc2234790d27d4e4814e02cdfda", size = 299418, upload-time = "2026-07-19T00:18:09.38Z" }, + { url = "https://files.pythonhosted.org/packages/65/38/c5bde94b4cedfd5850d64c3f08222d8e1600e84f6ee71d9b44b4b8163f74/regex-2026.7.19-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:f2e7f8e2ab6c2922be02c7ec45185aa5bd771e2e57b95455ee343a44d8130dff", size = 294486, upload-time = "2026-07-19T00:18:11.188Z" }, + { url = "https://files.pythonhosted.org/packages/d7/6a/2f5e107cb26c960b781967178899daf2787a7ab151844ed3c01d6fc95474/regex-2026.7.19-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fe31f28c94402043161876a258a9c6f757cb485905c7614ce8d6cd40e6b7bdc1", size = 811643, upload-time = "2026-07-19T00:18:12.975Z" }, + { url = "https://files.pythonhosted.org/packages/37/d4/a2f963406d7d73a62eed84ba05a258afb6cad1b21aa4517443ce40506b78/regex-2026.7.19-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f8f6fa298bb4f7f58a33334406218ba74716e68feddf5e4e54cd5d8082705abf", size = 871081, upload-time = "2026-07-19T00:18:14.733Z" }, + { url = "https://files.pythonhosted.org/packages/45/a3/44be546340bedb15f13063f5e7fe16793ea4d9ea2e805d09bd174ac27724/regex-2026.7.19-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:cc1b2440423a851fad781309dd87843868f4f66a6bcd1ddb9225cf4ec2c84732", size = 917372, upload-time = "2026-07-19T00:18:16.724Z" }, + { url = "https://files.pythonhosted.org/packages/f8/f6/e0870b0fd2a40dba0074e4b76e514b21313d37946c9248453e34ec43923e/regex-2026.7.19-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8ac59a0900474a52b7c04af8196affc22bd9842acb0950df12f7b813e983609a", size = 816089, upload-time = "2026-07-19T00:18:18.617Z" }, + { url = "https://files.pythonhosted.org/packages/ae/27/957e8e22690ad6634572b39b71f130a6105f4d0718bb16849eac00fff147/regex-2026.7.19-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4896db1f4ce0576765b8272aa922df324e0f5b9bb2c3d03044ff32a7234a9aba", size = 785206, upload-time = "2026-07-19T00:18:20.464Z" }, + { url = "https://files.pythonhosted.org/packages/76/a4/186e410941e731037c01166069ab86da9f65e8f8110c18009ccf4bd623ee/regex-2026.7.19-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:4e6883a021db30511d9fb8cfb0f222ce1f2c369f7d4d8b0448f449a93ba0bdfc", size = 800431, upload-time = "2026-07-19T00:18:22.716Z" }, + { url = "https://files.pythonhosted.org/packages/73/9f/e4e10e023d291d64a33e246610b724493bf1ce98e0e59c9b7c837e5acfb7/regex-2026.7.19-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:09523a592938aa9f587fb74467c63ff0cf88fc3df14c82ab0f0517dcf76aaa62", size = 864906, upload-time = "2026-07-19T00:18:24.772Z" }, + { url = "https://files.pythonhosted.org/packages/24/57/ccb20b6be5f1f52a053d1ba2a8f7a077edb9d918248b8490d7506c6832b3/regex-2026.7.19-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:1ebac3474b8589fce2f9b225b650afd61448f7c73a5d0255a10cc6366471aed1", size = 773559, upload-time = "2026-07-19T00:18:27.008Z" }, + { url = "https://files.pythonhosted.org/packages/a3/82/f3b263cf8fad927dc102891da8502e718b7ff9d19af7a2a07c03865d7188/regex-2026.7.19-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:4a0530bb1b8c1c985e7e2122e2b4d3aedd8a3c21c6bfddae6767c4405668b56e", size = 857739, upload-time = "2026-07-19T00:18:29.107Z" }, + { url = "https://files.pythonhosted.org/packages/47/2e/1687bd1b6c2aed5e672ccf845fc11557821fe7366d921b50889ea5ce57bf/regex-2026.7.19-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:2ef7eeb108c47ce7bcc9513e51bcb1bf57e8f483d52fce68a8642e3527141ae0", size = 804522, upload-time = "2026-07-19T00:18:31.362Z" }, + { url = "https://files.pythonhosted.org/packages/76/7c/cc4e7655181b2d9235b704f2c5e19d8eff002bbc437bae59baee0e381aca/regex-2026.7.19-cp313-cp313t-win32.whl", hash = "sha256:64b6ca7391a1395c2638dd5c7456d67bea44fc6c5e8e92c5dc8aa6a8f23292b4", size = 269141, upload-time = "2026-07-19T00:18:33.479Z" }, + { url = "https://files.pythonhosted.org/packages/bb/14/961b4c7b05a2391c32dbc85e27773076671ef8f97f36cec70fe414734c02/regex-2026.7.19-cp313-cp313t-win_amd64.whl", hash = "sha256:f04b9f56b0e0614c0126be12c2c2d9f8850c1e57af302bd0a63bed379d4af974", size = 280036, upload-time = "2026-07-19T00:18:35.419Z" }, + { url = "https://files.pythonhosted.org/packages/ce/67/795644550d788ddbb6dc458c95895f8009978ea6d6ea76b005eb3f45e8c9/regex-2026.7.19-cp313-cp313t-win_arm64.whl", hash = "sha256:fcee38cd8e5089d6d4f048ba1233b3ad76e5954f545382180889112ff5cb712d", size = 279394, upload-time = "2026-07-19T00:18:37.454Z" }, + { url = "https://files.pythonhosted.org/packages/d2/25/0c4c452f8ef3efe456745b2f33195f5904b573fb4c2ff3f0cb9ec188461e/regex-2026.7.19-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:a81758ed242b861b72e778ba34d41366441a2e10b16b472784c88da2dea7e2dd", size = 496750, upload-time = "2026-07-19T00:18:39.633Z" }, + { url = "https://files.pythonhosted.org/packages/24/9e/b70ca6c1704f6c7cd32a9e143c86cc5968d10981eca284bad670c245ea7d/regex-2026.7.19-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:4aa5435cdb3eb6f55fe98a171b05e3fbcd95fadaa4aa32acf62afd9b0cfdbcac", size = 297093, upload-time = "2026-07-19T00:18:41.583Z" }, + { url = "https://files.pythonhosted.org/packages/87/74/0b692da2520d51fbff19c88b83d97e4c702909dd02386c585998b7e2dbed/regex-2026.7.19-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:60be8693a1dadc210bbcbc0db3e26da5f7d01d1d5a3da594e99b4fa42df404f5", size = 292043, upload-time = "2026-07-19T00:18:43.347Z" }, + { url = "https://files.pythonhosted.org/packages/e3/a7/1d478e614016045a33feae57446215f9fd65b665a5ceb2f891fb3183bc52/regex-2026.7.19-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d19662dbedbe783d323196312d38f5ba53cf56296378252171985da6899887d3", size = 797214, upload-time = "2026-07-19T00:18:45.362Z" }, + { url = "https://files.pythonhosted.org/packages/aa/ae/11b9c9411d92c30e3d2db32df5a31133e4a99a8fc397a604fd08f6c4bffb/regex-2026.7.19-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d15df07081d91b76ff20d43f94592ee110330152d617b730fdbe5ef9fb680053", size = 866433, upload-time = "2026-07-19T00:18:47.315Z" }, + { url = "https://files.pythonhosted.org/packages/b1/62/2b2efc4992f91d6d204b24c647c9f9412e85379d92b7c0ab9fdae622327e/regex-2026.7.19-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:56ad4d9f77df871a99e25c37091052a02528ec0eb059de928ee33956b854b45b", size = 911360, upload-time = "2026-07-19T00:18:49.588Z" }, + { url = "https://files.pythonhosted.org/packages/14/71/986ceea9aa3da548bf1357cad89b63915ec6d21ec957c8113b29ece567df/regex-2026.7.19-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7322ec6cc9fba9d49ab888bb82d67ac5625627aa168f0165139b17018df3fb8a", size = 801275, upload-time = "2026-07-19T00:18:51.767Z" }, + { url = "https://files.pythonhosted.org/packages/15/be/ce9d9534b2cda96eab32c548261224b9b4e220a4126f098f60f42ae7b4cd/regex-2026.7.19-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9c7472192ebfad53a6be7c4a8bfb2d64b81c0e93a1fc8c57e1dd0b638297b5d1", size = 777131, upload-time = "2026-07-19T00:18:54.053Z" }, + { url = "https://files.pythonhosted.org/packages/61/2b/58b5c710f2c3929515a25f3a1ca0dad0dcd4518d4fff3cf23bc7adb8dcd2/regex-2026.7.19-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c10b82c2634df08dfb13b1f04e38fe310d086ee092f4f69c0c8da234251e556e", size = 785020, upload-time = "2026-07-19T00:18:56.579Z" }, + { url = "https://files.pythonhosted.org/packages/84/03/5fe091935b74f15fe0f97998c215cae418d1c0413f6258c7d4d2e83aa37f/regex-2026.7.19-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:17ed5692f6acc4183e98331101a5f9e4f64d72fe58b753da4d444a2c77d05b12", size = 861263, upload-time = "2026-07-19T00:18:58.64Z" }, + { url = "https://files.pythonhosted.org/packages/d8/fa/d60bf82e10841eef62a9e32aac401468f05fddfbcb2942e342b1ba3d2433/regex-2026.7.19-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:22a992de9a0d91bda927bf02b94351d737a0302905432c88a53de7c4b9ce62e2", size = 766199, upload-time = "2026-07-19T00:19:00.705Z" }, + { url = "https://files.pythonhosted.org/packages/bf/5d/11e64d151b0662b81d6bf644c74dc118d461df85bdf2577fadbbf751788a/regex-2026.7.19-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:618a0aed532be87294c4477b0481f3aa0f1520f4014a4374dd4cf789b4cd2c97", size = 851317, upload-time = "2026-07-19T00:19:03.015Z" }, + { url = "https://files.pythonhosted.org/packages/7c/34/532efb87488d90807bae6a443d357ee5e2728a478c597619c8aaa17cc0bd/regex-2026.7.19-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:2ce9e679f776649746729b6c86382da519ef649c8e34cc41df0d2e5e0f6c36d4", size = 789557, upload-time = "2026-07-19T00:19:05.338Z" }, + { url = "https://files.pythonhosted.org/packages/d6/90/3a8d5ca977171ec3ae21a71207d2228b2663bde14d7f7ef0e6363ecf9290/regex-2026.7.19-cp314-cp314-win32.whl", hash = "sha256:73f272fba87b8ccfe70a137d02a54af386f6d27aa509fbffdd978f5947aae1aa", size = 272531, upload-time = "2026-07-19T00:19:07.487Z" }, + { url = "https://files.pythonhosted.org/packages/96/e1/8862885e70409de70e8c005f57fb2e7be8d9ef0317250d60f4c9660a300d/regex-2026.7.19-cp314-cp314-win_amd64.whl", hash = "sha256:d721e53758b2cca74990185eb0671dd466d7a388a1a45d0c6f4c13cef41a68ac", size = 280831, upload-time = "2026-07-19T00:19:09.46Z" }, + { url = "https://files.pythonhosted.org/packages/08/82/2693e53e29f9104d9de95d37ce4dd826bd32d5f9c0085d3aa6ac042675c4/regex-2026.7.19-cp314-cp314-win_arm64.whl", hash = "sha256:65fa6cb38ed5e9c3637e68e544f598b39c3b86b808ed0627a67b68320384b459", size = 281099, upload-time = "2026-07-19T00:19:11.398Z" }, + { url = "https://files.pythonhosted.org/packages/92/b7/9a01aa16461a18cde9d7b9c3ab21e501db2ce33725f53014342b91df2b0a/regex-2026.7.19-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:5a2721c8720e2cb3c209925dfb9200199b4b07361c9e01d321719404b21458b3", size = 501121, upload-time = "2026-07-19T00:19:13.425Z" }, + { url = "https://files.pythonhosted.org/packages/f3/5e/bbaeca815dc9191c424c94a4fdc5c87c75748a64a6271821212ebdd4e1a3/regex-2026.7.19-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:199535629f25caf89698039af3d1ad5fcae7f933e2112c73f1cdf49165c99518", size = 299415, upload-time = "2026-07-19T00:19:15.43Z" }, + { url = "https://files.pythonhosted.org/packages/cd/d6/0dd1a321afaab95eb7ff44aa0f637301786f1dc71c6b797b9ed236ed8890/regex-2026.7.19-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:9b60d7814174f059e5de4ab98271cc5ba9259cfea55273a81544dceea32dc8d9", size = 294483, upload-time = "2026-07-19T00:19:17.879Z" }, + { url = "https://files.pythonhosted.org/packages/92/5f/40bacf91d0904f812e13bbbab3864604c463eced8afdc54aeaa50492ea95/regex-2026.7.19-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dbece16025afda5e3031af0c4059207e61dcf73ef13af844964f57f387d1c435", size = 811833, upload-time = "2026-07-19T00:19:20.102Z" }, + { url = "https://files.pythonhosted.org/packages/94/7c/4902744261f775aeede8b5627314b38482da29cf49a57b66a6fb753246c5/regex-2026.7.19-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d24ecb4f5e009ea0bd275ee37ad9953b32005e2e5e60f8bbae16da0dbbf0d3a0", size = 871270, upload-time = "2026-07-19T00:19:22.365Z" }, + { url = "https://files.pythonhosted.org/packages/16/70/6980c9be6bf21c0a60ed3e0aea39cf419ecf3b08d1d9947bc56e196ef186/regex-2026.7.19-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:8cae6fd77a5b72dae505084b1a2ee0360139faf72fedbab667cd7cc65aae7a6a", size = 917534, upload-time = "2026-07-19T00:19:24.529Z" }, + { url = "https://files.pythonhosted.org/packages/52/92/8b2bd872782ce8c42691e39acb38eb8efe014e5ddb78ad7d943d6f197ce9/regex-2026.7.19-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9724e6cb5e478cd7d8cabf027826178739cb18cf0e117d0e32814d479fa02276", size = 816135, upload-time = "2026-07-19T00:19:26.919Z" }, + { url = "https://files.pythonhosted.org/packages/de/2d/33a602f657bdc4041f17d79f92ab18261d255d91a06117a6e29df023e5e2/regex-2026.7.19-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:572fc57b0009c735ee56c175ea021b637a15551a312f56734277f923d6fd0f6c", size = 785492, upload-time = "2026-07-19T00:19:29.192Z" }, + { url = "https://files.pythonhosted.org/packages/9e/36/0987cf4cb271680064a70d24a475873775a151d0b7058698a006cb0cae4a/regex-2026.7.19-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:20568e182eb82d39a6bf7cff3fd58566f14c75c6f74b2c8c96537eecf9010e3a", size = 800658, upload-time = "2026-07-19T00:19:31.392Z" }, + { url = "https://files.pythonhosted.org/packages/a8/24/c14f31c135e1ba55fa4f9a58ca98d0842512bf6188230763c31c8f449e3b/regex-2026.7.19-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:1d58561843f0ff7dc78b4c28b5e2dc388f3eff94ebc8a232a3adba961fc00009", size = 865073, upload-time = "2026-07-19T00:19:33.485Z" }, + { url = "https://files.pythonhosted.org/packages/14/85/181a12211f22469f24d2de1ebddfe397d2396e2c29013b9a58134a91069a/regex-2026.7.19-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:61bb1bd45520aacd56dd80943bd34991fb5350afdd1f36f2282230fd5154a218", size = 773684, upload-time = "2026-07-19T00:19:35.599Z" }, + { url = "https://files.pythonhosted.org/packages/23/58/bd1a0c1a62251366f8d21f41b1ea3c76994962071b8b6ea42f72d505c0f0/regex-2026.7.19-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:cd3584591ea4429026cdb931b054342c2bcf189b44ff367f8d5c15bc092a2966", size = 857769, upload-time = "2026-07-19T00:19:37.738Z" }, + { url = "https://files.pythonhosted.org/packages/e4/4f/f7e2dad6756b2fe1fe75dd90a628c3b45f249d39f948dd90cd2476325417/regex-2026.7.19-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5cc26a66e212fa5d6c6170c3a40d99d888db3020c6fdab1523250d4341382e44", size = 804546, upload-time = "2026-07-19T00:19:40.229Z" }, + { url = "https://files.pythonhosted.org/packages/2b/d7/01d31d5bdb09bc026fab77f59a371fdf8f9b292e4810546c56182ca70498/regex-2026.7.19-cp314-cp314t-win32.whl", hash = "sha256:2c4e61e2e1be56f63ec3cc618aa9e0de81ef6f43d177205451840022e24f5b78", size = 274526, upload-time = "2026-07-19T00:19:42.398Z" }, + { url = "https://files.pythonhosted.org/packages/52/0e/cea4ce73bc0a8247a0748228ae6669984c7e1f8134b6fa66e59c0572e0ea/regex-2026.7.19-cp314-cp314t-win_amd64.whl", hash = "sha256:c639ea314df70a7b2811e8020448c75af8c9445f5a60f8a4ced81c306a9380c2", size = 283763, upload-time = "2026-07-19T00:19:44.644Z" }, + { url = "https://files.pythonhosted.org/packages/6f/b6/26e41975febae63b7a6e3e02f32cff6cff2e4f10d19c929082f56aebf7c6/regex-2026.7.19-cp314-cp314t-win_arm64.whl", hash = "sha256:9a15e785f244f3e07847b984ce8773fc3da10a9f3c131cc49a4c5b4d672b4547", size = 283451, upload-time = "2026-07-19T00:19:46.639Z" }, +] + [[package]] name = "requests" version = "2.33.0" @@ -2242,6 +2555,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8f/e8/726643a3ea68c727da31570bde48c7a10f1aa60eddd628d94078fec586ff/ruff-0.15.7-py3-none-win_arm64.whl", hash = "sha256:18e8d73f1c3fdf27931497972250340f92e8c861722161a9caeb89a58ead6ed2", size = 11023304, upload-time = "2026-03-19T16:26:51.669Z" }, ] +[[package]] +name = "s3transfer" +version = "0.19.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "botocore" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/76/43/35e4d8aa320bffe8287fe8f65f578fa2d2db0a64212f0e710dce58267854/s3transfer-0.19.2.tar.gz", hash = "sha256:ba0309fd86be3c27dbf78cdd813c13c5e1df16e5874b99d2535ebbdfb9892993", size = 165592, upload-time = "2026-07-22T19:30:44.432Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/bc/e7/5c595c75e9f41a44f30e526eda465ea0b4eec93470e074e4a111b253f13a/s3transfer-0.19.2-py3-none-any.whl", hash = "sha256:d8168eccca828cbb2cd573675333f3bddd254313a9c42494b84c76b539e8ba25", size = 90216, upload-time = "2026-07-22T19:30:43.251Z" }, +] + [[package]] name = "shellingham" version = "1.5.4" @@ -2342,6 +2667,80 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d7/c1/eb8f9debc45d3b7918a32ab756658a0904732f75e555402972246b0b8e71/tenacity-9.1.4-py3-none-any.whl", hash = "sha256:6095a360c919085f28c6527de529e76a06ad89b23659fa881ae0649b867a9d55", size = 28926, upload-time = "2026-02-07T10:45:32.24Z" }, ] +[[package]] +name = "tiktoken" +version = "0.14.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "regex" }, + { name = "requests" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/66/62/167a842aa0429d45f5e797354fd4343a96f6043d67d0513c675c7b8d36e6/tiktoken-0.14.0.tar.gz", hash = "sha256:231dec90efcdccf1b565a1416107736f1e09b1a08fe736ef9d6363e626d03874", size = 38898, upload-time = "2026-08-17T19:49:49.514Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/50/53/ee1453623bf65f019328721ccb6587846d2c5b7b82f34e73ca09101f072e/tiktoken-0.14.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e9c5fe393aab56469f04e432ff851216d3def3436cf5f07e442a240164bf500f", size = 1094198, upload-time = "2026-08-17T19:48:57.955Z" }, + { url = "https://files.pythonhosted.org/packages/ad/5f/6448cfe278c3664ba9ec5b5ac08344341f7dc3d42888476e215a14eda2be/tiktoken-0.14.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:cbe2cc3bba939bcdaf103e03df9d5039d33887080b315624be28ec69059e5f94", size = 1038820, upload-time = "2026-08-17T19:48:59.015Z" }, + { url = "https://files.pythonhosted.org/packages/69/3b/d67eac1bcce9dee3abe23aff5e3ded3116bbebaf67b80a0811c06d3806fc/tiktoken-0.14.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:2157f52e4b4d7ac5ecc7457b3716834706e7ef9a46f5144029bfeb7cf71f4e06", size = 1186175, upload-time = "2026-08-17T19:49:00.068Z" }, + { url = "https://files.pythonhosted.org/packages/37/62/cae690d9783146b0f81f564ada0f8f611de68178c0c9c7e1e969f0516b48/tiktoken-0.14.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:26e60f6a956ee171ab728b37b8439905d7ea1db435c30f9822f291e9861c861d", size = 1203884, upload-time = "2026-08-17T19:49:01.163Z" }, + { url = "https://files.pythonhosted.org/packages/b9/1e/633e30237b94e383cf814145499079f3bb9cdd4aeafc1bc42e01b0f810a6/tiktoken-0.14.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:380873f330b741c4435574f37edb20813d04603ace2d53e0a63560e1fec83010", size = 1250980, upload-time = "2026-08-17T19:49:02.274Z" }, + { url = "https://files.pythonhosted.org/packages/cb/56/4c12f07b812f84206f38d723eb1ebfdd34bad9309b5dbc0bee6bbcff4cbf/tiktoken-0.14.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:3fd7c14b1cb45b486c39fc9b3443bb341f3e2fc7e6f31247f3435a5836651632", size = 1315434, upload-time = "2026-08-17T19:49:03.434Z" }, + { url = "https://files.pythonhosted.org/packages/c9/e0/c65603f0c44811def666d3fbf611bf2af3b5e1ef613e06c19411419830b3/tiktoken-0.14.0-cp313-cp313-win_amd64.whl", hash = "sha256:90a762670c7f968184723769a06ed51f5cf5ce5dcd1e30164f25c72d85c2d1f1", size = 940883, upload-time = "2026-08-17T19:49:04.583Z" }, + { url = "https://files.pythonhosted.org/packages/59/b0/1cf129f4af8fc513931f931023def596b7c4bfc77026513cd9d851da9e88/tiktoken-0.14.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:e067f4cbcc5d036e8aff7fe7a6b530a8f4de2e4616ad9005a24a1879e24e6450", size = 1096273, upload-time = "2026-08-17T19:49:05.807Z" }, + { url = "https://files.pythonhosted.org/packages/62/85/2ae74575e321148484147e10b53c3b1717c59ebaa9edb4fe18b1f5c055f8/tiktoken-0.14.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:f2af4a336ea56d6c14f27741a0e1d8294a35dd0b038bcf990d232ebb54eb994b", size = 1040269, upload-time = "2026-08-17T19:49:06.943Z" }, + { url = "https://files.pythonhosted.org/packages/89/29/92a1120a12e4bcf2d5464350d1a91b68a433d63ce656bb7f806c27aec09c/tiktoken-0.14.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:f702e0aeeb6506e57687e881c59e844ebe8f0a6a097ddafe20e3ab25f387be4e", size = 1186101, upload-time = "2026-08-17T19:49:08.102Z" }, + { url = "https://files.pythonhosted.org/packages/5b/7d/144af98dc5ad68108451a82e2f5a17f80e2663f5115058b8dfd215c1ad02/tiktoken-0.14.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:e3442bbb2f0c588cec876061e37ae67b455b9df9978b003c8fe30e45f2ef5b42", size = 1204457, upload-time = "2026-08-17T19:49:09.28Z" }, + { url = "https://files.pythonhosted.org/packages/e6/1f/be7cb06ab2108f612f3e92e7b76cf391e192db0db37a984616f0cc32aafc/tiktoken-0.14.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:979c1524f753b662b0f3cd261b135afe6659cce33caaa7a5ea00dd1756b3055c", size = 1251716, upload-time = "2026-08-17T19:49:10.509Z" }, + { url = "https://files.pythonhosted.org/packages/ab/6b/81f158d0f90adb826cd704069c2129a046cb784a2a09861009519fc41cf4/tiktoken-0.14.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:2cc19ac87b41c9493c9778ff5847f0c8bbcf5bd0ec6b87ce06c1c802adc8a771", size = 1315432, upload-time = "2026-08-17T19:49:11.844Z" }, + { url = "https://files.pythonhosted.org/packages/fc/ec/f5fa35ec13f07279fdcaf3cc9c04bbb154ea591d23978651f2b672593e8a/tiktoken-0.14.0-cp314-cp314-win_amd64.whl", hash = "sha256:eceeff0c62419bc78d4b6e70a4762a4d25df3ae8f2d5946e3853ce93e7a57098", size = 988046, upload-time = "2026-08-17T19:49:13.282Z" }, + { url = "https://files.pythonhosted.org/packages/68/c9/7756717408d3d0dfea3f046c9466144b28afde39ff69d5808f2475dcd7f5/tiktoken-0.14.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:6eb94895c45f26bb8f5546e5fd8a069efcf6e3f108ea9d5cbe3bf6f7f3983438", size = 1096261, upload-time = "2026-08-17T19:49:14.351Z" }, + { url = "https://files.pythonhosted.org/packages/79/29/46ad8061f57bd9f8b2ea0aa82bf574e0f2aa040b0857a1582adba9957899/tiktoken-0.14.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:86951a971c53979ec857bd8c4a32dc227ab0fd33f6c12a3bd62d3fbf5f0bfcaa", size = 1040183, upload-time = "2026-08-17T19:49:15.707Z" }, + { url = "https://files.pythonhosted.org/packages/5a/7c/3184d17b868456f17b60b1a75f5ec0405618a43aa753336df341d8f11781/tiktoken-0.14.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:e2eca764c53490f8930dbce329e0769f11108d87d908282a80c5c130e26e7037", size = 1186719, upload-time = "2026-08-17T19:49:16.84Z" }, + { url = "https://files.pythonhosted.org/packages/0b/e8/46de4400d5bf859f640feee85bd7e32235f68ddf25db53c63be78e581e3a/tiktoken-0.14.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:26cc4b4840fa0e9f4b72ed489883e12f57e00d1021ca794720e3c29a12f0edef", size = 1204660, upload-time = "2026-08-17T19:49:17.987Z" }, + { url = "https://files.pythonhosted.org/packages/29/ce/af8964c38bc8226dd8950305b7a255fa33345d5572f78af7275a313d28e0/tiktoken-0.14.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:2fc834fbe3f6a0736905c36ab709537e6840dbd63b982dc9e0216ae7d305ba1a", size = 1250932, upload-time = "2026-08-17T19:49:19.28Z" }, + { url = "https://files.pythonhosted.org/packages/1d/4b/323631116fc986d9cc5bbeb2b8223c7c85e61a8bb94ea5ab4951023b149b/tiktoken-0.14.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:ca4db6ff5c5bf600f9b7761a0070ed44dfe5797a76bd432fb978bc480ef40c58", size = 1315190, upload-time = "2026-08-17T19:49:20.467Z" }, + { url = "https://files.pythonhosted.org/packages/18/8b/ba48a73729c9270989b36f37ab2ed5525e52690d715097c9fa791aaa5d05/tiktoken-0.14.0-cp314-cp314t-win_amd64.whl", hash = "sha256:7aab286a020660a039097912a088236b985d18a3090d73f136c4413d29d37ca0", size = 987717, upload-time = "2026-08-17T19:49:21.704Z" }, + { url = "https://files.pythonhosted.org/packages/1d/10/b73b7e319179e0f60b32475f783b044f9cece872c53b6662664e9084b0d0/tiktoken-0.14.0-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:14b47e3674f2624803a8acc8fb367b7e24fc53055f9df3296482fe9a3a34a232", size = 1096280, upload-time = "2026-08-17T19:49:22.779Z" }, + { url = "https://files.pythonhosted.org/packages/c2/6b/09999a9bf1d559670d1680e8f8e419ac0e2c5f6aac82e9bfdf70f260b30a/tiktoken-0.14.0-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:19d643d701fdaa70e5b9c7f8f96abcaffe77ca5e482a3a1a7dde46feb4284695", size = 1040433, upload-time = "2026-08-17T19:49:23.998Z" }, + { url = "https://files.pythonhosted.org/packages/cd/7b/8537be0836f3df99b2a636b44399bfa43cd757f2b8b4097dacb794cf24a7/tiktoken-0.14.0-cp315-cp315-manylinux_2_28_aarch64.whl", hash = "sha256:e4ddf863b59347deaa92302dcd90e5eb003cdc9be06ec2b692c38d1bdd9efd49", size = 1186989, upload-time = "2026-08-17T19:49:25.021Z" }, + { url = "https://files.pythonhosted.org/packages/7c/9d/f9c56d7a943a4468abf9ef37661bb9b8e0cd3aa8aa87368c7146cc3f3222/tiktoken-0.14.0-cp315-cp315-manylinux_2_28_x86_64.whl", hash = "sha256:60c47ca69ddda0dea8256fffd12e1b86f4b59734a20e4a70c61f63cc5f021df4", size = 1204615, upload-time = "2026-08-17T19:49:26.37Z" }, + { url = "https://files.pythonhosted.org/packages/4b/d2/98a38579db25c4a8a84e31dd95d9072ec5f21f7e70de591da0412e29b25b/tiktoken-0.14.0-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:728303a072163130c5b477b1f20d6211895569c1d5302c24ffc93a3009160871", size = 1251828, upload-time = "2026-08-17T19:49:27.423Z" }, + { url = "https://files.pythonhosted.org/packages/0c/83/467be424746c039c5493c0f4102feab16b9b48eb6f5c089b2a2438e3cde2/tiktoken-0.14.0-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:3c5349c9f916283bba32bec8af69b763e4faa304dc004d0eaaea66a3cf004c1f", size = 1316260, upload-time = "2026-08-17T19:49:29.101Z" }, + { url = "https://files.pythonhosted.org/packages/02/ee/ddf46ca78e371f5890e96b6e7d089a85b3536432be219851eb0481786ca8/tiktoken-0.14.0-cp315-cp315-win_amd64.whl", hash = "sha256:1b6e4adcfd285c44502aed51df98aaaca4f0fea028165dbf8a9e857b9f98d8ea", size = 988230, upload-time = "2026-08-17T19:49:30.246Z" }, + { url = "https://files.pythonhosted.org/packages/2a/00/5162e90c851a28da18ed382d34898b79a8022548e5619a64e14c03ce7c3d/tiktoken-0.14.0-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:11d8211b290855d2721334ff17dd9b3a17bfb26872be01f25d73612ef7ece890", size = 1096186, upload-time = "2026-08-17T19:49:31.656Z" }, + { url = "https://files.pythonhosted.org/packages/65/97/a5a7bfccf25b1bb65e82bae8edff11ac3c9c041c374b7b4a823d60c38133/tiktoken-0.14.0-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:d0781223705199b289faa59601bb9c2441712d4c600dd13c43d8fd6a33d22cd5", size = 1039947, upload-time = "2026-08-17T19:49:32.848Z" }, + { url = "https://files.pythonhosted.org/packages/fb/ba/ef427fc638f1439181c5e12dd26b70e881861f89c007aa7e5b36300f8342/tiktoken-0.14.0-cp315-cp315t-manylinux_2_28_aarch64.whl", hash = "sha256:2ea70afba6b9eddbf22c165142e5f0a2ad7aa36a452873c48b57bb2aeb8492ae", size = 1186997, upload-time = "2026-08-17T19:49:34.121Z" }, + { url = "https://files.pythonhosted.org/packages/3e/88/2f3f85a968cdc514152129af0a060ebcccb067005a2f29b0d5ef3c838514/tiktoken-0.14.0-cp315-cp315t-manylinux_2_28_x86_64.whl", hash = "sha256:78571efc311c30b73f31eb949a921d6dac39a5d9dc42d1cfa8f8db157b3447b1", size = 1205211, upload-time = "2026-08-17T19:49:35.284Z" }, + { url = "https://files.pythonhosted.org/packages/4e/f6/80760e98a08e6649d2d68afb6035af713121dfb615acce8c4f73810ec438/tiktoken-0.14.0-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:86f66c85e796f5d05d5c4a60ec1d40cbfebc47a32464053528c797163fa9ab89", size = 1251479, upload-time = "2026-08-17T19:49:36.419Z" }, + { url = "https://files.pythonhosted.org/packages/c5/84/50966fb6918a0fb9b32721277e5342bf729a2d74350074d662fbedf9772e/tiktoken-0.14.0-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:149d97453c4c98c04b081d64a85e635921269b532710d6faf81e9e82b790e7d3", size = 1316673, upload-time = "2026-08-17T19:49:37.756Z" }, + { url = "https://files.pythonhosted.org/packages/35/5e/9b01afd037bfa22a0033963fa091e0f75b6fb15cd85bffb42ff86e697323/tiktoken-0.14.0-cp315-cp315t-win_amd64.whl", hash = "sha256:561e7580f84a79859af1ef6f676968e9030fcc3fe195700b15235bca64f009c9", size = 987929, upload-time = "2026-08-17T19:49:38.947Z" }, +] + +[[package]] +name = "tokenizers" +version = "0.23.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "huggingface-hub" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c1/60/21f715d9faba5f5407ff759472ade058ec4a507ad62bcea47cb847239a73/tokenizers-0.23.1.tar.gz", hash = "sha256:1feeeadf865a7915adc25445dea30e9933e593c31bb96c277cee36de227c8bfa", size = 365748, upload-time = "2026-04-27T14:43:25.606Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/87/39/b87a87d5bb9470610b80a2d31df42fcffeaf35118b8b97952b2aff598cc7/tokenizers-0.23.1-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:e03d6ffcbe0d56ee9c1ccd070e70a13fa750727c0277e138152acbc0252c2224", size = 3146732, upload-time = "2026-04-27T14:43:15.427Z" }, + { url = "https://files.pythonhosted.org/packages/e2/6a/068ed9f6e444c9d7e9d55ce134181325700f3d7f30410721bdc8f848d727/tokenizers-0.23.1-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:e0948bbb1ac1d7cdfc9fb6d62c596e3b7550036ad60ecd654a66ad273326324e", size = 3054954, upload-time = "2026-04-27T14:43:13.745Z" }, + { url = "https://files.pythonhosted.org/packages/6c/36/e006edf031154cba92b8416057d92c3abe3635e4c4b0aa0b5b9bb39dde70/tokenizers-0.23.1-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1bf13402aff9bc533c89cb849ec3b412dc3fbeacc9744840e423d7bf3f7dc0e3", size = 3374081, upload-time = "2026-04-27T14:43:01.241Z" }, + { url = "https://files.pythonhosted.org/packages/a2/ef/7735d226f9c7f874a6bee5e3f27fb25ecabdf207d37b8cf45286d0795893/tokenizers-0.23.1-cp310-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f836ca703b89ae07919a309f9651f7a88fd5a33d5f718ba5ad0870ec0256bad6", size = 3247641, upload-time = "2026-04-27T14:43:03.856Z" }, + { url = "https://files.pythonhosted.org/packages/b9/d9/24827036f6e21297bfffda0768e58eb6096a4f411e932964a01707857931/tokenizers-0.23.1-cp310-abi3-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ae848657742035523fdf261773630cb819a26995fcd3d9ecae0c1daf6e5a4959", size = 3585624, upload-time = "2026-04-27T14:43:10.664Z" }, + { url = "https://files.pythonhosted.org/packages/0c/9a/22f3582b3a4f49358293a5206e25317621ee4526bfe9cdaa0f07a12e770e/tokenizers-0.23.1-cp310-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:53b09e85775d5187941e7bab30e941b4134ab4a7dd8c68e783d231fb7ca27c51", size = 3844062, upload-time = "2026-04-27T14:43:05.643Z" }, + { url = "https://files.pythonhosted.org/packages/7e/65/b8f8814eef95800f20721384136d9a1d22241d50b2874357cb70542c392f/tokenizers-0.23.1-cp310-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ea5a0ce170074329faaa8ea3f6400ecde604b6678192688533af80980daae71a", size = 3460098, upload-time = "2026-04-27T14:43:08.854Z" }, + { url = "https://files.pythonhosted.org/packages/0d/d5/1353e5f677ec27c2494fb6a6725e82d56c985f53e90ec511369e7e4f02c6/tokenizers-0.23.1-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5075b405006415ea148a992d093699c66eb01952bf59f4d5727089a98bda45a4", size = 3346235, upload-time = "2026-04-27T14:43:12.377Z" }, + { url = "https://files.pythonhosted.org/packages/71/89/39b6b8fc073fb6d413d0147aa333dc7eff7be65639ac9d19930a0b21bf33/tokenizers-0.23.1-cp310-abi3-manylinux_2_31_riscv64.whl", hash = "sha256:56f3a77de629917652f876294dc9fe6bad4a0c43bc229dc72e59bb23a0f4729a", size = 3426398, upload-time = "2026-04-27T14:43:07.264Z" }, + { url = "https://files.pythonhosted.org/packages/0f/80/127c854da64827e5b79264ce524993a90dddcb320e5cd42412c5c02f9e8a/tokenizers-0.23.1-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:9d10a6d957ef01896dc274e890eee27d41bd0e74ef31e60616f0fc311345184e", size = 9823279, upload-time = "2026-04-27T14:43:17.222Z" }, + { url = "https://files.pythonhosted.org/packages/fe/ba/44c2502feb1a058f096ddfb4e0996ef3225a01a388e1a9b094e91689fe93/tokenizers-0.23.1-cp310-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:1974288a609c343774f1b897c8b482c791ab17b75ab5c8c2b1737565c1d82288", size = 9644986, upload-time = "2026-04-27T14:43:19.45Z" }, + { url = "https://files.pythonhosted.org/packages/9e/c1/464019a9fb059870bfe4eebb4ba12208f3042035e258bf5e782906bd3847/tokenizers-0.23.1-cp310-abi3-musllinux_1_2_i686.whl", hash = "sha256:120468fb4c24faf0543c835a4fabafa4deb3f20a035c9b6e83d0b553a97615d4", size = 9976181, upload-time = "2026-04-27T14:43:21.463Z" }, + { url = "https://files.pythonhosted.org/packages/79/94/3ac1432bda31626071e9b6a12709b97ae05131c804b94c8f3ac622c5da32/tokenizers-0.23.1-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:e3d8f40ea6268047de7046906326abed5134f27d4e8447b23763afe5808c8a96", size = 10113853, upload-time = "2026-04-27T14:43:23.617Z" }, + { url = "https://files.pythonhosted.org/packages/6a/dd/631b21433c771b1382535326f0eca80b9c9cee2e64961dd993bc9ac4669e/tokenizers-0.23.1-cp310-abi3-win32.whl", hash = "sha256:93120a930b919416da7cd10a2f606ac9919cc69cacae7980fa2140e277660948", size = 2536263, upload-time = "2026-04-27T14:43:29.888Z" }, + { url = "https://files.pythonhosted.org/packages/97/c9/2553f72aaf65a2797d4229e37fa7fbe38ffbf3e32912d31bdd78b3323e59/tokenizers-0.23.1-cp310-abi3-win_amd64.whl", hash = "sha256:e7bfaf995c1bdbbd21d13539decb6650967013759318627d85daeb7881af16b7", size = 2798223, upload-time = "2026-04-27T14:43:28.51Z" }, + { url = "https://files.pythonhosted.org/packages/cd/2b/2be299bab55fc595e3d38567edb1a87f86e594842968fa9515a07bdcf422/tokenizers-0.23.1-cp310-abi3-win_arm64.whl", hash = "sha256:a26197957d8e4425dfba746315f3c425ea00cfa8367c5fbc4ec73447893dcea9", size = 2664127, upload-time = "2026-04-27T14:43:26.949Z" }, +] + [[package]] name = "tomli" version = "2.4.0"