fix(python): preserve explicit token limits for Responses API

py/autoevals/litellm.py

@@ -122,6 +122,8 @@ def _responses_params_to_chat_kwargs(kwargs: dict[str, Any]) -> dict[str, Any]:
     chat_kwargs = dict(kwargs)
     if "input" in chat_kwargs and "messages" not in chat_kwargs:
         chat_kwargs["messages"] = chat_kwargs.pop("input")
+    if "max_output_tokens" in chat_kwargs:
+        chat_kwargs["max_tokens"] = chat_kwargs.pop("max_output_tokens")
     # Responses-API tools use flat {type, name, description, parameters}; Chat-
     # Completions tools nest the schema under {type, function: {...}}.
     if "tools" in chat_kwargs:

py/autoevals/oai.py

@@ -323,6 +323,8 @@ def prepare_responses_params(kwargs: dict[str, Any]) -> dict[str, Any]:
                 # Copy supported parameters
                 if "temperature" in kwargs:
                     responses_params["temperature"] = kwargs["temperature"]
+                if "max_tokens" in kwargs:
+                    responses_params["max_output_tokens"] = kwargs["max_tokens"]
                 # The Responses API nests this under reasoning.effort, unlike Chat Completions.
                 if "reasoning_effort" in kwargs:
                     responses_params["reasoning"] = {"effort": kwargs["reasoning_effort"]}

py/autoevals/test_litellm.py

@@ -4,9 +4,11 @@
 from unittest.mock import AsyncMock, MagicMock
 
 import pytest
+from litellm import ModelResponse
 
 from autoevals import init
 from autoevals.litellm import AsyncLiteLLMClient, LiteLLMClient
+from autoevals.llm import LLMClassifier
 from autoevals.oai import LLMClient
 
 
@@ -162,3 +164,58 @@ def test_init_accepts_litellm_client(mocker):
     # Calling through the wrapper should dispatch to litellm.completion
     result = wrapper.complete(model="openai/gpt-4o-mini", messages=[{"role": "user", "content": "ping"}])
     assert result.choices[0].message.content == "init-ok"
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("is_async", [False, True], ids=["sync", "async"])
+@pytest.mark.parametrize(
+    "model,use_responses_api",
+    [("gpt-5-mini", False), ("openai/gpt-4o-mini", True), ("openai/gpt-4o-mini", False)],
+    ids=["automatic-responses", "explicit-responses", "chat"],
+)
+@pytest.mark.parametrize("max_tokens", [None, 256], ids=["default", "limited"])
+async def test_llm_classifier_preserves_litellm_token_limit(mocker, is_async, model, use_responses_api, max_tokens):
+    response = ModelResponse(
+        **{
+            "choices": [
+                {
+                    "message": {
+                        "role": "assistant",
+                        "tool_calls": [
+                            {"type": "function", "function": {"name": "select_choice", "arguments": '{"choice": "1"}'}}
+                        ],
+                    }
+                }
+            ]
+        }
+    )
+    if is_async:
+        completion = mocker.patch("litellm.acompletion", new=AsyncMock(return_value=response))
+        client = AsyncLiteLLMClient(api_key="test-api-key")
+    else:
+        completion = mocker.patch("litellm.completion", return_value=response)
+        client = LiteLLMClient(api_key="test-api-key")
+    classifier = LLMClassifier(
+        "test",
+        "Test prompt: {{output}}",
+        {"1": 1, "2": 0},
+        model=model,
+        use_responses_api=use_responses_api,
+        max_tokens=max_tokens,
+        client=client,
+    )
+
+    result = await classifier.eval_async(output="test output") if is_async else classifier.eval(output="test output")
+
+    assert result.score == 1
+    completion.assert_called_once()
+    if is_async:
+        completion.assert_awaited_once()
+    kwargs = completion.call_args.kwargs
+    assert "messages" in kwargs
+    assert "input" not in kwargs
+    assert "max_output_tokens" not in kwargs
+    if max_tokens is None:
+        assert "max_tokens" not in kwargs
+    else:
+        assert kwargs["max_tokens"] == max_tokens

py/autoevals/test_llm.py

@@ -5,7 +5,7 @@
 import pytest
 import respx
 from httpx import Response
-from openai import OpenAI
+from openai import AsyncOpenAI, OpenAI
 from pydantic import BaseModel
 
 from autoevals import init
@@ -402,7 +402,7 @@ def test_battle():
 
 @respx.mock
 def test_llm_classifier_omits_optional_parameters_when_not_specified():
-    """Test that temperature is not included in API request when not specified."""
+    """Optional temperature and token limits should not be added by default."""
     captured_request_body = None
 
     def capture_request(request):
@@ -441,13 +441,14 @@ def capture_request(request):
 
     classifier.eval(output="test output", expected="test expected")
 
-    # Verify that temperature is NOT in the request (Responses API doesn't support max_tokens)
     assert "temperature" not in captured_request_body
+    assert "max_output_tokens" not in captured_request_body
+    assert "max_tokens" not in captured_request_body
 
 
 @respx.mock
 def test_llm_classifier_includes_parameters_when_specified():
-    """Test that temperature is included in API request when specified (max_tokens not supported by Responses API)."""
+    """Explicit parameters should reach the Responses API using its field names."""
     captured_request_body = None
 
     def capture_request(request):
@@ -489,14 +490,71 @@ def capture_request(request):
 
     classifier.eval(output="test output", expected="test expected")
 
-    # Verify that temperature is in the request with correct value (max_tokens not supported by Responses API)
     assert captured_request_body["temperature"] == 0.5
+    assert captured_request_body["max_output_tokens"] == 256
     assert "max_tokens" not in captured_request_body
     # The Responses API nests reasoning effort under reasoning.effort.
     assert captured_request_body["reasoning"] == {"effort": "medium"}
     assert "reasoning_effort" not in captured_request_body
 
 
+@pytest.mark.asyncio
+@pytest.mark.parametrize("is_async", [False, True], ids=["sync", "async"])
+@pytest.mark.parametrize(
+    "model,use_responses_api",
+    [("gpt-5-mini", False), ("internal-proxy-model", True)],
+    ids=["automatic", "explicit"],
+)
+@pytest.mark.parametrize("max_tokens", [None, 256], ids=["default", "limited"])
+@respx.mock
+async def test_responses_token_limit(is_async, model, use_responses_api, max_tokens):
+    route = respx.post("https://api.openai.com/v1/responses").respond(
+        json={
+            "id": "resp-test",
+            "object": "response",
+            "created": 1234567890,
+            "model": model,
+            "output": [
+                {
+                    "type": "function_call",
+                    "call_id": "call_test",
+                    "name": "select_choice",
+                    "arguments": '{"choice": "1"}',
+                }
+            ],
+        }
+    )
+    client_class = AsyncOpenAI if is_async else OpenAI
+    client = client_class(api_key="test-api-key", base_url="https://api.openai.com/v1", max_retries=0)
+    classifier = LLMClassifier(
+        "test",
+        "Test prompt: {{output}}",
+        {"1": 1, "2": 0},
+        model=model,
+        use_responses_api=use_responses_api,
+        max_tokens=max_tokens,
+        client=client,
+    )
+    try:
+        result = (
+            await classifier.eval_async(output="test output") if is_async else classifier.eval(output="test output")
+        )
+    finally:
+        if is_async:
+            await client.close()
+        else:
+            client.close()
+
+    assert result.score == 1
+    assert route.call_count == 1
+    body = json.loads(route.calls[0].request.content)
+    assert "max_tokens" not in body
+    if max_tokens is None:
+        assert "max_output_tokens" not in body
+    else:
+        assert body["max_output_tokens"] == max_tokens
+
+
 @respx.mock
 def test_llm_classifier_uses_configured_default_model():
     """Test that LLMClassifier uses the configured default model."""