diff --git a/py/autoevals/litellm.py b/py/autoevals/litellm.py index fb8e74e..fba9e11 100644 --- a/py/autoevals/litellm.py +++ b/py/autoevals/litellm.py @@ -122,6 +122,8 @@ def _responses_params_to_chat_kwargs(kwargs: dict[str, Any]) -> dict[str, Any]: chat_kwargs = dict(kwargs) if "input" in chat_kwargs and "messages" not in chat_kwargs: chat_kwargs["messages"] = chat_kwargs.pop("input") + if "max_output_tokens" in chat_kwargs: + chat_kwargs["max_tokens"] = chat_kwargs.pop("max_output_tokens") # Responses-API tools use flat {type, name, description, parameters}; Chat- # Completions tools nest the schema under {type, function: {...}}. if "tools" in chat_kwargs: diff --git a/py/autoevals/oai.py b/py/autoevals/oai.py index 7a20e29..9fd9ba9 100644 --- a/py/autoevals/oai.py +++ b/py/autoevals/oai.py @@ -323,6 +323,8 @@ def prepare_responses_params(kwargs: dict[str, Any]) -> dict[str, Any]: # Copy supported parameters if "temperature" in kwargs: responses_params["temperature"] = kwargs["temperature"] + if "max_tokens" in kwargs: + responses_params["max_output_tokens"] = kwargs["max_tokens"] # The Responses API nests this under reasoning.effort, unlike Chat Completions. if "reasoning_effort" in kwargs: responses_params["reasoning"] = {"effort": kwargs["reasoning_effort"]} diff --git a/py/autoevals/test_litellm.py b/py/autoevals/test_litellm.py index 0327091..32d6ba6 100644 --- a/py/autoevals/test_litellm.py +++ b/py/autoevals/test_litellm.py @@ -4,9 +4,11 @@ from unittest.mock import AsyncMock, MagicMock import pytest +from litellm import ModelResponse from autoevals import init from autoevals.litellm import AsyncLiteLLMClient, LiteLLMClient +from autoevals.llm import LLMClassifier from autoevals.oai import LLMClient @@ -162,3 +164,58 @@ def test_init_accepts_litellm_client(mocker): # Calling through the wrapper should dispatch to litellm.completion result = wrapper.complete(model="openai/gpt-4o-mini", messages=[{"role": "user", "content": "ping"}]) assert result.choices[0].message.content == "init-ok" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("is_async", [False, True], ids=["sync", "async"]) +@pytest.mark.parametrize( + "model,use_responses_api", + [("gpt-5-mini", False), ("openai/gpt-4o-mini", True), ("openai/gpt-4o-mini", False)], + ids=["automatic-responses", "explicit-responses", "chat"], +) +@pytest.mark.parametrize("max_tokens", [None, 256], ids=["default", "limited"]) +async def test_llm_classifier_preserves_litellm_token_limit(mocker, is_async, model, use_responses_api, max_tokens): + response = ModelResponse( + **{ + "choices": [ + { + "message": { + "role": "assistant", + "tool_calls": [ + {"type": "function", "function": {"name": "select_choice", "arguments": '{"choice": "1"}'}} + ], + } + } + ] + } + ) + if is_async: + completion = mocker.patch("litellm.acompletion", new=AsyncMock(return_value=response)) + client = AsyncLiteLLMClient(api_key="test-api-key") + else: + completion = mocker.patch("litellm.completion", return_value=response) + client = LiteLLMClient(api_key="test-api-key") + classifier = LLMClassifier( + "test", + "Test prompt: {{output}}", + {"1": 1, "2": 0}, + model=model, + use_responses_api=use_responses_api, + max_tokens=max_tokens, + client=client, + ) + + result = await classifier.eval_async(output="test output") if is_async else classifier.eval(output="test output") + + assert result.score == 1 + completion.assert_called_once() + if is_async: + completion.assert_awaited_once() + kwargs = completion.call_args.kwargs + assert "messages" in kwargs + assert "input" not in kwargs + assert "max_output_tokens" not in kwargs + if max_tokens is None: + assert "max_tokens" not in kwargs + else: + assert kwargs["max_tokens"] == max_tokens diff --git a/py/autoevals/test_llm.py b/py/autoevals/test_llm.py index 8e987ef..659ff21 100644 --- a/py/autoevals/test_llm.py +++ b/py/autoevals/test_llm.py @@ -5,7 +5,7 @@ import pytest import respx from httpx import Response -from openai import OpenAI +from openai import AsyncOpenAI, OpenAI from pydantic import BaseModel from autoevals import init @@ -402,7 +402,7 @@ def test_battle(): @respx.mock def test_llm_classifier_omits_optional_parameters_when_not_specified(): - """Test that temperature is not included in API request when not specified.""" + """Optional temperature and token limits should not be added by default.""" captured_request_body = None def capture_request(request): @@ -441,13 +441,14 @@ def capture_request(request): classifier.eval(output="test output", expected="test expected") - # Verify that temperature is NOT in the request (Responses API doesn't support max_tokens) assert "temperature" not in captured_request_body + assert "max_output_tokens" not in captured_request_body + assert "max_tokens" not in captured_request_body @respx.mock def test_llm_classifier_includes_parameters_when_specified(): - """Test that temperature is included in API request when specified (max_tokens not supported by Responses API).""" + """Explicit parameters should reach the Responses API using its field names.""" captured_request_body = None def capture_request(request): @@ -489,14 +490,71 @@ def capture_request(request): classifier.eval(output="test output", expected="test expected") - # Verify that temperature is in the request with correct value (max_tokens not supported by Responses API) assert captured_request_body["temperature"] == 0.5 + assert captured_request_body["max_output_tokens"] == 256 assert "max_tokens" not in captured_request_body # The Responses API nests reasoning effort under reasoning.effort. assert captured_request_body["reasoning"] == {"effort": "medium"} assert "reasoning_effort" not in captured_request_body +@pytest.mark.asyncio +@pytest.mark.parametrize("is_async", [False, True], ids=["sync", "async"]) +@pytest.mark.parametrize( + "model,use_responses_api", + [("gpt-5-mini", False), ("internal-proxy-model", True)], + ids=["automatic", "explicit"], +) +@pytest.mark.parametrize("max_tokens", [None, 256], ids=["default", "limited"]) +@respx.mock +async def test_responses_token_limit(is_async, model, use_responses_api, max_tokens): + route = respx.post("https://api.openai.com/v1/responses").respond( + json={ + "id": "resp-test", + "object": "response", + "created": 1234567890, + "model": model, + "output": [ + { + "type": "function_call", + "call_id": "call_test", + "name": "select_choice", + "arguments": '{"choice": "1"}', + } + ], + } + ) + client_class = AsyncOpenAI if is_async else OpenAI + client = client_class(api_key="test-api-key", base_url="https://api.openai.com/v1", max_retries=0) + classifier = LLMClassifier( + "test", + "Test prompt: {{output}}", + {"1": 1, "2": 0}, + model=model, + use_responses_api=use_responses_api, + max_tokens=max_tokens, + client=client, + ) + try: + result = ( + await classifier.eval_async(output="test output") if is_async else classifier.eval(output="test output") + ) + finally: + if is_async: + await client.close() + else: + client.close() + + assert result.score == 1 + assert route.call_count == 1 + body = json.loads(route.calls[0].request.content) + assert "max_tokens" not in body + if max_tokens is None: + assert "max_output_tokens" not in body + else: + assert body["max_output_tokens"] == max_tokens + + @respx.mock def test_llm_classifier_uses_configured_default_model(): """Test that LLMClassifier uses the configured default model."""