diff --git a/lib/crewai/src/crewai/llm.py b/lib/crewai/src/crewai/llm.py index b0b5cd3a19..2886fb7454 100644 --- a/lib/crewai/src/crewai/llm.py +++ b/lib/crewai/src/crewai/llm.py @@ -165,6 +165,42 @@ def _ensure_litellm() -> bool: MAX_CONTEXT: Final[int] = 2097152 # Current max from gemini-1.5-pro ANTHROPIC_PREFIXES: Final[tuple[str, str, str]] = ("anthropic/", "claude-", "claude/") +# Static provider capability lists. These are checked *before* LiteLLM +# introspection so that well-known providers always receive correct capability +# detection even when LiteLLM's model-mapping table is unavailable or raises. +RESPONSE_FORMAT_SUPPORTED_PROVIDERS: Final[frozenset[str]] = frozenset( + { + "openai", + "azure", + "azure_openai", + "google", + "gemini", + "anthropic", + "bedrock", + "deepseek", + "openrouter", + "mistral", + "groq", + "fireworks_ai", + "together_ai", + "anyscale", + "cerebras", + "dashscope", + } +) + +REASONING_EFFORT_SUPPORTED_PROVIDERS: Final[frozenset[str]] = frozenset( + { + "openai", + "azure", + "azure_openai", + "anthropic", + "bedrock", + "deepseek", + "openrouter", + } +) + LLM_CONTEXT_WINDOW_SIZES: Final[dict[str, int]] = { "gpt-4": 8192, "gpt-4o": 128000, @@ -2373,27 +2409,59 @@ def _get_custom_llm_provider(self) -> str | None: return None def _validate_call_params(self) -> None: - """ - Validate parameters before making a call. Currently this only checks if - a response_format is provided and whether the model supports it. - The custom_llm_provider is dynamically determined from the model: - - E.g., "openrouter/deepseek/deepseek-chat" yields "openrouter" - - "gemini/gemini-1.5-pro" yields "gemini" - - If no slash is present, "openai" is assumed. + """Validate call parameters before executing a completion request. + + Performs two kinds of checks in order: + + 1. **Static capability checks** (always run): ``response_format`` and + ``reasoning_effort`` are tested against + :data:`RESPONSE_FORMAT_SUPPORTED_PROVIDERS` and + :data:`REASONING_EFFORT_SUPPORTED_PROVIDERS` respectively. These + module-level frozensets encode well-known provider support and run + unconditionally — regardless of whether LiteLLM is available. + + 2. **LiteLLM introspection** (best-effort): When the provider is *not* + in the static lists, ``supports_response_schema`` is called for + additional coverage. If introspection raises (e.g. an unmapped + custom model ID or proxy endpoint), the failure is logged at DEBUG + level and validation is skipped — permitting the request to proceed. Note: This validation only applies to the litellm fallback path. - Native providers have their own validation. + Native providers perform their own parameter validation. """ + provider = self._get_custom_llm_provider() or "openai" + + # --- Static reasoning_effort check (always runs) --- + if self.reasoning_effort is not None: + if provider not in REASONING_EFFORT_SUPPORTED_PROVIDERS: + logger.debug( + f"Provider '{provider}' does not appear in the static reasoning_effort " + "support list; the parameter will be forwarded and may be ignored." + ) + + # --- response_format checks --- + if self.response_format is None: + return + + # 1. Static check: known-supported providers skip LiteLLM introspection. + if provider in RESPONSE_FORMAT_SUPPORTED_PROVIDERS: + return + + # 2. LiteLLM introspection for providers not in the static list. if not _ensure_litellm() or supports_response_schema is None: - # When litellm is not available, skip validation - # (this path should only be reached for litellm fallback models) + # LiteLLM unavailable; static list was already consulted — allow. return - provider = self._get_custom_llm_provider() - if self.response_format is not None and not supports_response_schema( - model=self.model, - custom_llm_provider=provider, - ): + try: + is_supported = supports_response_schema( + model=self.model, + custom_llm_provider=provider, + ) + except Exception as e: + logger.debug(f"LiteLLM supports_response_schema check failed: {e!s}") + is_supported = True + + if not is_supported: raise ValueError( f"The model {self.model} does not support response_format for provider '{provider}'. " "Please remove response_format or use a supported model." diff --git a/lib/crewai/tests/test_llm.py b/lib/crewai/tests/test_llm.py index 1c98d751ef..11eaedfde6 100644 --- a/lib/crewai/tests/test_llm.py +++ b/lib/crewai/tests/test_llm.py @@ -233,10 +233,12 @@ def test_validate_call_params_not_supported(): class DummyResponse(BaseModel): a: int - # Patch supports_response_schema to simulate an unsupported model. + # Use an unknown provider (not in RESPONSE_FORMAT_SUPPORTED_PROVIDERS) so the + # LiteLLM introspection path is exercised and supports_response_schema can + # report False, which should raise ValueError. with patch("crewai.llm.supports_response_schema", return_value=False): llm = LLM( - model="gemini/gemini-1.5-pro", + model="unknown-llm-provider/some-model", response_format=DummyResponse, is_litellm=True, ) @@ -1169,6 +1171,7 @@ async def test_non_streaming_async_returns_tool_calls_when_text_also_present(): response = _build_response_with_text_and_tool_calls() async def _ret(*args, **kwargs): + """Return the pre-built mock response.""" return response with patch("crewai.llm.litellm.acompletion", side_effect=_ret): @@ -1177,3 +1180,80 @@ async def _ret(*args, **kwargs): assert isinstance(result, list) assert len(result) == 1 assert result[0].function.name == "search" + + +def test_validate_call_params_handles_introspection_error(caplog): + """Verify _validate_call_params gracefully handles LiteLLM introspection errors without crashing. + + When the provider is not in the static list, introspection is attempted. + An exception during introspection must be caught and treated as supported + so the request can proceed with custom/proxy models. The failure is + logged at DEBUG level so developers can trace the fallback. + """ + import logging + + llm = LLM(model="custom/unsupported-model", is_litellm=True, response_format={"type": "json_object"}) + + with caplog.at_level(logging.DEBUG, logger="crewai.llm"): + with patch("crewai.llm.supports_response_schema", side_effect=Exception("Introspection error")): + # Should not raise exception -- permissive fallback applies + llm._validate_call_params() + + assert any("LiteLLM supports_response_schema check failed" in r.message for r in caplog.records) + + +def test_validate_call_params_raises_for_confirmed_unsupported_model(): + """Verify _validate_call_params still raises ValueError when introspection confirms no support. + + This ensures the validation path is not silently skipped -- only introspection + errors are swallowed; a clear False return still produces a ValueError. + """ + llm = LLM(model="my-provider/my-model", is_litellm=True, response_format={"type": "json_object"}) + + with patch("crewai.llm.supports_response_schema", return_value=False): + with pytest.raises(ValueError, match="does not support response_format"): + llm._validate_call_params() + + +def test_validate_call_params_skips_check_when_no_response_format(): + """Verify _validate_call_params is a no-op for params that require no validation. + + When no response_format is set, the method must return without ever + calling the LiteLLM introspection function. + """ + llm = LLM(model="gpt-4o", is_litellm=True, temperature=0.5) + + with patch("crewai.llm.supports_response_schema", side_effect=Exception("Should not be called")): + # No exception -- response_format is None so introspection is never invoked + llm._validate_call_params() + + +def test_validate_call_params_reasoning_effort_supported_provider(): + """Verify _validate_call_params passes without error for reasoning_effort on a supported provider. + + openai is in REASONING_EFFORT_SUPPORTED_PROVIDERS, so no warning or + error should be raised and introspection must not be triggered. + """ + llm = LLM(model="openai/o3-mini", is_litellm=True, reasoning_effort="medium") + + with patch("crewai.llm.supports_response_schema", side_effect=Exception("Should not be called")): + # No response_format set -- introspection is never invoked + llm._validate_call_params() + + +def test_validate_call_params_reasoning_effort_unknown_provider_logs_debug(caplog): + """Verify _validate_call_params logs a debug message for unknown providers with reasoning_effort. + + Unknown providers are not rejected outright: parameters are forwarded and + may be silently ignored by the underlying provider. The debug log signals + this to developers without breaking the call. + """ + import logging + + llm = LLM(model="unknown-provider/some-model", is_litellm=True, reasoning_effort="high") + + with caplog.at_level(logging.DEBUG, logger="crewai.llm"): + llm._validate_call_params() + + assert any("reasoning_effort" in record.message for record in caplog.records) +