diff --git a/tests/ai_guard/conftest.py b/tests/ai_guard/conftest.py index 52af05d3aba..a989123be4c 100644 --- a/tests/ai_guard/conftest.py +++ b/tests/ai_guard/conftest.py @@ -1,13 +1,14 @@ +from collections.abc import Generator +from typing import Any import pytest -def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item]) -> None: - """Mark all ai_guard tests as xfail when generating cassettes.""" - if getattr(config.option, "generate_cassettes", False): - for item in items: - item.add_marker( - pytest.mark.xfail( - reason="Generating cassettes - test assertions are not evaluated", - strict=False, - ) - ) +@pytest.hookimpl(hookwrapper=True) +def pytest_runtest_makereport(item: pytest.Item, call: pytest.CallInfo) -> Generator[None, Any, None]: + """When generating cassettes, don't let setup/assertion failures fail the run.""" + outcome = yield + if item.config.option.generate_cassettes and call.when in ("setup", "call") and call.excinfo is not None: + report = outcome.get_result() + report.outcome = "skipped" + current_filename, lineno, _ = item.location + report.longrepr = (current_filename, lineno + 1, "Generating cassettes - test assertions are not evaluated") diff --git a/tests/integration_frameworks/conftest.py b/tests/integration_frameworks/conftest.py index a45e4f800b0..d9abd3873f5 100644 --- a/tests/integration_frameworks/conftest.py +++ b/tests/integration_frameworks/conftest.py @@ -1,5 +1,5 @@ from collections.abc import Generator - +from typing import Any import pytest from utils.docker_fixtures import ( @@ -10,16 +10,15 @@ from utils import context, scenarios, logger -def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item]) -> None: - """Mark all integration_frameworks tests as xfail when generating cassettes.""" - if config.option.generate_cassettes: - for item in items: - item.add_marker( - pytest.mark.xfail( - reason="Generating cassettes - test assertions are not evaluated", - strict=False, - ) - ) +@pytest.hookimpl(hookwrapper=True) +def pytest_runtest_makereport(item: pytest.Item, call: pytest.CallInfo) -> Generator[None, Any, None]: + """When generating cassettes, don't let setup/assertion failures fail the run.""" + outcome = yield + if item.config.option.generate_cassettes and call.when in ("setup", "call") and call.excinfo is not None: + report = outcome.get_result() + report.outcome = "skipped" + current_filename, lineno, _ = item.location + report.longrepr = (current_filename, lineno + 1, "Generating cassettes - test assertions are not evaluated") @pytest.fixture diff --git a/tests/parametric/test_llm_observability/conftest.py b/tests/parametric/test_llm_observability/conftest.py index 4a0e1d1b4fe..4e5dc727481 100644 --- a/tests/parametric/test_llm_observability/conftest.py +++ b/tests/parametric/test_llm_observability/conftest.py @@ -1,16 +1,17 @@ +from collections.abc import Generator +from typing import Any import pytest -def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item]) -> None: - """Mark all llm_observability tests as xfail when generating cassettes.""" - if config.option.generate_cassettes: - for item in items: - item.add_marker( - pytest.mark.xfail( - reason="Generating cassettes - test assertions are not evaluated", - strict=False, - ) - ) +@pytest.hookimpl(hookwrapper=True) +def pytest_runtest_makereport(item: pytest.Item, call: pytest.CallInfo) -> Generator[None, Any, None]: + """When generating cassettes, don't let setup/assertion failures fail the run.""" + outcome = yield + if item.config.option.generate_cassettes and call.when in ("setup", "call") and call.excinfo is not None: + report = outcome.get_result() + report.outcome = "skipped" + current_filename, lineno, _ = item.location + report.longrepr = (current_filename, lineno + 1, "Generating cassettes - test assertions are not evaluated") @pytest.fixture