From d66129f973c8bf7c80d2c20d9c552c165d003207 Mon Sep 17 00:00:00 2001 From: Charles de Beauchesne Date: Wed, 19 Aug 2026 10:44:49 +0200 Subject: [PATCH 1/2] Prevent the usage of xfail in tests files --- tests/ai_guard/conftest.py | 21 ++++++++++--------- tests/integration_frameworks/conftest.py | 21 +++++++++---------- .../test_llm_observability/conftest.py | 19 ++++++++--------- 3 files changed, 30 insertions(+), 31 deletions(-) diff --git a/tests/ai_guard/conftest.py b/tests/ai_guard/conftest.py index 52af05d3aba..e751036f78b 100644 --- a/tests/ai_guard/conftest.py +++ b/tests/ai_guard/conftest.py @@ -1,13 +1,14 @@ +from collections.abc import Generator +from typing import Any import pytest -def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item]) -> None: - """Mark all ai_guard tests as xfail when generating cassettes.""" - if getattr(config.option, "generate_cassettes", False): - for item in items: - item.add_marker( - pytest.mark.xfail( - reason="Generating cassettes - test assertions are not evaluated", - strict=False, - ) - ) +@pytest.hookimpl(hookwrapper=True) +def pytest_runtest_makereport(item: pytest.Item, call: pytest.CallInfo) -> Generator[None, Any, None]: + """When generating cassettes, don't let setup/assertion failures fail the run.""" + outcome = yield + if item.config.option.generate_cassettes and call.when in ("setup", "call") and call.excinfo is not None: + report = outcome.get_result() + report.outcome = "skipped" + current_filename, lineno, _ = item.location + report.longrepr = (current_filename, lineno, "Generating cassettes - test assertions are not evaluated") diff --git a/tests/integration_frameworks/conftest.py b/tests/integration_frameworks/conftest.py index a45e4f800b0..241c5b2baeb 100644 --- a/tests/integration_frameworks/conftest.py +++ b/tests/integration_frameworks/conftest.py @@ -1,5 +1,5 @@ from collections.abc import Generator - +from typing import Any import pytest from utils.docker_fixtures import ( @@ -10,16 +10,15 @@ from utils import context, scenarios, logger -def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item]) -> None: - """Mark all integration_frameworks tests as xfail when generating cassettes.""" - if config.option.generate_cassettes: - for item in items: - item.add_marker( - pytest.mark.xfail( - reason="Generating cassettes - test assertions are not evaluated", - strict=False, - ) - ) +@pytest.hookimpl(hookwrapper=True) +def pytest_runtest_makereport(item: pytest.Item, call: pytest.CallInfo) -> Generator[None, Any, None]: + """When generating cassettes, don't let setup/assertion failures fail the run.""" + outcome = yield + if item.config.option.generate_cassettes and call.when in ("setup", "call") and call.excinfo is not None: + report = outcome.get_result() + report.outcome = "skipped" + current_filename, lineno, _ = item.location + report.longrepr = (current_filename, lineno, "Generating cassettes - test assertions are not evaluated") @pytest.fixture diff --git a/tests/parametric/test_llm_observability/conftest.py b/tests/parametric/test_llm_observability/conftest.py index 4a0e1d1b4fe..bac29d345d9 100644 --- a/tests/parametric/test_llm_observability/conftest.py +++ b/tests/parametric/test_llm_observability/conftest.py @@ -1,16 +1,15 @@ import pytest -def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item]) -> None: - """Mark all llm_observability tests as xfail when generating cassettes.""" - if config.option.generate_cassettes: - for item in items: - item.add_marker( - pytest.mark.xfail( - reason="Generating cassettes - test assertions are not evaluated", - strict=False, - ) - ) +@pytest.hookimpl(hookwrapper=True) +def pytest_runtest_makereport(item: pytest.Item, call: pytest.CallInfo): + """When generating cassettes, don't let setup/assertion failures fail the run.""" + outcome = yield + if item.config.option.generate_cassettes and call.when in ("setup", "call") and call.excinfo is not None: + report = outcome.get_result() + report.outcome = "skipped" + current_filename, lineno, _ = item.location + report.longrepr = (current_filename, lineno, "Generating cassettes - test assertions are not evaluated") @pytest.fixture From 05d3d8e109834b5b4dce644f40e1081dd6fc45cb Mon Sep 17 00:00:00 2001 From: Charles de Beauchesne Date: Wed, 19 Aug 2026 11:05:11 +0200 Subject: [PATCH 2/2] Address comments --- tests/ai_guard/conftest.py | 2 +- tests/integration_frameworks/conftest.py | 2 +- tests/parametric/test_llm_observability/conftest.py | 6 ++++-- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/ai_guard/conftest.py b/tests/ai_guard/conftest.py index e751036f78b..a989123be4c 100644 --- a/tests/ai_guard/conftest.py +++ b/tests/ai_guard/conftest.py @@ -11,4 +11,4 @@ def pytest_runtest_makereport(item: pytest.Item, call: pytest.CallInfo) -> Gener report = outcome.get_result() report.outcome = "skipped" current_filename, lineno, _ = item.location - report.longrepr = (current_filename, lineno, "Generating cassettes - test assertions are not evaluated") + report.longrepr = (current_filename, lineno + 1, "Generating cassettes - test assertions are not evaluated") diff --git a/tests/integration_frameworks/conftest.py b/tests/integration_frameworks/conftest.py index 241c5b2baeb..d9abd3873f5 100644 --- a/tests/integration_frameworks/conftest.py +++ b/tests/integration_frameworks/conftest.py @@ -18,7 +18,7 @@ def pytest_runtest_makereport(item: pytest.Item, call: pytest.CallInfo) -> Gener report = outcome.get_result() report.outcome = "skipped" current_filename, lineno, _ = item.location - report.longrepr = (current_filename, lineno, "Generating cassettes - test assertions are not evaluated") + report.longrepr = (current_filename, lineno + 1, "Generating cassettes - test assertions are not evaluated") @pytest.fixture diff --git a/tests/parametric/test_llm_observability/conftest.py b/tests/parametric/test_llm_observability/conftest.py index bac29d345d9..4e5dc727481 100644 --- a/tests/parametric/test_llm_observability/conftest.py +++ b/tests/parametric/test_llm_observability/conftest.py @@ -1,15 +1,17 @@ +from collections.abc import Generator +from typing import Any import pytest @pytest.hookimpl(hookwrapper=True) -def pytest_runtest_makereport(item: pytest.Item, call: pytest.CallInfo): +def pytest_runtest_makereport(item: pytest.Item, call: pytest.CallInfo) -> Generator[None, Any, None]: """When generating cassettes, don't let setup/assertion failures fail the run.""" outcome = yield if item.config.option.generate_cassettes and call.when in ("setup", "call") and call.excinfo is not None: report = outcome.get_result() report.outcome = "skipped" current_filename, lineno, _ = item.location - report.longrepr = (current_filename, lineno, "Generating cassettes - test assertions are not evaluated") + report.longrepr = (current_filename, lineno + 1, "Generating cassettes - test assertions are not evaluated") @pytest.fixture