Prevent the usage of xfail in tests files

tests/ai_guard/conftest.py

@@ -1,13 +1,14 @@
+from collections.abc import Generator
+from typing import Any
 import pytest
 
 
-def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item]) -> None:
-    """Mark all ai_guard tests as xfail when generating cassettes."""
-    if getattr(config.option, "generate_cassettes", False):
-        for item in items:
-            item.add_marker(
-                pytest.mark.xfail(
-                    reason="Generating cassettes - test assertions are not evaluated",
-                    strict=False,
-                )
-            )
+@pytest.hookimpl(hookwrapper=True)
+def pytest_runtest_makereport(item: pytest.Item, call: pytest.CallInfo) -> Generator[None, Any, None]:
+    """When generating cassettes, don't let setup/assertion failures fail the run."""
+    outcome = yield
+    if item.config.option.generate_cassettes and call.when in ("setup", "call") and call.excinfo is not None:
+        report = outcome.get_result()
+        report.outcome = "skipped"
+        current_filename, lineno, _ = item.location
+        report.longrepr = (current_filename, lineno + 1, "Generating cassettes - test assertions are not evaluated")

tests/integration_frameworks/conftest.py

@@ -1,5 +1,5 @@
 from collections.abc import Generator
-
+from typing import Any
 import pytest
 
 from utils.docker_fixtures import (
@@ -10,16 +10,15 @@
 from utils import context, scenarios, logger
 
 
-def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item]) -> None:
-    """Mark all integration_frameworks tests as xfail when generating cassettes."""
-    if config.option.generate_cassettes:
-        for item in items:
-            item.add_marker(
-                pytest.mark.xfail(
-                    reason="Generating cassettes - test assertions are not evaluated",
-                    strict=False,
-                )
-            )
+@pytest.hookimpl(hookwrapper=True)
+def pytest_runtest_makereport(item: pytest.Item, call: pytest.CallInfo) -> Generator[None, Any, None]:
+    """When generating cassettes, don't let setup/assertion failures fail the run."""
+    outcome = yield
+    if item.config.option.generate_cassettes and call.when in ("setup", "call") and call.excinfo is not None:
+        report = outcome.get_result()
+        report.outcome = "skipped"
+        current_filename, lineno, _ = item.location
+        report.longrepr = (current_filename, lineno + 1, "Generating cassettes - test assertions are not evaluated")
 
 
 @pytest.fixture

tests/parametric/test_llm_observability/conftest.py

@@ -1,16 +1,17 @@
+from collections.abc import Generator
+from typing import Any
 import pytest
 
 
-def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item]) -> None:
-    """Mark all llm_observability tests as xfail when generating cassettes."""
-    if config.option.generate_cassettes:
-        for item in items:
-            item.add_marker(
-                pytest.mark.xfail(
-                    reason="Generating cassettes - test assertions are not evaluated",
-                    strict=False,
-                )
-            )
+@pytest.hookimpl(hookwrapper=True)
+def pytest_runtest_makereport(item: pytest.Item, call: pytest.CallInfo) -> Generator[None, Any, None]:
+    """When generating cassettes, don't let setup/assertion failures fail the run."""
+    outcome = yield
+    if item.config.option.generate_cassettes and call.when in ("setup", "call") and call.excinfo is not None:
+        report = outcome.get_result()
+        report.outcome = "skipped"
+        current_filename, lineno, _ = item.location
+        report.longrepr = (current_filename, lineno + 1, "Generating cassettes - test assertions are not evaluated")
 
 
 @pytest.fixture