From 54c74d3fa1eef8c0579d2e1d7d5ce2e3e247c5ee Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Tue, 28 Jul 2026 18:37:27 +0200 Subject: [PATCH 01/14] Creating pytest facade --- tests/ai_guard/conftest.py | 2 +- tests/appsec/waf/test_addresses.py | 2 +- tests/integration_frameworks/conftest.py | 2 +- .../llm/anthropic/test_anthropic_apm.py | 2 +- .../llm/anthropic/test_anthropic_llmobs.py | 2 +- .../llm/google_genai/test_google_genai_apm.py | 2 +- .../google_genai/test_google_genai_llmobs.py | 2 +- .../llm/openai/test_openai_ai_guard.py | 2 +- .../llm/openai/test_openai_apm.py | 2 +- .../llm/openai/test_openai_llmobs.py | 2 +- tests/parametric/conftest.py | 2 +- tests/parametric/test_128_bit_traceids.py | 2 +- tests/parametric/test_config_consistency.py | 2 +- tests/parametric/test_crashtracking.py | 2 +- .../parametric/test_dynamic_configuration.py | 2 +- tests/parametric/test_extract_behavior.py | 2 +- .../test_ffe/test_configuration_sources.py | 2 +- .../test_ffe/test_dynamic_evaluation.py | 2 +- .../test_ffe/test_span_enrichment.py | 2 +- tests/parametric/test_headers_b3.py | 2 +- tests/parametric/test_headers_b3multi.py | 2 +- tests/parametric/test_headers_baggage.py | 2 +- tests/parametric/test_headers_none.py | 2 +- tests/parametric/test_headers_opm.py | 2 +- tests/parametric/test_headers_precedence.py | 2 +- tests/parametric/test_headers_tracecontext.py | 2 +- .../parametric/test_headers_tracestate_dd.py | 2 +- tests/parametric/test_library_tracestats.py | 2 +- .../test_llm_observability/conftest.py | 2 +- .../test_llm_observability.py | 2 +- .../test_llm_observability_dne.py | 2 +- .../test_llm_observability/utils.py | 2 +- .../test_otel_api_interoperability.py | 2 +- tests/parametric/test_otel_env_vars.py | 2 +- tests/parametric/test_otel_logs.py | 2 +- tests/parametric/test_otel_metrics.py | 2 +- tests/parametric/test_otel_span_methods.py | 2 +- .../parametric/test_otel_span_with_baggage.py | 2 +- tests/parametric/test_otel_tracer.py | 2 +- tests/parametric/test_otlp_trace_metrics.py | 2 +- tests/parametric/test_parametric_endpoints.py | 2 +- tests/parametric/test_partial_flushing.py | 2 +- tests/parametric/test_process_discovery.py | 2 +- tests/parametric/test_sampling_delegation.py | 2 +- tests/parametric/test_sampling_manual.py | 2 +- tests/parametric/test_sampling_span_tags.py | 2 +- tests/parametric/test_span_events.py | 2 +- tests/parametric/test_span_links.py | 2 +- tests/parametric/test_span_sampling.py | 2 +- tests/parametric/test_startup_logs.py | 2 +- tests/parametric/test_telemetry.py | 2 +- tests/parametric/test_trace_filters.py | 2 +- tests/parametric/test_trace_sampling.py | 2 +- tests/parametric/test_tracer.py | 2 +- tests/parametric/test_tracer_flare.py | 2 +- tests/parametric/utils.py | 2 +- tests/stats/test_stats.py | 2 +- tests/test_library_conf.py | 2 +- tests/test_the_test/scenarios.json | 6 ++ tests/test_the_test/test_build_pipeline.py | 2 +- .../test_compute_libraries_and_scenarios.py | 2 +- tests/test_the_test/test_decorators.py | 13 ++++- tests/test_the_test/test_deserializer.py | 2 +- tests/test_the_test/test_docker_scenario.py | 2 +- tests/test_the_test/test_easy_win.py | 2 +- .../test_external_gitlab_pipeline.py | 2 +- tests/test_the_test/test_features.py | 2 +- tests/test_the_test/test_force_option.py | 2 +- tests/test_the_test/test_json_report.py | 2 +- tests/test_the_test/test_junit.py | 2 +- tests/test_the_test/test_manifest.py | 2 +- .../test_minimal_number_of_scenarios.py | 2 +- .../test_mock_ffe_agentless_backend.py | 2 +- tests/test_the_test/test_scrubber.py | 2 +- .../test_slow_and_crash_decorators.py | 36 ------------ .../test_the_test/test_telemetry_heartbeat.py | 2 +- tests/test_the_test/test_version.py | 2 +- utils/pytest.py | 56 +++++++++++++++++++ 78 files changed, 147 insertions(+), 112 deletions(-) create mode 100644 utils/pytest.py diff --git a/tests/ai_guard/conftest.py b/tests/ai_guard/conftest.py index 52af05d3aba..f75bcf973d5 100644 --- a/tests/ai_guard/conftest.py +++ b/tests/ai_guard/conftest.py @@ -1,4 +1,4 @@ -import pytest +from utils import pytest def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item]) -> None: diff --git a/tests/appsec/waf/test_addresses.py b/tests/appsec/waf/test_addresses.py index e77c66cae32..6be8a811eb3 100644 --- a/tests/appsec/waf/test_addresses.py +++ b/tests/appsec/waf/test_addresses.py @@ -3,7 +3,7 @@ # Copyright 2021 Datadog, Inc. import json -import pytest +from utils import pytest from utils import weblog, interfaces, rfc, scenarios, features, logger from utils.dd_types import DataDogLibrarySpan diff --git a/tests/integration_frameworks/conftest.py b/tests/integration_frameworks/conftest.py index 677647745f4..a65b0409c23 100644 --- a/tests/integration_frameworks/conftest.py +++ b/tests/integration_frameworks/conftest.py @@ -1,7 +1,7 @@ from collections.abc import Generator import uuid -import pytest +from utils import pytest from utils.docker_fixtures import ( FrameworkTestClientApi, diff --git a/tests/integration_frameworks/llm/anthropic/test_anthropic_apm.py b/tests/integration_frameworks/llm/anthropic/test_anthropic_apm.py index 50d825fa382..7557d600192 100644 --- a/tests/integration_frameworks/llm/anthropic/test_anthropic_apm.py +++ b/tests/integration_frameworks/llm/anthropic/test_anthropic_apm.py @@ -1,7 +1,7 @@ from utils import context, scenarios, features from utils.docker_fixtures import FrameworkTestClientApi, TestAgentAPI -import pytest +from utils import pytest from .utils import BaseAnthropicTest diff --git a/tests/integration_frameworks/llm/anthropic/test_anthropic_llmobs.py b/tests/integration_frameworks/llm/anthropic/test_anthropic_llmobs.py index 1bf59e370d2..77b089172d0 100644 --- a/tests/integration_frameworks/llm/anthropic/test_anthropic_llmobs.py +++ b/tests/integration_frameworks/llm/anthropic/test_anthropic_llmobs.py @@ -4,7 +4,7 @@ from .utils import BaseAnthropicTest -import pytest +from utils import pytest from unittest import mock import json diff --git a/tests/integration_frameworks/llm/google_genai/test_google_genai_apm.py b/tests/integration_frameworks/llm/google_genai/test_google_genai_apm.py index 0790ebd732d..29abba18b73 100644 --- a/tests/integration_frameworks/llm/google_genai/test_google_genai_apm.py +++ b/tests/integration_frameworks/llm/google_genai/test_google_genai_apm.py @@ -1,7 +1,7 @@ from utils import features, scenarios from utils.docker_fixtures import FrameworkTestClientApi, TestAgentAPI -import pytest +from utils import pytest from .utils import BaseGoogleGenaiTest diff --git a/tests/integration_frameworks/llm/google_genai/test_google_genai_llmobs.py b/tests/integration_frameworks/llm/google_genai/test_google_genai_llmobs.py index e297d556afa..ec3673e6b0b 100644 --- a/tests/integration_frameworks/llm/google_genai/test_google_genai_llmobs.py +++ b/tests/integration_frameworks/llm/google_genai/test_google_genai_llmobs.py @@ -3,7 +3,7 @@ from utils import features, scenarios from utils.docker_fixtures import FrameworkTestClientApi, TestAgentAPI -import pytest +from utils import pytest from unittest import mock from typing import Any diff --git a/tests/integration_frameworks/llm/openai/test_openai_ai_guard.py b/tests/integration_frameworks/llm/openai/test_openai_ai_guard.py index 4c8aba64122..2559a96f243 100644 --- a/tests/integration_frameworks/llm/openai/test_openai_ai_guard.py +++ b/tests/integration_frameworks/llm/openai/test_openai_ai_guard.py @@ -29,7 +29,7 @@ import time -import pytest +from utils import pytest import requests from utils import features, scenarios diff --git a/tests/integration_frameworks/llm/openai/test_openai_apm.py b/tests/integration_frameworks/llm/openai/test_openai_apm.py index 4b374fcbb38..8fce0d17894 100644 --- a/tests/integration_frameworks/llm/openai/test_openai_apm.py +++ b/tests/integration_frameworks/llm/openai/test_openai_apm.py @@ -1,7 +1,7 @@ from utils import features, scenarios from .utils import TOOLS, BaseOpenaiTest -import pytest +from utils import pytest from utils.docker_fixtures import FrameworkTestClientApi, TestAgentAPI diff --git a/tests/integration_frameworks/llm/openai/test_openai_llmobs.py b/tests/integration_frameworks/llm/openai/test_openai_llmobs.py index de79c2bc85b..b81a43af33c 100644 --- a/tests/integration_frameworks/llm/openai/test_openai_llmobs.py +++ b/tests/integration_frameworks/llm/openai/test_openai_llmobs.py @@ -1,7 +1,7 @@ import json from utils import features, scenarios -import pytest +from utils import pytest from unittest import mock from utils.docker_fixtures import FrameworkTestClientApi, TestAgentAPI diff --git a/tests/parametric/conftest.py b/tests/parametric/conftest.py index 0c8d37874a2..782bf238476 100644 --- a/tests/parametric/conftest.py +++ b/tests/parametric/conftest.py @@ -6,7 +6,7 @@ import subprocess import uuid -import pytest +from utils import pytest import yaml from utils import scenarios, logger diff --git a/tests/parametric/test_128_bit_traceids.py b/tests/parametric/test_128_bit_traceids.py index bdd9c6381e2..40de09b84a1 100644 --- a/tests/parametric/test_128_bit_traceids.py +++ b/tests/parametric/test_128_bit_traceids.py @@ -1,4 +1,4 @@ -import pytest +from utils import pytest from utils.docker_fixtures.spec.trace import find_first_span_in_trace_payload, find_trace, find_only_span from utils import scenarios, features diff --git a/tests/parametric/test_config_consistency.py b/tests/parametric/test_config_consistency.py index 1868052ea2a..bf6e56dc338 100644 --- a/tests/parametric/test_config_consistency.py +++ b/tests/parametric/test_config_consistency.py @@ -2,7 +2,7 @@ from urllib.parse import urlparse -import pytest +from utils import pytest import yaml from utils import ( scenarios, diff --git a/tests/parametric/test_crashtracking.py b/tests/parametric/test_crashtracking.py index 06b94ad898f..4f1569732f9 100644 --- a/tests/parametric/test_crashtracking.py +++ b/tests/parametric/test_crashtracking.py @@ -2,7 +2,7 @@ import base64 import json -import pytest +from utils import pytest from utils import features, scenarios, logger from utils.docker_fixtures import TestAgentAPI, ParametricTestClientApi as APMLibrary diff --git a/tests/parametric/test_dynamic_configuration.py b/tests/parametric/test_dynamic_configuration.py index 57be43391bd..b00777d5f24 100644 --- a/tests/parametric/test_dynamic_configuration.py +++ b/tests/parametric/test_dynamic_configuration.py @@ -6,7 +6,7 @@ from pathlib import Path from typing import Any -import pytest +from utils import pytest import yaml from utils import ( diff --git a/tests/parametric/test_extract_behavior.py b/tests/parametric/test_extract_behavior.py index 0781883b6d3..9da62dcb2b5 100644 --- a/tests/parametric/test_extract_behavior.py +++ b/tests/parametric/test_extract_behavior.py @@ -1,4 +1,4 @@ -import pytest +from utils import pytest from utils import features, scenarios from utils.docker_fixtures import TestAgentAPI diff --git a/tests/parametric/test_ffe/test_configuration_sources.py b/tests/parametric/test_ffe/test_configuration_sources.py index 195edddd1ad..618016044b3 100644 --- a/tests/parametric/test_ffe/test_configuration_sources.py +++ b/tests/parametric/test_ffe/test_configuration_sources.py @@ -18,7 +18,7 @@ import time from typing import Any -import pytest +from utils import pytest from tests.parametric.conftest import APMLibrary from tests.parametric.test_ffe.test_dynamic_evaluation import _set_and_wait_ffe_rc, _ffe_evaluate_with_rc_retry diff --git a/tests/parametric/test_ffe/test_dynamic_evaluation.py b/tests/parametric/test_ffe/test_dynamic_evaluation.py index 7d48a1e0b7c..35524d7aac0 100644 --- a/tests/parametric/test_ffe/test_dynamic_evaluation.py +++ b/tests/parametric/test_ffe/test_dynamic_evaluation.py @@ -1,7 +1,7 @@ """Test FFE (Feature Flags & Experimentation) functionality via parametric tests.""" import json -import pytest +from utils import pytest import time from pathlib import Path from typing import Any diff --git a/tests/parametric/test_ffe/test_span_enrichment.py b/tests/parametric/test_ffe/test_span_enrichment.py index 7b8168a3eb1..6a4e54bdacb 100644 --- a/tests/parametric/test_ffe/test_span_enrichment.py +++ b/tests/parametric/test_ffe/test_span_enrichment.py @@ -20,7 +20,7 @@ """ import json -import pytest +from utils import pytest from pathlib import Path from typing import Any diff --git a/tests/parametric/test_headers_b3.py b/tests/parametric/test_headers_b3.py index 334ea3d9e1b..832ab9d4f4d 100644 --- a/tests/parametric/test_headers_b3.py +++ b/tests/parametric/test_headers_b3.py @@ -1,4 +1,4 @@ -import pytest +from utils import pytest from utils.docker_fixtures.spec.trace import SAMPLING_PRIORITY_KEY, ORIGIN from utils.docker_fixtures.spec.trace import span_has_no_parent diff --git a/tests/parametric/test_headers_b3multi.py b/tests/parametric/test_headers_b3multi.py index 3a294c5cc92..c3390aba984 100644 --- a/tests/parametric/test_headers_b3multi.py +++ b/tests/parametric/test_headers_b3multi.py @@ -1,4 +1,4 @@ -import pytest +from utils import pytest from utils.docker_fixtures.spec.trace import SAMPLING_PRIORITY_KEY, ORIGIN from utils.docker_fixtures.spec.trace import span_has_no_parent diff --git a/tests/parametric/test_headers_baggage.py b/tests/parametric/test_headers_baggage.py index 6796fef5d98..0a0c912b3fc 100644 --- a/tests/parametric/test_headers_baggage.py +++ b/tests/parametric/test_headers_baggage.py @@ -2,7 +2,7 @@ from utils import features, scenarios from utils.docker_fixtures import TestAgentAPI -import pytest +from utils import pytest from .conftest import APMLibrary diff --git a/tests/parametric/test_headers_none.py b/tests/parametric/test_headers_none.py index 6968e612d5e..2b650f527d0 100644 --- a/tests/parametric/test_headers_none.py +++ b/tests/parametric/test_headers_none.py @@ -1,4 +1,4 @@ -import pytest +from utils import pytest from utils.docker_fixtures.spec.trace import SAMPLING_PRIORITY_KEY, ORIGIN from utils.docker_fixtures.spec.trace import find_only_span diff --git a/tests/parametric/test_headers_opm.py b/tests/parametric/test_headers_opm.py index bdb0d0fa24c..331485d0fd6 100644 --- a/tests/parametric/test_headers_opm.py +++ b/tests/parametric/test_headers_opm.py @@ -20,7 +20,7 @@ RFC: https://docs.google.com/document/d/1SzZWivVWT79lJe80ZulEra6AARszjEYVEwWJ7IhJ6Xo/edit?tab=t.0 """ -import pytest +from utils import pytest from utils import features, scenarios, rfc from utils.docker_fixtures import TestAgentAPI diff --git a/tests/parametric/test_headers_precedence.py b/tests/parametric/test_headers_precedence.py index 16a6c750f33..7cef84066d4 100644 --- a/tests/parametric/test_headers_precedence.py +++ b/tests/parametric/test_headers_precedence.py @@ -1,4 +1,4 @@ -import pytest +from utils import pytest from utils.docker_fixtures.spec.tracecontext import get_tracecontext from utils import scenarios, features diff --git a/tests/parametric/test_headers_tracecontext.py b/tests/parametric/test_headers_tracecontext.py index 540eef4cd18..fa827306f93 100644 --- a/tests/parametric/test_headers_tracecontext.py +++ b/tests/parametric/test_headers_tracecontext.py @@ -8,7 +8,7 @@ # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -import pytest +from utils import pytest from utils.docker_fixtures.spec.tracecontext import get_tracecontext from utils.docker_fixtures.spec.trace import find_span_in_traces, find_only_span diff --git a/tests/parametric/test_headers_tracestate_dd.py b/tests/parametric/test_headers_tracestate_dd.py index 22542689d7a..5324aa94ef7 100644 --- a/tests/parametric/test_headers_tracestate_dd.py +++ b/tests/parametric/test_headers_tracestate_dd.py @@ -1,4 +1,4 @@ -import pytest +from utils import pytest from utils.docker_fixtures.spec.tracecontext import get_tracecontext from utils import scenarios, features diff --git a/tests/parametric/test_library_tracestats.py b/tests/parametric/test_library_tracestats.py index b57375b4043..05d71b43186 100644 --- a/tests/parametric/test_library_tracestats.py +++ b/tests/parametric/test_library_tracestats.py @@ -1,7 +1,7 @@ import base64 import msgpack -import pytest +from utils import pytest from utils.docker_fixtures.spec.trace import SPAN_MEASURED_KEY diff --git a/tests/parametric/test_llm_observability/conftest.py b/tests/parametric/test_llm_observability/conftest.py index 4a0e1d1b4fe..3ccb5c1387b 100644 --- a/tests/parametric/test_llm_observability/conftest.py +++ b/tests/parametric/test_llm_observability/conftest.py @@ -1,4 +1,4 @@ -import pytest +from utils import pytest def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item]) -> None: diff --git a/tests/parametric/test_llm_observability/test_llm_observability.py b/tests/parametric/test_llm_observability/test_llm_observability.py index a253a89ce63..72757d1c260 100644 --- a/tests/parametric/test_llm_observability/test_llm_observability.py +++ b/tests/parametric/test_llm_observability/test_llm_observability.py @@ -1,4 +1,4 @@ -import pytest +from utils import pytest from utils import scenarios, features from utils.docker_fixtures import TestAgentAPI diff --git a/tests/parametric/test_llm_observability/test_llm_observability_dne.py b/tests/parametric/test_llm_observability/test_llm_observability_dne.py index f4176e02ee3..504e6c8df29 100644 --- a/tests/parametric/test_llm_observability/test_llm_observability_dne.py +++ b/tests/parametric/test_llm_observability/test_llm_observability_dne.py @@ -1,5 +1,5 @@ from typing import TYPE_CHECKING -import pytest +from utils import pytest from tests.parametric.test_llm_observability.utils import check_and_get_api_key from utils import features, scenarios diff --git a/tests/parametric/test_llm_observability/utils.py b/tests/parametric/test_llm_observability/utils.py index 648f5a6c5cb..bc174ee1292 100644 --- a/tests/parametric/test_llm_observability/utils.py +++ b/tests/parametric/test_llm_observability/utils.py @@ -1,5 +1,5 @@ import os -import pytest +from utils import pytest def check_and_get_api_key(api_key_name: str, *, generate_cassettes: bool = False) -> str | None: diff --git a/tests/parametric/test_otel_api_interoperability.py b/tests/parametric/test_otel_api_interoperability.py index 38258383c6a..a96416cb0b7 100644 --- a/tests/parametric/test_otel_api_interoperability.py +++ b/tests/parametric/test_otel_api_interoperability.py @@ -1,4 +1,4 @@ -import pytest +from utils import pytest from utils import scenarios, features from opentelemetry.trace import SpanKind diff --git a/tests/parametric/test_otel_env_vars.py b/tests/parametric/test_otel_env_vars.py index 9621540ed86..8ac7605c388 100644 --- a/tests/parametric/test_otel_env_vars.py +++ b/tests/parametric/test_otel_env_vars.py @@ -1,4 +1,4 @@ -import pytest +from utils import pytest from utils import context, scenarios, features from utils.docker_fixtures import TestAgentAPI from utils.docker_fixtures.spec.trace import find_only_span diff --git a/tests/parametric/test_otel_logs.py b/tests/parametric/test_otel_logs.py index 1dfa70eddc8..4905a752bf0 100644 --- a/tests/parametric/test_otel_logs.py +++ b/tests/parametric/test_otel_logs.py @@ -2,7 +2,7 @@ from collections.abc import Generator from urllib.parse import urlparse -import pytest +from utils import pytest from utils import scenarios, features, logger from utils.docker_fixtures.parametric import LogLevel diff --git a/tests/parametric/test_otel_metrics.py b/tests/parametric/test_otel_metrics.py index b4a075e9436..0fe32fa4eb7 100644 --- a/tests/parametric/test_otel_metrics.py +++ b/tests/parametric/test_otel_metrics.py @@ -1,5 +1,5 @@ from urllib.parse import urlparse -import pytest +from utils import pytest from utils import features, scenarios diff --git a/tests/parametric/test_otel_span_methods.py b/tests/parametric/test_otel_span_methods.py index 5837fddff8a..5676b1c0969 100644 --- a/tests/parametric/test_otel_span_methods.py +++ b/tests/parametric/test_otel_span_methods.py @@ -1,6 +1,6 @@ import time -import pytest +from utils import pytest from opentelemetry.trace import StatusCode from opentelemetry.trace import SpanKind diff --git a/tests/parametric/test_otel_span_with_baggage.py b/tests/parametric/test_otel_span_with_baggage.py index d0dda9653c6..e113505bcc6 100644 --- a/tests/parametric/test_otel_span_with_baggage.py +++ b/tests/parametric/test_otel_span_with_baggage.py @@ -1,4 +1,4 @@ -import pytest +from utils import pytest from utils import scenarios, features from .conftest import APMLibrary diff --git a/tests/parametric/test_otel_tracer.py b/tests/parametric/test_otel_tracer.py index 95c09f347f4..1f1ec7817a6 100644 --- a/tests/parametric/test_otel_tracer.py +++ b/tests/parametric/test_otel_tracer.py @@ -1,4 +1,4 @@ -import pytest +from utils import pytest from utils.docker_fixtures.spec.trace import find_trace from utils.docker_fixtures.spec.trace import find_span diff --git a/tests/parametric/test_otlp_trace_metrics.py b/tests/parametric/test_otlp_trace_metrics.py index bab1f18630e..28dadf676da 100644 --- a/tests/parametric/test_otlp_trace_metrics.py +++ b/tests/parametric/test_otlp_trace_metrics.py @@ -59,7 +59,7 @@ import time from typing import Any -import pytest +from utils import pytest from google.protobuf.json_format import MessageToDict from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ExportTraceServiceRequest diff --git a/tests/parametric/test_parametric_endpoints.py b/tests/parametric/test_parametric_endpoints.py index 50aa0b905a7..ab3656763a8 100644 --- a/tests/parametric/test_parametric_endpoints.py +++ b/tests/parametric/test_parametric_endpoints.py @@ -8,7 +8,7 @@ from typing import Any -import pytest +from utils import pytest import time from opentelemetry.trace import SpanKind diff --git a/tests/parametric/test_partial_flushing.py b/tests/parametric/test_partial_flushing.py index 8743587f934..246e7e20a61 100644 --- a/tests/parametric/test_partial_flushing.py +++ b/tests/parametric/test_partial_flushing.py @@ -1,4 +1,4 @@ -import pytest +from utils import pytest from utils.docker_fixtures.spec.trace import find_first_span_in_trace_payload, find_span, find_trace from utils import features, scenarios from utils.docker_fixtures import TestAgentAPI diff --git a/tests/parametric/test_process_discovery.py b/tests/parametric/test_process_discovery.py index b7586d79110..4bd9f0fbefd 100644 --- a/tests/parametric/test_process_discovery.py +++ b/tests/parametric/test_process_discovery.py @@ -1,6 +1,6 @@ """Test the instrumented process discovery mechanism feature.""" -import pytest +from utils import pytest import json import msgpack import re diff --git a/tests/parametric/test_sampling_delegation.py b/tests/parametric/test_sampling_delegation.py index 621324c0ced..b75fde4d5ed 100644 --- a/tests/parametric/test_sampling_delegation.py +++ b/tests/parametric/test_sampling_delegation.py @@ -6,7 +6,7 @@ [1]: https://github.com/DataDog/architecture/tree/master/rfcs/apm/integrations/sampling-delegation """ -import pytest +from utils import pytest from utils import features, rfc, scenarios from utils.docker_fixtures import TestAgentAPI from .conftest import APMLibrary diff --git a/tests/parametric/test_sampling_manual.py b/tests/parametric/test_sampling_manual.py index 4c6d879562e..077363c6203 100644 --- a/tests/parametric/test_sampling_manual.py +++ b/tests/parametric/test_sampling_manual.py @@ -6,7 +6,7 @@ Manual keep sampling should take precedence over any other sampling decision. """ -import pytest +from utils import pytest from utils import features, rfc, scenarios from utils.dd_constants import SamplingMechanism from utils.dd_constants import SamplingPriority diff --git a/tests/parametric/test_sampling_span_tags.py b/tests/parametric/test_sampling_span_tags.py index 629e06dc11f..061b71ec208 100644 --- a/tests/parametric/test_sampling_span_tags.py +++ b/tests/parametric/test_sampling_span_tags.py @@ -1,6 +1,6 @@ import json -import pytest +from utils import pytest from utils import scenarios, features from utils.docker_fixtures.spec.trace import MANUAL_DROP_KEY from utils.docker_fixtures.spec.trace import MANUAL_KEEP_KEY diff --git a/tests/parametric/test_span_events.py b/tests/parametric/test_span_events.py index 2cb0d05464d..43fbed17482 100644 --- a/tests/parametric/test_span_events.py +++ b/tests/parametric/test_span_events.py @@ -1,5 +1,5 @@ import json -import pytest +from utils import pytest from utils import scenarios, features, rfc from utils.docker_fixtures.spec.trace import find_span, find_trace diff --git a/tests/parametric/test_span_links.py b/tests/parametric/test_span_links.py index b4c7fffd248..df477885347 100644 --- a/tests/parametric/test_span_links.py +++ b/tests/parametric/test_span_links.py @@ -1,5 +1,5 @@ import json -import pytest +from utils import pytest from utils.docker_fixtures.spec.trace import ORIGIN from utils.docker_fixtures.spec.trace import SAMPLING_PRIORITY_KEY diff --git a/tests/parametric/test_span_sampling.py b/tests/parametric/test_span_sampling.py index ddad4be025e..ade592d2015 100644 --- a/tests/parametric/test_span_sampling.py +++ b/tests/parametric/test_span_sampling.py @@ -1,6 +1,6 @@ import time import json -import pytest +from utils import pytest from utils.docker_fixtures.spec.trace import SAMPLING_PRIORITY_KEY from utils.docker_fixtures.spec.trace import SINGLE_SPAN_SAMPLING_MAX_PER_SEC from utils.docker_fixtures.spec.trace import SINGLE_SPAN_SAMPLING_MECHANISM diff --git a/tests/parametric/test_startup_logs.py b/tests/parametric/test_startup_logs.py index e50399b3e73..6760fb9a904 100644 --- a/tests/parametric/test_startup_logs.py +++ b/tests/parametric/test_startup_logs.py @@ -2,7 +2,7 @@ import re -import pytest +from utils import pytest from utils import scenarios, features, context, logger from .conftest import APMLibrary diff --git a/tests/parametric/test_telemetry.py b/tests/parametric/test_telemetry.py index 410d76d8c34..5cbc7ccc0bf 100644 --- a/tests/parametric/test_telemetry.py +++ b/tests/parametric/test_telemetry.py @@ -5,7 +5,7 @@ import time import uuid -import pytest +from utils import pytest from .conftest import StableConfigWriter from utils.telemetry_utils import TelemetryUtils diff --git a/tests/parametric/test_trace_filters.py b/tests/parametric/test_trace_filters.py index 9aec46740c7..7785270b84e 100644 --- a/tests/parametric/test_trace_filters.py +++ b/tests/parametric/test_trace_filters.py @@ -1,5 +1,5 @@ import json -import pytest +from utils import pytest from .conftest import APMLibrary from .utils import MIN_AGENT_VERSION_FOR_CSS, enable_tracestats diff --git a/tests/parametric/test_trace_sampling.py b/tests/parametric/test_trace_sampling.py index 2fd68ca787c..d38925b752e 100644 --- a/tests/parametric/test_trace_sampling.py +++ b/tests/parametric/test_trace_sampling.py @@ -1,6 +1,6 @@ import json -import pytest +from utils import pytest import random from utils.docker_fixtures.spec.trace import find_only_span, find_span_in_traces diff --git a/tests/parametric/test_tracer.py b/tests/parametric/test_tracer.py index 21be477630c..fae0f54c895 100644 --- a/tests/parametric/test_tracer.py +++ b/tests/parametric/test_tracer.py @@ -1,4 +1,4 @@ -import pytest +from utils import pytest from utils.docker_fixtures.spec.trace import find_trace from utils.docker_fixtures.spec.trace import find_span diff --git a/tests/parametric/test_tracer_flare.py b/tests/parametric/test_tracer_flare.py index 0852f6d2313..c93a75b3f44 100644 --- a/tests/parametric/test_tracer_flare.py +++ b/tests/parametric/test_tracer_flare.py @@ -7,7 +7,7 @@ from typing import Any from uuid import uuid4 -import pytest +from utils import pytest from utils import rfc, scenarios, features, context from utils.dd_constants import RemoteConfigApplyState diff --git a/tests/parametric/utils.py b/tests/parametric/utils.py index 8a962dc035e..951cb919421 100644 --- a/tests/parametric/utils.py +++ b/tests/parametric/utils.py @@ -1,4 +1,4 @@ -import pytest +from utils import pytest from utils import context diff --git a/tests/stats/test_stats.py b/tests/stats/test_stats.py index 0ce6e338aef..c6ca6f3834b 100644 --- a/tests/stats/test_stats.py +++ b/tests/stats/test_stats.py @@ -1,5 +1,5 @@ import contextlib -import pytest +from utils import pytest from utils import features, interfaces, logger, scenarios, weblog diff --git a/tests/test_library_conf.py b/tests/test_library_conf.py index 00da9e1c0ee..3581614b761 100644 --- a/tests/test_library_conf.py +++ b/tests/test_library_conf.py @@ -1,7 +1,7 @@ # Unless explicitly stated otherwise all files in this repository are licensed under the the Apache License Version 2.0. # This product includes software developed at Datadog (https://www.datadoghq.com/). # Copyright 2021 Datadog, Inc. -import pytest +from utils import pytest from utils import weblog, interfaces, scenarios, features from utils.dd_types import DataDogAgentSpan from utils._context.header_tag_vars import ( diff --git a/tests/test_the_test/scenarios.json b/tests/test_the_test/scenarios.json index 5870b1e19e6..a087e14e7cd 100644 --- a/tests/test_the_test/scenarios.json +++ b/tests/test_the_test/scenarios.json @@ -5656,6 +5656,12 @@ "tests/test_the_test/test_decorators.py::Test_Skips::test_regular": [ "TEST_THE_TEST" ], + "tests/test_the_test/test_decorators.py::Test_PytestProxy::test_exposes_approved_apis": [ + "TEST_THE_TEST" + ], + "tests/test_the_test/test_decorators.py::Test_PytestProxy::test_hides_internal_force_skip_marker": [ + "TEST_THE_TEST" + ], "tests/test_the_test/test_decorators.py::test_version_range": [ "TEST_THE_TEST" ], diff --git a/tests/test_the_test/test_build_pipeline.py b/tests/test_the_test/test_build_pipeline.py index bf8571f48ab..75c02d0a510 100644 --- a/tests/test_the_test/test_build_pipeline.py +++ b/tests/test_the_test/test_build_pipeline.py @@ -4,7 +4,7 @@ import re from pathlib import Path -import pytest +from utils import pytest import yaml from utils import scenarios diff --git a/tests/test_the_test/test_compute_libraries_and_scenarios.py b/tests/test_the_test/test_compute_libraries_and_scenarios.py index 25eadecbacc..75b726d2c47 100644 --- a/tests/test_the_test/test_compute_libraries_and_scenarios.py +++ b/tests/test_the_test/test_compute_libraries_and_scenarios.py @@ -4,7 +4,7 @@ from functools import wraps -import pytest +from utils import pytest from utils.scripts.compute_libraries_and_scenarios import Inputs, process from utils import scenarios diff --git a/tests/test_the_test/test_decorators.py b/tests/test_the_test/test_decorators.py index aae54c195f4..b3bf15fa989 100644 --- a/tests/test_the_test/test_decorators.py +++ b/tests/test_the_test/test_decorators.py @@ -1,9 +1,8 @@ import sys import logging from typing import Any -import pytest -from utils import irrelevant, missing_feature, flaky, rfc, logger +from utils import irrelevant, missing_feature, flaky, pytest, rfc, logger from utils._decorators import add_pytest_marker from utils.manifest import TestDeclaration @@ -84,5 +83,15 @@ def test_invalid() -> None: add_pytest_marker(test_invalid, TestDeclaration.BUG, "APPSEC-123 & APPSEC-456") +class Test_PytestProxy: + def test_exposes_approved_apis(self): + assert callable(pytest.fixture) + assert callable(pytest.raises) + assert callable(pytest.mark.parametrize) + + def test_hides_internal_force_skip_marker(self): + assert not hasattr(pytest.mark, "skip_if_xfail") + + if __name__ == "__main__": sys.exit("Usage: pytest utils/test_the_test.py") diff --git a/tests/test_the_test/test_deserializer.py b/tests/test_the_test/test_deserializer.py index 85c3fd6c4de..74a7d097985 100644 --- a/tests/test_the_test/test_deserializer.py +++ b/tests/test_the_test/test_deserializer.py @@ -8,7 +8,7 @@ ) import base64 import msgpack -import pytest +from utils import pytest @scenarios.test_the_test diff --git a/tests/test_the_test/test_docker_scenario.py b/tests/test_the_test/test_docker_scenario.py index 51e0cebf268..84f90a55fde 100644 --- a/tests/test_the_test/test_docker_scenario.py +++ b/tests/test_the_test/test_docker_scenario.py @@ -1,7 +1,7 @@ from threading import RLock from unittest.mock import MagicMock -import pytest +from utils import pytest from utils import interfaces, scenarios from utils._context._scenarios.endtoend import DdTraceEndToEndScenario, DockerScenario diff --git a/tests/test_the_test/test_easy_win.py b/tests/test_the_test/test_easy_win.py index 45498cbb888..ccdb9bef629 100644 --- a/tests/test_the_test/test_easy_win.py +++ b/tests/test_the_test/test_easy_win.py @@ -3,7 +3,7 @@ from pathlib import Path -import pytest +from utils import pytest import yaml from utils.manifest._internal.types import Condition, SkipDeclaration, SemverRange diff --git a/tests/test_the_test/test_external_gitlab_pipeline.py b/tests/test_the_test/test_external_gitlab_pipeline.py index 6dd7fee9fed..4e5b3e1ebf9 100644 --- a/tests/test_the_test/test_external_gitlab_pipeline.py +++ b/tests/test_the_test/test_external_gitlab_pipeline.py @@ -1,6 +1,6 @@ """Tests for utils/scripts/ci_orchestrators/external_gitlab_pipeline.py.""" -import pytest +from utils import pytest from utils import scenarios from utils.scripts.ci_orchestrators.external_gitlab_pipeline import ( diff --git a/tests/test_the_test/test_features.py b/tests/test_the_test/test_features.py index 0ad561dc831..6439543b48c 100644 --- a/tests/test_the_test/test_features.py +++ b/tests/test_the_test/test_features.py @@ -1,4 +1,4 @@ -import pytest +from utils import pytest from collections.abc import Callable from utils import scenarios, features, logger from utils._features import NOT_REPORTED_ID diff --git a/tests/test_the_test/test_force_option.py b/tests/test_the_test/test_force_option.py index 42e6a1539c2..62c04318147 100644 --- a/tests/test_the_test/test_force_option.py +++ b/tests/test_the_test/test_force_option.py @@ -1,5 +1,5 @@ import os -import pytest +from utils import pytest from utils import bug, irrelevant, scenarios, features from utils._context._scenarios import Scenario diff --git a/tests/test_the_test/test_json_report.py b/tests/test_the_test/test_json_report.py index 0a83819736c..08221b5e921 100644 --- a/tests/test_the_test/test_json_report.py +++ b/tests/test_the_test/test_json_report.py @@ -1,6 +1,6 @@ import os import json -import pytest +from utils import pytest from utils import missing_feature, irrelevant, scenarios, rfc, features, bug, flaky, logger diff --git a/tests/test_the_test/test_junit.py b/tests/test_the_test/test_junit.py index 3106bfdece1..2ea8cc3084e 100644 --- a/tests/test_the_test/test_junit.py +++ b/tests/test_the_test/test_junit.py @@ -4,7 +4,7 @@ import xml.etree.ElementTree as ET from xml.dom import minidom -import pytest +from utils import pytest from utils import scenarios, features, irrelevant, bug, flaky, missing_feature, slow from .utils import run_system_tests diff --git a/tests/test_the_test/test_manifest.py b/tests/test_the_test/test_manifest.py index 1e5ba8be614..2708533e352 100644 --- a/tests/test_the_test/test_manifest.py +++ b/tests/test_the_test/test_manifest.py @@ -2,7 +2,7 @@ import shutil import tempfile import textwrap -import pytest +from utils import pytest from utils import scenarios from utils._context.component_version import Version from utils.manifest import Manifest, SkipDeclaration, TestDeclaration diff --git a/tests/test_the_test/test_minimal_number_of_scenarios.py b/tests/test_the_test/test_minimal_number_of_scenarios.py index a352cd7996c..89ade95e3b7 100644 --- a/tests/test_the_test/test_minimal_number_of_scenarios.py +++ b/tests/test_the_test/test_minimal_number_of_scenarios.py @@ -2,7 +2,7 @@ import json from typing import Any -import pytest +from utils import pytest from utils._context._scenarios import get_all_scenarios, scenarios from utils._context._scenarios.endtoend import EndToEndScenario diff --git a/tests/test_the_test/test_mock_ffe_agentless_backend.py b/tests/test_the_test/test_mock_ffe_agentless_backend.py index eefba0368aa..b0f7e6ac412 100644 --- a/tests/test_the_test/test_mock_ffe_agentless_backend.py +++ b/tests/test_the_test/test_mock_ffe_agentless_backend.py @@ -4,7 +4,7 @@ from unittest.mock import MagicMock import requests -import pytest +from utils import pytest from utils import scenarios from utils._context._scenarios import agentless_endtoend as agentless_endtoend_scenarios diff --git a/tests/test_the_test/test_scrubber.py b/tests/test_the_test/test_scrubber.py index 96915ef4fa2..650b1fd02f4 100644 --- a/tests/test_the_test/test_scrubber.py +++ b/tests/test_the_test/test_scrubber.py @@ -3,7 +3,7 @@ import os from pathlib import Path import subprocess -import pytest +from utils import pytest from utils import scenarios, missing_feature, logger diff --git a/tests/test_the_test/test_slow_and_crash_decorators.py b/tests/test_the_test/test_slow_and_crash_decorators.py index 34f6260b7aa..bb40c4fb702 100644 --- a/tests/test_the_test/test_slow_and_crash_decorators.py +++ b/tests/test_the_test/test_slow_and_crash_decorators.py @@ -2,8 +2,6 @@ import textwrap from pathlib import Path -import pytest - from utils import bug, missing_feature, scenarios, features, slow, scenario_crash from .utils import run_system_tests @@ -135,25 +133,6 @@ def test_scenario_crash_on_class_with_declaration_is_skipped(self): assert tests[nodeid]["outcome"] == "skipped" -@scenarios.test_the_test -class Test_SkipIfXfail: - """Test that tests with both skip_if_xfail and declaration markers are skipped.""" - - def test_skip_if_xfail_with_declaration_is_skipped(self): - """Test that a test marked with both skip_if_xfail and a declaration marker is skipped.""" - tests = run_system_tests(test_path=FILENAME) - - nodeid = f"{FILENAME}::Test_SkipIfXfailMock::test_with_both_markers" - assert tests[nodeid]["outcome"] == "skipped" - - def test_skip_if_xfail_without_declaration_is_not_skipped(self): - """Test that a test marked with only skip_if_xfail (no declaration) is not skipped.""" - tests = run_system_tests(test_path=FILENAME) - - nodeid = f"{FILENAME}::Test_SkipIfXfailMock::test_skip_if_xfail_only" - assert tests[nodeid]["outcome"] == "passed" - - # Mock test classes used by the test scenarios above @@ -187,21 +166,6 @@ def test_scenario_crash_with_missing_feature(self): assert True -@scenarios.mock_the_test -@features.adaptive_sampling -class Test_SkipIfXfailMock: - @bug(condition=True, reason="FAKE-001") - @pytest.mark.skip_if_xfail - def test_with_both_markers(self): - """Test with both skip_if_xfail and declaration markers - should be skipped.""" - assert True - - @pytest.mark.skip_if_xfail - def test_skip_if_xfail_only(self): - """Test with only skip_if_xfail marker, no declaration - should NOT be skipped.""" - assert True - - @scenarios.mock_the_test @features.adaptive_sampling class Test_SlowManifestMock: diff --git a/tests/test_the_test/test_telemetry_heartbeat.py b/tests/test_the_test/test_telemetry_heartbeat.py index d5f1e1b8dcc..60dfe496c34 100644 --- a/tests/test_the_test/test_telemetry_heartbeat.py +++ b/tests/test_the_test/test_telemetry_heartbeat.py @@ -1,7 +1,7 @@ from datetime import datetime, timedelta, UTC from typing import Any -import pytest +from utils import pytest from tests.test_telemetry_heartbeat_utils import heartbeat_delays_by_runtime diff --git a/tests/test_the_test/test_version.py b/tests/test_the_test/test_version.py index d46f36b38d7..bd1ac277a7e 100644 --- a/tests/test_the_test/test_version.py +++ b/tests/test_the_test/test_version.py @@ -1,4 +1,4 @@ -import pytest +from utils import pytest import semantic_version as semver from utils.manifest._internal.types import SemverRange as CustomSpec from utils._context.component_version import ComponentVersion, Version diff --git a/utils/pytest.py b/utils/pytest.py new file mode 100644 index 00000000000..53180f416f0 --- /dev/null +++ b/utils/pytest.py @@ -0,0 +1,56 @@ +"""Approved pytest API for system tests. + +Test modules should import this facade with ``from utils import pytest``. +The facade intentionally exposes only the pytest APIs used by this repository. +In particular, force-skip markers are available only through the semantic +decorators exported by :mod:`utils`. +""" + +import pytest as _pytest # noqa: PT013 - keep the underlying pytest module private + + +Config = _pytest.Config +FixtureRequest = _pytest.FixtureRequest +Item = _pytest.Item +Mark = _pytest.Mark +MarkDecorator = _pytest.MarkDecorator +MonkeyPatch = _pytest.MonkeyPatch +Session = _pytest.Session + +approx = _pytest.approx +exit = _pytest.exit # noqa: A001 - preserve the public pytest API name +fail = _pytest.fail +fixture = _pytest.fixture +param = _pytest.param +raises = _pytest.raises +skip = _pytest.skip + + +class _AllowedMarks: + """Pytest markers approved for direct use by system tests.""" + + features = _pytest.mark.features + parametrize = _pytest.mark.parametrize + scenario = _pytest.mark.scenario + xfail = _pytest.mark.xfail + + +mark = _AllowedMarks() + +__all__ = [ + "Config", + "FixtureRequest", + "Item", + "Mark", + "MarkDecorator", + "MonkeyPatch", + "Session", + "approx", + "exit", + "fail", + "fixture", + "mark", + "param", + "raises", + "skip", +] From a8bdb28a23bb16b6b2c61437bee8976d3524a133 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Wed, 29 Jul 2026 10:01:14 +0200 Subject: [PATCH 02/14] Fixes --- format.sh | 2 +- utils/scripts/shellcheck.sh | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/format.sh b/format.sh index a548303ee1e..208c9bd7902 100755 --- a/format.sh +++ b/format.sh @@ -152,7 +152,7 @@ else fi echo "Running yamllint checks..." -if ! ./venv/bin/yamllint -s manifests/ utils/ci/gitlab/ .gitlab-ci.yml; then +if ! yamllint -s manifests/ utils/ci/gitlab/ .gitlab-ci.yml; then echo "yamllint checks failed. Please fix the errors above. 💥 💔 💥" exit 1 fi diff --git a/utils/scripts/shellcheck.sh b/utils/scripts/shellcheck.sh index 7066b14999d..eec76c873e1 100755 --- a/utils/scripts/shellcheck.sh +++ b/utils/scripts/shellcheck.sh @@ -41,9 +41,9 @@ function lint() { fi files+=("$f") - done < <( find utils -type d -name 'node_modules' -prune -o -name '*.sh' -print; ls -1 -- *.sh ) + done < <( find utils -type d -name 'node_modules' -prune -o -type f -name '*.sh' -print; ls -1 -- *.sh ) - ./venv/bin/shellcheck "${files[@]}" + shellcheck "${files[@]}" } function root() { @@ -71,5 +71,5 @@ function main() { } if [[ "${0}" == "${BASH_SOURCE[0]}" ]]; then - main "${@}" + main fi From ef7234138150dce83db6dc3ab7375560656d2c8d Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Wed, 29 Jul 2026 10:01:38 +0200 Subject: [PATCH 03/14] Replacing skip_if_xfail by the correct decorator --- tests/ffe/test_flag_eval_evp.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/ffe/test_flag_eval_evp.py b/tests/ffe/test_flag_eval_evp.py index 6f909598a6b..997a9f5ee77 100644 --- a/tests/ffe/test_flag_eval_evp.py +++ b/tests/ffe/test_flag_eval_evp.py @@ -4,7 +4,6 @@ from concurrent.futures import ThreadPoolExecutor from typing import cast - from tests.ffe.utils.fixtures import JSON, make_ufc_fixture from utils import HttpResponse from utils import features From c4d4b7b9314187a2c8709d0552a67e7b92b60db4 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Wed, 29 Jul 2026 10:02:25 +0200 Subject: [PATCH 04/14] Adding import check rules for test in test imports and using pytest directly --- .cursor/rules/code-format-standards.mdc | 4 ++ .cursor/rules/pr-review.mdc | 8 ++- .cursor/rules/repository-structure.mdc | 9 ++- .cursor/rules/test-activation.mdc | 14 +++++ .importlinter | 29 +++++++++ format.sh | 6 ++ import_linter_contracts.py | 80 +++++++++++++++++++++++++ pyproject.toml | 2 +- requirements.txt | 1 + 9 files changed, 150 insertions(+), 3 deletions(-) create mode 100644 .importlinter create mode 100644 import_linter_contracts.py diff --git a/.cursor/rules/code-format-standards.mdc b/.cursor/rules/code-format-standards.mdc index 534ff7804cc..70485727fe3 100644 --- a/.cursor/rules/code-format-standards.mdc +++ b/.cursor/rules/code-format-standards.mdc @@ -7,5 +7,9 @@ alwaysApply: true * All shell/bash scripts must be compatible with shellcheck. All shell/bash scripts you create or modify must pass shellcheck without errors or warnings. * Follow Python type annotation best practices that are compatible with mypy strict checking, using Python 3.12 standards as defined in pyproject.toml. Always provide explicit type annotations for all function arguments and return values. For collections, use precise types (e.g., list[str], dict[str, int]). If a value can be None, always use Optional[...] explicitly. Do not rely on implicit Optional types (e.g., avoid using x: int = None—instead, use x: Optional[int] = None). Do not omit type annotations, and do not use untyped or partially typed collections. +* Test modules must access pytest through the approved facade: `from utils import pytest`. Never use `import pytest` or `from pytest import ...` under `tests/`. + * If a pytest API is missing, first decide whether it belongs in the facade. Add only the narrow, reviewed API to `utils/pytest.py`; do not bypass the facade. + * Do not expose `pytest.mark.skip_if_xfail`. Use the semantic `slow` or `scenario_crash` decorators from `utils`, together with a manifest declaration, as described in the test activation rules. + * Run `lint-imports` (also included in `./format.sh`) to verify this policy. * Always run [format](mdc:format.sh) before committing changes to ensure code follows the project's style guidelines, including proper Path usage instead of os.path, no unused variables, complete type annotations, and efficient code patterns that satisfy mypy and ruff checks. if the format.sh script fails, try to fix the format mistakes. * All YAML files you create or modify must pass both yamllint and yamlfmt checks before being committed. diff --git a/.cursor/rules/pr-review.mdc b/.cursor/rules/pr-review.mdc index b04dc179d43..daa126839fc 100644 --- a/.cursor/rules/pr-review.mdc +++ b/.cursor/rules/pr-review.mdc @@ -58,7 +58,13 @@ When adding a new weblog, verify the name is unique across all languages. Search - Ref: [build.md](mdc:docs/execute/build.md) -## 10. Manifest YAML Syntax +## 10. No Cross-Test-File Imports + +A `test_*.py` file must never import from another `test_*.py` file. If logic is shared between test files, it must be moved to a non-test utility module instead. Flag any PR that adds a cross-test import or a new exception to `.importlinter`. + +- Ref: [repository-structure.mdc](mdc:.cursor/rules/repository-structure.mdc), enforced by the `Test files do not import other test files` Import Linter contract + +## 11. Manifest YAML Syntax - `bug` and `flaky` markers must include a JIRA ticket (e.g., `bug (JIRA-123)`) - Values with special YAML characters (`>`, `<`, `:`, `#`) must be quoted diff --git a/.cursor/rules/repository-structure.mdc b/.cursor/rules/repository-structure.mdc index a96fcad25a3..86d31ea6630 100644 --- a/.cursor/rules/repository-structure.mdc +++ b/.cursor/rules/repository-structure.mdc @@ -119,4 +119,11 @@ system-tests/ def test_XYZ(self): ... ``` -- Never define a setup method without a matching test method. \ No newline at end of file +- Never define a setup method without a matching test method. + +## 6. No Cross-Test-File Imports + +- A test file (`test_*.py`) must never import anything from another test file, whether through an absolute import such as `from tests.xxx.test_yyy import ...` or a relative import such as `from .test_yyy import ...`. +- If logic needs to be shared between test files, move it into a non-test utility module and have both test files import from there. +- Existing exceptions are explicitly listed in `.importlinter`. Do not add a new exception; refactor the shared logic instead. +- This is enforced by the `Test files do not import other test files` Import Linter contract run by `./format.sh`. diff --git a/.cursor/rules/test-activation.mdc b/.cursor/rules/test-activation.mdc index 918630e476e..b6083bc1ba3 100644 --- a/.cursor/rules/test-activation.mdc +++ b/.cursor/rules/test-activation.mdc @@ -97,6 +97,20 @@ which version contains the change. ## Decorator Rules +### Pytest Facade + +Test modules must import pytest with `from utils import pytest`. Direct imports of +the external `pytest` package are forbidden by Import Linter. + +`pytest.mark.skip_if_xfail` is an internal implementation detail and is +intentionally absent from the facade. Never recreate an alias for it. If an +expected failure must not execute: + +1. Keep its declaration in the appropriate manifest. +2. Add `@slow` when running the test would consume excessive CI time. +3. Add `@scenario_crash` only when running the test could crash the scenario or + disrupt other tests. + ### Version Format **CRITICAL**: Always use `library@version` format: diff --git a/.importlinter b/.importlinter new file mode 100644 index 00000000000..6c5e26d7992 --- /dev/null +++ b/.importlinter @@ -0,0 +1,29 @@ +[importlinter] +root_packages = + tests + utils +include_external_packages = True +contract_types = + no-cross-test-imports: import_linter_contracts.NoCrossTestImportsContract + +[importlinter:contract:pytest-facade] +name = Tests use the approved pytest facade +type = forbidden +source_modules = + tests +forbidden_modules = + pytest +allow_indirect_imports = True + +[importlinter:contract:no-cross-test-imports] +name = Test files do not import other test files +type = no-cross-test-imports +ignore_imports = + tests.appsec.api_security.test_apisecurity_telemetry -> tests.appsec.api_security.test_schemas + tests.appsec.waf.test_blocking -> tests.appsec.waf.test_blocking_security_response_id + tests.docker_ssi.test_docker_ssi -> tests.parametric.test_telemetry + tests.docker_ssi.test_docker_ssi_appsec -> tests.parametric.test_telemetry + tests.parametric.test_ffe.test_configuration_sources -> tests.parametric.test_ffe.test_dynamic_evaluation + tests.parametric.test_parametric_endpoints -> tests.parametric.test_dynamic_configuration + tests.test_telemetry -> tests.test_telemetry_heartbeat_utils + tests.test_the_test.test_telemetry_heartbeat -> tests.test_telemetry_heartbeat_utils diff --git a/format.sh b/format.sh index 208c9bd7902..9747d50ac16 100755 --- a/format.sh +++ b/format.sh @@ -46,6 +46,12 @@ if ! mypy --config pyproject.toml; then exit 1 fi +echo "Running import policy checks..." +if ! lint-imports; then + echo "Import policy checks failed. Please fix the errors above. 💥 💔 💥" + exit 1 +fi + echo "Running ruff formatter..." if [ "$COMMAND" == "fix" ]; then ruff format diff --git a/import_linter_contracts.py b/import_linter_contracts.py new file mode 100644 index 00000000000..094ad1eafc3 --- /dev/null +++ b/import_linter_contracts.py @@ -0,0 +1,80 @@ +"""Repository-specific Import Linter contracts.""" + +from typing import TYPE_CHECKING, TypedDict, cast + +from grimp import ImportGraph +from importlinter import Contract, ContractCheck, fields, output + +if TYPE_CHECKING: + from importlinter.domain.imports import ImportExpression + + +class _CrossTestImport(TypedDict): + importer: str + imported: str + line_numbers: tuple[int, ...] + + +def _is_test_module(module: str) -> bool: + return module.startswith("tests.") and module.rsplit(".", maxsplit=1)[-1].startswith("test_") + + +class NoCrossTestImportsContract(Contract): + """Prevent test modules from importing other test modules directly.""" + + ignore_imports = fields.SetField(subfield=fields.ImportExpressionField(), required=False) + + def check(self, graph: ImportGraph, verbose: bool) -> ContractCheck: # noqa: ARG002, FBT001 + ignored_imports = self._resolve_ignored_imports(graph) + existing_cross_test_imports: set[tuple[str, str]] = set() + violations: list[_CrossTestImport] = [] + + for importer in sorted(graph.modules): + if not _is_test_module(importer): + continue + + for imported in sorted(graph.find_modules_directly_imported_by(importer)): + if not _is_test_module(imported) or importer == imported: + continue + existing_cross_test_imports.add((importer, imported)) + if (importer, imported) in ignored_imports: + continue + + details = graph.get_import_details(importer=importer, imported=imported) + violations.append( + { + "importer": importer, + "imported": imported, + "line_numbers": tuple(detail["line_number"] for detail in details), + } + ) + + unused_ignores = sorted(ignored_imports - existing_cross_test_imports) + return ContractCheck( + kept=not violations and not unused_ignores, + metadata={"unused_ignores": unused_ignores, "violations": violations}, + ) + + def render_broken_contract(self, check: ContractCheck) -> None: + violations = cast("list[_CrossTestImport]", check.metadata["violations"]) + for violation in violations: + lines = ", ".join(f"l.{line_number}" for line_number in violation["line_numbers"]) + output.print_error( + f"{violation['importer']} imports {violation['imported']} ({lines})", + bold=False, + ) + + unused_ignores = cast("list[tuple[str, str]]", check.metadata["unused_ignores"]) + for importer, imported in unused_ignores: + output.print_error(f"Unused exception: {importer} -> {imported}", bold=False) + + def _resolve_ignored_imports(self, graph: ImportGraph) -> set[tuple[str, str]]: + ignored_imports: set[tuple[str, str]] = set() + expressions = cast("set[ImportExpression] | None", self.ignore_imports) + + for expression in expressions or set(): + importers = graph.find_matching_modules(expression.importer.expression) + imported_modules = graph.find_matching_modules(expression.imported.expression) + ignored_imports.update((importer, imported) for importer in importers for imported in imported_modules) + + return ignored_imports diff --git a/pyproject.toml b/pyproject.toml index 7f7b509dae2..e0eac127dd7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,6 +41,7 @@ python_files = "test_*.py *utils.py" [tool.mypy] python_version = "3.12" files = [ + "import_linter_contracts.py", "utils/", "tests/", "conftest.py", @@ -280,4 +281,3 @@ ignore = [ "S506", # unsafe-yaml-load: TODO ] "utils/waf_rules.py" = ["N801"] # generated file - diff --git a/requirements.txt b/requirements.txt index 9f58d5a27d5..73c072deb0b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,6 +6,7 @@ ddapm-test-agent==1.18.0 dictdiffer==0.9.0 # for parametric tests docker==7.1.0 filelock==3.12.2 # for parametric tests +import-linter==2.13 jsonschema==4.16.0 kubernetes==29.0.0 #lib-injection kubernetes mitmproxy==9.0.1 From 7dfc221cebc5405b863ccc3208afc62a1c2ba9f4 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Thu, 30 Jul 2026 09:52:19 +0200 Subject: [PATCH 05/14] Update runner image tag --- utils/ci/gitlab/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/ci/gitlab/main.yml b/utils/ci/gitlab/main.yml index 889b4d41a92..4b9bf29a9fe 100644 --- a/utils/ci/gitlab/main.yml +++ b/utils/ci/gitlab/main.yml @@ -75,7 +75,7 @@ variables: # Tag = first 12 chars of sha256(utils/ci/gitlab/docker/system-tests.Dockerfile + requirements.txt) # Update this when either file changes: # cat utils/ci/gitlab/docker/system-tests.Dockerfile requirements.txt | sha256sum | cut -c1-12 - CI_IMAGE: "registry.ddbuild.io/system-tests/ci-runner:75a57a4b6391" + CI_IMAGE: "registry.ddbuild.io/system-tests/ci-runner:082a514c5331" SYSTEM_TESTS_SPLIT_PIPELINE: "$[[ inputs.split_pipeline ]]" .system_tests_param_base: From 8e2d950d92fc99c05a7c08579e12e95a163232dc Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Wed, 12 Aug 2026 14:40:45 +0200 Subject: [PATCH 06/14] pytest import cleanup --- tests/test_the_test/test_github_nightly.py | 2 +- utils/pytest.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_the_test/test_github_nightly.py b/tests/test_the_test/test_github_nightly.py index 910d768e44a..64af46691ee 100644 --- a/tests/test_the_test/test_github_nightly.py +++ b/tests/test_the_test/test_github_nightly.py @@ -4,7 +4,7 @@ from dataclasses import dataclass from pathlib import Path -import pytest +from utils import pytest from utils import scenarios from utils.ci.github import nightly diff --git a/utils/pytest.py b/utils/pytest.py index 53180f416f0..b9008b31187 100644 --- a/utils/pytest.py +++ b/utils/pytest.py @@ -15,6 +15,7 @@ Mark = _pytest.Mark MarkDecorator = _pytest.MarkDecorator MonkeyPatch = _pytest.MonkeyPatch +CaptureFixture = _pytest.CaptureFixture Session = _pytest.Session approx = _pytest.approx From 0d9e711cbe7be50b007868faba4f44df66a1b3ef Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Wed, 26 Aug 2026 15:16:48 +0200 Subject: [PATCH 07/14] Restore shellcheck --- utils/scripts/shellcheck.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/utils/scripts/shellcheck.sh b/utils/scripts/shellcheck.sh index eec76c873e1..7066b14999d 100755 --- a/utils/scripts/shellcheck.sh +++ b/utils/scripts/shellcheck.sh @@ -41,9 +41,9 @@ function lint() { fi files+=("$f") - done < <( find utils -type d -name 'node_modules' -prune -o -type f -name '*.sh' -print; ls -1 -- *.sh ) + done < <( find utils -type d -name 'node_modules' -prune -o -name '*.sh' -print; ls -1 -- *.sh ) - shellcheck "${files[@]}" + ./venv/bin/shellcheck "${files[@]}" } function root() { @@ -71,5 +71,5 @@ function main() { } if [[ "${0}" == "${BASH_SOURCE[0]}" ]]; then - main + main "${@}" fi From 75cb418965b12ab894a962f1cd01a400f69db932 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Wed, 26 Aug 2026 15:18:33 +0200 Subject: [PATCH 08/14] Ban xfail --- utils/pytest.py | 1 - 1 file changed, 1 deletion(-) diff --git a/utils/pytest.py b/utils/pytest.py index d13d510245f..2cd50f721c4 100644 --- a/utils/pytest.py +++ b/utils/pytest.py @@ -35,7 +35,6 @@ class _AllowedMarks: features = _pytest.mark.features parametrize = _pytest.mark.parametrize scenario = _pytest.mark.scenario - xfail = _pytest.mark.xfail mark = _AllowedMarks() From 82b2043050ffef9783a50150caf14f7601213b6d Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Wed, 26 Aug 2026 15:23:13 +0200 Subject: [PATCH 09/14] Move importlinter policy to dedicated utils dir --- .importlinter | 2 +- pyproject.toml | 1 - .../format/import_linter_contracts.py | 0 3 files changed, 1 insertion(+), 2 deletions(-) rename import_linter_contracts.py => utils/format/import_linter_contracts.py (100%) diff --git a/.importlinter b/.importlinter index 6c5e26d7992..6d10a203df1 100644 --- a/.importlinter +++ b/.importlinter @@ -4,7 +4,7 @@ root_packages = utils include_external_packages = True contract_types = - no-cross-test-imports: import_linter_contracts.NoCrossTestImportsContract + no-cross-test-imports: utils.format.import_linter_contracts.NoCrossTestImportsContract [importlinter:contract:pytest-facade] name = Tests use the approved pytest facade diff --git a/pyproject.toml b/pyproject.toml index 56dfb4af4cd..ed26590ad58 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,7 +41,6 @@ python_files = "test_*.py *utils.py" [tool.mypy] python_version = "3.12" files = [ - "import_linter_contracts.py", "utils/", "tests/", "conftest.py", diff --git a/import_linter_contracts.py b/utils/format/import_linter_contracts.py similarity index 100% rename from import_linter_contracts.py rename to utils/format/import_linter_contracts.py From 51327847e94ff2ad962f6c679cd589444b558bbb Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Wed, 26 Aug 2026 15:44:05 +0200 Subject: [PATCH 10/14] Adding the facade to new imports --- .../llm/anthropic/test_anthropic_ai_guard.py | 2 +- tests/parametric/test_otel_tracestate_sampling.py | 2 +- tests/test_the_test/test_ai_guard_span_helpers.py | 2 +- tests/test_the_test/test_docker_run_cleanup.py | 2 +- tests/test_the_test/test_get_image_list.py | 2 +- tests/test_the_test/test_update_mirror_images.py | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/integration_frameworks/llm/anthropic/test_anthropic_ai_guard.py b/tests/integration_frameworks/llm/anthropic/test_anthropic_ai_guard.py index 86e1eec91d8..549762797f5 100644 --- a/tests/integration_frameworks/llm/anthropic/test_anthropic_ai_guard.py +++ b/tests/integration_frameworks/llm/anthropic/test_anthropic_ai_guard.py @@ -2,7 +2,7 @@ /create request and its tool_use blocks. After-model needs the stream path: not yet cross-language. """ -import pytest +from utils import pytest from utils import features, scenarios from utils.docker_fixtures import FrameworkTestClientApi, TestAgentAPI diff --git a/tests/parametric/test_otel_tracestate_sampling.py b/tests/parametric/test_otel_tracestate_sampling.py index 51208894752..c494fd160be 100644 --- a/tests/parametric/test_otel_tracestate_sampling.py +++ b/tests/parametric/test_otel_tracestate_sampling.py @@ -2,7 +2,7 @@ import json -import pytest +from utils import pytest from utils import features, scenarios from utils.dd_constants import SamplingPriority diff --git a/tests/test_the_test/test_ai_guard_span_helpers.py b/tests/test_the_test/test_ai_guard_span_helpers.py index 6af27eea5b3..113fc536f2d 100644 --- a/tests/test_the_test/test_ai_guard_span_helpers.py +++ b/tests/test_the_test/test_ai_guard_span_helpers.py @@ -6,7 +6,7 @@ import re from typing import Any -import pytest +from utils import pytest import requests from tests.integration_frameworks.llm import utils as llm_utils diff --git a/tests/test_the_test/test_docker_run_cleanup.py b/tests/test_the_test/test_docker_run_cleanup.py index 2af4b165bc7..ea5f222269e 100644 --- a/tests/test_the_test/test_docker_run_cleanup.py +++ b/tests/test_the_test/test_docker_run_cleanup.py @@ -4,7 +4,7 @@ from _pytest.outcomes import Failed from docker.errors import APIError, NotFound -import pytest +from utils import pytest from utils import scenarios from utils.docker_fixtures import _core as docker_core diff --git a/tests/test_the_test/test_get_image_list.py b/tests/test_the_test/test_get_image_list.py index 1e7b40016c4..dfdbd780c0a 100644 --- a/tests/test_the_test/test_get_image_list.py +++ b/tests/test_the_test/test_get_image_list.py @@ -5,7 +5,7 @@ import subprocess import sys -import pytest +from utils import pytest import yaml from utils import scenarios diff --git a/tests/test_the_test/test_update_mirror_images.py b/tests/test_the_test/test_update_mirror_images.py index 5503a55d466..eab22a393ef 100644 --- a/tests/test_the_test/test_update_mirror_images.py +++ b/tests/test_the_test/test_update_mirror_images.py @@ -3,7 +3,7 @@ from pathlib import Path import subprocess -import pytest +from utils import pytest from utils import scenarios from utils.scripts import update_mirror_images From f9c238ab0eeb560983b36a2b4d467d69e2a9558a Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Wed, 26 Aug 2026 15:45:09 +0200 Subject: [PATCH 11/14] format --- utils/format/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 utils/format/__init__.py diff --git a/utils/format/__init__.py b/utils/format/__init__.py new file mode 100644 index 00000000000..e69de29bb2d From 7a286e05568a4c35fc718866be658ad296eda840 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Tue, 8 Sep 2026 17:06:20 +0200 Subject: [PATCH 12/14] Fix --- tests/parametric/otel_env_vars/test_otel_sdk_disabled.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/parametric/otel_env_vars/test_otel_sdk_disabled.py b/tests/parametric/otel_env_vars/test_otel_sdk_disabled.py index f3b5284ea65..52abf821b82 100644 --- a/tests/parametric/otel_env_vars/test_otel_sdk_disabled.py +++ b/tests/parametric/otel_env_vars/test_otel_sdk_disabled.py @@ -1,4 +1,4 @@ -import pytest +from utils import pytest from tests.parametric.conftest import APMLibrary, nodejs_telemetry_value from utils import features, scenarios From 6f4240c709b613214e7a827df715fa54e0862227 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Fri, 11 Sep 2026 14:38:37 +0200 Subject: [PATCH 13/14] Fixing new imports --- tests/parametric/otel_env_vars/test_otel_service_name.py | 2 +- tests/test_the_test/test_mock_backend_v2.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/parametric/otel_env_vars/test_otel_service_name.py b/tests/parametric/otel_env_vars/test_otel_service_name.py index 76926432f89..52ee89087f9 100644 --- a/tests/parametric/otel_env_vars/test_otel_service_name.py +++ b/tests/parametric/otel_env_vars/test_otel_service_name.py @@ -1,4 +1,4 @@ -import pytest +from utils import pytest from tests.parametric.conftest import APMLibrary from utils import features, scenarios diff --git a/tests/test_the_test/test_mock_backend_v2.py b/tests/test_the_test/test_mock_backend_v2.py index c8f8419c770..f64a18918a8 100644 --- a/tests/test_the_test/test_mock_backend_v2.py +++ b/tests/test_the_test/test_mock_backend_v2.py @@ -5,7 +5,7 @@ from collections.abc import Generator from pathlib import Path -import pytest +from utils import pytest import requests import zstandard From 5d02d6fc5ae4ca6d53213fbd52209fee5b72389e Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Mon, 14 Sep 2026 15:43:28 +0200 Subject: [PATCH 14/14] fix --- tests/test_the_test/test_build_base_images.py | 2 +- utils/pytest.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_the_test/test_build_base_images.py b/tests/test_the_test/test_build_base_images.py index e28f703b849..7deeebcec8f 100644 --- a/tests/test_the_test/test_build_base_images.py +++ b/tests/test_the_test/test_build_base_images.py @@ -8,7 +8,7 @@ import tempfile import textwrap -import pytest +from utils import pytest from utils import scenarios from utils.base_images import build_base_images diff --git a/utils/pytest.py b/utils/pytest.py index 2cd50f721c4..4e4fdead182 100644 --- a/utils/pytest.py +++ b/utils/pytest.py @@ -18,6 +18,7 @@ CaptureFixture = _pytest.CaptureFixture Session = _pytest.Session CallInfo = _pytest.CallInfo +TempPathFactory = _pytest.TempPathFactory approx = _pytest.approx exit = _pytest.exit # noqa: A001 - preserve the public pytest API name