diff --git a/.cursor/rules/code-format-standards.mdc b/.cursor/rules/code-format-standards.mdc index 534ff7804cc..70485727fe3 100644 --- a/.cursor/rules/code-format-standards.mdc +++ b/.cursor/rules/code-format-standards.mdc @@ -7,5 +7,9 @@ alwaysApply: true * All shell/bash scripts must be compatible with shellcheck. All shell/bash scripts you create or modify must pass shellcheck without errors or warnings. * Follow Python type annotation best practices that are compatible with mypy strict checking, using Python 3.12 standards as defined in pyproject.toml. Always provide explicit type annotations for all function arguments and return values. For collections, use precise types (e.g., list[str], dict[str, int]). If a value can be None, always use Optional[...] explicitly. Do not rely on implicit Optional types (e.g., avoid using x: int = None—instead, use x: Optional[int] = None). Do not omit type annotations, and do not use untyped or partially typed collections. +* Test modules must access pytest through the approved facade: `from utils import pytest`. Never use `import pytest` or `from pytest import ...` under `tests/`. + * If a pytest API is missing, first decide whether it belongs in the facade. Add only the narrow, reviewed API to `utils/pytest.py`; do not bypass the facade. + * Do not expose `pytest.mark.skip_if_xfail`. Use the semantic `slow` or `scenario_crash` decorators from `utils`, together with a manifest declaration, as described in the test activation rules. + * Run `lint-imports` (also included in `./format.sh`) to verify this policy. * Always run [format](mdc:format.sh) before committing changes to ensure code follows the project's style guidelines, including proper Path usage instead of os.path, no unused variables, complete type annotations, and efficient code patterns that satisfy mypy and ruff checks. if the format.sh script fails, try to fix the format mistakes. * All YAML files you create or modify must pass both yamllint and yamlfmt checks before being committed. diff --git a/.cursor/rules/pr-review.mdc b/.cursor/rules/pr-review.mdc index b04dc179d43..daa126839fc 100644 --- a/.cursor/rules/pr-review.mdc +++ b/.cursor/rules/pr-review.mdc @@ -58,7 +58,13 @@ When adding a new weblog, verify the name is unique across all languages. Search - Ref: [build.md](mdc:docs/execute/build.md) -## 10. Manifest YAML Syntax +## 10. No Cross-Test-File Imports + +A `test_*.py` file must never import from another `test_*.py` file. If logic is shared between test files, it must be moved to a non-test utility module instead. Flag any PR that adds a cross-test import or a new exception to `.importlinter`. + +- Ref: [repository-structure.mdc](mdc:.cursor/rules/repository-structure.mdc), enforced by the `Test files do not import other test files` Import Linter contract + +## 11. Manifest YAML Syntax - `bug` and `flaky` markers must include a JIRA ticket (e.g., `bug (JIRA-123)`) - Values with special YAML characters (`>`, `<`, `:`, `#`) must be quoted diff --git a/.cursor/rules/repository-structure.mdc b/.cursor/rules/repository-structure.mdc index a96fcad25a3..86d31ea6630 100644 --- a/.cursor/rules/repository-structure.mdc +++ b/.cursor/rules/repository-structure.mdc @@ -119,4 +119,11 @@ system-tests/ def test_XYZ(self): ... ``` -- Never define a setup method without a matching test method. \ No newline at end of file +- Never define a setup method without a matching test method. + +## 6. No Cross-Test-File Imports + +- A test file (`test_*.py`) must never import anything from another test file, whether through an absolute import such as `from tests.xxx.test_yyy import ...` or a relative import such as `from .test_yyy import ...`. +- If logic needs to be shared between test files, move it into a non-test utility module and have both test files import from there. +- Existing exceptions are explicitly listed in `.importlinter`. Do not add a new exception; refactor the shared logic instead. +- This is enforced by the `Test files do not import other test files` Import Linter contract run by `./format.sh`. diff --git a/.cursor/rules/test-activation.mdc b/.cursor/rules/test-activation.mdc index 918630e476e..b6083bc1ba3 100644 --- a/.cursor/rules/test-activation.mdc +++ b/.cursor/rules/test-activation.mdc @@ -97,6 +97,20 @@ which version contains the change. ## Decorator Rules +### Pytest Facade + +Test modules must import pytest with `from utils import pytest`. Direct imports of +the external `pytest` package are forbidden by Import Linter. + +`pytest.mark.skip_if_xfail` is an internal implementation detail and is +intentionally absent from the facade. Never recreate an alias for it. If an +expected failure must not execute: + +1. Keep its declaration in the appropriate manifest. +2. Add `@slow` when running the test would consume excessive CI time. +3. Add `@scenario_crash` only when running the test could crash the scenario or + disrupt other tests. + ### Version Format **CRITICAL**: Always use `library@version` format: diff --git a/.importlinter b/.importlinter new file mode 100644 index 00000000000..6d10a203df1 --- /dev/null +++ b/.importlinter @@ -0,0 +1,29 @@ +[importlinter] +root_packages = + tests + utils +include_external_packages = True +contract_types = + no-cross-test-imports: utils.format.import_linter_contracts.NoCrossTestImportsContract + +[importlinter:contract:pytest-facade] +name = Tests use the approved pytest facade +type = forbidden +source_modules = + tests +forbidden_modules = + pytest +allow_indirect_imports = True + +[importlinter:contract:no-cross-test-imports] +name = Test files do not import other test files +type = no-cross-test-imports +ignore_imports = + tests.appsec.api_security.test_apisecurity_telemetry -> tests.appsec.api_security.test_schemas + tests.appsec.waf.test_blocking -> tests.appsec.waf.test_blocking_security_response_id + tests.docker_ssi.test_docker_ssi -> tests.parametric.test_telemetry + tests.docker_ssi.test_docker_ssi_appsec -> tests.parametric.test_telemetry + tests.parametric.test_ffe.test_configuration_sources -> tests.parametric.test_ffe.test_dynamic_evaluation + tests.parametric.test_parametric_endpoints -> tests.parametric.test_dynamic_configuration + tests.test_telemetry -> tests.test_telemetry_heartbeat_utils + tests.test_the_test.test_telemetry_heartbeat -> tests.test_telemetry_heartbeat_utils diff --git a/format.sh b/format.sh index 5ebed45dd81..6f634787937 100755 --- a/format.sh +++ b/format.sh @@ -46,6 +46,12 @@ if ! mypy --config pyproject.toml; then exit 1 fi +echo "Running import policy checks..." +if ! lint-imports; then + echo "Import policy checks failed. Please fix the errors above. 💥 💔 💥" + exit 1 +fi + echo "Running ruff formatter..." if [ "$COMMAND" == "fix" ]; then ruff format diff --git a/requirements.txt b/requirements.txt index 7e0dd733656..9e6148e40c8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,6 +7,7 @@ ddapm-test-agent==1.18.0 dictdiffer==0.9.0 # for parametric tests docker==7.1.0 filelock==3.12.2 # for parametric tests +import-linter==2.13 jsonschema==4.16.0 kubernetes==29.0.0 #lib-injection kubernetes mitmproxy==9.0.1 diff --git a/tests/ai_guard/conftest.py b/tests/ai_guard/conftest.py index a989123be4c..168631baaa6 100644 --- a/tests/ai_guard/conftest.py +++ b/tests/ai_guard/conftest.py @@ -1,6 +1,6 @@ from collections.abc import Generator from typing import Any -import pytest +from utils import pytest @pytest.hookimpl(hookwrapper=True) diff --git a/tests/appsec/waf/test_addresses.py b/tests/appsec/waf/test_addresses.py index e77c66cae32..6be8a811eb3 100644 --- a/tests/appsec/waf/test_addresses.py +++ b/tests/appsec/waf/test_addresses.py @@ -3,7 +3,7 @@ # Copyright 2021 Datadog, Inc. import json -import pytest +from utils import pytest from utils import weblog, interfaces, rfc, scenarios, features, logger from utils.dd_types import DataDogLibrarySpan diff --git a/tests/ffe/test_flag_eval_evp.py b/tests/ffe/test_flag_eval_evp.py index 71f3a0cf73f..90b73743e1b 100644 --- a/tests/ffe/test_flag_eval_evp.py +++ b/tests/ffe/test_flag_eval_evp.py @@ -4,7 +4,6 @@ from concurrent.futures import ThreadPoolExecutor from typing import cast - from tests.ffe.utils.fixtures import JSON, make_ufc_fixture from utils import HttpResponse from utils import features diff --git a/tests/integration_frameworks/conftest.py b/tests/integration_frameworks/conftest.py index d9abd3873f5..262d61177ee 100644 --- a/tests/integration_frameworks/conftest.py +++ b/tests/integration_frameworks/conftest.py @@ -1,6 +1,6 @@ from collections.abc import Generator from typing import Any -import pytest +from utils import pytest from utils.docker_fixtures import ( FrameworkTestClientApi, diff --git a/tests/integration_frameworks/llm/anthropic/test_anthropic_ai_guard.py b/tests/integration_frameworks/llm/anthropic/test_anthropic_ai_guard.py index 86e1eec91d8..549762797f5 100644 --- a/tests/integration_frameworks/llm/anthropic/test_anthropic_ai_guard.py +++ b/tests/integration_frameworks/llm/anthropic/test_anthropic_ai_guard.py @@ -2,7 +2,7 @@ /create request and its tool_use blocks. After-model needs the stream path: not yet cross-language. """ -import pytest +from utils import pytest from utils import features, scenarios from utils.docker_fixtures import FrameworkTestClientApi, TestAgentAPI diff --git a/tests/integration_frameworks/llm/anthropic/test_anthropic_apm.py b/tests/integration_frameworks/llm/anthropic/test_anthropic_apm.py index 50d825fa382..7557d600192 100644 --- a/tests/integration_frameworks/llm/anthropic/test_anthropic_apm.py +++ b/tests/integration_frameworks/llm/anthropic/test_anthropic_apm.py @@ -1,7 +1,7 @@ from utils import context, scenarios, features from utils.docker_fixtures import FrameworkTestClientApi, TestAgentAPI -import pytest +from utils import pytest from .utils import BaseAnthropicTest diff --git a/tests/integration_frameworks/llm/anthropic/test_anthropic_llmobs.py b/tests/integration_frameworks/llm/anthropic/test_anthropic_llmobs.py index 8514b76ef8c..6caee2f5ce4 100644 --- a/tests/integration_frameworks/llm/anthropic/test_anthropic_llmobs.py +++ b/tests/integration_frameworks/llm/anthropic/test_anthropic_llmobs.py @@ -4,7 +4,7 @@ from .utils import TOOLS, BaseAnthropicTest -import pytest +from utils import pytest from unittest import mock import json diff --git a/tests/integration_frameworks/llm/google_genai/test_google_genai_apm.py b/tests/integration_frameworks/llm/google_genai/test_google_genai_apm.py index 0790ebd732d..29abba18b73 100644 --- a/tests/integration_frameworks/llm/google_genai/test_google_genai_apm.py +++ b/tests/integration_frameworks/llm/google_genai/test_google_genai_apm.py @@ -1,7 +1,7 @@ from utils import features, scenarios from utils.docker_fixtures import FrameworkTestClientApi, TestAgentAPI -import pytest +from utils import pytest from .utils import BaseGoogleGenaiTest diff --git a/tests/integration_frameworks/llm/google_genai/test_google_genai_llmobs.py b/tests/integration_frameworks/llm/google_genai/test_google_genai_llmobs.py index e297d556afa..ec3673e6b0b 100644 --- a/tests/integration_frameworks/llm/google_genai/test_google_genai_llmobs.py +++ b/tests/integration_frameworks/llm/google_genai/test_google_genai_llmobs.py @@ -3,7 +3,7 @@ from utils import features, scenarios from utils.docker_fixtures import FrameworkTestClientApi, TestAgentAPI -import pytest +from utils import pytest from unittest import mock from typing import Any diff --git a/tests/integration_frameworks/llm/openai/test_openai_ai_guard.py b/tests/integration_frameworks/llm/openai/test_openai_ai_guard.py index 6ba0a1bc76b..5a5c3ad69ae 100644 --- a/tests/integration_frameworks/llm/openai/test_openai_ai_guard.py +++ b/tests/integration_frameworks/llm/openai/test_openai_ai_guard.py @@ -2,7 +2,7 @@ /chat/completions request and its tool calls. After-model needs the stream path: not yet cross-language. """ -import pytest +from utils import pytest from utils import features, scenarios from utils.docker_fixtures import FrameworkTestClientApi, TestAgentAPI diff --git a/tests/integration_frameworks/llm/openai/test_openai_apm.py b/tests/integration_frameworks/llm/openai/test_openai_apm.py index 4b374fcbb38..8fce0d17894 100644 --- a/tests/integration_frameworks/llm/openai/test_openai_apm.py +++ b/tests/integration_frameworks/llm/openai/test_openai_apm.py @@ -1,7 +1,7 @@ from utils import features, scenarios from .utils import TOOLS, BaseOpenaiTest -import pytest +from utils import pytest from utils.docker_fixtures import FrameworkTestClientApi, TestAgentAPI diff --git a/tests/integration_frameworks/llm/openai/test_openai_llmobs.py b/tests/integration_frameworks/llm/openai/test_openai_llmobs.py index de79c2bc85b..b81a43af33c 100644 --- a/tests/integration_frameworks/llm/openai/test_openai_llmobs.py +++ b/tests/integration_frameworks/llm/openai/test_openai_llmobs.py @@ -1,7 +1,7 @@ import json from utils import features, scenarios -import pytest +from utils import pytest from unittest import mock from utils.docker_fixtures import FrameworkTestClientApi, TestAgentAPI diff --git a/tests/parametric/conftest.py b/tests/parametric/conftest.py index 0ff995d5030..a931748f743 100644 --- a/tests/parametric/conftest.py +++ b/tests/parametric/conftest.py @@ -5,7 +5,7 @@ import shutil import subprocess -import pytest +from utils import pytest import yaml from utils import scenarios, logger diff --git a/tests/parametric/otel_env_vars/test_otel_sdk_disabled.py b/tests/parametric/otel_env_vars/test_otel_sdk_disabled.py index 9f50bd65ae7..d09395411a7 100644 --- a/tests/parametric/otel_env_vars/test_otel_sdk_disabled.py +++ b/tests/parametric/otel_env_vars/test_otel_sdk_disabled.py @@ -1,4 +1,4 @@ -import pytest +from utils import pytest from tests.parametric.conftest import APMLibrary, nodejs_telemetry_value from utils import features, scenarios diff --git a/tests/parametric/otel_env_vars/test_otel_service_name.py b/tests/parametric/otel_env_vars/test_otel_service_name.py index 76926432f89..52ee89087f9 100644 --- a/tests/parametric/otel_env_vars/test_otel_service_name.py +++ b/tests/parametric/otel_env_vars/test_otel_service_name.py @@ -1,4 +1,4 @@ -import pytest +from utils import pytest from tests.parametric.conftest import APMLibrary from utils import features, scenarios diff --git a/tests/parametric/test_128_bit_traceids.py b/tests/parametric/test_128_bit_traceids.py index bdd9c6381e2..40de09b84a1 100644 --- a/tests/parametric/test_128_bit_traceids.py +++ b/tests/parametric/test_128_bit_traceids.py @@ -1,4 +1,4 @@ -import pytest +from utils import pytest from utils.docker_fixtures.spec.trace import find_first_span_in_trace_payload, find_trace, find_only_span from utils import scenarios, features diff --git a/tests/parametric/test_config_consistency.py b/tests/parametric/test_config_consistency.py index 1868052ea2a..bf6e56dc338 100644 --- a/tests/parametric/test_config_consistency.py +++ b/tests/parametric/test_config_consistency.py @@ -2,7 +2,7 @@ from urllib.parse import urlparse -import pytest +from utils import pytest import yaml from utils import ( scenarios, diff --git a/tests/parametric/test_crashtracking.py b/tests/parametric/test_crashtracking.py index 06b94ad898f..4f1569732f9 100644 --- a/tests/parametric/test_crashtracking.py +++ b/tests/parametric/test_crashtracking.py @@ -2,7 +2,7 @@ import base64 import json -import pytest +from utils import pytest from utils import features, scenarios, logger from utils.docker_fixtures import TestAgentAPI, ParametricTestClientApi as APMLibrary diff --git a/tests/parametric/test_dynamic_configuration.py b/tests/parametric/test_dynamic_configuration.py index 57be43391bd..b00777d5f24 100644 --- a/tests/parametric/test_dynamic_configuration.py +++ b/tests/parametric/test_dynamic_configuration.py @@ -6,7 +6,7 @@ from pathlib import Path from typing import Any -import pytest +from utils import pytest import yaml from utils import ( diff --git a/tests/parametric/test_extract_behavior.py b/tests/parametric/test_extract_behavior.py index 0781883b6d3..9da62dcb2b5 100644 --- a/tests/parametric/test_extract_behavior.py +++ b/tests/parametric/test_extract_behavior.py @@ -1,4 +1,4 @@ -import pytest +from utils import pytest from utils import features, scenarios from utils.docker_fixtures import TestAgentAPI diff --git a/tests/parametric/test_ffe/test_configuration_sources.py b/tests/parametric/test_ffe/test_configuration_sources.py index 195edddd1ad..618016044b3 100644 --- a/tests/parametric/test_ffe/test_configuration_sources.py +++ b/tests/parametric/test_ffe/test_configuration_sources.py @@ -18,7 +18,7 @@ import time from typing import Any -import pytest +from utils import pytest from tests.parametric.conftest import APMLibrary from tests.parametric.test_ffe.test_dynamic_evaluation import _set_and_wait_ffe_rc, _ffe_evaluate_with_rc_retry diff --git a/tests/parametric/test_ffe/test_dynamic_evaluation.py b/tests/parametric/test_ffe/test_dynamic_evaluation.py index 7d48a1e0b7c..35524d7aac0 100644 --- a/tests/parametric/test_ffe/test_dynamic_evaluation.py +++ b/tests/parametric/test_ffe/test_dynamic_evaluation.py @@ -1,7 +1,7 @@ """Test FFE (Feature Flags & Experimentation) functionality via parametric tests.""" import json -import pytest +from utils import pytest import time from pathlib import Path from typing import Any diff --git a/tests/parametric/test_ffe/test_span_enrichment.py b/tests/parametric/test_ffe/test_span_enrichment.py index 7b8168a3eb1..6a4e54bdacb 100644 --- a/tests/parametric/test_ffe/test_span_enrichment.py +++ b/tests/parametric/test_ffe/test_span_enrichment.py @@ -20,7 +20,7 @@ """ import json -import pytest +from utils import pytest from pathlib import Path from typing import Any diff --git a/tests/parametric/test_headers_b3.py b/tests/parametric/test_headers_b3.py index 334ea3d9e1b..832ab9d4f4d 100644 --- a/tests/parametric/test_headers_b3.py +++ b/tests/parametric/test_headers_b3.py @@ -1,4 +1,4 @@ -import pytest +from utils import pytest from utils.docker_fixtures.spec.trace import SAMPLING_PRIORITY_KEY, ORIGIN from utils.docker_fixtures.spec.trace import span_has_no_parent diff --git a/tests/parametric/test_headers_b3multi.py b/tests/parametric/test_headers_b3multi.py index 3a294c5cc92..c3390aba984 100644 --- a/tests/parametric/test_headers_b3multi.py +++ b/tests/parametric/test_headers_b3multi.py @@ -1,4 +1,4 @@ -import pytest +from utils import pytest from utils.docker_fixtures.spec.trace import SAMPLING_PRIORITY_KEY, ORIGIN from utils.docker_fixtures.spec.trace import span_has_no_parent diff --git a/tests/parametric/test_headers_baggage.py b/tests/parametric/test_headers_baggage.py index 6796fef5d98..0a0c912b3fc 100644 --- a/tests/parametric/test_headers_baggage.py +++ b/tests/parametric/test_headers_baggage.py @@ -2,7 +2,7 @@ from utils import features, scenarios from utils.docker_fixtures import TestAgentAPI -import pytest +from utils import pytest from .conftest import APMLibrary diff --git a/tests/parametric/test_headers_none.py b/tests/parametric/test_headers_none.py index 6968e612d5e..2b650f527d0 100644 --- a/tests/parametric/test_headers_none.py +++ b/tests/parametric/test_headers_none.py @@ -1,4 +1,4 @@ -import pytest +from utils import pytest from utils.docker_fixtures.spec.trace import SAMPLING_PRIORITY_KEY, ORIGIN from utils.docker_fixtures.spec.trace import find_only_span diff --git a/tests/parametric/test_headers_opm.py b/tests/parametric/test_headers_opm.py index bdb0d0fa24c..331485d0fd6 100644 --- a/tests/parametric/test_headers_opm.py +++ b/tests/parametric/test_headers_opm.py @@ -20,7 +20,7 @@ RFC: https://docs.google.com/document/d/1SzZWivVWT79lJe80ZulEra6AARszjEYVEwWJ7IhJ6Xo/edit?tab=t.0 """ -import pytest +from utils import pytest from utils import features, scenarios, rfc from utils.docker_fixtures import TestAgentAPI diff --git a/tests/parametric/test_headers_precedence.py b/tests/parametric/test_headers_precedence.py index 16a6c750f33..7cef84066d4 100644 --- a/tests/parametric/test_headers_precedence.py +++ b/tests/parametric/test_headers_precedence.py @@ -1,4 +1,4 @@ -import pytest +from utils import pytest from utils.docker_fixtures.spec.tracecontext import get_tracecontext from utils import scenarios, features diff --git a/tests/parametric/test_headers_tracecontext.py b/tests/parametric/test_headers_tracecontext.py index 540eef4cd18..fa827306f93 100644 --- a/tests/parametric/test_headers_tracecontext.py +++ b/tests/parametric/test_headers_tracecontext.py @@ -8,7 +8,7 @@ # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -import pytest +from utils import pytest from utils.docker_fixtures.spec.tracecontext import get_tracecontext from utils.docker_fixtures.spec.trace import find_span_in_traces, find_only_span diff --git a/tests/parametric/test_headers_tracestate_dd.py b/tests/parametric/test_headers_tracestate_dd.py index d1801bf82cb..cbae4621fbc 100644 --- a/tests/parametric/test_headers_tracestate_dd.py +++ b/tests/parametric/test_headers_tracestate_dd.py @@ -1,4 +1,4 @@ -import pytest +from utils import pytest from utils.docker_fixtures.spec.tracecontext import get_tracecontext from utils import scenarios, features diff --git a/tests/parametric/test_library_tracestats.py b/tests/parametric/test_library_tracestats.py index b57375b4043..05d71b43186 100644 --- a/tests/parametric/test_library_tracestats.py +++ b/tests/parametric/test_library_tracestats.py @@ -1,7 +1,7 @@ import base64 import msgpack -import pytest +from utils import pytest from utils.docker_fixtures.spec.trace import SPAN_MEASURED_KEY diff --git a/tests/parametric/test_llm_observability/conftest.py b/tests/parametric/test_llm_observability/conftest.py index 4e5dc727481..2f5c7e41399 100644 --- a/tests/parametric/test_llm_observability/conftest.py +++ b/tests/parametric/test_llm_observability/conftest.py @@ -1,6 +1,6 @@ from collections.abc import Generator from typing import Any -import pytest +from utils import pytest @pytest.hookimpl(hookwrapper=True) diff --git a/tests/parametric/test_llm_observability/test_llm_observability.py b/tests/parametric/test_llm_observability/test_llm_observability.py index a253a89ce63..72757d1c260 100644 --- a/tests/parametric/test_llm_observability/test_llm_observability.py +++ b/tests/parametric/test_llm_observability/test_llm_observability.py @@ -1,4 +1,4 @@ -import pytest +from utils import pytest from utils import scenarios, features from utils.docker_fixtures import TestAgentAPI diff --git a/tests/parametric/test_llm_observability/test_llm_observability_dne.py b/tests/parametric/test_llm_observability/test_llm_observability_dne.py index f4176e02ee3..504e6c8df29 100644 --- a/tests/parametric/test_llm_observability/test_llm_observability_dne.py +++ b/tests/parametric/test_llm_observability/test_llm_observability_dne.py @@ -1,5 +1,5 @@ from typing import TYPE_CHECKING -import pytest +from utils import pytest from tests.parametric.test_llm_observability.utils import check_and_get_api_key from utils import features, scenarios diff --git a/tests/parametric/test_llm_observability/utils.py b/tests/parametric/test_llm_observability/utils.py index 648f5a6c5cb..bc174ee1292 100644 --- a/tests/parametric/test_llm_observability/utils.py +++ b/tests/parametric/test_llm_observability/utils.py @@ -1,5 +1,5 @@ import os -import pytest +from utils import pytest def check_and_get_api_key(api_key_name: str, *, generate_cassettes: bool = False) -> str | None: diff --git a/tests/parametric/test_otel_api_interoperability.py b/tests/parametric/test_otel_api_interoperability.py index 38258383c6a..a96416cb0b7 100644 --- a/tests/parametric/test_otel_api_interoperability.py +++ b/tests/parametric/test_otel_api_interoperability.py @@ -1,4 +1,4 @@ -import pytest +from utils import pytest from utils import scenarios, features from opentelemetry.trace import SpanKind diff --git a/tests/parametric/test_otel_env_vars.py b/tests/parametric/test_otel_env_vars.py index ff3a0aec4db..7e07b575891 100644 --- a/tests/parametric/test_otel_env_vars.py +++ b/tests/parametric/test_otel_env_vars.py @@ -1,4 +1,4 @@ -import pytest +from utils import pytest from utils import context, scenarios, features from utils.docker_fixtures import TestAgentAPI from utils.docker_fixtures.spec.trace import find_only_span diff --git a/tests/parametric/test_otel_logs.py b/tests/parametric/test_otel_logs.py index 1dfa70eddc8..4905a752bf0 100644 --- a/tests/parametric/test_otel_logs.py +++ b/tests/parametric/test_otel_logs.py @@ -2,7 +2,7 @@ from collections.abc import Generator from urllib.parse import urlparse -import pytest +from utils import pytest from utils import scenarios, features, logger from utils.docker_fixtures.parametric import LogLevel diff --git a/tests/parametric/test_otel_metrics.py b/tests/parametric/test_otel_metrics.py index b4a075e9436..0fe32fa4eb7 100644 --- a/tests/parametric/test_otel_metrics.py +++ b/tests/parametric/test_otel_metrics.py @@ -1,5 +1,5 @@ from urllib.parse import urlparse -import pytest +from utils import pytest from utils import features, scenarios diff --git a/tests/parametric/test_otel_span_methods.py b/tests/parametric/test_otel_span_methods.py index 5837fddff8a..5676b1c0969 100644 --- a/tests/parametric/test_otel_span_methods.py +++ b/tests/parametric/test_otel_span_methods.py @@ -1,6 +1,6 @@ import time -import pytest +from utils import pytest from opentelemetry.trace import StatusCode from opentelemetry.trace import SpanKind diff --git a/tests/parametric/test_otel_span_with_baggage.py b/tests/parametric/test_otel_span_with_baggage.py index d0dda9653c6..e113505bcc6 100644 --- a/tests/parametric/test_otel_span_with_baggage.py +++ b/tests/parametric/test_otel_span_with_baggage.py @@ -1,4 +1,4 @@ -import pytest +from utils import pytest from utils import scenarios, features from .conftest import APMLibrary diff --git a/tests/parametric/test_otel_tracer.py b/tests/parametric/test_otel_tracer.py index 95c09f347f4..1f1ec7817a6 100644 --- a/tests/parametric/test_otel_tracer.py +++ b/tests/parametric/test_otel_tracer.py @@ -1,4 +1,4 @@ -import pytest +from utils import pytest from utils.docker_fixtures.spec.trace import find_trace from utils.docker_fixtures.spec.trace import find_span diff --git a/tests/parametric/test_otel_tracestate_sampling.py b/tests/parametric/test_otel_tracestate_sampling.py index 51208894752..c494fd160be 100644 --- a/tests/parametric/test_otel_tracestate_sampling.py +++ b/tests/parametric/test_otel_tracestate_sampling.py @@ -2,7 +2,7 @@ import json -import pytest +from utils import pytest from utils import features, scenarios from utils.dd_constants import SamplingPriority diff --git a/tests/parametric/test_otlp_trace_metrics.py b/tests/parametric/test_otlp_trace_metrics.py index 7d47b86b0df..1ef00786002 100644 --- a/tests/parametric/test_otlp_trace_metrics.py +++ b/tests/parametric/test_otlp_trace_metrics.py @@ -59,7 +59,7 @@ import time from typing import Any -import pytest +from utils import pytest from google.protobuf.json_format import MessageToDict from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ExportTraceServiceRequest diff --git a/tests/parametric/test_parametric_endpoints.py b/tests/parametric/test_parametric_endpoints.py index 50aa0b905a7..ab3656763a8 100644 --- a/tests/parametric/test_parametric_endpoints.py +++ b/tests/parametric/test_parametric_endpoints.py @@ -8,7 +8,7 @@ from typing import Any -import pytest +from utils import pytest import time from opentelemetry.trace import SpanKind diff --git a/tests/parametric/test_partial_flushing.py b/tests/parametric/test_partial_flushing.py index 8743587f934..246e7e20a61 100644 --- a/tests/parametric/test_partial_flushing.py +++ b/tests/parametric/test_partial_flushing.py @@ -1,4 +1,4 @@ -import pytest +from utils import pytest from utils.docker_fixtures.spec.trace import find_first_span_in_trace_payload, find_span, find_trace from utils import features, scenarios from utils.docker_fixtures import TestAgentAPI diff --git a/tests/parametric/test_process_discovery.py b/tests/parametric/test_process_discovery.py index b7586d79110..4bd9f0fbefd 100644 --- a/tests/parametric/test_process_discovery.py +++ b/tests/parametric/test_process_discovery.py @@ -1,6 +1,6 @@ """Test the instrumented process discovery mechanism feature.""" -import pytest +from utils import pytest import json import msgpack import re diff --git a/tests/parametric/test_sampling_delegation.py b/tests/parametric/test_sampling_delegation.py index 621324c0ced..b75fde4d5ed 100644 --- a/tests/parametric/test_sampling_delegation.py +++ b/tests/parametric/test_sampling_delegation.py @@ -6,7 +6,7 @@ [1]: https://github.com/DataDog/architecture/tree/master/rfcs/apm/integrations/sampling-delegation """ -import pytest +from utils import pytest from utils import features, rfc, scenarios from utils.docker_fixtures import TestAgentAPI from .conftest import APMLibrary diff --git a/tests/parametric/test_sampling_manual.py b/tests/parametric/test_sampling_manual.py index 4c6d879562e..077363c6203 100644 --- a/tests/parametric/test_sampling_manual.py +++ b/tests/parametric/test_sampling_manual.py @@ -6,7 +6,7 @@ Manual keep sampling should take precedence over any other sampling decision. """ -import pytest +from utils import pytest from utils import features, rfc, scenarios from utils.dd_constants import SamplingMechanism from utils.dd_constants import SamplingPriority diff --git a/tests/parametric/test_sampling_span_tags.py b/tests/parametric/test_sampling_span_tags.py index 629e06dc11f..061b71ec208 100644 --- a/tests/parametric/test_sampling_span_tags.py +++ b/tests/parametric/test_sampling_span_tags.py @@ -1,6 +1,6 @@ import json -import pytest +from utils import pytest from utils import scenarios, features from utils.docker_fixtures.spec.trace import MANUAL_DROP_KEY from utils.docker_fixtures.spec.trace import MANUAL_KEEP_KEY diff --git a/tests/parametric/test_span_events.py b/tests/parametric/test_span_events.py index 2cb0d05464d..43fbed17482 100644 --- a/tests/parametric/test_span_events.py +++ b/tests/parametric/test_span_events.py @@ -1,5 +1,5 @@ import json -import pytest +from utils import pytest from utils import scenarios, features, rfc from utils.docker_fixtures.spec.trace import find_span, find_trace diff --git a/tests/parametric/test_span_links.py b/tests/parametric/test_span_links.py index 521bf4fb904..020290d5fe8 100644 --- a/tests/parametric/test_span_links.py +++ b/tests/parametric/test_span_links.py @@ -1,5 +1,5 @@ import json -import pytest +from utils import pytest from utils.docker_fixtures.spec.trace import ORIGIN from utils.docker_fixtures.spec.trace import SAMPLING_PRIORITY_KEY diff --git a/tests/parametric/test_span_sampling.py b/tests/parametric/test_span_sampling.py index ddad4be025e..ade592d2015 100644 --- a/tests/parametric/test_span_sampling.py +++ b/tests/parametric/test_span_sampling.py @@ -1,6 +1,6 @@ import time import json -import pytest +from utils import pytest from utils.docker_fixtures.spec.trace import SAMPLING_PRIORITY_KEY from utils.docker_fixtures.spec.trace import SINGLE_SPAN_SAMPLING_MAX_PER_SEC from utils.docker_fixtures.spec.trace import SINGLE_SPAN_SAMPLING_MECHANISM diff --git a/tests/parametric/test_startup_logs.py b/tests/parametric/test_startup_logs.py index e50399b3e73..6760fb9a904 100644 --- a/tests/parametric/test_startup_logs.py +++ b/tests/parametric/test_startup_logs.py @@ -2,7 +2,7 @@ import re -import pytest +from utils import pytest from utils import scenarios, features, context, logger from .conftest import APMLibrary diff --git a/tests/parametric/test_telemetry.py b/tests/parametric/test_telemetry.py index 410d76d8c34..5cbc7ccc0bf 100644 --- a/tests/parametric/test_telemetry.py +++ b/tests/parametric/test_telemetry.py @@ -5,7 +5,7 @@ import time import uuid -import pytest +from utils import pytest from .conftest import StableConfigWriter from utils.telemetry_utils import TelemetryUtils diff --git a/tests/parametric/test_trace_filters.py b/tests/parametric/test_trace_filters.py index 9aec46740c7..7785270b84e 100644 --- a/tests/parametric/test_trace_filters.py +++ b/tests/parametric/test_trace_filters.py @@ -1,5 +1,5 @@ import json -import pytest +from utils import pytest from .conftest import APMLibrary from .utils import MIN_AGENT_VERSION_FOR_CSS, enable_tracestats diff --git a/tests/parametric/test_trace_sampling.py b/tests/parametric/test_trace_sampling.py index 2fd68ca787c..d38925b752e 100644 --- a/tests/parametric/test_trace_sampling.py +++ b/tests/parametric/test_trace_sampling.py @@ -1,6 +1,6 @@ import json -import pytest +from utils import pytest import random from utils.docker_fixtures.spec.trace import find_only_span, find_span_in_traces diff --git a/tests/parametric/test_tracer.py b/tests/parametric/test_tracer.py index 21be477630c..fae0f54c895 100644 --- a/tests/parametric/test_tracer.py +++ b/tests/parametric/test_tracer.py @@ -1,4 +1,4 @@ -import pytest +from utils import pytest from utils.docker_fixtures.spec.trace import find_trace from utils.docker_fixtures.spec.trace import find_span diff --git a/tests/parametric/test_tracer_flare.py b/tests/parametric/test_tracer_flare.py index 0852f6d2313..c93a75b3f44 100644 --- a/tests/parametric/test_tracer_flare.py +++ b/tests/parametric/test_tracer_flare.py @@ -7,7 +7,7 @@ from typing import Any from uuid import uuid4 -import pytest +from utils import pytest from utils import rfc, scenarios, features, context from utils.dd_constants import RemoteConfigApplyState diff --git a/tests/parametric/utils.py b/tests/parametric/utils.py index 8a962dc035e..951cb919421 100644 --- a/tests/parametric/utils.py +++ b/tests/parametric/utils.py @@ -1,4 +1,4 @@ -import pytest +from utils import pytest from utils import context diff --git a/tests/stats/test_stats.py b/tests/stats/test_stats.py index 68c6ee2b91b..3f44513b209 100644 --- a/tests/stats/test_stats.py +++ b/tests/stats/test_stats.py @@ -1,5 +1,5 @@ import contextlib -import pytest +from utils import pytest from utils import features, interfaces, logger, scenarios, weblog diff --git a/tests/test_library_conf.py b/tests/test_library_conf.py index 00da9e1c0ee..3581614b761 100644 --- a/tests/test_library_conf.py +++ b/tests/test_library_conf.py @@ -1,7 +1,7 @@ # Unless explicitly stated otherwise all files in this repository are licensed under the the Apache License Version 2.0. # This product includes software developed at Datadog (https://www.datadoghq.com/). # Copyright 2021 Datadog, Inc. -import pytest +from utils import pytest from utils import weblog, interfaces, scenarios, features from utils.dd_types import DataDogAgentSpan from utils._context.header_tag_vars import ( diff --git a/tests/test_the_test/scenarios.json b/tests/test_the_test/scenarios.json index 7030b60d222..10a01ebe3ca 100644 --- a/tests/test_the_test/scenarios.json +++ b/tests/test_the_test/scenarios.json @@ -5698,6 +5698,12 @@ "tests/test_the_test/test_decorators.py::Test_Skips::test_regular": [ "TEST_THE_TEST" ], + "tests/test_the_test/test_decorators.py::Test_PytestProxy::test_exposes_approved_apis": [ + "TEST_THE_TEST" + ], + "tests/test_the_test/test_decorators.py::Test_PytestProxy::test_hides_internal_force_skip_marker": [ + "TEST_THE_TEST" + ], "tests/test_the_test/test_decorators.py::test_version_range": [ "TEST_THE_TEST" ], diff --git a/tests/test_the_test/test_ai_guard_span_helpers.py b/tests/test_the_test/test_ai_guard_span_helpers.py index 6af27eea5b3..113fc536f2d 100644 --- a/tests/test_the_test/test_ai_guard_span_helpers.py +++ b/tests/test_the_test/test_ai_guard_span_helpers.py @@ -6,7 +6,7 @@ import re from typing import Any -import pytest +from utils import pytest import requests from tests.integration_frameworks.llm import utils as llm_utils diff --git a/tests/test_the_test/test_build_base_images.py b/tests/test_the_test/test_build_base_images.py index e28f703b849..7deeebcec8f 100644 --- a/tests/test_the_test/test_build_base_images.py +++ b/tests/test_the_test/test_build_base_images.py @@ -8,7 +8,7 @@ import tempfile import textwrap -import pytest +from utils import pytest from utils import scenarios from utils.base_images import build_base_images diff --git a/tests/test_the_test/test_build_pipeline.py b/tests/test_the_test/test_build_pipeline.py index bf8571f48ab..75c02d0a510 100644 --- a/tests/test_the_test/test_build_pipeline.py +++ b/tests/test_the_test/test_build_pipeline.py @@ -4,7 +4,7 @@ import re from pathlib import Path -import pytest +from utils import pytest import yaml from utils import scenarios diff --git a/tests/test_the_test/test_compute_libraries_and_scenarios.py b/tests/test_the_test/test_compute_libraries_and_scenarios.py index 4108b2784cd..ada69313f54 100644 --- a/tests/test_the_test/test_compute_libraries_and_scenarios.py +++ b/tests/test_the_test/test_compute_libraries_and_scenarios.py @@ -4,7 +4,7 @@ from functools import wraps -import pytest +from utils import pytest from utils.scripts.compute_libraries_and_scenarios import Inputs, process from utils import scenarios diff --git a/tests/test_the_test/test_decorators.py b/tests/test_the_test/test_decorators.py index aae54c195f4..b3bf15fa989 100644 --- a/tests/test_the_test/test_decorators.py +++ b/tests/test_the_test/test_decorators.py @@ -1,9 +1,8 @@ import sys import logging from typing import Any -import pytest -from utils import irrelevant, missing_feature, flaky, rfc, logger +from utils import irrelevant, missing_feature, flaky, pytest, rfc, logger from utils._decorators import add_pytest_marker from utils.manifest import TestDeclaration @@ -84,5 +83,15 @@ def test_invalid() -> None: add_pytest_marker(test_invalid, TestDeclaration.BUG, "APPSEC-123 & APPSEC-456") +class Test_PytestProxy: + def test_exposes_approved_apis(self): + assert callable(pytest.fixture) + assert callable(pytest.raises) + assert callable(pytest.mark.parametrize) + + def test_hides_internal_force_skip_marker(self): + assert not hasattr(pytest.mark, "skip_if_xfail") + + if __name__ == "__main__": sys.exit("Usage: pytest utils/test_the_test.py") diff --git a/tests/test_the_test/test_deserializer.py b/tests/test_the_test/test_deserializer.py index 85c3fd6c4de..74a7d097985 100644 --- a/tests/test_the_test/test_deserializer.py +++ b/tests/test_the_test/test_deserializer.py @@ -8,7 +8,7 @@ ) import base64 import msgpack -import pytest +from utils import pytest @scenarios.test_the_test diff --git a/tests/test_the_test/test_docker_run_cleanup.py b/tests/test_the_test/test_docker_run_cleanup.py index 2af4b165bc7..ea5f222269e 100644 --- a/tests/test_the_test/test_docker_run_cleanup.py +++ b/tests/test_the_test/test_docker_run_cleanup.py @@ -4,7 +4,7 @@ from _pytest.outcomes import Failed from docker.errors import APIError, NotFound -import pytest +from utils import pytest from utils import scenarios from utils.docker_fixtures import _core as docker_core diff --git a/tests/test_the_test/test_docker_scenario.py b/tests/test_the_test/test_docker_scenario.py index 51e0cebf268..84f90a55fde 100644 --- a/tests/test_the_test/test_docker_scenario.py +++ b/tests/test_the_test/test_docker_scenario.py @@ -1,7 +1,7 @@ from threading import RLock from unittest.mock import MagicMock -import pytest +from utils import pytest from utils import interfaces, scenarios from utils._context._scenarios.endtoend import DdTraceEndToEndScenario, DockerScenario diff --git a/tests/test_the_test/test_easy_win.py b/tests/test_the_test/test_easy_win.py index 382ef7eca3b..873a79c3732 100644 --- a/tests/test_the_test/test_easy_win.py +++ b/tests/test_the_test/test_easy_win.py @@ -5,7 +5,7 @@ from pathlib import Path -import pytest +from utils import pytest import yaml from utils.manifest._internal.types import Condition, SkipDeclaration, SemverRange diff --git a/tests/test_the_test/test_external_gitlab_pipeline.py b/tests/test_the_test/test_external_gitlab_pipeline.py index 6dd7fee9fed..4e5b3e1ebf9 100644 --- a/tests/test_the_test/test_external_gitlab_pipeline.py +++ b/tests/test_the_test/test_external_gitlab_pipeline.py @@ -1,6 +1,6 @@ """Tests for utils/scripts/ci_orchestrators/external_gitlab_pipeline.py.""" -import pytest +from utils import pytest from utils import scenarios from utils.scripts.ci_orchestrators.external_gitlab_pipeline import ( diff --git a/tests/test_the_test/test_features.py b/tests/test_the_test/test_features.py index 0ad561dc831..6439543b48c 100644 --- a/tests/test_the_test/test_features.py +++ b/tests/test_the_test/test_features.py @@ -1,4 +1,4 @@ -import pytest +from utils import pytest from collections.abc import Callable from utils import scenarios, features, logger from utils._features import NOT_REPORTED_ID diff --git a/tests/test_the_test/test_force_option.py b/tests/test_the_test/test_force_option.py index 42e6a1539c2..62c04318147 100644 --- a/tests/test_the_test/test_force_option.py +++ b/tests/test_the_test/test_force_option.py @@ -1,5 +1,5 @@ import os -import pytest +from utils import pytest from utils import bug, irrelevant, scenarios, features from utils._context._scenarios import Scenario diff --git a/tests/test_the_test/test_get_image_list.py b/tests/test_the_test/test_get_image_list.py index 1e7b40016c4..dfdbd780c0a 100644 --- a/tests/test_the_test/test_get_image_list.py +++ b/tests/test_the_test/test_get_image_list.py @@ -5,7 +5,7 @@ import subprocess import sys -import pytest +from utils import pytest import yaml from utils import scenarios diff --git a/tests/test_the_test/test_github_nightly.py b/tests/test_the_test/test_github_nightly.py index 01fb374435d..a915a34f8e6 100644 --- a/tests/test_the_test/test_github_nightly.py +++ b/tests/test_the_test/test_github_nightly.py @@ -4,7 +4,7 @@ from dataclasses import dataclass from pathlib import Path -import pytest +from utils import pytest from utils import scenarios from utils.ci.github import nightly diff --git a/tests/test_the_test/test_json_report.py b/tests/test_the_test/test_json_report.py index 0a83819736c..08221b5e921 100644 --- a/tests/test_the_test/test_json_report.py +++ b/tests/test_the_test/test_json_report.py @@ -1,6 +1,6 @@ import os import json -import pytest +from utils import pytest from utils import missing_feature, irrelevant, scenarios, rfc, features, bug, flaky, logger diff --git a/tests/test_the_test/test_junit.py b/tests/test_the_test/test_junit.py index 3106bfdece1..2ea8cc3084e 100644 --- a/tests/test_the_test/test_junit.py +++ b/tests/test_the_test/test_junit.py @@ -4,7 +4,7 @@ import xml.etree.ElementTree as ET from xml.dom import minidom -import pytest +from utils import pytest from utils import scenarios, features, irrelevant, bug, flaky, missing_feature, slow from .utils import run_system_tests diff --git a/tests/test_the_test/test_manifest.py b/tests/test_the_test/test_manifest.py index 1e5ba8be614..2708533e352 100644 --- a/tests/test_the_test/test_manifest.py +++ b/tests/test_the_test/test_manifest.py @@ -2,7 +2,7 @@ import shutil import tempfile import textwrap -import pytest +from utils import pytest from utils import scenarios from utils._context.component_version import Version from utils.manifest import Manifest, SkipDeclaration, TestDeclaration diff --git a/tests/test_the_test/test_minimal_number_of_scenarios.py b/tests/test_the_test/test_minimal_number_of_scenarios.py index 32f397dc3b2..06cf0bbe1ba 100644 --- a/tests/test_the_test/test_minimal_number_of_scenarios.py +++ b/tests/test_the_test/test_minimal_number_of_scenarios.py @@ -2,7 +2,7 @@ import json from typing import Any -import pytest +from utils import pytest from utils._context._scenarios import get_all_scenarios, scenarios from utils._context._scenarios.endtoend import EndToEndScenario diff --git a/tests/test_the_test/test_mock_backend_v2.py b/tests/test_the_test/test_mock_backend_v2.py index c8f8419c770..f64a18918a8 100644 --- a/tests/test_the_test/test_mock_backend_v2.py +++ b/tests/test_the_test/test_mock_backend_v2.py @@ -5,7 +5,7 @@ from collections.abc import Generator from pathlib import Path -import pytest +from utils import pytest import requests import zstandard diff --git a/tests/test_the_test/test_mock_ffe_agentless_backend.py b/tests/test_the_test/test_mock_ffe_agentless_backend.py index d3568744b2b..0a13ed2f9df 100644 --- a/tests/test_the_test/test_mock_ffe_agentless_backend.py +++ b/tests/test_the_test/test_mock_ffe_agentless_backend.py @@ -5,7 +5,7 @@ from unittest.mock import MagicMock import requests -import pytest +from utils import pytest from utils import features, scenarios from utils._context.containers import ServerlessInitContainer diff --git a/tests/test_the_test/test_scrubber.py b/tests/test_the_test/test_scrubber.py index 96915ef4fa2..650b1fd02f4 100644 --- a/tests/test_the_test/test_scrubber.py +++ b/tests/test_the_test/test_scrubber.py @@ -3,7 +3,7 @@ import os from pathlib import Path import subprocess -import pytest +from utils import pytest from utils import scenarios, missing_feature, logger diff --git a/tests/test_the_test/test_slow_and_crash_decorators.py b/tests/test_the_test/test_slow_and_crash_decorators.py index 34f6260b7aa..bb40c4fb702 100644 --- a/tests/test_the_test/test_slow_and_crash_decorators.py +++ b/tests/test_the_test/test_slow_and_crash_decorators.py @@ -2,8 +2,6 @@ import textwrap from pathlib import Path -import pytest - from utils import bug, missing_feature, scenarios, features, slow, scenario_crash from .utils import run_system_tests @@ -135,25 +133,6 @@ def test_scenario_crash_on_class_with_declaration_is_skipped(self): assert tests[nodeid]["outcome"] == "skipped" -@scenarios.test_the_test -class Test_SkipIfXfail: - """Test that tests with both skip_if_xfail and declaration markers are skipped.""" - - def test_skip_if_xfail_with_declaration_is_skipped(self): - """Test that a test marked with both skip_if_xfail and a declaration marker is skipped.""" - tests = run_system_tests(test_path=FILENAME) - - nodeid = f"{FILENAME}::Test_SkipIfXfailMock::test_with_both_markers" - assert tests[nodeid]["outcome"] == "skipped" - - def test_skip_if_xfail_without_declaration_is_not_skipped(self): - """Test that a test marked with only skip_if_xfail (no declaration) is not skipped.""" - tests = run_system_tests(test_path=FILENAME) - - nodeid = f"{FILENAME}::Test_SkipIfXfailMock::test_skip_if_xfail_only" - assert tests[nodeid]["outcome"] == "passed" - - # Mock test classes used by the test scenarios above @@ -187,21 +166,6 @@ def test_scenario_crash_with_missing_feature(self): assert True -@scenarios.mock_the_test -@features.adaptive_sampling -class Test_SkipIfXfailMock: - @bug(condition=True, reason="FAKE-001") - @pytest.mark.skip_if_xfail - def test_with_both_markers(self): - """Test with both skip_if_xfail and declaration markers - should be skipped.""" - assert True - - @pytest.mark.skip_if_xfail - def test_skip_if_xfail_only(self): - """Test with only skip_if_xfail marker, no declaration - should NOT be skipped.""" - assert True - - @scenarios.mock_the_test @features.adaptive_sampling class Test_SlowManifestMock: diff --git a/tests/test_the_test/test_telemetry_heartbeat.py b/tests/test_the_test/test_telemetry_heartbeat.py index d5f1e1b8dcc..60dfe496c34 100644 --- a/tests/test_the_test/test_telemetry_heartbeat.py +++ b/tests/test_the_test/test_telemetry_heartbeat.py @@ -1,7 +1,7 @@ from datetime import datetime, timedelta, UTC from typing import Any -import pytest +from utils import pytest from tests.test_telemetry_heartbeat_utils import heartbeat_delays_by_runtime diff --git a/tests/test_the_test/test_update_mirror_images.py b/tests/test_the_test/test_update_mirror_images.py index 235d750a372..ae8af935525 100644 --- a/tests/test_the_test/test_update_mirror_images.py +++ b/tests/test_the_test/test_update_mirror_images.py @@ -3,7 +3,7 @@ from pathlib import Path import subprocess -import pytest +from utils import pytest import yaml from utils import scenarios diff --git a/tests/test_the_test/test_version.py b/tests/test_the_test/test_version.py index d46f36b38d7..bd1ac277a7e 100644 --- a/tests/test_the_test/test_version.py +++ b/tests/test_the_test/test_version.py @@ -1,4 +1,4 @@ -import pytest +from utils import pytest import semantic_version as semver from utils.manifest._internal.types import SemverRange as CustomSpec from utils._context.component_version import ComponentVersion, Version diff --git a/utils/ci/gitlab/main.yml b/utils/ci/gitlab/main.yml index cec5958ba53..a3340447534 100644 --- a/utils/ci/gitlab/main.yml +++ b/utils/ci/gitlab/main.yml @@ -79,7 +79,7 @@ variables: # Tag = first 12 chars of sha256(utils/ci/gitlab/docker/system-tests.Dockerfile + requirements.txt) # Update this when either file changes: # cat utils/ci/gitlab/docker/system-tests.Dockerfile requirements.txt | sha256sum | cut -c1-12 - CI_IMAGE: "registry.ddbuild.io/system-tests/ci-runner:1728376181ec" + CI_IMAGE: "registry.ddbuild.io/system-tests/ci-runner:65e2f06534fd" SYSTEM_TESTS_SPLIT_PIPELINE: "$[[ inputs.split_pipeline ]]" .system_tests_param_base: diff --git a/utils/format/__init__.py b/utils/format/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/utils/format/import_linter_contracts.py b/utils/format/import_linter_contracts.py new file mode 100644 index 00000000000..094ad1eafc3 --- /dev/null +++ b/utils/format/import_linter_contracts.py @@ -0,0 +1,80 @@ +"""Repository-specific Import Linter contracts.""" + +from typing import TYPE_CHECKING, TypedDict, cast + +from grimp import ImportGraph +from importlinter import Contract, ContractCheck, fields, output + +if TYPE_CHECKING: + from importlinter.domain.imports import ImportExpression + + +class _CrossTestImport(TypedDict): + importer: str + imported: str + line_numbers: tuple[int, ...] + + +def _is_test_module(module: str) -> bool: + return module.startswith("tests.") and module.rsplit(".", maxsplit=1)[-1].startswith("test_") + + +class NoCrossTestImportsContract(Contract): + """Prevent test modules from importing other test modules directly.""" + + ignore_imports = fields.SetField(subfield=fields.ImportExpressionField(), required=False) + + def check(self, graph: ImportGraph, verbose: bool) -> ContractCheck: # noqa: ARG002, FBT001 + ignored_imports = self._resolve_ignored_imports(graph) + existing_cross_test_imports: set[tuple[str, str]] = set() + violations: list[_CrossTestImport] = [] + + for importer in sorted(graph.modules): + if not _is_test_module(importer): + continue + + for imported in sorted(graph.find_modules_directly_imported_by(importer)): + if not _is_test_module(imported) or importer == imported: + continue + existing_cross_test_imports.add((importer, imported)) + if (importer, imported) in ignored_imports: + continue + + details = graph.get_import_details(importer=importer, imported=imported) + violations.append( + { + "importer": importer, + "imported": imported, + "line_numbers": tuple(detail["line_number"] for detail in details), + } + ) + + unused_ignores = sorted(ignored_imports - existing_cross_test_imports) + return ContractCheck( + kept=not violations and not unused_ignores, + metadata={"unused_ignores": unused_ignores, "violations": violations}, + ) + + def render_broken_contract(self, check: ContractCheck) -> None: + violations = cast("list[_CrossTestImport]", check.metadata["violations"]) + for violation in violations: + lines = ", ".join(f"l.{line_number}" for line_number in violation["line_numbers"]) + output.print_error( + f"{violation['importer']} imports {violation['imported']} ({lines})", + bold=False, + ) + + unused_ignores = cast("list[tuple[str, str]]", check.metadata["unused_ignores"]) + for importer, imported in unused_ignores: + output.print_error(f"Unused exception: {importer} -> {imported}", bold=False) + + def _resolve_ignored_imports(self, graph: ImportGraph) -> set[tuple[str, str]]: + ignored_imports: set[tuple[str, str]] = set() + expressions = cast("set[ImportExpression] | None", self.ignore_imports) + + for expression in expressions or set(): + importers = graph.find_matching_modules(expression.importer.expression) + imported_modules = graph.find_matching_modules(expression.imported.expression) + ignored_imports.update((importer, imported) for importer in importers for imported in imported_modules) + + return ignored_imports diff --git a/utils/pytest.py b/utils/pytest.py new file mode 100644 index 00000000000..4e4fdead182 --- /dev/null +++ b/utils/pytest.py @@ -0,0 +1,61 @@ +"""Approved pytest API for system tests. + +Test modules should import this facade with ``from utils import pytest``. +The facade intentionally exposes only the pytest APIs used by this repository. +In particular, force-skip markers are available only through the semantic +decorators exported by :mod:`utils`. +""" + +import pytest as _pytest # noqa: PT013 - keep the underlying pytest module private + + +Config = _pytest.Config +FixtureRequest = _pytest.FixtureRequest +Item = _pytest.Item +Mark = _pytest.Mark +MarkDecorator = _pytest.MarkDecorator +MonkeyPatch = _pytest.MonkeyPatch +CaptureFixture = _pytest.CaptureFixture +Session = _pytest.Session +CallInfo = _pytest.CallInfo +TempPathFactory = _pytest.TempPathFactory + +approx = _pytest.approx +exit = _pytest.exit # noqa: A001 - preserve the public pytest API name +fail = _pytest.fail +fixture = _pytest.fixture +param = _pytest.param +raises = _pytest.raises +skip = _pytest.skip +hookimpl = _pytest.hookimpl + + +class _AllowedMarks: + """Pytest markers approved for direct use by system tests.""" + + features = _pytest.mark.features + parametrize = _pytest.mark.parametrize + scenario = _pytest.mark.scenario + + +mark = _AllowedMarks() + +__all__ = [ + "CallInfo", + "Config", + "FixtureRequest", + "Item", + "Mark", + "MarkDecorator", + "MonkeyPatch", + "Session", + "approx", + "exit", + "fail", + "fixture", + "hookimpl", + "mark", + "param", + "raises", + "skip", +]