From 8d3d2fe028539de1979e1d431b8eee6365212212 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Thu, 6 Aug 2026 13:48:31 +0200 Subject: [PATCH 01/12] Adding buildx remote cache to the build jobs in gitlab --- tests/test_the_test/test_build_pipeline.py | 64 ++++++++++++++++++++++ utils/ci/gitlab/build_pipeline.py | 46 ++++++++++++++-- utils/ci/gitlab/main.yml | 6 +- utils/ci/gitlab/system-tests.yml.j2 | 24 +------- 4 files changed, 107 insertions(+), 33 deletions(-) diff --git a/tests/test_the_test/test_build_pipeline.py b/tests/test_the_test/test_build_pipeline.py index bf8571f48ab..98760623fae 100644 --- a/tests/test_the_test/test_build_pipeline.py +++ b/tests/test_the_test/test_build_pipeline.py @@ -134,3 +134,67 @@ def test_c_pipeline_renders_three_scenarios_and_package_artifact(self, tmp_path: for job_name in expected_run_jobs: assert ".system_tests_base" in pipeline[job_name]["extends"] + + def test_buildx_cache_update_from_main(self, tmp_path: Path) -> None: + params = { + "endtoend_defs": { + "parallel_weblogs": [{"name": "perl-mojolicious"}], + "parallel_jobs": [ + { + "weblog": "perl-mojolicious", + "scenarios": ["DEFAULT", "SAMPLING", "IPV6"], + "weblog_build_required": True, + } + ], + }, + "miscs": {"binaries_artifact": ""}, + "parametric": {"enable": False, "parallel_jobs": []}, + } + (tmp_path / "params_c.json").write_text(json.dumps(params)) + out = tmp_path / "out" + + build( + ["c"], + tmp_path, + out, + stage="e2e", + ci_image="myimage", + chunks=1, + binaries_artifacts="system_tests_package_refs", + binaries_artifact_path="system-tests-binaries", + ref="main", + ) + + assert re.search("--cache-to=type=registry,ref=", (out / "generated-pipeline-chunk-0.yml").read_text()) + + def test_buildx_cache_does_not_update_from_not_main(self, tmp_path: Path) -> None: + params = { + "endtoend_defs": { + "parallel_weblogs": [{"name": "perl-mojolicious"}], + "parallel_jobs": [ + { + "weblog": "perl-mojolicious", + "scenarios": ["DEFAULT", "SAMPLING", "IPV6"], + "weblog_build_required": True, + } + ], + }, + "miscs": {"binaries_artifact": ""}, + "parametric": {"enable": False, "parallel_jobs": []}, + } + (tmp_path / "params_c.json").write_text(json.dumps(params)) + out = tmp_path / "out" + + build( + ["c"], + tmp_path, + out, + stage="e2e", + ci_image="myimage", + chunks=1, + binaries_artifacts="system_tests_package_refs", + binaries_artifact_path="system-tests-binaries", + ref="some-branch", + ) + + assert not re.search("--cache-to=type=registry,ref=", (out / "generated-pipeline-chunk-0.yml").read_text()) diff --git a/utils/ci/gitlab/build_pipeline.py b/utils/ci/gitlab/build_pipeline.py index 6f8984416bf..6d24c229ee6 100644 --- a/utils/ci/gitlab/build_pipeline.py +++ b/utils/ci/gitlab/build_pipeline.py @@ -26,6 +26,45 @@ def noop_stub(stage: str) -> str: """ +def _ensure_quoted(s: str) -> str: + ret = s + if not s.startswith('"'): + ret = '"' + ret + if not s.endswith('"'): + ret = ret + '"' + return ret + + +def _trace(name: str, command: str, library: str, weblog: str = "", scenario: str = "") -> str: + return "".join( + [ + "datadog-ci trace ", + f"--name {_ensure_quoted(name)} ", + f"--tags system-tests.library:{library} ", + f"--tags system-tests.weblog:{weblog} " if weblog else "", + f"--tags system-tests.scenario:{scenario} " if scenario else "", + f"-- {command}", + ] + ) + + +def _render_build(library: str, weblog: str, *, push: bool = False) -> str: + registry_base = "registry.ddbuild.io/system-tests/cache" + ref = f"{registry_base}/{library}/{weblog}:main" + command = "".join( + [ + f"./build.sh {library} ", + "-i weblog --save-to-binaries ", + f"--weblog-variant {weblog} ", + '--extra-docker-args "', + f"--cache-from=type=registry,ref={ref} ", + f"--cache-to=type=registry,ref={ref},mode=max" if push else "", + '"', + ] + ) + return _trace(f'"build {library} {weblog}"', command, library, weblog) + + def render_library( library: str, params: dict, @@ -35,7 +74,6 @@ def render_library( ci_image: str, ref: str, push_to_test_optimization: bool, - docker_auth: bool, binaries_artifact_path: str, binaries_artifacts: str, pipeline_start_time: str, @@ -71,8 +109,9 @@ def render_library( ref=ref, push_to_test_optimization=push_to_test_optimization, skip_header=skip_header, - docker_auth_enabled=docker_auth, pipeline_start_time=pipeline_start_time, + render_build=_render_build, + trace=_trace, ) @@ -86,7 +125,6 @@ def build( ref: str = "", push_to_test_optimization: bool = False, chunks: int = 3, - docker_auth: bool = False, binaries_artifact_path: str = "", binaries_artifacts: str = "", pipeline_start_time: str = "", @@ -126,7 +164,6 @@ def build( ci_image=ci_image, ref=ref, push_to_test_optimization=push_to_test_optimization, - docker_auth=docker_auth, binaries_artifact_path=binaries_artifact_path, binaries_artifacts=binaries_artifacts, pipeline_start_time=pipeline_start_time, @@ -181,7 +218,6 @@ def main(argv: list[str] | None = None) -> int: ref=args.ref, push_to_test_optimization=args.push_to_test_optimization == "true", chunks=args.chunks, - docker_auth=args.docker_auth == "true", binaries_artifact_path=args.binaries_artifact_path, binaries_artifacts=args.binaries_artifacts, pipeline_start_time=args.pipeline_start_time, diff --git a/utils/ci/gitlab/main.yml b/utils/ci/gitlab/main.yml index 889b4d41a92..011f083cde7 100644 --- a/utils/ci/gitlab/main.yml +++ b/utils/ci/gitlab/main.yml @@ -47,9 +47,6 @@ spec: condition: description: "GitLab CI rule expression controlling whether system-tests jobs run (e.g. '$NIGHTLY_BUILD == \"true\"'). Defaults to always-run." default: "null == null" - docker_auth: - description: "Whether to authenticate calls to docker hub" - default: "false" split_pipeline: description: "Split jobs across multiple child pipelines" type: boolean @@ -122,7 +119,6 @@ system_tests_build_pipeline: force_execute=$(echo "$[[ inputs.force_execute ]],$FORCE_EXECUTE" | tr ',' '\n' | sed '/^$/d' | tr '\n' ',' | sed 's/,$//') parametric_job_count="${SYSTEM_TESTS_PARAMETRIC_JOB_COUNT:-$[[ inputs.parametric_job_count ]]}" push_to_test_optimization="${SYSTEM_TESTS_PUSH_TO_TEST_OPTIMIZATION:-$[[ inputs.push_to_test_optimization ]]}" - docker_auth="${SYSTEM_TESTS_DOCKER_AUTH:-$[[ inputs.docker_auth ]]}" skip_empty_scenario="${SYSTEM_TESTS_SKIP_EMPTY_SCENARIO:-$[[ inputs.skip_empty_scenarios ]]}" echo "libraries: $libraries" echo "scenarios: $scenarios" @@ -141,7 +137,7 @@ system_tests_build_pipeline: done chunks=1 if [ "$SYSTEM_TESTS_SPLIT_PIPELINE" = "true" ]; then chunks=3; fi - python3 utils/ci/gitlab/build_pipeline.py --stage $[[ inputs.stage ]] --ci-image "$CI_IMAGE" --ref "$[[ inputs.ref ]]" --push-to-test-optimization "$push_to_test_optimization" --libraries "$libraries" --params-dir . --output-dir . --chunks $chunks --docker-auth "$docker_auth" --binaries-artifact-path "$binaries_artifact_path" --binaries-artifacts "$binaries_artifacts" --pipeline-start-time "$SYSTEM_TESTS_PIPELINE_START_TIME" + python3 utils/ci/gitlab/build_pipeline.py --stage $[[ inputs.stage ]] --ci-image "$CI_IMAGE" --ref "$[[ inputs.ref ]]" --push-to-test-optimization "$push_to_test_optimization" --libraries "$libraries" --params-dir . --output-dir . --chunks $chunks --binaries-artifact-path "$binaries_artifact_path" --binaries-artifacts "$binaries_artifacts" --pipeline-start-time "$SYSTEM_TESTS_PIPELINE_START_TIME" artifacts: paths: - system-tests/generated-pipeline-chunk-*.yml diff --git a/utils/ci/gitlab/system-tests.yml.j2 b/utils/ci/gitlab/system-tests.yml.j2 index 695aa868f83..40def8353bf 100644 --- a/utils/ci/gitlab/system-tests.yml.j2 +++ b/utils/ci/gitlab/system-tests.yml.j2 @@ -1,16 +1,3 @@ -{% macro docker_auth() %} - - section_start "docker_auth" "Docker hub auth" - - export DOCKER_LOGIN=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-write --with-decryption --query "Parameter.Value" --out text) - - export DOCKER_LOGIN_PASS=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-pass-write --with-decryption --query "Parameter.Value" --out text) - - | - for i in 1 2 3; do - echo "$DOCKER_LOGIN_PASS" | docker login --username "$DOCKER_LOGIN" --password-stdin && break - if [ "$i" -eq 3 ]; then echo "docker login failed after 3 attempts"; exit 1; fi - echo "docker login failed (attempt $i), retrying in $((i*5))s..." - sleep $((i*5)) - done - - section_end "docker_auth" -{% endmacro %} {% macro copy_binaries(path) %} - section_start "copy_binaries" "Copying pre-built binaries into binaries/" - mkdir -p binaries @@ -30,9 +17,6 @@ echo "SYSTEM_TESTS_GENERATED_PIPELINE_START_TIME not set or not numeric ('$SYSTEM_TESTS_GENERATED_PIPELINE_START_TIME'), skipping metric emission" fi {% endmacro %} -{% macro trace(name, command, library, weblog=none, scenario=none) -%} -datadog-ci trace --name "{{name}}" --tags system-tests.library:{{library}}{% if weblog %} --tags system-tests.weblog:{{weblog}}{% endif %}{% if scenario %} --tags system-tests.scenario:{{scenario}}{% endif %} -- {{command}} -{%- endmacro %} {% if not skip_header %} workflow: name: "System-tests end to end" @@ -104,15 +88,12 @@ system_tests_build_{{library}}_{{variant}}: {% endfor %} {% endif %} script: - {% if docker_auth_enabled %} - {{ docker_auth() }} - {% endif %} {% if binaries_artifacts_list and binaries_artifact_path %} {{ copy_binaries(binaries_artifact_path) }} {% endif %} - section_start "build" "Building weblog" false {{ job_tag("build") }} - - {{ trace("build " ~ library ~ " " ~ variant, "./build.sh " ~ library ~ " -i weblog --save-to-binaries --weblog-variant " ~ variant, library, weblog=variant) }} + - {{ render_build(library, variant, push = ref=="main") }} - mv binaries .. - section_end "build" artifacts: @@ -146,9 +127,6 @@ system_tests_run_{{library}}_{{scenario}}_{{variant}}: {% endfor %} {% endif %} script: - {% if docker_auth_enabled %} - {{ docker_auth() }} - {% endif %} - section_start "weblog_setup" "Setting up the weblog" {% if build_required %} - mv ../binaries/* binaries/ From 92ce5b4d277c41589d4064796c74ecec79f16fa4 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Mon, 10 Aug 2026 11:54:48 +0200 Subject: [PATCH 02/12] Test --- utils/ci/gitlab/system-tests.yml.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/ci/gitlab/system-tests.yml.j2 b/utils/ci/gitlab/system-tests.yml.j2 index 40def8353bf..6cae8ad4554 100644 --- a/utils/ci/gitlab/system-tests.yml.j2 +++ b/utils/ci/gitlab/system-tests.yml.j2 @@ -93,7 +93,7 @@ system_tests_build_{{library}}_{{variant}}: {% endif %} - section_start "build" "Building weblog" false {{ job_tag("build") }} - - {{ render_build(library, variant, push = ref=="main") }} + - {{ render_build(library, variant, push = true) }} - mv binaries .. - section_end "build" artifacts: From 3cbb9a314c8bb9f5192f74356c3df1ca0a7e2460 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Mon, 10 Aug 2026 14:32:44 +0200 Subject: [PATCH 03/12] Add cache entry for lib main branch --- tests/test_the_test/test_build_pipeline.py | 61 +++++++++++++++++++++- utils/ci/gitlab/build_pipeline.py | 60 +++++++++++++++------ utils/ci/gitlab/system-tests.yml.j2 | 2 +- 3 files changed, 104 insertions(+), 19 deletions(-) diff --git a/tests/test_the_test/test_build_pipeline.py b/tests/test_the_test/test_build_pipeline.py index 98760623fae..2da343723c9 100644 --- a/tests/test_the_test/test_build_pipeline.py +++ b/tests/test_the_test/test_build_pipeline.py @@ -135,7 +135,7 @@ def test_c_pipeline_renders_three_scenarios_and_package_artifact(self, tmp_path: for job_name in expected_run_jobs: assert ".system_tests_base" in pipeline[job_name]["extends"] - def test_buildx_cache_update_from_main(self, tmp_path: Path) -> None: + def test_buildx_cache_updates_system_tests_main(self, tmp_path: Path) -> None: params = { "endtoend_defs": { "parallel_weblogs": [{"name": "perl-mojolicious"}], @@ -163,9 +163,63 @@ def test_buildx_cache_update_from_main(self, tmp_path: Path) -> None: binaries_artifacts="system_tests_package_refs", binaries_artifact_path="system-tests-binaries", ref="main", + ci_project_name="system-tests", + ci_commit_branch="main", + ci_default_branch="main", ) - assert re.search("--cache-to=type=registry,ref=", (out / "generated-pipeline-chunk-0.yml").read_text()) + text = (out / "generated-pipeline-chunk-0.yml").read_text() + assert ( + "--cache-to=type=registry,ref=registry.ddbuild.io/system-tests/cache/c/perl-mojolicious:main,mode=max" + in text + ) + assert ( + "--cache-to=type=registry,ref=registry.ddbuild.io/system-tests/cache/c/perl-mojolicious:lib_main,mode=max" + not in text + ) + + def test_buildx_cache_updates_library_default_branch(self, tmp_path: Path) -> None: + params = { + "endtoend_defs": { + "parallel_weblogs": [{"name": "perl-mojolicious"}], + "parallel_jobs": [ + { + "weblog": "perl-mojolicious", + "scenarios": ["DEFAULT", "SAMPLING", "IPV6"], + "weblog_build_required": True, + } + ], + }, + "miscs": {"binaries_artifact": ""}, + "parametric": {"enable": False, "parallel_jobs": []}, + } + (tmp_path / "params_c.json").write_text(json.dumps(params)) + out = tmp_path / "out" + + build( + ["c"], + tmp_path, + out, + stage="e2e", + ci_image="myimage", + chunks=1, + binaries_artifacts="system_tests_package_refs", + binaries_artifact_path="system-tests-binaries", + ref="main", + ci_project_name="dd-trace-rb", + ci_commit_branch="master", + ci_default_branch="master", + ) + + text = (out / "generated-pipeline-chunk-0.yml").read_text() + assert ( + "--cache-to=type=registry,ref=registry.ddbuild.io/system-tests/cache/c/perl-mojolicious:lib_main,mode=max" + in text + ) + assert ( + "--cache-to=type=registry,ref=registry.ddbuild.io/system-tests/cache/c/perl-mojolicious:main,mode=max" + not in text + ) def test_buildx_cache_does_not_update_from_not_main(self, tmp_path: Path) -> None: params = { @@ -195,6 +249,9 @@ def test_buildx_cache_does_not_update_from_not_main(self, tmp_path: Path) -> Non binaries_artifacts="system_tests_package_refs", binaries_artifact_path="system-tests-binaries", ref="some-branch", + ci_project_name="system-tests", + ci_commit_branch="some-branch", + ci_default_branch="main", ) assert not re.search("--cache-to=type=registry,ref=", (out / "generated-pipeline-chunk-0.yml").read_text()) diff --git a/utils/ci/gitlab/build_pipeline.py b/utils/ci/gitlab/build_pipeline.py index 6d24c229ee6..55581f82bf7 100644 --- a/utils/ci/gitlab/build_pipeline.py +++ b/utils/ci/gitlab/build_pipeline.py @@ -2,11 +2,16 @@ import argparse import json +import os import sys from pathlib import Path +from typing import TYPE_CHECKING from jinja2 import Environment, FileSystemLoader, select_autoescape +if TYPE_CHECKING: + from collections.abc import Callable + _env = Environment(loader=FileSystemLoader(Path(__file__).resolve().parent), autoescape=select_autoescape()) _template = _env.get_template("system-tests.yml.j2") @@ -48,21 +53,26 @@ def _trace(name: str, command: str, library: str, weblog: str = "", scenario: st ) -def _render_build(library: str, weblog: str, *, push: bool = False) -> str: - registry_base = "registry.ddbuild.io/system-tests/cache" - ref = f"{registry_base}/{library}/{weblog}:main" - command = "".join( - [ - f"./build.sh {library} ", - "-i weblog --save-to-binaries ", - f"--weblog-variant {weblog} ", - '--extra-docker-args "', - f"--cache-from=type=registry,ref={ref} ", - f"--cache-to=type=registry,ref={ref},mode=max" if push else "", - '"', - ] - ) - return _trace(f'"build {library} {weblog}"', command, library, weblog) +def _generate_build_renderer(*, push_main: bool = False, push_lib_main: bool = False) -> Callable[[str, str], str]: + def _render_build(library: str, weblog: str) -> str: + registry_base = "registry.ddbuild.io/system-tests/cache" + ref = f"{registry_base}/{library}/{weblog}" + command = "".join( + [ + f"./build.sh {library} ", + "-i weblog --save-to-binaries ", + f"--weblog-variant {weblog} ", + '--extra-docker-args "', + f"--cache-from=type=registry,ref={ref}:main ", + f"--cache-from=type=registry,ref={ref}:lib_main ", + f"--cache-to=type=registry,ref={ref}:main,mode=max " if push_main else "", + f"--cache-to=type=registry,ref={ref}:lib_main,mode=max" if push_lib_main else "", + '"', + ] + ) + return _trace(f'"build {library} {weblog}"', command, library, weblog) + + return _render_build def render_library( @@ -77,6 +87,9 @@ def render_library( binaries_artifact_path: str, binaries_artifacts: str, pipeline_start_time: str, + ci_project_name: str = "", + ci_commit_branch: str = "", + ci_default_branch: str = "", ) -> str: parallel_weblogs = params.get("endtoend_defs", {}).get("parallel_weblogs", []) parallel_jobs = params.get("endtoend_defs", {}).get("parallel_jobs", []) @@ -96,6 +109,12 @@ def render_library( binaries_artifacts_list = [binaries_artifact] else: binaries_artifacts_list = [] + + render_build = _generate_build_renderer( + push_main=ci_project_name == "system-tests" and ci_commit_branch == "main", + push_lib_main=(ci_project_name != "system-tests" and ci_commit_branch in {"main", "master", ci_default_branch}), + ) + return _template.render( scenario_pairs=scenario_pairs, stage=stage, @@ -110,7 +129,7 @@ def render_library( push_to_test_optimization=push_to_test_optimization, skip_header=skip_header, pipeline_start_time=pipeline_start_time, - render_build=_render_build, + render_build=render_build, trace=_trace, ) @@ -128,6 +147,9 @@ def build( binaries_artifact_path: str = "", binaries_artifacts: str = "", pipeline_start_time: str = "", + ci_project_name: str = "", + ci_commit_branch: str = "", + ci_default_branch: str = "", ) -> None: """Render pipeline chunk files into *output_dir*, one per chunk.""" output_dir.mkdir(parents=True, exist_ok=True) @@ -167,6 +189,9 @@ def build( binaries_artifact_path=binaries_artifact_path, binaries_artifacts=binaries_artifacts, pipeline_start_time=pipeline_start_time, + ci_project_name=ci_project_name, + ci_commit_branch=ci_commit_branch, + ci_default_branch=ci_default_branch, ) ) @@ -221,6 +246,9 @@ def main(argv: list[str] | None = None) -> int: binaries_artifact_path=args.binaries_artifact_path, binaries_artifacts=args.binaries_artifacts, pipeline_start_time=args.pipeline_start_time, + ci_project_name=os.getenv("CI_PROJECT_NAME", ""), + ci_commit_branch=os.getenv("CI_COMMIT_BRANCH", ""), + ci_default_branch=os.getenv("CI_DEFAULT_BRANCH", ""), ) return 0 diff --git a/utils/ci/gitlab/system-tests.yml.j2 b/utils/ci/gitlab/system-tests.yml.j2 index 6cae8ad4554..3268f4dc5dd 100644 --- a/utils/ci/gitlab/system-tests.yml.j2 +++ b/utils/ci/gitlab/system-tests.yml.j2 @@ -93,7 +93,7 @@ system_tests_build_{{library}}_{{variant}}: {% endif %} - section_start "build" "Building weblog" false {{ job_tag("build") }} - - {{ render_build(library, variant, push = true) }} + - {{ render_build(library, variant) }} - mv binaries .. - section_end "build" artifacts: From c9651b42df0c17fbe66ea34ec1b2ec314c46ce4a Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Mon, 10 Aug 2026 14:35:11 +0200 Subject: [PATCH 04/12] test --- utils/ci/gitlab/build_pipeline.py | 2 +- utils/scripts/compute_libraries_and_scenarios.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/utils/ci/gitlab/build_pipeline.py b/utils/ci/gitlab/build_pipeline.py index 55581f82bf7..4a7b9389959 100644 --- a/utils/ci/gitlab/build_pipeline.py +++ b/utils/ci/gitlab/build_pipeline.py @@ -111,7 +111,7 @@ def render_library( binaries_artifacts_list = [] render_build = _generate_build_renderer( - push_main=ci_project_name == "system-tests" and ci_commit_branch == "main", + push_main=True, push_lib_main=(ci_project_name != "system-tests" and ci_commit_branch in {"main", "master", ci_default_branch}), ) diff --git a/utils/scripts/compute_libraries_and_scenarios.py b/utils/scripts/compute_libraries_and_scenarios.py index 7337ebabc2c..eb5fe204a67 100644 --- a/utils/scripts/compute_libraries_and_scenarios.py +++ b/utils/scripts/compute_libraries_and_scenarios.py @@ -30,8 +30,8 @@ OTEL_LIBRARIES = COMPONENT_GROUPS.otel - {"nodejs_otel"} # nodejs_otel intentionally excluded ALL_LIBRARIES = LIBRARIES | OTEL_LIBRARIES GITHUB_EXCLUDED_LIBRARIES = {"c"} -GITLAB_PR_LIBRARIES = {"c"} -GITLAB_MAIN = {"python"} +GITLAB_PR_LIBRARIES = {"c", "python"} +GITLAB_MAIN = {} def check_scenarios(scenarios: set[str]) -> bool: From e36782ae0d28ff4100944db45747d76069e76356 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Tue, 11 Aug 2026 11:13:23 +0200 Subject: [PATCH 05/12] Implement target artifact staging --- docs/CI/README.md | 6 + docs/CI/system-tests-ci.md | 12 +- docs/execute/binaries.md | 23 + docs/execute/build.md | 9 +- docs/glossary.md | 12 + docs/internals/README.md | 1 + .../internals/target-artifact-staging-spec.md | 158 +++ tests/test_the_test/test_build_pipeline.py | 60 + tests/test_the_test/test_load_binary.py | 111 +- tests/test_the_test/test_target_artifacts.py | 1079 +++++++++++++++++ utils/__init__.py | 77 +- utils/build/build.sh | 59 + utils/build/docker/c/artifact.py | 89 ++ .../docker/c/perl-mojolicious.Dockerfile | 6 +- utils/build/docker/cpp/artifact.py | 42 + utils/build/docker/cpp_httpd/artifact.py | 56 + .../build/docker/cpp_httpd/install_ddtrace.sh | 19 +- utils/build/docker/cpp_kong/artifact.py | 63 + .../build/docker/cpp_kong/install_ddtrace.sh | 18 +- utils/build/docker/cpp_nginx/artifact.py | 72 ++ .../build/docker/cpp_nginx/install_ddprof.sh | 29 +- .../build/docker/cpp_nginx/install_ddtrace.sh | 93 +- utils/build/docker/dotnet/artifact.py | 49 + utils/build/docker/dotnet/install_ddtrace.sh | 22 +- utils/build/docker/golang/artifact.py | 114 ++ utils/build/docker/java/artifact.py | 39 + utils/build/docker/java/install_ddtrace.sh | 33 +- .../docker/java/parametric/install_ddtrace.sh | 13 +- utils/build/docker/java_lambda/artifact.py | 39 + utils/build/docker/java_otel/artifact.py | 33 + .../docker/java_otel/install_opentelemetry.sh | 10 +- utils/build/docker/nodejs/artifact.py | 41 + utils/build/docker/nodejs_lambda/artifact.py | 57 + .../nodejs_lambda/install_datadog_lambda.sh | 37 +- utils/build/docker/nodejs_otel/artifact.py | 35 + .../nodejs_otel/express4-otel.Dockerfile | 2 + utils/build/docker/otel_collector/artifact.py | 39 + utils/build/docker/php/artifact.py | 55 + .../docker/php/common/install_ddtrace.sh | 39 +- utils/build/docker/python/artifact.py | 40 + utils/build/docker/python_lambda/artifact.py | 57 + .../python_lambda/install_datadog_lambda.sh | 30 +- utils/build/docker/python_otel/artifact.py | 35 + .../python_otel/flask-poc-otel.Dockerfile | 9 +- utils/build/docker/ruby/artifact.py | 52 + utils/build/docker/ruby_lambda/artifact.py | 44 + .../ruby_lambda/install_datadog_lambda.sh | 19 +- utils/build/docker/rust/artifact.py | 45 + utils/build/docker/rust/install_ddtrace.sh | 15 +- utils/ci/gitlab/build_pipeline.py | 7 +- utils/ci/gitlab/system-tests.yml.j2 | 9 + .../compute_libraries_and_scenarios.py | 4 +- utils/scripts/docker_base_image.sh | 16 +- utils/scripts/load-binary.sh | 421 +------ utils/scripts/stage-target-artifacts.py | 13 + utils/target_artifacts/__init__.py | 32 + utils/target_artifacts/__main__.py | 3 + utils/target_artifacts/cli.py | 41 + utils/target_artifacts/compat.py | 36 + utils/target_artifacts/entry_helpers.py | 38 + utils/target_artifacts/env.py | 46 + utils/target_artifacts/models.py | 109 ++ utils/target_artifacts/orchestrator.py | 169 +++ utils/target_artifacts/resolvers.py | 378 ++++++ 64 files changed, 3854 insertions(+), 565 deletions(-) create mode 100644 docs/internals/target-artifact-staging-spec.md create mode 100644 tests/test_the_test/test_target_artifacts.py create mode 100644 utils/build/docker/c/artifact.py create mode 100644 utils/build/docker/cpp/artifact.py create mode 100644 utils/build/docker/cpp_httpd/artifact.py create mode 100644 utils/build/docker/cpp_kong/artifact.py create mode 100644 utils/build/docker/cpp_nginx/artifact.py create mode 100644 utils/build/docker/dotnet/artifact.py create mode 100644 utils/build/docker/golang/artifact.py create mode 100644 utils/build/docker/java/artifact.py create mode 100644 utils/build/docker/java_lambda/artifact.py create mode 100644 utils/build/docker/java_otel/artifact.py create mode 100644 utils/build/docker/nodejs/artifact.py create mode 100644 utils/build/docker/nodejs_lambda/artifact.py create mode 100644 utils/build/docker/nodejs_otel/artifact.py create mode 100644 utils/build/docker/otel_collector/artifact.py create mode 100644 utils/build/docker/php/artifact.py create mode 100644 utils/build/docker/python/artifact.py create mode 100644 utils/build/docker/python_lambda/artifact.py create mode 100644 utils/build/docker/python_otel/artifact.py create mode 100644 utils/build/docker/ruby/artifact.py create mode 100644 utils/build/docker/ruby_lambda/artifact.py create mode 100644 utils/build/docker/rust/artifact.py create mode 100755 utils/scripts/stage-target-artifacts.py create mode 100644 utils/target_artifacts/__init__.py create mode 100644 utils/target_artifacts/__main__.py create mode 100644 utils/target_artifacts/cli.py create mode 100644 utils/target_artifacts/compat.py create mode 100644 utils/target_artifacts/entry_helpers.py create mode 100644 utils/target_artifacts/env.py create mode 100644 utils/target_artifacts/models.py create mode 100644 utils/target_artifacts/orchestrator.py create mode 100644 utils/target_artifacts/resolvers.py diff --git a/docs/CI/README.md b/docs/CI/README.md index 6ece16c0386..635290c00df 100644 --- a/docs/CI/README.md +++ b/docs/CI/README.md @@ -19,6 +19,12 @@ For the system-tests own CI, see also: * [CI test selection](./ci-test-selection.md): how the CI decides which libraries and scenarios to run based on modified files +### Target artifact staging in generated GitLab jobs + +Generated GitLab child-pipeline jobs run target artifact staging directly where the artifact entries are consumed. Build jobs stage before building a weblog when no upstream binaries bundle takes precedence. Parametric jobs stage before `./run.sh PARAMETRIC` when no build artifact bundle exists. Custom jobs that receive an upstream binaries bundle skip staging because that bundle is the selected source of truth. + +GitHub workflows keep their existing compatibility path during the migration. The `load-binary.sh` command remains available for local and workflow compatibility, but test target selection is routed through the Python staging model. + ### GitLab CI secrets setup 1. Install aws-cli diff --git a/docs/CI/system-tests-ci.md b/docs/CI/system-tests-ci.md index 7f9be6d21af..ee9b6fd0df5 100644 --- a/docs/CI/system-tests-ci.md +++ b/docs/CI/system-tests-ci.md @@ -26,10 +26,20 @@ Each library in the CI matrix will use its own specified branch. Libraries witho As a security measure, the "Fail if target branch is specified" job always fails if a target branch is selected. +### Target artifact staging + +GitLab generated jobs stage target artifacts at the point where the generated `binaries/` entries are needed: + +- Build jobs run `python3 utils/scripts/stage-target-artifacts.py ` before building a weblog when no upstream binaries bundle takes precedence. +- Parametric jobs run the same command before `./run.sh PARAMETRIC` when no build artifact bundle exists. +- Run jobs that consume a build job's artifact bundle do not repeat staging. +- Custom jobs with upstream binaries bundles skip staging because the upstream bundle is the selected artifact source of truth. + +The command accepts `custom` as a no-op, so generated templates can use one command shape safely. GitHub workflows keep using existing compatibility behavior until they choose to consume the new production artifact entries directly. + ### Scenario detection in CI When a modification is made in system tests, the CI tries to detect which scenario to run: 1. based on modified files in `tests/`, by extracting scenarios targeted by those files 2. based on any modification in a `tests/**/utils.py`, and applying the logic 1. on any sub file in `tests/**` - diff --git a/docs/execute/binaries.md b/docs/execute/binaries.md index 193c147bc6f..71954cda43c 100644 --- a/docs/execute/binaries.md +++ b/docs/execute/binaries.md @@ -2,10 +2,31 @@ By default, system tests will build a [weblog](../edit/weblog.md) image that shi But we often want to run system tests against unmerged changes. The general approach is to identify the git commit hash that contains your changes and use this commit hash to download a targeted build of the tracer. Note: ensure that the commit is pushed to a remote branch first, and when taking the commit hash, ensure you use the full hash. You can identify the commit hash using `git log` or from the github UI. +## Target artifact staging + +Target artifact staging is the preferred way to generate the text files consumed from `binaries/`. +Run it with: + +```bash +python3 utils/scripts/stage-target-artifacts.py +``` + +The legacy compatibility command still works and delegates test targets to the same staging model: + +```bash +./utils/scripts/load-binary.sh +``` + +Generated artifact entries are text-only. They select an artifact by a bounded artifact selector, such as a commit SHA, release tag, package version, or OCI digest. Generated entries must not use unbounded rolling selectors such as `latest`. + +Manual payload overrides remain supported. If you put a jar, wheel, archive, native module, local checkout, or explicit marker file in `binaries/`, the installer behavior documented below still applies. Staging refuses to overwrite unowned files so local payload overrides are not silently replaced. + +Some providers require an installer-facing fetch selector that is not itself bounded. In that case staging also writes a selection marker containing the bounded selector used for cache identity. The generated `binaries/.target-artifacts-manifest.json` file records which target owns generated entries and lets later staging refresh stale entries safely. ## Agent * Add a file `agent-image` in `binaries/`. The content must be a valid docker image name containing the datadog agent, like `datadog/agent` or `datadog/agent-dev:master-py3`. +* Compatibility command: `./utils/scripts/load-binary.sh agent dev` ## C++ library @@ -31,6 +52,7 @@ There are three ways to run system-tests with a custom Kong plugin: ```bash ./utils/scripts/load-binary.sh cpp_kong ``` + The command now stages bounded references and metadata; the Docker build fetches the selected payload when needed. To test with a custom dd-trace-cpp C binding, you can additionally: * Create a file `cpp-load-from-git` in `binaries/` (e.g. `https://github.com/DataDog/dd-trace-cpp@main`) @@ -240,6 +262,7 @@ You can also use `utils/scripts/watch.sh` script to sync your local `dd-trace-rs ## WAF rule set * copy a file `waf_rule_set` in `binaries/` +* Compatibility command: `./utils/scripts/load-binary.sh waf_rule_set dev` #### After Testing with a Custom Tracer: Most of the ways to run system-tests with a custom tracer version involve modifying the binaries directory. Modifying the binaries will alter the tracer version used across your local computer. Once you're done testing with the custom tracer, ensure you **remove** it. For example for Python: diff --git a/docs/execute/build.md b/docs/execute/build.md index a4ae354c60a..f521666f8f5 100644 --- a/docs/execute/build.md +++ b/docs/execute/build.md @@ -59,14 +59,15 @@ Build the native C tracer workload with: ./build.sh c -w perl-mojolicious ``` -The production build uses the published `apm-library-c-package:latest` and -`apm-inject-package:latest` images from `install.datadoghq.com`. Run +The production build starts from the published `apm-library-c-package:latest` +and `apm-inject-package:latest` images from `install.datadoghq.com`, then +records immutable digest references in `binaries/`. Run `./utils/scripts/load-binary.sh c` to validate and record both production image -references in `binaries/`. For a development build, set +references. For a development build, set `LIBRARY_TARGET_BRANCH`, `AUTO_INJECT_TARGET_BRANCH`, or both before running the loader. Each branch override is resolved to an immutable commit-SHA tag from `installtesting.datad0g.com` (with a zero in `datad0g`); components without a -branch override continue to use the production `latest` image. +branch override continue to use production image digest references. The `perl-mojolicious` workload supports `DEFAULT`, `SAMPLING`, and `IPV6`. It uses Perl and Mojolicious without a Datadog Perl tracer; all tracing comes from diff --git a/docs/glossary.md b/docs/glossary.md index 32b5afeca05..5543748414c 100644 --- a/docs/glossary.md +++ b/docs/glossary.md @@ -1,5 +1,17 @@ # Glossary +## Target artifact staging + +- target artifact: The library, layer, image, module, release, or workflow artifact selected for a system-tests target such as `python`, `java`, `c`, or `cpp_nginx`. +- dependency artifact: A supporting artifact that is not the selected test target, such as the Datadog Agent image. +- overlay artifact: A supplemental artifact layered onto tests without being the selected test target, such as the WAF rule set. +- artifact staging: The step that resolves target artifact inputs and writes generated artifact entries into `binaries/` before a Docker build or test run consumes them. +- artifact entry: A generated text file in `binaries/` that tells an installer which target artifact to use. +- bounded artifact selector: A selector with stable meaning, such as a commit SHA, release tag, package version, or OCI digest. +- selection marker: A generated artifact entry that records the bounded selector when another entry must use a provider-specific fetch selector. +- payload override: A manual payload placed in `binaries/`, such as a jar, wheel, archive, native module, or local checkout, that takes precedence over generated artifact entries. +- artifact manifest: The generated `binaries/.target-artifacts-manifest.json` file that tracks ownership and hashes for generated artifact entries. + ## Test activation/deactivation - successful: A test is successful if none of its assertions are failing diff --git a/docs/internals/README.md b/docs/internals/README.md index 96c3989f17b..ca5e34e09dd 100644 --- a/docs/internals/README.md +++ b/docs/internals/README.md @@ -11,6 +11,7 @@ All about system-tests deep internals. For those of you who are not afraid of ge - [MITM certificate](recreating_MITM_certificate.md) -- how to recreate the proxy certificate - [Core dump generation](generate-core-dump.md) -- generating core dumps for debugging +- [Target artifact staging](target-artifact-staging-spec.md) -- target-owned artifact selection model, manifest behavior, and maintainer contract ### Recreating protobuf schemas diff --git a/docs/internals/target-artifact-staging-spec.md b/docs/internals/target-artifact-staging-spec.md new file mode 100644 index 00000000000..8c1126f117e --- /dev/null +++ b/docs/internals/target-artifact-staging-spec.md @@ -0,0 +1,158 @@ +# Target Artifact Staging Spec + +## Problem Statement + +System-tests currently has two different mechanisms for choosing the target artifact to test. +Development artifact selection is mostly centralized in the legacy binary-loading script, while production artifact selection often happens dynamically inside Dockerfiles or installer scripts. This makes ownership unclear, makes non-library test targets fit awkwardly into library-oriented workflows, and makes Docker layer cache behavior hard to reason about when production releases change. + +The user wants each test target to own its target artifact selection logic for both development and production. Artifact staging should happen before the Docker build or test run consumes the selected artifact entries. The Docker build should not dynamically discover the latest production release for the target artifact, because a mutable selector such as `latest` can match every version ever released and does not provide a bounded contract for cache invalidation. + +## Solution + +Introduce a Python-based artifact staging mechanism where every test target defines two top-level environment classes, one for development and one for production. Both classes implement a common Protocol. Each environment class declares the artifact inputs it needs, then maps resolved artifact inputs to generated artifact entries. + +The orchestrator owns side effects: loading local environment configuration, resolving declared artifact inputs through shared resolvers, writing artifact entries, and maintaining the artifact manifest. Target environment classes remain side-effect-free and only return text artifact entries. + +Generated artifact entries must represent bounded artifact selectors. They may use version numbers, commit SHAs, release tags, package versions, or image digests. They must not use unbounded rolling selectors such as `latest`. When a provider-specific fetch selector cannot be bounded, the target must also emit a visible selection marker containing the bounded selector used for cache identity. + +The legacy binary-loading command remains available as a compatibility wrapper. The canonical behavior becomes target artifact staging, but existing local and CI invocations keep working transparently. + +## User Stories + +1. As a system-tests user, I want each test target to define its own target artifact selection, so that target-specific behavior is easy to find and review. +2. As a system-tests user, I want development and production target artifact selection to live together, so that both environments follow the same model. +3. As a system-tests user, I want production artifact selection to happen before Docker builds dynamically install a tracer, so that Docker layer cache invalidation is easier to reason about. +4. As a system-tests user, I want production target artifact entries to avoid `latest`, so that a staged artifact selection has a bounded meaning. +5. As a system-tests user, I want version selectors to remain valid when they match multiple architecture-specific payloads, so that runtime-specific installers can still choose the correct artifact. +6. As a system-tests user, I want development branch inputs to resolve to bounded selectors when possible, so that testing a branch still has a stable cache identity. +7. As a system-tests user, I want providers that require branch-based fetching to write an additional selection marker, so that cache invalidation still tracks the resolved commit. +8. As a system-tests user, I want selection markers to be documented clearly, so that I understand when they are required and why. +9. As a system-tests user, I want generated artifact entries to be text files, so that the staging contract stays simple and inspectable. +10. As a system-tests user, I want local payload overrides to remain supported outside generated artifact staging, so that I can still test a local jar, wheel, archive, native library, or checkout. +11. As a system-tests user, I want generated artifact staging to avoid overwriting my manual payload overrides, so that local testing artifacts are not destroyed. +12. As a system-tests user, I want generated artifact staging to avoid overwriting my manual marker files, so that explicit local selections are not silently replaced. +13. As a system-tests user, I want a clear error when a generated artifact entry conflicts with an unowned file, so that I know which local file to remove. +14. As a system-tests user, I want generated artifact entries from previous runs to be refreshed safely, so that stale generated files do not keep affecting builds. +15. As a system-tests user, I want staging one environment for a target to replace the previous environment for that same target, so that development and production selections cannot be active for the same target at the same time. +16. As a system-tests user, I want staging one test target to leave other staged targets alone, so that dependency artifacts and multi-target workflows can coexist. +17. As a system-tests user, I want artifact staging to track generated ownership in a manifest, so that the orchestrator can distinguish generated files from manual files. +18. As a system-tests user, I want the artifact manifest to avoid duplicating artifact entry contents, so that the actual staged files remain the source of truth. +19. As a system-tests user, I want structured artifact entries to use JSON, so that multi-field references are not encoded with fragile delimiters. +20. As a system-tests user, I want JSON artifact entries to use a JSON file extension, so that format expectations are obvious. +21. As a target maintainer, I want my target's artifact inputs to be explicitly declared, so that repo names, defaults, and external providers are not hidden in orchestration code. +22. As a target maintainer, I want shared resolvers for common external metadata lookups, so that target files do not duplicate GitHub, registry, or environment parsing logic. +23. As a target maintainer, I want target environment classes to receive resolved inputs, so that I can unit test target mapping without network, filesystem, or environment side effects. +24. As a target maintainer, I want the target module to own ecosystem-specific selection semantics, so that Java, Go, PHP, C, Lambda, and native-web-server targets can express their different needs. +25. As a target maintainer, I want authenticated artifact entries to contain only non-secret metadata, so that uploaded artifact bundles do not leak credentials. +26. As a CI maintainer, I want GitLab build jobs to run artifact staging directly, so that small workloads do not pay the startup cost of a separate staging job. +27. As a CI maintainer, I want GitLab parametric jobs to run artifact staging directly, so that parametric runs have target artifacts even when no weblog build job exists. +28. As a CI maintainer, I want GitLab custom runs with upstream artifact bundles to skip artifact staging, so that external artifacts remain the source of truth. +29. As a CI maintainer, I want the staging command to accept custom as a no-op, so that templates can call a single command shape safely. +30. As a CI maintainer, I want GitHub workflows to keep working transparently during migration, so that the refactor does not require immediate GitHub production-flow changes. +31. As a CI maintainer, I want the legacy command name to keep working, so that existing workflows and local habits do not break during migration. +32. As a system-tests maintainer, I want every test target to define real development and production staging behavior, so that there are no placeholder loaders. +33. As a system-tests maintainer, I want dependency artifacts such as the agent to stay outside the test target protocol, so that test targets and dependencies remain separate domain concepts. +34. As a system-tests maintainer, I want WAF rule set loading to remain outside the test target protocol, so that overlay artifacts do not blur target ownership. +35. As a system-tests maintainer, I want the target artifact context to stay target-level, so that one staged artifact bundle can be reused across many weblog variants. +36. As a system-tests maintainer, I want architecture and runtime-specific payload selection to remain in installers when needed, so that the artifact staging phase does not need weblog-specific facts. +37. As a system-tests maintainer, I want remote lookups to fetch as little data as possible, so that staging resolves metadata or bounded references without downloading payloads unnecessarily. +38. As a system-tests maintainer, I want GitHub Actions artifact staging to return metadata rather than payloads, so that Docker builds fetch the selected artifact only once. +39. As a system-tests maintainer, I want OCI production image selectors to resolve to digests, so that image-based target artifacts do not use mutable tags. +40. As a system-tests maintainer, I want the bounded-selector rule documented as a contract rather than enforced by expensive runtime checks, so that the system remains practical. + +## Implementation Decisions + +- The canonical operation is artifact staging, not binary loading. The legacy command remains as a compatibility entrypoint. +- Every test target must provide real development and production artifact staging behavior. Checked-in placeholder behavior is not acceptable. +- A test target is the component selected as the CI matrix library value. Dependencies and overlays are not test targets. +- Dependency artifacts, including the agent, remain compatibility-only behavior in the orchestrator and are outside the per-test-target Protocol. +- WAF rule set loading remains compatibility-only behavior in the orchestrator and is outside the per-test-target Protocol. +- Each test target owns a target artifact module in its existing target-specific Docker area. +- Shared protocol, models, resolvers, and orchestration live in a normal importable Python package outside the Docker build asset tree. +- The orchestrator imports each target artifact module dynamically by file location instead of turning every Docker target directory into a Python package. +- Each target artifact module exposes two top-level classes, `Dev` and `Prod`. +- `Dev` and `Prod` explicitly inherit from the shared Protocol type supplied by the typing module. +- `Dev` and `Prod` have no constructor arguments. Runtime data is passed through context and resolved artifact inputs. +- The Protocol exposes one method for declaring artifact inputs and one method for returning artifact entries. +- Artifact input declarations are explicit for both development and production. The orchestrator does not infer repository names, default branches, production release sources, or ecosystem semantics. +- Resolved artifact inputs are accessed as a mapping keyed by input name. +- Resolved artifact input values are typed frozen dataclasses, not plain strings. +- Target artifact functions are side-effect-free. They do not read environment variables, read or write files, call subprocesses, or perform network requests. +- The orchestrator owns side effects: environment loading, remote metadata resolution, artifact entry writing, and manifest maintenance. +- Generated artifact entries are text-only. Payload bytes and local checkouts remain manual payload overrides outside this loader protocol. +- Single-value artifact entries use plain text. +- Multi-field artifact entries use JSON and their filenames indicate the JSON format. +- Artifact entries must use bounded artifact selectors by contract. The shared implementation should document this rule but should not try to prove arbitrary content is valid. +- Mutable development inputs such as branch names should resolve to bounded selectors before artifact entries are generated when the provider supports that. +- When a provider requires an unbounded or provider-specific fetch selector, the loader must also emit a selection marker that contains the bounded selector used for artifact selection identity. +- Selection markers are visible, documented artifact entries. They are required when the installer-facing entry cannot itself be bounded. +- A shared helper creates provider-fetch entries with the required selection marker, reducing the chance that a target forgets it. +- Production OCI image references resolve to digests. +- GitHub latest release inputs return minimal release metadata by default. Asset metadata is included only when a target requests it. +- GitHub Actions artifact inputs resolve to stable artifact metadata, not downloaded payloads. +- Artifact entries that require authenticated downloads contain only non-secret metadata. Credentials are supplied separately by the build environment. +- The command loads local environment configuration by default using the Python dotenv package. Process environment variables override dotenv values. +- The `custom` environment is an orchestrator-only no-op. It is not represented in target modules. +- The artifact manifest is a single generated manifest for all staged targets in the staging directory. +- The manifest stores versioned ownership metadata and content hashes, not duplicate artifact entry contents. +- The manifest forbids two owners from owning the same artifact entry filename. +- Staging a target in one environment removes previously owned entries for other environments of the same target. +- Staging a target does not remove generated entries owned by other targets. +- Staging removes stale previously owned entries for the same target when the new run no longer emits them. +- Staging refuses to overwrite unowned existing files. There is no force mode in the first implementation. +- There is no clean subcommand in the first implementation. +- GitHub integration is kept transparent for now. Existing development artifact preparation continues through the compatibility command, and new production outputs are ignored until GitHub workflows choose to consume them. +- GitLab generated build jobs run artifact staging directly using the job's CI environment. +- GitLab generated parametric jobs run artifact staging directly using the job's CI environment. +- GitLab custom jobs with upstream artifact bundles skip artifact staging because the upstream bundle is the selected artifact source of truth. +- GitLab accepts the small risk that per-job production resolution could differ if a release changes mid-pipeline. This race is considered extremely unlikely and preferable to adding job startup overhead. + +## Testing Decisions + +- The main test seam is the artifact staging CLI/orchestrator. Tests should execute the staging flow at the command boundary with fake or stubbed resolvers and inspect the staged artifact entries plus manifest behavior. +- Target environment classes should be tested through their public Protocol methods by passing fake resolved inputs and asserting returned artifact entries. Tests should not inspect target implementation internals. +- Manifest behavior should be tested through observable filesystem effects: safe overwrite of owned files, refusal to overwrite unowned files, cleanup of stale owned entries, replacement of a target's previous environment, preservation of other targets, and owner conflict failures. +- GitLab integration should be tested through the existing pipeline rendering seam. Generated jobs should include artifact staging in build and parametric jobs, skip it for custom upstream artifact bundles, and avoid adding a separate staging job. +- Compatibility behavior should be tested through the legacy command entrypoint, showing that existing invocations continue to route to the new staging behavior. +- Custom environment behavior should be tested at the CLI/orchestrator seam as a successful no-op that does not load target modules or write manifest changes. +- Expected user/configuration failures should raise the shared domain exception and produce clear CLI errors. +- Tests should not perform real network calls. GitHub release metadata, GitHub Actions artifact metadata, branch-to-SHA resolution, and OCI digest resolution should be exercised through resolver fakes. +- Tests should not assert private helper call sequences when the same behavior can be verified through generated artifact entries, manifest contents, and rendered CI commands. +- Prior art exists in current tests for the legacy binary-loading command and GitLab pipeline rendering. The new tests should reuse those high-level seams where possible rather than spreading assertions across many low-level helpers. + +## Out of Scope + +- Publishing this spec to an issue tracker. +- Adding GitHub production artifact staging to workflows immediately. +- Adding runtime purity enforcement, monkeypatch-based side-effect tests, or AST guards. +- Adding a force option. +- Adding a clean subcommand. +- Turning dependency artifacts into test target loaders. +- Turning WAF rule set loading into a test target loader. +- Generating payload artifact entries. +- Removing support for manual payload overrides. +- Making the target artifact context weblog-specific, runtime-specific, or architecture-specific. +- Proving bounded-selector validity for arbitrary strings at runtime. +- Adding broad generic downloader abstractions before a target needs them. + +## Further Notes + +- All test targets currently have a matching target-specific Docker directory. Extra Docker directories such as shared support, dependency, and proxy directories are not test targets. +- The current C target is a real test target even though it is exercised through GitLab rather than GitHub. +- The agent is a dependency artifact, not a test target. +- The accepted GitLab consistency tradeoff is deliberate: per-job staging may theoretically resolve different production selectors if a release changes mid-pipeline, but the risk is very low and avoids disproportionate job startup cost. +- Documentation should make selection markers highly visible because missing them breaks the cache identity contract when the installer-facing selector is not bounded. +- Teams can ask questions about system-tests behavior in `#apm-shared-testing`. + +## Maintainer Checklist + +When adding or changing a target's artifact staging behavior: + +1. Add or update `utils/build/docker//artifact.py`. +2. Define top-level `Dev` and `Prod` classes that implement `TargetArtifactEnvironment`. +3. Keep both classes side-effect-free: declare `ArtifactInput` values in `artifact_inputs`, and turn resolved values into text or JSON `ArtifactEntry` values in `artifact_entries`. +4. Put provider lookups in shared resolvers instead of target modules. +5. Emit bounded artifact selectors whenever possible. If an installer-facing entry must use a provider-specific fetch selector, emit a selection marker with `provider_fetch_entries`. +6. Keep local payload override handling in the installer script. Staging should write selectors and metadata, not jar, wheel, zip, tarball, or checkout payloads. +7. Use JSON entries for multi-field references, and give those files a `.json` extension. +8. Add or update `TEST_THE_TEST` coverage that exercises the target through the public Protocol methods with fake resolved inputs. diff --git a/tests/test_the_test/test_build_pipeline.py b/tests/test_the_test/test_build_pipeline.py index 2da343723c9..36ea8d80b33 100644 --- a/tests/test_the_test/test_build_pipeline.py +++ b/tests/test_the_test/test_build_pipeline.py @@ -135,6 +135,66 @@ def test_c_pipeline_renders_three_scenarios_and_package_artifact(self, tmp_path: for job_name in expected_run_jobs: assert ".system_tests_base" in pipeline[job_name]["extends"] + def test_build_job_stages_target_artifacts_without_upstream_bundle(self, tmp_path: Path) -> None: + params = { + "endtoend_defs": { + "parallel_weblogs": [{"name": "flask"}], + "parallel_jobs": [{"weblog": "flask", "scenarios": ["DEFAULT"], "weblog_build_required": True}], + }, + "miscs": {"binaries_artifact": "", "ci_environment": "prod"}, + "parametric": {"enable": False, "parallel_jobs": []}, + } + (tmp_path / "params_python.json").write_text(json.dumps(params)) + out = tmp_path / "out" + + build(["python"], tmp_path, out, stage="e2e", ci_image="myimage", chunks=1) + + pipeline = yaml.safe_load((out / "generated-pipeline-chunk-0.yml").read_text()) + build_script = pipeline["system_tests_build_python_flask"]["script"] + assert "python3 utils/scripts/stage-target-artifacts.py python prod" in build_script + assert not any(job_name.startswith("system_tests_stage") for job_name in pipeline) + + def test_parametric_job_stages_target_artifacts_without_upstream_bundle(self, tmp_path: Path) -> None: + params = { + "endtoend_defs": {"parallel_weblogs": [], "parallel_jobs": []}, + "miscs": {"binaries_artifact": "", "ci_environment": "dev"}, + "parametric": {"enable": True, "job_count": 1, "job_matrix": [1]}, + } + (tmp_path / "params_nodejs.json").write_text(json.dumps(params)) + out = tmp_path / "out" + + build(["nodejs"], tmp_path, out, stage="e2e", ci_image="myimage", chunks=1) + + pipeline = yaml.safe_load((out / "generated-pipeline-chunk-0.yml").read_text()) + run_script = pipeline["system_tests_run_nodejs_PARAMETRIC_1"]["script"] + assert "python3 utils/scripts/stage-target-artifacts.py nodejs dev" in run_script + + def test_upstream_artifact_bundle_skips_target_artifact_staging(self, tmp_path: Path) -> None: + params = { + "endtoend_defs": { + "parallel_weblogs": [{"name": "flask"}], + "parallel_jobs": [{"weblog": "flask", "scenarios": ["DEFAULT"], "weblog_build_required": True}], + }, + "miscs": {"binaries_artifact": "", "ci_environment": "custom"}, + "parametric": {"enable": True, "job_count": 1, "job_matrix": [1]}, + } + (tmp_path / "params_python.json").write_text(json.dumps(params)) + out = tmp_path / "out" + + build( + ["python"], + tmp_path, + out, + stage="e2e", + ci_image="myimage", + chunks=1, + binaries_artifacts="upstream-binaries", + binaries_artifact_path="system-tests-binaries", + ) + + text = (out / "generated-pipeline-chunk-0.yml").read_text() + assert "stage-target-artifacts.py" not in text + def test_buildx_cache_updates_system_tests_main(self, tmp_path: Path) -> None: params = { "endtoend_defs": { diff --git a/tests/test_the_test/test_load_binary.py b/tests/test_the_test/test_load_binary.py index 149f2f47bb3..7b5e26927e6 100644 --- a/tests/test_the_test/test_load_binary.py +++ b/tests/test_the_test/test_load_binary.py @@ -1,15 +1,19 @@ from __future__ import annotations +import json import os from pathlib import Path import subprocess from utils import scenarios +from utils.target_artifacts.orchestrator import MANIFEST_FILENAME SCRIPT = Path("utils/scripts/load-binary.sh") -C_LIBRARY_PROD_IMAGE = "install.datadoghq.com/apm-library-c-package:latest" -C_INJECTOR_PROD_IMAGE = "install.datadoghq.com/apm-inject-package:latest" +C_LIBRARY_DIGEST = "sha256:" + ("a" * 64) +C_INJECTOR_DIGEST = "sha256:" + ("b" * 64) +C_LIBRARY_PROD_IMAGE = f"install.datadoghq.com/apm-library-c-package@{C_LIBRARY_DIGEST}" +C_INJECTOR_PROD_IMAGE = f"install.datadoghq.com/apm-inject-package@{C_INJECTOR_DIGEST}" C_LIBRARY_SHA = "1" * 40 C_INJECTOR_SHA = "2" * 40 @@ -23,6 +27,7 @@ def _run_loader( tmp_path: Path, version: str, *, + target: str = "c", extra_env: dict[str, str] | None = None, ) -> subprocess.CompletedProcess[str]: bin_dir = tmp_path / "bin" @@ -31,29 +36,33 @@ def _run_loader( binaries_dir.mkdir() _write_executable( - bin_dir / "curl", + bin_dir / "docker", f"""#!/usr/bin/env bash set -eu -url="${{!#}}" -printf '%s\\n' "$url" >> "$CURL_CALLS" -if [[ "${{MISSING_BRANCH:-}}" != "" && "$url" == *"${{MISSING_BRANCH}}"* ]]; then - exit 22 +printf '%s\\n' "$*" >> "$DOCKER_CALLS" +if [[ "${{FAIL_IMAGE:-}}" != "" && "$*" == *"$FAIL_IMAGE"* ]]; then + exit 1 fi -if [[ "$url" == *"DataDog/dd-trace-c"* ]]; then - printf '%s\\n' '{{"commit":{{"sha":"{C_LIBRARY_SHA}"}}}}' +if [[ "$*" == *"apm-library-c-package"* ]]; then + printf 'Name: apm-library-c-package\\nDigest: {C_LIBRARY_DIGEST}\\n' else - printf '%s\\n' '{{"commit":{{"sha":"{C_INJECTOR_SHA}"}}}}' + printf 'Name: apm-inject-package\\nDigest: {C_INJECTOR_DIGEST}\\n' fi """, ) _write_executable( - bin_dir / "docker", + bin_dir / "curl", """#!/usr/bin/env bash set -eu -printf '%s\\n' "$*" >> "$DOCKER_CALLS" -if [[ "${FAIL_IMAGE:-}" != "" && "$*" == *"$FAIL_IMAGE"* ]]; then - exit 1 -fi +output="" +while [ "$#" -gt 0 ]; do + if [ "$1" = "--output" ]; then + shift + output="$1" + fi + shift +done +printf '{"rules":[]}\\n' > "$output" """, ) @@ -61,14 +70,13 @@ def _run_loader( **os.environ, "PATH": f"{bin_dir}:{os.environ['PATH']}", "BINARIES_DIR": str(binaries_dir), - "CURL_CALLS": str(tmp_path / "curl-calls"), "DOCKER_CALLS": str(tmp_path / "docker-calls"), } env.pop("LIBRARY_TARGET_BRANCH", None) env.pop("AUTO_INJECT_TARGET_BRANCH", None) env.update(extra_env or {}) return subprocess.run( - ["bash", str(SCRIPT), "c", version], + ["bash", str(SCRIPT), target, version], check=False, capture_output=True, text=True, @@ -78,6 +86,21 @@ def _run_loader( @scenarios.test_the_test class Test_LoadBinaryC: + def test_stage_target_artifacts_entrypoint_imports_without_pythonpath(self) -> None: + env = dict(os.environ) + env.pop("PYTHONPATH", None) + + result = subprocess.run( + ["python3", "utils/scripts/stage-target-artifacts.py", "--help"], + check=False, + capture_output=True, + text=True, + env=env, + ) + + assert result.returncode == 0, result.stderr + assert "usage: stage-target-artifacts" in result.stdout + def test_native_library_is_loaded_by_auto_inject(self) -> None: dockerfile = Path("utils/build/docker/c/perl-mojolicious.Dockerfile").read_text(encoding="utf-8") launcher = Path("utils/build/docker/c/perl-mojolicious/app.sh").read_text(encoding="utf-8") @@ -94,22 +117,21 @@ def test_production_package_defaults(self, tmp_path: Path) -> None: assert (tmp_path / "binaries/c-library-image").read_text(encoding="utf-8").strip() == C_LIBRARY_PROD_IMAGE assert (tmp_path / "binaries/c-injector-image").read_text(encoding="utf-8").strip() == C_INJECTOR_PROD_IMAGE docker_calls = (tmp_path / "docker-calls").read_text(encoding="utf-8") - assert C_LIBRARY_PROD_IMAGE in docker_calls - assert C_INJECTOR_PROD_IMAGE in docker_calls + assert "install.datadoghq.com/apm-library-c-package:latest" in docker_calls + assert "install.datadoghq.com/apm-inject-package:latest" in docker_calls def test_development_package_defaults_to_production_without_overrides(self, tmp_path: Path) -> None: result = _run_loader(tmp_path, "dev") assert result.returncode == 0, result.stderr - assert not (tmp_path / "curl-calls").exists() assert (tmp_path / "binaries/c-library-image").read_text(encoding="utf-8").strip() == C_LIBRARY_PROD_IMAGE assert (tmp_path / "binaries/c-injector-image").read_text(encoding="utf-8").strip() == C_INJECTOR_PROD_IMAGE docker_calls = (tmp_path / "docker-calls").read_text(encoding="utf-8") - assert C_LIBRARY_PROD_IMAGE in docker_calls - assert C_INJECTOR_PROD_IMAGE in docker_calls + assert "install.datadoghq.com/apm-library-c-package:latest" in docker_calls + assert "install.datadoghq.com/apm-inject-package:latest" in docker_calls def test_single_branch_override_keeps_other_component_on_production(self, tmp_path: Path) -> None: - result = _run_loader(tmp_path, "dev", extra_env={"LIBRARY_TARGET_BRANCH": "feature/c-client"}) + result = _run_loader(tmp_path, "dev", extra_env={"LIBRARY_TARGET_BRANCH": C_LIBRARY_SHA}) assert result.returncode == 0, result.stderr assert (tmp_path / "binaries/c-library-image").read_text(encoding="utf-8").strip() == ( @@ -122,15 +144,13 @@ def test_independent_branch_overrides_resolve_to_sha_tags(self, tmp_path: Path) tmp_path, "dev", extra_env={ - "LIBRARY_TARGET_BRANCH": "feature/c-client", - "AUTO_INJECT_TARGET_BRANCH": "feature/injector", + "LIBRARY_TARGET_BRANCH": C_LIBRARY_SHA, + "AUTO_INJECT_TARGET_BRANCH": C_INJECTOR_SHA, }, ) assert result.returncode == 0, result.stderr - curl_calls = (tmp_path / "curl-calls").read_text(encoding="utf-8") - assert "feature%2Fc-client" in curl_calls - assert "feature%2Finjector" in curl_calls + assert not (tmp_path / "docker-calls").exists() assert (tmp_path / "binaries/c-library-image").read_text(encoding="utf-8").strip() == ( f"installtesting.datad0g.com/apm-library-c-package:{C_LIBRARY_SHA}" ) @@ -138,15 +158,15 @@ def test_independent_branch_overrides_resolve_to_sha_tags(self, tmp_path: Path) f"installtesting.datad0g.com/apm-inject-package:{C_INJECTOR_SHA}" ) - def test_missing_branch_fails_before_package_validation(self, tmp_path: Path) -> None: + def test_production_rejects_branch_overrides_before_package_validation(self, tmp_path: Path) -> None: result = _run_loader( tmp_path, - "dev", - extra_env={"LIBRARY_TARGET_BRANCH": "missing", "MISSING_BRANCH": "missing"}, + "prod", + extra_env={"LIBRARY_TARGET_BRANCH": C_LIBRARY_SHA}, ) assert result.returncode != 0 - assert "Unable to resolve branch 'missing' in DataDog/dd-trace-c" in result.stderr + assert "Target branches can only be used with the development c packages" in result.stderr assert not (tmp_path / "docker-calls").exists() def test_missing_package_fails_with_clear_error(self, tmp_path: Path) -> None: @@ -157,4 +177,29 @@ def test_missing_package_fails_with_clear_error(self, tmp_path: Path) -> None: ) assert result.returncode != 0 - assert "OCI package does not exist or is not accessible" in result.stderr + assert "Unable to resolve OCI digest" in result.stderr + + def test_agent_dependency_uses_explicit_compatibility_path(self, tmp_path: Path) -> None: + result = _run_loader( + tmp_path, + "dev", + target="agent", + extra_env={"AGENT_TARGET_BRANCH": "feature-agent"}, + ) + + assert result.returncode == 0, result.stderr + binaries_dir = tmp_path / "binaries" + assert (binaries_dir / "agent-image").read_text(encoding="utf-8").strip() == ("datadog/agent-dev:feature-agent") + manifest = json.loads((binaries_dir / MANIFEST_FILENAME).read_text(encoding="utf-8")) + assert manifest["entries"]["agent-image"]["owner"] == { + "target": "agent", + "environment": "dependency", + } + + def test_waf_rule_set_overlay_stays_outside_target_manifest(self, tmp_path: Path) -> None: + result = _run_loader(tmp_path, "dev", target="waf_rule_set") + + assert result.returncode == 0, result.stderr + binaries_dir = tmp_path / "binaries" + assert json.loads((binaries_dir / "waf_rule_set.json").read_text(encoding="utf-8")) == {"rules": []} + assert not (binaries_dir / MANIFEST_FILENAME).exists() diff --git a/tests/test_the_test/test_target_artifacts.py b/tests/test_the_test/test_target_artifacts.py new file mode 100644 index 00000000000..367b77d714e --- /dev/null +++ b/tests/test_the_test/test_target_artifacts.py @@ -0,0 +1,1079 @@ +from __future__ import annotations + +import json +import subprocess +from pathlib import Path +from typing import TYPE_CHECKING, Any + +import pytest +import requests + +from utils import scenarios +from utils.const import COMPONENT_GROUPS +from utils.target_artifacts.models import ( + ArtifactResolver, + BranchReference, + GitHubActionsArtifactReference, + GitHubReleaseReference, + LiteralValue, + ModuleVersion, + OciImageReference, + ReleaseAsset, + ResolvedArtifactInput, + TargetArtifactError, +) +from utils.target_artifacts.orchestrator import MANIFEST_FILENAME, load_target_environment, stage_target +from utils.target_artifacts.resolvers import ( + CratesLatestResolver, + EnvResolver, + GitHubActionsArtifactResolver, + GitHubBranchResolver, + GitHubLatestReleaseResolver, + GoModuleLatestResolver, + NpmLatestResolver, + OciDigestResolver, + PypiLatestResolver, + RubygemsLatestResolver, +) + +if TYPE_CHECKING: + from collections.abc import Callable + +SHA = "1" * 40 +DIGEST = "sha256:" + ("2" * 64) +OTHER_SHA = "3" * 40 + + +class StubResponse: + def __init__( + self, + payload: object, + *, + status_error: requests.RequestException | None = None, + json_error: ValueError | None = None, + ) -> None: + self.payload = payload + self.status_error = status_error + self.json_error = json_error + + def raise_for_status(self) -> None: + if self.status_error is not None: + raise self.status_error + + def json(self) -> object: + if self.json_error is not None: + raise self.json_error + return self.payload + + +def _stub_get_json( + monkeypatch: pytest.MonkeyPatch, + payloads: dict[str, dict[str, Any]] | Callable[[str, dict[str, str]], dict[str, Any]], +) -> list[tuple[str, dict[str, str]]]: + calls: list[tuple[str, dict[str, str]]] = [] + + def fake_get_json(url: str, headers: dict[str, str]) -> dict[str, Any]: + calls.append((url, dict(headers))) + if callable(payloads): + return payloads(url, headers) + return payloads[url] + + monkeypatch.setattr("utils.target_artifacts.resolvers._get_json", fake_get_json) + return calls + + +def _completed_process( + args: list[str], + *, + returncode: int = 0, + stdout: str = "", + stderr: str = "", +) -> subprocess.CompletedProcess[str]: + return subprocess.CompletedProcess(args=args, returncode=returncode, stdout=stdout, stderr=stderr) + + +class FakeResolver: + def resolve(self, artifact_resolver: ArtifactResolver, env: dict[str, str]) -> ResolvedArtifactInput: + if isinstance(artifact_resolver, EnvResolver): + return LiteralValue( + name=artifact_resolver.name, + value=env.get(artifact_resolver.variable_name, artifact_resolver.default_value), + ) + if isinstance(artifact_resolver, GitHubBranchResolver): + return BranchReference( + name=artifact_resolver.name, + repository=artifact_resolver.repository, + branch=env.get(artifact_resolver.variable_name, artifact_resolver.default_value), + sha=SHA, + ) + if isinstance(artifact_resolver, GitHubLatestReleaseResolver): + return GitHubReleaseReference( + name=artifact_resolver.name, + repository=artifact_resolver.repository, + tag_name="v1.2.3", + ) + if isinstance(artifact_resolver, GitHubActionsArtifactResolver): + return GitHubActionsArtifactReference( + name=artifact_resolver.name, + repository=artifact_resolver.repository, + workflow=artifact_resolver.workflow, + branch=env.get(artifact_resolver.variable_name, artifact_resolver.default_value), + commit_sha=SHA, + run_id=123, + run_url="https://github.example/run", + artifact_id=456, + artifact_name=artifact_resolver.artifact_name, + archive_download_url="https://github.example/artifact.zip", + ) + if isinstance(artifact_resolver, OciDigestResolver): + image = env.get( + artifact_resolver.variable_name, + artifact_resolver.image or artifact_resolver.default_value, + ) + last_slash = image.rfind("/") + last_colon = image.rfind(":") + repository = image[:last_colon] if last_colon > last_slash else image + return OciImageReference( + name=artifact_resolver.name, + image=image, + digest=DIGEST, + reference=f"{repository}@{DIGEST}", + ) + if isinstance( + artifact_resolver, + (NpmLatestResolver, PypiLatestResolver, RubygemsLatestResolver, CratesLatestResolver), + ): + return ModuleVersion(name=artifact_resolver.name, module=artifact_resolver.package, version="1.2.3") + if isinstance(artifact_resolver, GoModuleLatestResolver): + return ModuleVersion(name=artifact_resolver.name, module=artifact_resolver.module, version="v1.2.3") + raise AssertionError(f"Unhandled input resolver: {type(artifact_resolver).__name__}") + + +def _write_target_module(repo_root: Path, body: str) -> None: + target_dir = repo_root / "utils" / "build" / "docker" / "fake" + target_dir.mkdir(parents=True) + (target_dir / "artifact.py").write_text(body, encoding="utf-8") + + +def _manifest_entries(binaries_dir: Path) -> dict[str, object]: + manifest = json.loads((binaries_dir / MANIFEST_FILENAME).read_text(encoding="utf-8")) + return manifest["entries"] + + +@scenarios.test_the_test +class Test_TargetArtifactStaging: + def test_custom_environment_is_noop(self, tmp_path: Path) -> None: + binaries_dir = tmp_path / "binaries" + + stage_target( + "does-not-exist", + "custom", + repo_root=tmp_path, + binaries_dir=binaries_dir, + process_env={}, + ) + + assert not binaries_dir.exists() + + def test_dotenv_values_are_loaded_and_process_environment_wins(self, tmp_path: Path) -> None: + _write_target_module( + tmp_path, + """ +from utils.target_artifacts.entry_helpers import text_entry +from utils.target_artifacts.resolvers import EnvResolver + +class Dev: + def artifact_inputs(self, env): + return (EnvResolver(name="value", variable_name="STAGED_VALUE", default_value="default"),) + + def artifact_entries(self, resolved_inputs): + return (text_entry("value", resolved_inputs["value"].value),) + +class Prod(Dev): + pass +""", + ) + (tmp_path / ".env").write_text("STAGED_VALUE=dotenv\n", encoding="utf-8") + + stage_target( + "fake", + "dev", + repo_root=tmp_path, + binaries_dir=tmp_path / "binaries", + process_env={"STAGED_VALUE": "process"}, + ) + + assert (tmp_path / "binaries" / "value").read_text(encoding="utf-8") == "process\n" + + def test_manifest_refreshes_owned_files_and_preserves_other_targets(self, tmp_path: Path) -> None: + module_path = tmp_path / "utils" / "build" / "docker" / "fake" + module_path.mkdir(parents=True) + artifact_module = module_path / "artifact.py" + artifact_module.write_text( + """ +from utils.target_artifacts.entry_helpers import text_entry + +class Dev: + def artifact_inputs(self, env): + return () + + def artifact_entries(self, resolved_inputs): + return (text_entry("kept", "one"), text_entry("stale", "old")) + +class Prod: + def artifact_inputs(self, env): + return () + + def artifact_entries(self, resolved_inputs): + return (text_entry("kept", "two"),) +""", + encoding="utf-8", + ) + other_module = tmp_path / "utils" / "build" / "docker" / "other" / "artifact.py" + other_module.parent.mkdir(parents=True) + other_module.write_text( + """ +from utils.target_artifacts.entry_helpers import text_entry + +class Dev: + def artifact_inputs(self, env): + return () + + def artifact_entries(self, resolved_inputs): + return (text_entry("other", "target"),) + +class Prod(Dev): + pass +""", + encoding="utf-8", + ) + + binaries_dir = tmp_path / "binaries" + stage_target("fake", "dev", repo_root=tmp_path, binaries_dir=binaries_dir) + stage_target("other", "dev", repo_root=tmp_path, binaries_dir=binaries_dir) + stage_target("fake", "prod", repo_root=tmp_path, binaries_dir=binaries_dir) + + assert (binaries_dir / "kept").read_text(encoding="utf-8") == "two\n" + assert not (binaries_dir / "stale").exists() + assert (binaries_dir / "other").read_text(encoding="utf-8") == "target\n" + assert set(_manifest_entries(binaries_dir)) == {"kept", "other"} + + def test_unowned_file_is_not_overwritten(self, tmp_path: Path) -> None: + _write_target_module( + tmp_path, + """ +from utils.target_artifacts.entry_helpers import text_entry + +class Dev: + def artifact_inputs(self, env): + return () + + def artifact_entries(self, resolved_inputs): + return (text_entry("manual", "generated"),) + +class Prod(Dev): + pass +""", + ) + binaries_dir = tmp_path / "binaries" + binaries_dir.mkdir() + (binaries_dir / "manual").write_text("user\n", encoding="utf-8") + + with pytest.raises(Exception, match="Refusing to overwrite unowned artifact entry 'manual'"): + stage_target("fake", "dev", repo_root=tmp_path, binaries_dir=binaries_dir) + + assert (binaries_dir / "manual").read_text(encoding="utf-8") == "user\n" + + def test_github_release_resolver_wraps_request_failures(self, monkeypatch: pytest.MonkeyPatch) -> None: + def fail_get(*_args: object, **_kwargs: object) -> object: + raise requests.ConnectionError("network unavailable") + + monkeypatch.setattr("utils.target_artifacts.resolvers.requests.get", fail_get) + resolver = GitHubLatestReleaseResolver(name="release", repository="DataDog/dd-trace-py") + + with pytest.raises(TargetArtifactError, match="Unable to resolve artifact metadata"): + resolver.resolve({}) + + def test_env_resolver_resolves_env_input(self) -> None: + resolver = EnvResolver(name="value", variable_name="STAGED_VALUE", default_value="default") + + resolved = resolver.resolve({"STAGED_VALUE": "from-env"}) + + assert resolved == LiteralValue(name="value", value="from-env") + + @pytest.mark.parametrize( + ("resolver_type", "resolved_type"), + [ + (EnvResolver, LiteralValue.__name__), + (GitHubBranchResolver, BranchReference.__name__), + (GitHubLatestReleaseResolver, GitHubReleaseReference.__name__), + (GitHubActionsArtifactResolver, GitHubActionsArtifactReference.__name__), + (OciDigestResolver, OciImageReference.__name__), + (NpmLatestResolver, ModuleVersion.__name__), + (PypiLatestResolver, ModuleVersion.__name__), + (RubygemsLatestResolver, ModuleVersion.__name__), + (CratesLatestResolver, ModuleVersion.__name__), + (GoModuleLatestResolver, ModuleVersion.__name__), + ], + ) + def test_artifact_resolver_docstring_names_resolved_input_type( + self, + resolver_type: type[ArtifactResolver], + resolved_type: str, + ) -> None: + assert resolver_type.__doc__ is not None + assert resolved_type in resolver_type.__doc__ + + +@scenarios.test_the_test +class Test_TargetArtifactResolvers: + def test_github_requests_include_auth_header_when_token_is_provided( + self, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + calls: list[tuple[str, dict[str, str], int]] = [] + + def fake_get(url: str, *, headers: dict[str, str], timeout: int) -> StubResponse: + calls.append((url, dict(headers), timeout)) + return StubResponse({"tag_name": "v1.2.3"}) + + monkeypatch.setattr("utils.target_artifacts.resolvers.requests.get", fake_get) + + resolved = GitHubLatestReleaseResolver(name="release", repository="DataDog/example").resolve( + {"GITHUB_TOKEN": "secret-token"}, + ) + + assert resolved.tag_name == "v1.2.3" + assert calls == [ + ( + "https://api.github.com/repos/DataDog/example/releases/latest", + { + "Accept": "application/vnd.github.v3+json", + "Authorization": "Bearer secret-token", + }, + 30, + ), + ] + + def test_get_json_wraps_non_json_payloads(self, monkeypatch: pytest.MonkeyPatch) -> None: + def fake_get(_url: str, *, headers: dict[str, str], timeout: int) -> StubResponse: + assert headers == {"Accept": "application/vnd.github.v3+json"} + assert timeout == 30 + return StubResponse({}, json_error=ValueError("invalid json")) + + monkeypatch.setattr("utils.target_artifacts.resolvers.requests.get", fake_get) + resolver = GitHubLatestReleaseResolver(name="release", repository="DataDog/example") + + with pytest.raises(TargetArtifactError, match="Unable to parse artifact metadata"): + resolver.resolve({}) + + def test_github_branch_resolver_accepts_full_sha_without_network(self, monkeypatch: pytest.MonkeyPatch) -> None: + def fail_get_json(url: str, headers: dict[str, str]) -> dict[str, Any]: + raise AssertionError(f"Unexpected GitHub request to {url} with {headers}") + + monkeypatch.setattr("utils.target_artifacts.resolvers._get_json", fail_get_json) + resolver = GitHubBranchResolver( + name="library_branch", + repository="DataDog/dd-trace-py", + variable_name="LIBRARY_TARGET_BRANCH", + ) + + resolved = resolver.resolve({"LIBRARY_TARGET_BRANCH": SHA}) + + assert resolved == BranchReference( + name="library_branch", + repository="DataDog/dd-trace-py", + branch=SHA, + sha=SHA, + ) + + def test_github_branch_resolver_resolves_quoted_branch_name(self, monkeypatch: pytest.MonkeyPatch) -> None: + branch = "feature/space branch" + expected_url = "https://api.github.com/repos/DataDog/dd-trace-py/branches/feature%2Fspace%20branch" + calls = _stub_get_json( + monkeypatch, + { + expected_url: { + "commit": { + "sha": OTHER_SHA, + }, + }, + }, + ) + resolver = GitHubBranchResolver( + name="library_branch", + repository="DataDog/dd-trace-py", + variable_name="LIBRARY_TARGET_BRANCH", + ) + + resolved = resolver.resolve({"GITHUB_TOKEN": "secret-token", "LIBRARY_TARGET_BRANCH": branch}) + + assert resolved == BranchReference( + name="library_branch", + repository="DataDog/dd-trace-py", + branch=branch, + sha=OTHER_SHA, + ) + assert calls == [ + ( + expected_url, + { + "Accept": "application/vnd.github.v3+json", + "Authorization": "Bearer secret-token", + }, + ), + ] + + def test_github_branch_resolver_rejects_missing_branch(self) -> None: + resolver = GitHubBranchResolver(name="library_branch", repository="DataDog/dd-trace-py") + + with pytest.raises(TargetArtifactError, match="Missing branch for input 'library_branch'"): + resolver.resolve({}) + + def test_github_branch_resolver_rejects_invalid_sha(self, monkeypatch: pytest.MonkeyPatch) -> None: + _stub_get_json( + monkeypatch, + { + "https://api.github.com/repos/DataDog/dd-trace-py/branches/main": { + "commit": { + "sha": "not-a-sha", + }, + }, + }, + ) + resolver = GitHubBranchResolver( + name="library_branch", + repository="DataDog/dd-trace-py", + default_value="main", + ) + + with pytest.raises(TargetArtifactError, match="did not resolve to a commit SHA"): + resolver.resolve({}) + + def test_github_latest_release_resolver_includes_assets(self, monkeypatch: pytest.MonkeyPatch) -> None: + _stub_get_json( + monkeypatch, + { + "https://api.github.com/repos/DataDog/dd-trace-java/releases/latest": { + "tag_name": "v1.2.3", + "assets": [ + { + "name": "dd-java-agent.jar", + "browser_download_url": "https://github.example/dd-java-agent.jar", + }, + ], + }, + }, + ) + resolver = GitHubLatestReleaseResolver( + name="release", + repository="DataDog/dd-trace-java", + include_assets=True, + ) + + resolved = resolver.resolve({}) + + assert resolved == GitHubReleaseReference( + name="release", + repository="DataDog/dd-trace-java", + tag_name="v1.2.3", + assets=( + ReleaseAsset( + name="dd-java-agent.jar", + browser_download_url="https://github.example/dd-java-agent.jar", + ), + ), + ) + + def test_github_latest_release_resolver_rejects_missing_assets(self, monkeypatch: pytest.MonkeyPatch) -> None: + _stub_get_json( + monkeypatch, + { + "https://api.github.com/repos/DataDog/dd-trace-java/releases/latest": { + "tag_name": "v1.2.3", + }, + }, + ) + resolver = GitHubLatestReleaseResolver( + name="release", + repository="DataDog/dd-trace-java", + include_assets=True, + ) + + with pytest.raises(TargetArtifactError, match="did not include assets"): + resolver.resolve({}) + + def test_github_actions_artifact_resolver_selects_matching_artifact( + self, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + expected_runs_url = ( + "https://api.github.com/repos/DataDog/httpd-datadog/actions/workflows/dev.yml/runs" + "?branch=feature%2Fbranch&status=completed&per_page=100" + ) + expected_artifacts_url = "https://api.github.example/runs/123/artifacts?per_page=100" + calls = _stub_get_json( + monkeypatch, + { + expected_runs_url: { + "workflow_runs": [ + { + "conclusion": "failure", + }, + { + "conclusion": "success", + "artifacts_url": "https://api.github.example/runs/123/artifacts", + "head_sha": SHA, + "id": 123, + "html_url": "https://github.example/DataDog/httpd-datadog/actions/runs/123", + }, + ], + }, + expected_artifacts_url: { + "artifacts": [ + { + "id": 456, + "name": "logs", + "archive_download_url": "https://github.example/logs.zip", + }, + { + "id": 789, + "name": "mod_datadog_artifact.zip", + "archive_download_url": "https://github.example/mod_datadog_artifact.zip", + }, + ], + }, + }, + ) + resolver = GitHubActionsArtifactResolver( + name="workflow_artifact", + repository="DataDog/httpd-datadog", + workflow="dev.yml", + artifact_name="mod_datadog_artifact", + variable_name="LIBRARY_TARGET_BRANCH", + ) + + resolved = resolver.resolve({"LIBRARY_TARGET_BRANCH": "feature/branch"}) + + assert resolved == GitHubActionsArtifactReference( + name="workflow_artifact", + repository="DataDog/httpd-datadog", + workflow="dev.yml", + branch="feature/branch", + commit_sha=SHA, + run_id=123, + run_url="https://github.example/DataDog/httpd-datadog/actions/runs/123", + artifact_id=789, + artifact_name="mod_datadog_artifact.zip", + archive_download_url="https://github.example/mod_datadog_artifact.zip", + ) + assert [url for url, _headers in calls] == [expected_runs_url, expected_artifacts_url] + + def test_github_actions_artifact_resolver_errors_when_only_failed_runs_exist( + self, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + _stub_get_json( + monkeypatch, + { + "https://api.github.com/repos/DataDog/httpd-datadog/actions/workflows/dev.yml/runs" + "?branch=main&status=completed&per_page=100": { + "workflow_runs": [ + { + "conclusion": "failure", + }, + ], + }, + }, + ) + resolver = GitHubActionsArtifactResolver( + name="workflow_artifact", + repository="DataDog/httpd-datadog", + workflow="dev.yml", + artifact_name="mod_datadog_artifact", + default_value="main", + ) + + with pytest.raises(TargetArtifactError, match="No completed workflow run found"): + resolver.resolve({}) + + def test_github_actions_artifact_resolver_errors_when_artifact_is_missing( + self, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + _stub_get_json( + monkeypatch, + { + "https://api.github.com/repos/DataDog/httpd-datadog/actions/workflows/dev.yml/runs" + "?branch=main&status=completed&per_page=100": { + "workflow_runs": [ + { + "conclusion": "success", + "artifacts_url": "https://api.github.example/runs/123/artifacts", + "head_sha": SHA, + "id": 123, + "html_url": "https://github.example/DataDog/httpd-datadog/actions/runs/123", + }, + ], + }, + "https://api.github.example/runs/123/artifacts?per_page=100": { + "artifacts": [ + { + "id": 456, + "name": "logs", + "archive_download_url": "https://github.example/logs.zip", + }, + ], + }, + }, + ) + resolver = GitHubActionsArtifactResolver( + name="workflow_artifact", + repository="DataDog/httpd-datadog", + workflow="dev.yml", + artifact_name="mod_datadog_artifact", + default_value="main", + ) + + with pytest.raises(TargetArtifactError, match="No artifact containing 'mod_datadog_artifact' found"): + resolver.resolve({}) + + def test_oci_digest_resolver_accepts_pinned_digest_without_docker(self, monkeypatch: pytest.MonkeyPatch) -> None: + def fail_run( + args: list[str], *, capture_output: bool, check: bool, text: bool + ) -> subprocess.CompletedProcess[str]: + raise AssertionError(f"Unexpected docker invocation: {args}, {capture_output}, {check}, {text}") + + monkeypatch.setattr("utils.target_artifacts.resolvers.subprocess.run", fail_run) + image = f"registry.example.com/team/app@{DIGEST}" + resolver = OciDigestResolver(name="image", image=image) + + resolved = resolver.resolve({}) + + assert resolved == OciImageReference( + name="image", + image=image, + digest=DIGEST, + reference=image, + ) + + def test_oci_digest_resolver_builds_digest_reference_for_registry_with_port( + self, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + image = "registry.example.com:5000/team/app:latest" + + def fake_run( + args: list[str], + *, + capture_output: bool, + check: bool, + text: bool, + ) -> subprocess.CompletedProcess[str]: + assert args == ["docker", "buildx", "imagetools", "inspect", image] + assert capture_output is True + assert check is False + assert text is True + return _completed_process(args, stdout=f"Name: {image}\nDigest: {DIGEST}\n") + + monkeypatch.setattr("utils.target_artifacts.resolvers.subprocess.run", fake_run) + resolver = OciDigestResolver(name="image", image=image) + + resolved = resolver.resolve({}) + + assert resolved == OciImageReference( + name="image", + image=image, + digest=DIGEST, + reference=f"registry.example.com:5000/team/app@{DIGEST}", + ) + + @pytest.mark.parametrize( + ("run_result", "match"), + [ + (FileNotFoundError(), "docker was not found"), + (_completed_process([], returncode=1, stderr="denied"), "denied"), + (_completed_process([], stdout="Name: registry.example.com/app\n"), "Unable to find OCI digest"), + ], + ) + def test_oci_digest_resolver_wraps_docker_failures( + self, + monkeypatch: pytest.MonkeyPatch, + run_result: subprocess.CompletedProcess[str] | FileNotFoundError, + match: str, + ) -> None: + def fake_run( + args: list[str], + *, + capture_output: bool, + check: bool, + text: bool, + ) -> subprocess.CompletedProcess[str]: + assert capture_output is True + assert check is False + assert text is True + if isinstance(run_result, FileNotFoundError): + raise run_result + return _completed_process( + args, returncode=run_result.returncode, stdout=run_result.stdout, stderr=run_result.stderr + ) + + monkeypatch.setattr("utils.target_artifacts.resolvers.subprocess.run", fake_run) + resolver = OciDigestResolver(name="image", image="registry.example.com/app:latest") + + with pytest.raises(TargetArtifactError, match=match): + resolver.resolve({}) + + @pytest.mark.parametrize( + ("resolver", "payload", "expected"), + [ + ( + NpmLatestResolver(name="package", package="@datadog/browser-core"), + {"version": "1.2.3"}, + ModuleVersion(name="package", module="@datadog/browser-core", version="1.2.3"), + ), + ( + PypiLatestResolver(name="package", package="ddtrace"), + {"info": {"version": "2.3.4"}}, + ModuleVersion(name="package", module="ddtrace", version="2.3.4"), + ), + ( + RubygemsLatestResolver(name="package", package="datadog"), + {"version": "3.4.5"}, + ModuleVersion(name="package", module="datadog", version="3.4.5"), + ), + ( + CratesLatestResolver(name="package", package="datadog-opentelemetry"), + {"crate": {"max_stable_version": "0.1.0", "max_version": "0.2.0"}}, + ModuleVersion(name="package", module="datadog-opentelemetry", version="0.1.0"), + ), + ], + ) + def test_package_registry_resolvers_return_versions( + self, + monkeypatch: pytest.MonkeyPatch, + resolver: NpmLatestResolver | PypiLatestResolver | RubygemsLatestResolver | CratesLatestResolver, + payload: dict[str, Any], + expected: ModuleVersion, + ) -> None: + calls = _stub_get_json(monkeypatch, lambda _url, _headers: payload) + + resolved = resolver.resolve({}) + + assert resolved == expected + assert len(calls) == 1 + + def test_crates_latest_resolver_falls_back_to_max_version(self, monkeypatch: pytest.MonkeyPatch) -> None: + _stub_get_json( + monkeypatch, + lambda _url, _headers: { + "crate": { + "max_stable_version": None, + "max_version": "0.2.0", + }, + }, + ) + resolver = CratesLatestResolver(name="package", package="datadog-opentelemetry") + + resolved = resolver.resolve({}) + + assert resolved == ModuleVersion(name="package", module="datadog-opentelemetry", version="0.2.0") + + def test_crates_latest_resolver_sends_descriptive_user_agent(self, monkeypatch: pytest.MonkeyPatch) -> None: + calls = _stub_get_json( + monkeypatch, + lambda _url, _headers: { + "crate": { + "max_stable_version": "0.1.0", + }, + }, + ) + resolver = CratesLatestResolver(name="package", package="datadog-opentelemetry") + + resolver.resolve({}) + + assert calls == [ + ( + "https://crates.io/api/v1/crates/datadog-opentelemetry", + { + "Accept": "application/json", + "User-Agent": "system-tests-target-artifacts (https://github.com/DataDog/system-tests)", + }, + ), + ] + + @pytest.mark.parametrize( + ("resolver", "payload", "match"), + [ + ( + NpmLatestResolver(name="package", package="dd-trace"), + {}, + "NPM package dd-trace did not include a version", + ), + ( + PypiLatestResolver(name="package", package="ddtrace"), + {}, + "Expected PyPI package ddtrace info to be an object", + ), + ( + RubygemsLatestResolver(name="package", package="datadog"), + {"version": ""}, + "RubyGems package datadog did not include a version", + ), + ( + CratesLatestResolver(name="package", package="datadog-opentelemetry"), + {"crate": {}}, + "crate datadog-opentelemetry did not include a version", + ), + ], + ) + def test_package_registry_resolvers_reject_missing_versions( + self, + monkeypatch: pytest.MonkeyPatch, + resolver: NpmLatestResolver | PypiLatestResolver | RubygemsLatestResolver | CratesLatestResolver, + payload: dict[str, Any], + match: str, + ) -> None: + _stub_get_json(monkeypatch, lambda _url, _headers: payload) + + with pytest.raises(TargetArtifactError, match=match): + resolver.resolve({}) + + def test_go_module_latest_resolver_returns_version(self, monkeypatch: pytest.MonkeyPatch) -> None: + module = "github.com/DataDog/dd-trace-go/v2" + + def fake_run( + args: list[str], + *, + capture_output: bool, + check: bool, + text: bool, + ) -> subprocess.CompletedProcess[str]: + assert args == ["go", "list", "-m", "-json", f"{module}@latest"] + assert capture_output is True + assert check is False + assert text is True + return _completed_process(args, stdout='{"Version": "v1.2.3"}') + + monkeypatch.setattr("utils.target_artifacts.resolvers.subprocess.run", fake_run) + resolver = GoModuleLatestResolver(name="module", module=module) + + resolved = resolver.resolve({}) + + assert resolved == ModuleVersion(name="module", module=module, version="v1.2.3") + + @pytest.mark.parametrize( + ("run_result", "match"), + [ + (FileNotFoundError(), "go was not found"), + (_completed_process([], returncode=1, stderr="module not found"), "module not found"), + (_completed_process([], stdout="{not-json"), "Unable to parse Go module metadata"), + ( + _completed_process([], stdout='{"Path": "github.com/DataDog/dd-trace-go/v2"}'), + "did not include a version", + ), + ], + ) + def test_go_module_latest_resolver_wraps_go_failures( + self, + monkeypatch: pytest.MonkeyPatch, + run_result: subprocess.CompletedProcess[str] | FileNotFoundError, + match: str, + ) -> None: + def fake_run( + args: list[str], + *, + capture_output: bool, + check: bool, + text: bool, + ) -> subprocess.CompletedProcess[str]: + assert capture_output is True + assert check is False + assert text is True + if isinstance(run_result, FileNotFoundError): + raise run_result + return _completed_process( + args, returncode=run_result.returncode, stdout=run_result.stdout, stderr=run_result.stderr + ) + + monkeypatch.setattr("utils.target_artifacts.resolvers.subprocess.run", fake_run) + resolver = GoModuleLatestResolver(name="module", module="github.com/DataDog/dd-trace-go/v2") + + with pytest.raises(TargetArtifactError, match=match): + resolver.resolve({}) + + +@scenarios.test_the_test +class Test_TargetArtifactExternalContracts: + def test_public_github_branch_contract(self) -> None: + resolved = GitHubBranchResolver( + name="library_branch", + repository="DataDog/dd-trace-py", + default_value="main", + ).resolve({}) + + assert resolved.branch == "main" + assert resolved.repository == "DataDog/dd-trace-py" + assert len(resolved.sha) == 40 + assert all(character in "0123456789abcdef" for character in resolved.sha) + + def test_public_github_latest_release_contract_includes_assets(self) -> None: + resolved = GitHubLatestReleaseResolver( + name="release", + repository="DataDog/datadog-lambda-python", + include_assets=True, + ).resolve({}) + + assert resolved.repository == "DataDog/datadog-lambda-python" + assert resolved.tag_name.startswith("v") + assert resolved.assets + assert all(asset.name for asset in resolved.assets) + assert all( + asset.browser_download_url.startswith( + "https://github.com/DataDog/datadog-lambda-python/releases/download/", + ) + for asset in resolved.assets + ) + + def test_public_github_actions_artifact_contract_uses_unauthenticated_request(self) -> None: + resolved = GitHubActionsArtifactResolver( + name="workflow_artifact", + repository="DataDog/httpd-datadog", + workflow="dev.yml", + artifact_name="mod_datadog_artifact", + default_value="main", + ).resolve({}) + + assert resolved.repository == "DataDog/httpd-datadog" + assert resolved.workflow == "dev.yml" + assert resolved.branch == "main" + assert len(resolved.commit_sha) == 40 + assert all(character in "0123456789abcdef" for character in resolved.commit_sha) + assert resolved.run_url.startswith("https://github.com/DataDog/httpd-datadog/actions/runs/") + assert "mod_datadog_artifact" in resolved.artifact_name + assert resolved.archive_download_url.startswith( + "https://api.github.com/repos/DataDog/httpd-datadog/actions/artifacts/", + ) + + @pytest.mark.parametrize( + "resolver", + [ + NpmLatestResolver(name="package", package="dd-trace"), + PypiLatestResolver(name="package", package="ddtrace"), + RubygemsLatestResolver(name="package", package="datadog"), + CratesLatestResolver(name="package", package="datadog-opentelemetry"), + ], + ) + def test_public_package_registry_contracts_return_versions( + self, + resolver: NpmLatestResolver | PypiLatestResolver | RubygemsLatestResolver | CratesLatestResolver, + ) -> None: + resolved = resolver.resolve({}) + + assert resolved.module + assert resolved.version + assert any(character.isdigit() for character in resolved.version) + + +@scenarios.test_the_test +class Test_TargetArtifactModules: + @pytest.mark.parametrize("target", sorted(COMPONENT_GROUPS.all)) + @pytest.mark.parametrize("environment", ["dev", "prod"]) + def test_every_target_has_real_staging_behavior(self, target: str, environment: str) -> None: + target_environment = load_target_environment(Path.cwd(), target, environment) + env = {} + if environment == "dev": + env = { + "AUTO_INJECT_TARGET_BRANCH": "auto-inject-branch", + "LIBRARY_TARGET_BRANCH": "library-branch", + "ORCHESTRION_TARGET_BRANCH": "orchestrion-branch", + } + resolver = FakeResolver() + resolved = { + artifact_resolver.name: resolver.resolve(artifact_resolver, env) + for artifact_resolver in target_environment.artifact_inputs(env) + } + + entries = target_environment.artifact_entries(resolved) + + assert entries, f"{target} {environment} did not emit artifact entries" + assert all(entry.content.endswith("\n") for entry in entries) + assert all("placeholder" not in entry.content.lower() for entry in entries) + if environment == "prod": + assert all(":latest" not in entry.content for entry in entries) + assert all("@latest" not in entry.content for entry in entries) + + def test_c_dev_supports_independent_branch_overrides(self) -> None: + target_environment = load_target_environment(Path.cwd(), "c", "dev") + env = { + "AUTO_INJECT_TARGET_BRANCH": "auto-inject-branch", + "LIBRARY_TARGET_BRANCH": "library-branch", + } + resolver = FakeResolver() + resolved = { + artifact_resolver.name: resolver.resolve(artifact_resolver, env) + for artifact_resolver in target_environment.artifact_inputs(env) + } + + entries = {entry.filename: entry.content.strip() for entry in target_environment.artifact_entries(resolved)} + + assert entries == { + "c-injector-image": f"installtesting.datad0g.com/apm-inject-package:{SHA}", + "c-library-image": f"installtesting.datad0g.com/apm-library-c-package:{SHA}", + } + + def test_workflow_artifact_entries_are_credential_free_json(self) -> None: + target_environment = load_target_environment(Path.cwd(), "python_lambda", "dev") + env: dict[str, str] = {} + resolver = FakeResolver() + resolved = { + artifact_resolver.name: resolver.resolve(artifact_resolver, env) + for artifact_resolver in target_environment.artifact_inputs(env) + } + + entry = target_environment.artifact_entries(resolved)[0] + payload = json.loads(entry.content) + + assert entry.filename.endswith(".json") + assert payload["commit_sha"] == SHA + assert "token" not in entry.content.lower() + + def test_provider_package_selectors_have_build_consumers(self) -> None: + build_script = Path("utils/build/build.sh").read_text(encoding="utf-8") + + assert "binaries/dotnet-package-image" in build_script + assert "datadog-dotnet-apm*.tar.gz" in build_script + assert "binaries/php-package-image" in build_script + assert "dd-library-php-*-linux-gnu.tar.gz" in build_script + assert "datadog-setup.php" in build_script + + def test_staged_java_otel_selector_has_installer_consumer(self) -> None: + target_environment = load_target_environment(Path.cwd(), "java_otel", "dev") + env = {"LIBRARY_TARGET_BRANCH": "ignored"} + resolver = FakeResolver() + resolved = { + artifact_resolver.name: resolver.resolve(artifact_resolver, env) + for artifact_resolver in target_environment.artifact_inputs(env) + } + + entries = target_environment.artifact_entries(resolved) + installer = Path("utils/build/docker/java_otel/install_opentelemetry.sh").read_text(encoding="utf-8") + + assert {entry.filename for entry in entries} == {"java-otel-load-from-release"} + assert "java-otel-load-from-release" in installer + + def test_lambda_workflow_metadata_is_parsed_with_jq(self) -> None: + for installer_path, metadata_filename in ( + ( + Path("utils/build/docker/python_lambda/install_datadog_lambda.sh"), + "python-lambda-github-actions-artifact.json", + ), + ( + Path("utils/build/docker/nodejs_lambda/install_datadog_lambda.sh"), + "nodejs-lambda-github-actions-artifact.json", + ), + ): + installer = installer_path.read_text(encoding="utf-8") + + assert metadata_filename in installer + assert "jq -r '.archive_download_url'" in installer diff --git a/utils/__init__.py b/utils/__init__.py index 56fe17b8fae..9bad69eba76 100644 --- a/utils/__init__.py +++ b/utils/__init__.py @@ -2,25 +2,31 @@ # This product includes software developed at Datadog (https://www.datadoghq.com/). # Copyright 2021 Datadog, Inc. -# singletons -from utils._weblog import weblog, HttpResponse -from utils._context.core import context -from utils._context._scenarios import scenarios, scenario_groups -from utils._decorators import ( - bug, - irrelevant, - missing_feature, - rfc, - flaky, - incomplete_test_app, - slow, - scenario_crash, - auxiliary_test, -) -from utils._logger import logger -from utils import interfaces, _remote_config as remote_config -from utils.interfaces._core import ValidationError -from utils._features import features +from __future__ import annotations + +from importlib import import_module +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + import utils._remote_config as remote_config + from utils import interfaces + from utils._context._scenarios import scenario_groups as scenario_groups + from utils._context._scenarios import scenarios as scenarios + from utils._context.core import context as context + from utils._decorators import auxiliary_test as auxiliary_test + from utils._decorators import bug as bug + from utils._decorators import flaky as flaky + from utils._decorators import incomplete_test_app as incomplete_test_app + from utils._decorators import irrelevant as irrelevant + from utils._decorators import missing_feature as missing_feature + from utils._decorators import rfc as rfc + from utils._decorators import scenario_crash as scenario_crash + from utils._decorators import slow as slow + from utils._features import features as features + from utils._logger import logger as logger + from utils._weblog import HttpResponse as HttpResponse + from utils._weblog import weblog as weblog + from utils.interfaces._core import ValidationError as ValidationError __all__ = [ "HttpResponse", @@ -43,3 +49,36 @@ "slow", "weblog", ] + +_LAZY_EXPORTS = { + "HttpResponse": ("utils._weblog", "HttpResponse"), + "ValidationError": ("utils.interfaces._core", "ValidationError"), + "auxiliary_test": ("utils._decorators", "auxiliary_test"), + "bug": ("utils._decorators", "bug"), + "context": ("utils._context.core", "context"), + "features": ("utils._features", "features"), + "flaky": ("utils._decorators", "flaky"), + "incomplete_test_app": ("utils._decorators", "incomplete_test_app"), + "interfaces": ("utils.interfaces", None), + "irrelevant": ("utils._decorators", "irrelevant"), + "logger": ("utils._logger", "logger"), + "missing_feature": ("utils._decorators", "missing_feature"), + "remote_config": ("utils._remote_config", None), + "rfc": ("utils._decorators", "rfc"), + "scenario_crash": ("utils._decorators", "scenario_crash"), + "scenario_groups": ("utils._context._scenarios", "scenario_groups"), + "scenarios": ("utils._context._scenarios", "scenarios"), + "slow": ("utils._decorators", "slow"), + "weblog": ("utils._weblog", "weblog"), +} + + +def __getattr__(name: str) -> object: + if name not in _LAZY_EXPORTS: + raise AttributeError(f"module 'utils' has no attribute '{name}'") + + module_name, attribute_name = _LAZY_EXPORTS[name] + module = import_module(module_name) + value = module if attribute_name is None else getattr(module, attribute_name) + globals()[name] = value + return value diff --git a/utils/build/build.sh b/utils/build/build.sh index 85f217423f8..f3d22b847fa 100755 --- a/utils/build/build.sh +++ b/utils/build/build.sh @@ -143,6 +143,63 @@ run_build_command() { return "${exit_code}" } +copy_staged_package_files() { + local source_dir=$1 + local package_pattern=$2 + local setup_pattern=${3:-} + local package_count + local setup_count=1 + + package_count=$(find "$source_dir" -type f -name "$package_pattern" | wc -l) + if [[ "$package_count" -eq 0 ]]; then + echo "ERROR: extracted staged package image did not contain $package_pattern" >&2 + exit 1 + fi + + if [[ -n "$setup_pattern" ]]; then + setup_count=$(find "$source_dir" -type f -name "$setup_pattern" | wc -l) + if [[ "$setup_count" -eq 0 ]]; then + echo "ERROR: extracted staged package image did not contain $setup_pattern" >&2 + exit 1 + fi + find "$source_dir" -type f -name "$setup_pattern" -exec cp {} binaries/ \; + fi + + find "$source_dir" -type f -name "$package_pattern" -exec cp {} binaries/ \; +} + +materialize_staged_package_image() { + local selector_file=$1 + local package_pattern=$2 + local setup_pattern=${3:-} + local temp_dir + local image + + image=$(<"$selector_file") + temp_dir=$(mktemp -d "${TMPDIR:-/tmp}/system-tests-staged-package.XXXXXX") + run_build_command utils/scripts/docker_base_image.sh "$image" "$temp_dir" + copy_staged_package_files "$temp_dir" "$package_pattern" "$setup_pattern" + rm -rf "$temp_dir" +} + +materialize_staged_provider_artifacts() { + if [[ $TEST_LIBRARY == dotnet ]] && [[ -f binaries/dotnet-package-image ]]; then + if [[ "$(find binaries -maxdepth 1 \( -name 'datadog-dotnet-apm*.tar.gz' -o -name 'Datadog.Trace.ClrProfiler.Native.so' \) | wc -l)" -gt 0 ]]; then + echo "Skipping staged .NET package image because local .NET artifacts already exist in binaries/" + else + materialize_staged_package_image binaries/dotnet-package-image 'datadog-dotnet-apm*.tar.gz' + fi + fi + + if [[ $TEST_LIBRARY == php ]] && [[ -f binaries/php-package-image ]]; then + if [[ "$(find binaries -maxdepth 1 \( -name 'dd-library-php-*-linux-gnu.tar.gz' -o -name 'datadog-setup.php' \) | wc -l)" -gt 0 ]]; then + echo "Skipping staged PHP package image because local PHP artifacts already exist in binaries/" + else + materialize_staged_package_image binaries/php-package-image 'dd-library-php-*-linux-gnu.tar.gz' 'datadog-setup.php' + fi + fi +} + build() { echo "==================================" @@ -302,6 +359,8 @@ build() { run_build_command docker run ${DOCKER_PLATFORM_ARGS} -v ./binaries/:/app -w /app ghcr.io/datadog/dd-trace-py/testrunner bash -c "pyenv global $PYTHON_VERSION; pip wheel --no-deps -w . /app/dd-trace-py" fi + materialize_staged_provider_artifacts + DOCKERFILE=utils/build/docker/${TEST_LIBRARY}/${WEBLOG_VARIANT}.Dockerfile # When the image mirror is enabled, create (or reuse) a buildx diff --git a/utils/build/docker/c/artifact.py b/utils/build/docker/c/artifact.py new file mode 100644 index 00000000000..255b229c309 --- /dev/null +++ b/utils/build/docker/c/artifact.py @@ -0,0 +1,89 @@ +from __future__ import annotations + +from typing import cast + +from utils.target_artifacts.entry_helpers import text_entry +from utils.target_artifacts.models import ( + ArtifactEntry, + BranchReference, + OciImageReference, + TargetArtifactError, +) +from utils.target_artifacts.resolvers import GitHubBranchResolver, OciDigestResolver + +type ArtifactInputResolver = GitHubBranchResolver | OciDigestResolver +type ResolvedCInput = BranchReference | OciImageReference + +PROD_LIBRARY_IMAGE = "install.datadoghq.com/apm-library-c-package:latest" +PROD_INJECTOR_IMAGE = "install.datadoghq.com/apm-inject-package:latest" +DEV_LIBRARY_IMAGE = "installtesting.datad0g.com/apm-library-c-package" +DEV_INJECTOR_IMAGE = "installtesting.datad0g.com/apm-inject-package" + + +class Dev: + def artifact_inputs(self, env: dict[str, str]) -> tuple[ArtifactInputResolver, ...]: + inputs: list[ArtifactInputResolver] = [] + if env.get("LIBRARY_TARGET_BRANCH"): + inputs.append( + GitHubBranchResolver( + name="library_branch", + repository="DataDog/dd-trace-c", + variable_name="LIBRARY_TARGET_BRANCH", + ) + ) + else: + inputs.append(OciDigestResolver(name="library_image", image=PROD_LIBRARY_IMAGE)) + + if env.get("AUTO_INJECT_TARGET_BRANCH"): + inputs.append( + GitHubBranchResolver( + name="injector_branch", + repository="DataDog/auto_inject", + variable_name="AUTO_INJECT_TARGET_BRANCH", + ) + ) + else: + inputs.append(OciDigestResolver(name="injector_image", image=PROD_INJECTOR_IMAGE)) + return tuple(inputs) + + def artifact_entries( + self, + resolved_inputs: dict[str, ResolvedCInput], + ) -> tuple[ArtifactEntry, ArtifactEntry]: + if "library_branch" in resolved_inputs: + library_branch = cast(BranchReference, resolved_inputs["library_branch"]) + library_ref = f"{DEV_LIBRARY_IMAGE}:{library_branch.sha}" + else: + library_image = cast(OciImageReference, resolved_inputs["library_image"]) + library_ref = library_image.reference + + if "injector_branch" in resolved_inputs: + injector_branch = cast(BranchReference, resolved_inputs["injector_branch"]) + injector_ref = f"{DEV_INJECTOR_IMAGE}:{injector_branch.sha}" + else: + injector_image = cast(OciImageReference, resolved_inputs["injector_image"]) + injector_ref = injector_image.reference + + return ( + text_entry("c-library-image", library_ref), + text_entry("c-injector-image", injector_ref), + ) + + +class Prod: + def artifact_inputs(self, env: dict[str, str]) -> tuple[OciDigestResolver, OciDigestResolver]: + if env.get("LIBRARY_TARGET_BRANCH") or env.get("AUTO_INJECT_TARGET_BRANCH"): + raise TargetArtifactError("Target branches can only be used with the development c packages") + return ( + OciDigestResolver(name="library_image", image=PROD_LIBRARY_IMAGE), + OciDigestResolver(name="injector_image", image=PROD_INJECTOR_IMAGE), + ) + + def artifact_entries( + self, + resolved_inputs: dict[str, OciImageReference], + ) -> tuple[ArtifactEntry, ArtifactEntry]: + return ( + text_entry("c-library-image", resolved_inputs["library_image"].reference), + text_entry("c-injector-image", resolved_inputs["injector_image"].reference), + ) diff --git a/utils/build/docker/c/perl-mojolicious.Dockerfile b/utils/build/docker/c/perl-mojolicious.Dockerfile index c784ded3313..1e36a65e96e 100644 --- a/utils/build/docker/c/perl-mojolicious.Dockerfile +++ b/utils/build/docker/c/perl-mojolicious.Dockerfile @@ -17,7 +17,11 @@ RUN apk add --no-cache jq zstd \ output="$2"; \ manifest="$(oras manifest fetch --platform "linux/${TARGETARCH}" "$reference")"; \ digest="$(printf '%s' "$manifest" | jq -er '.layers[0].digest')"; \ - repository="${reference%:*}"; \ + if printf '%s' "$reference" | grep -q '@'; then \ + repository="${reference%%@*}"; \ + else \ + repository="${reference%:*}"; \ + fi; \ mkdir -p "$output"; \ oras blob fetch --output /tmp/package.tar.zst "${repository}@${digest}"; \ zstd --decompress --stdout /tmp/package.tar.zst | tar -x -C "$output"; \ diff --git a/utils/build/docker/cpp/artifact.py b/utils/build/docker/cpp/artifact.py new file mode 100644 index 00000000000..e008184b4b8 --- /dev/null +++ b/utils/build/docker/cpp/artifact.py @@ -0,0 +1,42 @@ +from __future__ import annotations + + +from utils.target_artifacts.entry_helpers import text_entry +from utils.target_artifacts.models import ( + ArtifactEntry, + BranchReference, + GitHubReleaseReference, +) +from utils.target_artifacts.resolvers import GitHubBranchResolver, GitHubLatestReleaseResolver + +REPOSITORY = "DataDog/dd-trace-cpp" +GIT_URL = "https://github.com/DataDog/dd-trace-cpp" + + +class Dev: + def artifact_inputs(self, env: dict[str, str]) -> tuple[GitHubBranchResolver]: + return ( + GitHubBranchResolver( + name="library_branch", + repository=REPOSITORY, + variable_name="LIBRARY_TARGET_BRANCH", + default_value="main", + ), + ) + + def artifact_entries( + self, + resolved_inputs: dict[str, BranchReference], + ) -> tuple[ArtifactEntry]: + return (text_entry("cpp-load-from-git", f"{GIT_URL}@{resolved_inputs['library_branch'].sha}"),) + + +class Prod: + def artifact_inputs(self, env: dict[str, str]) -> tuple[GitHubLatestReleaseResolver]: + return (GitHubLatestReleaseResolver(name="release", repository=REPOSITORY),) + + def artifact_entries( + self, + resolved_inputs: dict[str, GitHubReleaseReference], + ) -> tuple[ArtifactEntry]: + return (text_entry("cpp-load-from-git", f"{GIT_URL}@{resolved_inputs['release'].tag_name}"),) diff --git a/utils/build/docker/cpp_httpd/artifact.py b/utils/build/docker/cpp_httpd/artifact.py new file mode 100644 index 00000000000..e8b69d7eb21 --- /dev/null +++ b/utils/build/docker/cpp_httpd/artifact.py @@ -0,0 +1,56 @@ +from __future__ import annotations + + +from utils.target_artifacts.entry_helpers import json_entry, text_entry +from utils.target_artifacts.models import ( + ArtifactEntry, + GitHubActionsArtifactReference, + GitHubReleaseReference, +) +from utils.target_artifacts.resolvers import GitHubActionsArtifactResolver, GitHubLatestReleaseResolver + + +class Dev: + def artifact_inputs(self, env: dict[str, str]) -> tuple[GitHubActionsArtifactResolver]: + return ( + GitHubActionsArtifactResolver( + name="workflow_artifact", + repository="DataDog/httpd-datadog", + workflow="dev.yml", + artifact_name="mod_datadog_artifact", + variable_name="LIBRARY_TARGET_BRANCH", + default_value="main", + ), + ) + + def artifact_entries( + self, + resolved_inputs: dict[str, GitHubActionsArtifactReference], + ) -> tuple[ArtifactEntry]: + artifact = resolved_inputs["workflow_artifact"] + return ( + json_entry( + "cpp-httpd-github-actions-artifact.json", + { + "archive_download_url": artifact.archive_download_url, + "artifact_id": artifact.artifact_id, + "artifact_name": artifact.artifact_name, + "commit_sha": artifact.commit_sha, + "repository": artifact.repository, + "run_id": artifact.run_id, + "run_url": artifact.run_url, + "workflow": artifact.workflow, + }, + ), + ) + + +class Prod: + def artifact_inputs(self, env: dict[str, str]) -> tuple[GitHubLatestReleaseResolver]: + return (GitHubLatestReleaseResolver(name="release", repository="DataDog/httpd-datadog"),) + + def artifact_entries( + self, + resolved_inputs: dict[str, GitHubReleaseReference], + ) -> tuple[ArtifactEntry]: + return (text_entry("cpp-httpd-load-from-release", resolved_inputs["release"].tag_name),) diff --git a/utils/build/docker/cpp_httpd/install_ddtrace.sh b/utils/build/docker/cpp_httpd/install_ddtrace.sh index dece759f857..3dcbe0c5619 100755 --- a/utils/build/docker/cpp_httpd/install_ddtrace.sh +++ b/utils/build/docker/cpp_httpd/install_ddtrace.sh @@ -12,9 +12,23 @@ cd /binaries if [ -f "$FILENAME" ]; then echo "Install HTTPD plugin from binaries/$FILENAME" HTTPD_DATADOG_VERSION="v99.99.99" # TODO: get version from the binary. Right now, use the "big-version" trick - cp $FILENAME "$DEST_FOLDER/$FILENAME" + cp "$FILENAME" "$DEST_FOLDER/$FILENAME" +elif [ -f cpp-httpd-github-actions-artifact.json ]; then + echo "Install HTTPD plugin from staged GitHub Actions artifact metadata" + auth_header=$(get_authentication_header) + ARCHIVE_URL=$(jq -r '.archive_download_url' cpp-httpd-github-actions-artifact.json) + curl_cmd="curl -Lf $auth_header -o mod_datadog_artifact.zip ${ARCHIVE_URL}" + eval "$curl_cmd" + mkdir -p /tmp/mod-datadog-artifact + unzip -o mod_datadog_artifact.zip -d /tmp/mod-datadog-artifact + cp "$(find /tmp/mod-datadog-artifact -name "$FILENAME" | head -1)" "$DEST_FOLDER/$FILENAME" + HTTPD_DATADOG_VERSION="$(jq -r '.commit_sha' cpp-httpd-github-actions-artifact.json | cut -c1-12)" else - HTTPD_DATADOG_VERSION="$(get_latest_release DataDog/httpd-datadog)" + if [ -f cpp-httpd-load-from-release ]; then + HTTPD_DATADOG_VERSION=$(cat cpp-httpd-load-from-release) + else + HTTPD_DATADOG_VERSION="$(get_latest_release DataDog/httpd-datadog)" + fi TARBALL="mod_datadog_artifact.zip" URL="https://github.com/DataDog/httpd-datadog/releases/download/${HTTPD_DATADOG_VERSION}/${TARBALL}" echo "Get APACHE plugin from $URL" @@ -26,4 +40,3 @@ fi echo '{"status": "ok", "library": {"name": "cpp_httpd", "version": "'"$HTTPD_DATADOG_VERSION"'"}}' > /app/healthcheck.json echo "$HTTPD_DATADOG_VERSION" > SYSTEM_TESTS_LIBRARY_VERSION cat /app/healthcheck.json - diff --git a/utils/build/docker/cpp_kong/artifact.py b/utils/build/docker/cpp_kong/artifact.py new file mode 100644 index 00000000000..7335571c91c --- /dev/null +++ b/utils/build/docker/cpp_kong/artifact.py @@ -0,0 +1,63 @@ +from __future__ import annotations + + +from utils.target_artifacts.entry_helpers import text_entry +from utils.target_artifacts.models import ( + ArtifactEntry, + BranchReference, + GitHubReleaseReference, +) +from utils.target_artifacts.resolvers import GitHubBranchResolver, GitHubLatestReleaseResolver + + +class Dev: + def artifact_inputs(self, env: dict[str, str]) -> tuple[GitHubBranchResolver, GitHubBranchResolver]: + return ( + GitHubBranchResolver( + name="cpp_branch", + repository="DataDog/dd-trace-cpp", + variable_name="DD_TRACE_CPP_TARGET_BRANCH", + default_value="main", + ), + GitHubBranchResolver( + name="plugin_branch", + repository="DataDog/kong-plugin-ddtrace", + variable_name="LIBRARY_TARGET_BRANCH", + default_value="main", + ), + ) + + def artifact_entries( + self, + resolved_inputs: dict[str, BranchReference], + ) -> tuple[ArtifactEntry, ArtifactEntry]: + return ( + text_entry( + "cpp-load-from-git", + f"https://github.com/DataDog/dd-trace-cpp@{resolved_inputs['cpp_branch'].sha}", + ), + text_entry( + "cpp-kong-plugin-git", + f"https://github.com/DataDog/kong-plugin-ddtrace@{resolved_inputs['plugin_branch'].sha}", + ), + ) + + +class Prod: + def artifact_inputs(self, env: dict[str, str]) -> tuple[GitHubLatestReleaseResolver, GitHubLatestReleaseResolver]: + return ( + GitHubLatestReleaseResolver(name="cpp_release", repository="DataDog/dd-trace-cpp"), + GitHubLatestReleaseResolver(name="plugin_release", repository="DataDog/kong-plugin-ddtrace"), + ) + + def artifact_entries( + self, + resolved_inputs: dict[str, GitHubReleaseReference], + ) -> tuple[ArtifactEntry, ArtifactEntry]: + return ( + text_entry( + "cpp-load-from-git", + f"https://github.com/DataDog/dd-trace-cpp@{resolved_inputs['cpp_release'].tag_name}", + ), + text_entry("cpp-kong-load-from-release", resolved_inputs["plugin_release"].tag_name), + ) diff --git a/utils/build/docker/cpp_kong/install_ddtrace.sh b/utils/build/docker/cpp_kong/install_ddtrace.sh index 11ef895c154..3c448074b74 100755 --- a/utils/build/docker/cpp_kong/install_ddtrace.sh +++ b/utils/build/docker/cpp_kong/install_ddtrace.sh @@ -31,8 +31,9 @@ elif [ -f cpp-load-from-git ]; then echo "Build libdd_trace_c.so from cpp-load-from-git" TARGET=$(cat cpp-load-from-git) URL=$(echo "$TARGET" | cut -d "@" -f 1) - BRANCH=$(echo "$TARGET" | cut -d "@" -f 2) - git clone --depth 1 --branch "$BRANCH" "$URL" dd-trace-cpp + REF=$(echo "$TARGET" | cut -d "@" -f 2) + git clone "$URL" dd-trace-cpp + git -C dd-trace-cpp checkout "$REF" cd dd-trace-cpp cmake -S . -B build \ -DDD_TRACE_BUILD_C_BINDING=ON \ @@ -96,8 +97,19 @@ if [ -n "$rock_file" ]; then elif [ -d kong-plugin-ddtrace ]; then echo "Using Kong plugin from binaries/kong-plugin-ddtrace" +elif [ -f cpp-kong-plugin-git ]; then + TARGET=$(cat cpp-kong-plugin-git) + URL=$(echo "$TARGET" | cut -d "@" -f 1) + REF=$(echo "$TARGET" | cut -d "@" -f 2) + git clone "$URL" kong-plugin-ddtrace + git -C kong-plugin-ddtrace checkout "$REF" + else - TAG=$(get_latest_release "DataDog/kong-plugin-ddtrace") + if [ -f cpp-kong-load-from-release ]; then + TAG=$(cat cpp-kong-load-from-release) + else + TAG=$(get_latest_release "DataDog/kong-plugin-ddtrace") + fi echo "Installing kong-plugin-ddtrace from latest release ${TAG}" curl -sL "https://github.com/DataDog/kong-plugin-ddtrace/archive/refs/tags/${TAG}.tar.gz" \ | tar -xz diff --git a/utils/build/docker/cpp_nginx/artifact.py b/utils/build/docker/cpp_nginx/artifact.py new file mode 100644 index 00000000000..b0dcdef45c5 --- /dev/null +++ b/utils/build/docker/cpp_nginx/artifact.py @@ -0,0 +1,72 @@ +from __future__ import annotations + +from typing import cast + +from utils.target_artifacts.entry_helpers import json_entry, text_entry +from utils.target_artifacts.models import ( + ArtifactEntry, + GitHubActionsArtifactReference, + GitHubReleaseReference, +) +from utils.target_artifacts.resolvers import GitHubActionsArtifactResolver, GitHubLatestReleaseResolver + +type ResolvedNginxInput = GitHubActionsArtifactReference | GitHubReleaseReference + + +class Dev: + def artifact_inputs( + self, + env: dict[str, str], + ) -> tuple[GitHubActionsArtifactResolver, GitHubLatestReleaseResolver]: + return ( + GitHubActionsArtifactResolver( + name="workflow_artifact", + repository="DataDog/nginx-datadog", + workflow="system-tests.yml", + artifact_name="binaries", + variable_name="LIBRARY_TARGET_BRANCH", + default_value="master", + ignore_failed_workflow=False, + ), + GitHubLatestReleaseResolver(name="ddprof_release", repository="DataDog/ddprof"), + ) + + def artifact_entries( + self, + resolved_inputs: dict[str, ResolvedNginxInput], + ) -> tuple[ArtifactEntry, ArtifactEntry]: + artifact = cast(GitHubActionsArtifactReference, resolved_inputs["workflow_artifact"]) + ddprof_release = cast(GitHubReleaseReference, resolved_inputs["ddprof_release"]) + return ( + json_entry( + "cpp-nginx-github-actions-artifact.json", + { + "archive_download_url": artifact.archive_download_url, + "artifact_id": artifact.artifact_id, + "artifact_name": artifact.artifact_name, + "commit_sha": artifact.commit_sha, + "repository": artifact.repository, + "run_id": artifact.run_id, + "run_url": artifact.run_url, + "workflow": artifact.workflow, + }, + ), + text_entry("cpp-nginx-ddprof-load-from-release", ddprof_release.tag_name), + ) + + +class Prod: + def artifact_inputs(self, env: dict[str, str]) -> tuple[GitHubLatestReleaseResolver, GitHubLatestReleaseResolver]: + return ( + GitHubLatestReleaseResolver(name="release", repository="DataDog/nginx-datadog"), + GitHubLatestReleaseResolver(name="ddprof_release", repository="DataDog/ddprof"), + ) + + def artifact_entries( + self, + resolved_inputs: dict[str, GitHubReleaseReference], + ) -> tuple[ArtifactEntry, ArtifactEntry]: + return ( + text_entry("cpp-nginx-load-from-release", resolved_inputs["release"].tag_name), + text_entry("cpp-nginx-ddprof-load-from-release", resolved_inputs["ddprof_release"].tag_name), + ) diff --git a/utils/build/docker/cpp_nginx/install_ddprof.sh b/utils/build/docker/cpp_nginx/install_ddprof.sh index 15e51085ef8..9f4063cbf01 100755 --- a/utils/build/docker/cpp_nginx/install_ddprof.sh +++ b/utils/build/docker/cpp_nginx/install_ddprof.sh @@ -4,39 +4,44 @@ set -euo pipefail # Checks in binary folder otherwise download from github ddprof_name=$(ls -1 ddprof*.xz 2> /dev/null || true) -if [ "$(echo $ddprof_name | wc -l)" -ge "2" ]; then +if [ "$(echo "$ddprof_name" | wc -l)" -ge "2" ]; then echo "Clean up the folder in ${PWD}" exit 1 fi -curl_install=$(which curl 2> /dev/null || true) +curl_install=$(command -v curl 2> /dev/null || true) -if [ -z $curl_install ]; then +if [ -z "$curl_install" ]; then echo "please install curl" exit 1 fi if [ -z "${ddprof_name}" ] || [ ! -e "${ddprof_name}" ]; then - url_releases="https://api.github.com/repos/DataDog/ddprof/releases/latest" - echo "Could not find a version of ddprof in ${PWD}, get last release in ${url_releases}" - tag_name=$(curl -s --retry 3 "${url_releases}" | jq -r '.tag_name' | cut -c2-) + if [ -f /binaries/cpp-nginx-ddprof-load-from-release ]; then + tag_name=$(cut -c2- /binaries/cpp-nginx-ddprof-load-from-release) + else + url_releases="https://api.github.com/repos/DataDog/ddprof/releases/latest" + echo "Could not find a version of ddprof in ${PWD}, get last release in ${url_releases}" + tag_name=$(curl -s --retry 3 "${url_releases}" | jq -r '.tag_name' | cut -c2-) + fi url_release="https://github.com/DataDog/ddprof/releases/download/v${tag_name}/ddprof-${tag_name}-amd64-linux.tar.xz" echo "Using $url_release" - curl -L -O -s --retry 3 ${url_release} + curl -L -O -s --retry 3 "${url_release}" ddprof_name=$(ls ddprof*.xz) else echo "using existing ddprof ${ddprof_name}" fi ddprof_install_path=${1-""} -if [ -z ${ddprof_install_path-:""} ]; then +if [ -z "$ddprof_install_path" ]; then echo "Specify install path" ddprof_install_path="/usr/local/bin/" echo "Override install path to: ${ddprof_install_path}" fi -tar xvf ${ddprof_name} ddprof/bin/ddprof -O > ${ddprof_install_path}/ddprof -chmod +x ${ddprof_install_path}/ddprof +ddprof_binary="${ddprof_install_path%/}/ddprof" +tar xvf "${ddprof_name}" ddprof/bin/ddprof -O > "$ddprof_binary" +chmod +x "$ddprof_binary" -SYSTEM_TESTS_PROFILER_VERSION=$(${ddprof_install_path}/ddprof --version) -echo "Profiler version: $(echo ${SYSTEM_TESTS_PROFILER_VERSION})" +SYSTEM_TESTS_PROFILER_VERSION=$("$ddprof_binary" --version) +echo "Profiler version: ${SYSTEM_TESTS_PROFILER_VERSION}" diff --git a/utils/build/docker/cpp_nginx/install_ddtrace.sh b/utils/build/docker/cpp_nginx/install_ddtrace.sh index 078afb5c30a..b70a89e56d2 100755 --- a/utils/build/docker/cpp_nginx/install_ddtrace.sh +++ b/utils/build/docker/cpp_nginx/install_ddtrace.sh @@ -41,12 +41,10 @@ function epilogue { } version_first_is_greater() { - local v1=(${1//./ }) - local v2=(${2//./ }) - - # Remove the 'v' prefix from the version numbers - v1[0]=${v1[0]//v/} - v2[0]=${v2[0]//v/} + local v1=() + local v2=() + IFS='.' read -r -a v1 <<< "${1#v}" + IFS='.' read -r -a v2 <<< "${2#v}" # Compare the major, minor, and patch numbers for i in {0..2}; do @@ -61,43 +59,69 @@ version_first_is_greater() { return 1 } -if [[ $(find /binaries -name 'ngx_http_datadog_module-*.so.tgz' | wc -l) -gt 0 ]]; then - echo "Found module in /binaries" +function install_staged_binaries { + if [[ $(find /binaries -name 'ngx_http_datadog_module-*.so.tgz' | wc -l) -gt 0 ]]; then + echo "Found module in /binaries" - if [[ $(find /binaries -name 'ngx_http_datadog_module-*.so.tgz' | wc -l) -gt 1 ]]; then - echo "ERROR: Found several ngx_http_datadog_module-*.so.tgz files in binaries/, abort." - exit 1 - fi + if [[ $(find /binaries -name 'ngx_http_datadog_module-*.so.tgz' | wc -l) -gt 1 ]]; then + echo "ERROR: Found several ngx_http_datadog_module-*.so.tgz files in binaries/, abort." + exit 1 + fi - NGINX_VERSION_OF_MODULE=$(find /binaries -name 'ngx_http_datadog_module-*.so.tgz' | grep -Po '(\d+\.\d+\.\d+)') - if [[ $NGINX_VERSION_OF_MODULE != $NGINX_VERSION ]]; then - echo "ERROR: nginx mismatch: module for $NGINX_VERSION_OF_MODULE, but base image of $NGINX_VERSION" - exit 1 - fi + NGINX_VERSION_OF_MODULE=$(find /binaries -name 'ngx_http_datadog_module-*.so.tgz' | grep -Po '(\d+\.\d+\.\d+)') + if [[ $NGINX_VERSION_OF_MODULE != "$NGINX_VERSION" ]]; then + echo "ERROR: nginx mismatch: module for $NGINX_VERSION_OF_MODULE, but base image of $NGINX_VERSION" + exit 1 + fi + + MAIN_TARBALL=$(find /binaries -name 'ngx_http_datadog_module-*.so.tgz') + tar -xzvf "$MAIN_TARBALL" -C /usr/lib/nginx/modules + if [[ $(find /binaries -name 'ngx_http_datadog_module-*.so.debug.tgz' | wc -l) -eq 1 ]]; then + tar -xzvf /binaries/ngx_http_datadog_module-*.so.debug.tgz -C /usr/lib/nginx/modules + fi - MAIN_TARBALL=$(find /binaries -name 'ngx_http_datadog_module-*.so.tgz') - tar -xzvf "$MAIN_TARBALL" -C /usr/lib/nginx/modules - if [[ $(find /binaries -name 'ngx_http_datadog_module-*.so.debug.tgz' | wc -l) -eq 1 ]]; then - tar -xzvf /binaries/ngx_http_datadog_module-*.so.debug.tgz -C /usr/lib/nginx/modules + epilogue unknown_mod_version + exit 0 fi - epilogue unknown_mod_version - exit 0 -fi + if [[ -f /binaries/ngx_http_datadog_module.so ]]; then + cp -v /binaries/ngx_http_datadog_module.so /usr/lib/nginx/modules + if [[ -f /binaries/ngx_http_datadog_module.so.debug ]]; then + cp -v /binaries/ngx_http_datadog_module.so.debug /usr/lib/nginx/modules + fi -if [[ -f /binaries/ngx_http_datadog_module.so ]]; then - cp -v /binaries/ngx_http_datadog_module.so /usr/lib/nginx/modules - if [[ -f /binaries/ngx_http_datadog_module.so.debug ]]; then - cp -v /binaries/ngx_http_datadog_module.so.debug /usr/lib/nginx/modules + epilogue unknown_mod_version + exit 0 fi +} + +install_staged_binaries - epilogue unknown_mod_version - exit 0 +if [[ -f /binaries/cpp-nginx-github-actions-artifact.json ]]; then + echo "Install NGINX plugin from staged GitHub Actions artifact metadata" + ARCHIVE_URL=$(jq -r '.archive_download_url' /binaries/cpp-nginx-github-actions-artifact.json) + AUTH_HEADER=() + if [[ -f /run/secrets/github_token ]]; then + AUTH_HEADER=(-H "Authorization: Bearer $(cat /run/secrets/github_token)") + fi + curl -Lf "${AUTH_HEADER[@]}" -o /tmp/nginx-datadog-artifact.zip "$ARCHIVE_URL" + mkdir -p /tmp/nginx-datadog-artifact + unzip -o /tmp/nginx-datadog-artifact.zip -d /tmp/nginx-datadog-artifact + if [[ -f /tmp/nginx-datadog-artifact/binaries.zip ]]; then + unzip -o /tmp/nginx-datadog-artifact/binaries.zip -d /binaries + else + find /tmp/nginx-datadog-artifact -type f -name 'ngx_http_datadog_module*' -exec cp '{}' /binaries/ ';' + fi + install_staged_binaries fi get_latest_release() { + if [[ -f /binaries/cpp-nginx-load-from-release ]]; then + cat /binaries/cpp-nginx-load-from-release + else wget -qO- "https://api.github.com/repos/DataDog/nginx-datadog/releases/latest" \ | jq -r '.tag_name' + fi } get_architecture() { @@ -105,12 +129,13 @@ get_architecture() { } -if [ NGINX_VERSION == "" ]; then +if [[ -z ${NGINX_VERSION:-} ]]; then echo 1>&2 "ERROR: Missing NGINX_VERSION." exit 1 fi -readonly ARCH=$(get_architecture) +ARCH=$(get_architecture) +readonly ARCH if [[ $ARCH != "amd64" && $ARCH != "arm64" ]]; then echo 1>&2 "ERROR: Architecture ${ARCH} is not supported." @@ -121,10 +146,10 @@ FILENAME=ngx_http_datadog_module-appsec-$ARCH-$NGINX_VERSION.so if [ -f "$FILENAME" ]; then echo "Install NGINX plugin from binaries/$FILENAME" - cp $FILENAME /usr/lib/nginx/modules/ngx_http_datadog_module.so + cp "$FILENAME" /usr/lib/nginx/modules/ngx_http_datadog_module.so NGINX_DATADOG_VERSION="v99.99.99" # TODO: get version from the binary. Right now, use the "big-version" trick else - readonly NGINX_DATADOG_VERSION="$(get_latest_release)" + NGINX_DATADOG_VERSION="$(get_latest_release)" if version_first_is_greater "$NGINX_DATADOG_VERSION" "v1.1.0"; then TARBALLS=( diff --git a/utils/build/docker/dotnet/artifact.py b/utils/build/docker/dotnet/artifact.py new file mode 100644 index 00000000000..9e94eac4269 --- /dev/null +++ b/utils/build/docker/dotnet/artifact.py @@ -0,0 +1,49 @@ +from __future__ import annotations + + +from utils.target_artifacts.entry_helpers import provider_fetch_entries, text_entry +from utils.target_artifacts.models import ( + ArtifactEntry, + BranchReference, + GitHubReleaseReference, +) +from utils.target_artifacts.resolvers import GitHubBranchResolver, GitHubLatestReleaseResolver + + +def _normalize_branch_for_image_tag(branch_name: str) -> str: + return branch_name.replace("/", "_") + + +class Dev: + def artifact_inputs(self, env: dict[str, str]) -> tuple[GitHubBranchResolver]: + return ( + GitHubBranchResolver( + name="library_branch", + repository="DataDog/dd-trace-dotnet", + variable_name="LIBRARY_TARGET_BRANCH", + default_value="master", + ), + ) + + def artifact_entries(self, resolved_inputs: dict[str, BranchReference]) -> tuple[ArtifactEntry, ArtifactEntry]: + resolved_branch = resolved_inputs["library_branch"] + fetch_selector = ( + f"ghcr.io/datadog/dd-trace-dotnet/dd-trace-dotnet:{_normalize_branch_for_image_tag(resolved_branch.branch)}" + ) + return provider_fetch_entries( + fetch_filename="dotnet-package-image", + fetch_selector=fetch_selector, + marker_filename="dotnet-package-selection", + bounded_selector=resolved_branch.sha, + ) + + +class Prod: + def artifact_inputs(self, env: dict[str, str]) -> tuple[GitHubLatestReleaseResolver]: + return (GitHubLatestReleaseResolver(name="release", repository="DataDog/dd-trace-dotnet"),) + + def artifact_entries( + self, + resolved_inputs: dict[str, GitHubReleaseReference], + ) -> tuple[ArtifactEntry]: + return (text_entry("dotnet-load-from-release", resolved_inputs["release"].tag_name),) diff --git a/utils/build/docker/dotnet/install_ddtrace.sh b/utils/build/docker/dotnet/install_ddtrace.sh index f785cf445cd..e8943d6470c 100755 --- a/utils/build/docker/dotnet/install_ddtrace.sh +++ b/utils/build/docker/dotnet/install_ddtrace.sh @@ -17,33 +17,39 @@ get_latest_release() { echo "Failed to get latest release" exit 1 fi - echo $releases | grep '"tag_name":' | sed -E 's/.*"v([^"]+)".*/\1/'; + echo "$releases" | grep '"tag_name":' | sed -E 's/.*"v([^"]+)".*/\1/'; } mkdir -p /opt/datadog -if [ $(ls /binaries/Datadog.Trace.ClrProfiler.Native.so | wc -l) = 1 ]; then +native_file_count=$(find /binaries -maxdepth 1 -name 'Datadog.Trace.ClrProfiler.Native.so' | wc -l) +if [ "$native_file_count" = 1 ]; then echo "Install ddtrace from local folder" cp -r /binaries/* /opt/datadog/ else - if [ $(ls datadog-dotnet-apm*.tar.gz | wc -l) = 1 ]; then - echo "Install ddtrace from $(ls datadog-dotnet-apm*.tar.gz)" + tarball_count=$(find /binaries -maxdepth 1 -name 'datadog-dotnet-apm*.tar.gz' | wc -l) + if [ "$tarball_count" = 1 ]; then + echo "Install ddtrace from $(find /binaries -maxdepth 1 -name 'datadog-dotnet-apm*.tar.gz')" else echo "Install ddtrace from github releases" - if ! DDTRACE_VERSION="$(get_latest_release DataDog/dd-trace-dotnet)"; then + if [ -f /binaries/dotnet-load-from-release ]; then + DDTRACE_VERSION="$(cat /binaries/dotnet-load-from-release)" + DDTRACE_VERSION="${DDTRACE_VERSION#v}" + elif ! DDTRACE_VERSION="$(get_latest_release DataDog/dd-trace-dotnet)"; then echo "Failed to get latest release version" exit 1 fi - if [ $(uname -m) = "aarch64" ]; then + if [ "$(uname -m)" = "aarch64" ]; then artifact=datadog-dotnet-apm-${DDTRACE_VERSION}.arm64.tar.gz else artifact=datadog-dotnet-apm-${DDTRACE_VERSION}.tar.gz fi echo "Using artifact ${artifact}" - curl -L --fail "${GITHUB_AUTH_HEADER[@]}" https://github.com/DataDog/dd-trace-dotnet/releases/download/v${DDTRACE_VERSION}/${artifact} --output ${artifact} + curl -L --fail "${GITHUB_AUTH_HEADER[@]}" "https://github.com/DataDog/dd-trace-dotnet/releases/download/v${DDTRACE_VERSION}/${artifact}" --output "${artifact}" fi - tar xzf $(ls datadog-dotnet-apm*.tar.gz) -C /opt/datadog + tarball=$(find /binaries -maxdepth 1 -name 'datadog-dotnet-apm*.tar.gz') + tar xzf "$tarball" -C /opt/datadog fi diff --git a/utils/build/docker/golang/artifact.py b/utils/build/docker/golang/artifact.py new file mode 100644 index 00000000000..9ef119ebdf3 --- /dev/null +++ b/utils/build/docker/golang/artifact.py @@ -0,0 +1,114 @@ +from __future__ import annotations + +from typing import cast + +from utils.target_artifacts.entry_helpers import text_entry +from utils.target_artifacts.models import ( + ArtifactEntry, + BranchReference, + ModuleVersion, + OciImageReference, +) +from utils.target_artifacts.resolvers import GitHubBranchResolver, GoModuleLatestResolver, OciDigestResolver + +type ArtifactInputResolver = GitHubBranchResolver | GoModuleLatestResolver | OciDigestResolver +type ResolvedGoInput = BranchReference | ModuleVersion | OciImageReference + +GO_MODULES = ( + "github.com/DataDog/dd-trace-go/v2", + "github.com/DataDog/dd-trace-go/contrib/database/sql/v2", + "github.com/DataDog/dd-trace-go/contrib/net/http/v2", + "github.com/DataDog/dd-trace-go/contrib/google.golang.org/grpc/v2", + "github.com/DataDog/dd-trace-go/contrib/99designs/gqlgen/v2", + "github.com/DataDog/dd-trace-go/contrib/gin-gonic/gin/v2", + "github.com/DataDog/dd-trace-go/contrib/graphql-go/graphql/v2", + "github.com/DataDog/dd-trace-go/contrib/graph-gophers/graphql-go/v2", + "github.com/DataDog/dd-trace-go/contrib/go-chi/chi.v5/v2", + "github.com/DataDog/dd-trace-go/contrib/IBM/sarama/v2", + "github.com/DataDog/dd-trace-go/contrib/labstack/echo.v4/v2", + "github.com/DataDog/dd-trace-go/contrib/sirupsen/logrus/v2", +) + +DEV_SERVICE_EXTENSIONS_IMAGE = "ghcr.io/datadog/dd-trace-go/service-extensions-callout:dev" +DEV_HAPROXY_SPOA_IMAGE = "ghcr.io/datadog/dd-trace-go/haproxy-spoa:dev" +PROD_SERVICE_EXTENSIONS_IMAGE = "ghcr.io/datadog/dd-trace-go/service-extensions-callout:latest" +PROD_HAPROXY_SPOA_IMAGE = "ghcr.io/datadog/dd-trace-go/haproxy-spoa:latest" + + +class Dev: + def artifact_inputs(self, env: dict[str, str]) -> tuple[ArtifactInputResolver, ...]: + inputs: list[ArtifactInputResolver] = [ + GitHubBranchResolver( + name="library_branch", + repository="DataDog/dd-trace-go", + variable_name="LIBRARY_TARGET_BRANCH", + default_value="main", + ), + OciDigestResolver(name="service_extensions_image", image=DEV_SERVICE_EXTENSIONS_IMAGE), + OciDigestResolver(name="haproxy_spoa_image", image=DEV_HAPROXY_SPOA_IMAGE), + ] + if env.get("ORCHESTRION_TARGET_BRANCH"): + inputs.append( + GitHubBranchResolver( + name="orchestrion_branch", + repository="DataDog/orchestrion", + variable_name="ORCHESTRION_TARGET_BRANCH", + ) + ) + else: + inputs.append(GoModuleLatestResolver(name="orchestrion_version", module="github.com/DataDog/orchestrion")) + return tuple(inputs) + + def artifact_entries( + self, + resolved_inputs: dict[str, ResolvedGoInput], + ) -> tuple[ArtifactEntry, ArtifactEntry, ArtifactEntry, ArtifactEntry]: + library_branch = cast(BranchReference, resolved_inputs["library_branch"]) + sha = library_branch.sha + return _entries_for_go_ref(resolved_inputs, sha) + + +class Prod: + def artifact_inputs( + self, + env: dict[str, str], + ) -> tuple[GoModuleLatestResolver, GoModuleLatestResolver, OciDigestResolver, OciDigestResolver]: + return ( + GoModuleLatestResolver(name="library_version", module="github.com/DataDog/dd-trace-go/v2"), + GoModuleLatestResolver(name="orchestrion_version", module="github.com/DataDog/orchestrion"), + OciDigestResolver(name="service_extensions_image", image=PROD_SERVICE_EXTENSIONS_IMAGE), + OciDigestResolver(name="haproxy_spoa_image", image=PROD_HAPROXY_SPOA_IMAGE), + ) + + def artifact_entries( + self, + resolved_inputs: dict[str, ResolvedGoInput], + ) -> tuple[ArtifactEntry, ArtifactEntry, ArtifactEntry, ArtifactEntry]: + library_version = cast(ModuleVersion, resolved_inputs["library_version"]) + version = library_version.version + return _entries_for_go_ref(resolved_inputs, version) + + +def _entries_for_go_ref( + resolved_inputs: dict[str, ResolvedGoInput], + go_ref: str, +) -> tuple[ArtifactEntry, ArtifactEntry, ArtifactEntry, ArtifactEntry]: + if "orchestrion_branch" in resolved_inputs: + orchestrion_branch = cast(BranchReference, resolved_inputs["orchestrion_branch"]) + orchestrion_ref = orchestrion_branch.sha + else: + orchestrion_version = cast(ModuleVersion, resolved_inputs["orchestrion_version"]) + orchestrion_ref = orchestrion_version.version + + service_extensions_image = cast(OciImageReference, resolved_inputs["service_extensions_image"]) + haproxy_spoa_image = cast(OciImageReference, resolved_inputs["haproxy_spoa_image"]) + + return ( + text_entry("golang-load-from-go-get", "\n".join(f"{module}@{go_ref}" for module in GO_MODULES)), + text_entry("orchestrion-load-from-go-get", f"github.com/DataDog/orchestrion@{orchestrion_ref}"), + text_entry( + "golang-service-extensions-callout-image", + service_extensions_image.reference, + ), + text_entry("golang-haproxy-spoa-image", haproxy_spoa_image.reference), + ) diff --git a/utils/build/docker/java/artifact.py b/utils/build/docker/java/artifact.py new file mode 100644 index 00000000000..395a3809101 --- /dev/null +++ b/utils/build/docker/java/artifact.py @@ -0,0 +1,39 @@ +from __future__ import annotations + + +from utils.target_artifacts.entry_helpers import text_entry +from utils.target_artifacts.models import ( + ArtifactEntry, + BranchReference, + GitHubReleaseReference, +) +from utils.target_artifacts.resolvers import GitHubBranchResolver, GitHubLatestReleaseResolver + + +class Dev: + def artifact_inputs(self, env: dict[str, str]) -> tuple[GitHubBranchResolver]: + return ( + GitHubBranchResolver( + name="library_branch", + repository="DataDog/dd-trace-java", + variable_name="LIBRARY_TARGET_BRANCH", + default_value="master", + ), + ) + + def artifact_entries( + self, + resolved_inputs: dict[str, BranchReference], + ) -> tuple[ArtifactEntry]: + return (text_entry("java-load-from-s3", resolved_inputs["library_branch"].sha),) + + +class Prod: + def artifact_inputs(self, env: dict[str, str]) -> tuple[GitHubLatestReleaseResolver]: + return (GitHubLatestReleaseResolver(name="release", repository="DataDog/dd-trace-java"),) + + def artifact_entries( + self, + resolved_inputs: dict[str, GitHubReleaseReference], + ) -> tuple[ArtifactEntry]: + return (text_entry("java-load-from-release", resolved_inputs["release"].tag_name),) diff --git a/utils/build/docker/java/install_ddtrace.sh b/utils/build/docker/java/install_ddtrace.sh index 20d67c955ac..13eb98509ca 100755 --- a/utils/build/docker/java/install_ddtrace.sh +++ b/utils/build/docker/java/install_ddtrace.sh @@ -14,10 +14,15 @@ install_custom_jar() { echo "Using default $artifact_id" elif [ "$jar_count" = 1 ]; then [[ "$#" -lt 3 ]] && MVN_OPTS= || MVN_OPTS="$3" + local mvn_args=() + if [[ -n "$MVN_OPTS" ]]; then + read -r -a mvn_args <<< "$MVN_OPTS" + fi local custom_jar custom_jar=$(find /binaries/ -name "${jar_pattern}") echo "Using custom $artifact_id: ${custom_jar}" - mvn -Dfile="$custom_jar" -DgroupId=com.datadoghq -DartifactId="$artifact_id" -Dversion=9999 -Dpackaging=jar $MVN_OPTS install:install-file + mvn -Dfile="$custom_jar" -DgroupId=com.datadoghq -DartifactId="$artifact_id" \ + -Dversion=9999 -Dpackaging=jar "${mvn_args[@]}" install:install-file else echo "Too many $artifact_id within binaries folder" exit 1 @@ -33,14 +38,23 @@ install_custom_jar "dd-trace-api*.jar" "dd-trace-api" "$MVN_OPTS" install_custom_jar "dd-openfeature*.jar" "dd-openfeature" "$MVN_OPTS" # Look for custom dd-trace-java jar in custom binaries folder -if [ $(ls /binaries/dd-java-agent*.jar | wc -l) = 0 ]; then - BUILD_URL="https://github.com/DataDog/dd-trace-java/releases/latest/download/dd-java-agent.jar" - echo "install from Github release: $BUILD_URL" - curl -Lf -o /dd-tracer/dd-java-agent.jar $BUILD_URL +if [ "$(find /binaries -maxdepth 1 -name 'dd-java-agent*.jar' | wc -l)" = 0 ]; then + if [ -f /binaries/java-load-from-s3 ]; then + GIT_REF=$(cat /binaries/java-load-from-s3) + BUILD_URL="https://s3.us-east-1.amazonaws.com/dd-trace-java-builds/${GIT_REF}/dd-java-agent.jar" + elif [ -f /binaries/java-load-from-release ]; then + RELEASE_TAG=$(cat /binaries/java-load-from-release) + BUILD_URL="https://github.com/DataDog/dd-trace-java/releases/download/${RELEASE_TAG}/dd-java-agent.jar" + else + BUILD_URL="https://github.com/DataDog/dd-trace-java/releases/latest/download/dd-java-agent.jar" + fi + echo "install from reference: $BUILD_URL" + curl -Lf -o /dd-tracer/dd-java-agent.jar "$BUILD_URL" -elif [ $(ls /binaries/dd-java-agent*.jar | wc -l) = 1 ]; then - echo "Install local file $(ls /binaries/dd-java-agent*.jar)" - cp $(ls /binaries/dd-java-agent*.jar) /dd-tracer/dd-java-agent.jar +elif [ "$(find /binaries -maxdepth 1 -name 'dd-java-agent*.jar' | wc -l)" = 1 ]; then + CUSTOM_JAR=$(find /binaries -maxdepth 1 -name 'dd-java-agent*.jar') + echo "Install local file $CUSTOM_JAR" + cp "$CUSTOM_JAR" /dd-tracer/dd-java-agent.jar else echo "Too many jar files in binaries" @@ -51,7 +65,4 @@ java -jar /dd-tracer/dd-java-agent.jar > /binaries/SYSTEM_TESTS_LIBRARY_VERSION echo "Installed $(cat /binaries/SYSTEM_TESTS_LIBRARY_VERSION) java library" -SYSTEM_TESTS_LIBRARY_VERSION=$(cat /binaries/SYSTEM_TESTS_LIBRARY_VERSION) - echo "dd-trace version: $(cat /binaries/SYSTEM_TESTS_LIBRARY_VERSION)" - diff --git a/utils/build/docker/java/parametric/install_ddtrace.sh b/utils/build/docker/java/parametric/install_ddtrace.sh index 580bfc7c6e4..a40c0229634 100755 --- a/utils/build/docker/java/parametric/install_ddtrace.sh +++ b/utils/build/docker/java/parametric/install_ddtrace.sh @@ -31,8 +31,17 @@ configure_custom_jar "dd-openfeature*.jar" "dd-openfeature" "customDdOpenfeature # Look for custom dd-java-agent jar in custom binaries folder CUSTOM_DD_JAVA_AGENT_COUNT=$(find /binaries/dd-java-agent*.jar 2>/dev/null | wc -l) if [ "$CUSTOM_DD_JAVA_AGENT_COUNT" = 0 ]; then - echo "Using latest dd-java-agent" - wget -O /client/tracer/dd-java-agent.jar --no-cache https://github.com/DataDog/dd-trace-java/releases/latest/download/dd-java-agent.jar + if [ -f /binaries/java-load-from-s3 ]; then + GIT_REF=$(cat /binaries/java-load-from-s3) + BUILD_URL="https://s3.us-east-1.amazonaws.com/dd-trace-java-builds/${GIT_REF}/dd-java-agent.jar" + elif [ -f /binaries/java-load-from-release ]; then + RELEASE_TAG=$(cat /binaries/java-load-from-release) + BUILD_URL="https://github.com/DataDog/dd-trace-java/releases/download/${RELEASE_TAG}/dd-java-agent.jar" + else + BUILD_URL="https://github.com/DataDog/dd-trace-java/releases/latest/download/dd-java-agent.jar" + fi + echo "Using dd-java-agent from $BUILD_URL" + wget -O /client/tracer/dd-java-agent.jar --no-cache "$BUILD_URL" elif [ "$CUSTOM_DD_JAVA_AGENT_COUNT" = 1 ]; then CUSTOM_DD_JAVA_AGENT=$(find /binaries/dd-java-agent*.jar) echo "Using custom dd-java-agent: ${CUSTOM_DD_JAVA_AGENT}" diff --git a/utils/build/docker/java_lambda/artifact.py b/utils/build/docker/java_lambda/artifact.py new file mode 100644 index 00000000000..395a3809101 --- /dev/null +++ b/utils/build/docker/java_lambda/artifact.py @@ -0,0 +1,39 @@ +from __future__ import annotations + + +from utils.target_artifacts.entry_helpers import text_entry +from utils.target_artifacts.models import ( + ArtifactEntry, + BranchReference, + GitHubReleaseReference, +) +from utils.target_artifacts.resolvers import GitHubBranchResolver, GitHubLatestReleaseResolver + + +class Dev: + def artifact_inputs(self, env: dict[str, str]) -> tuple[GitHubBranchResolver]: + return ( + GitHubBranchResolver( + name="library_branch", + repository="DataDog/dd-trace-java", + variable_name="LIBRARY_TARGET_BRANCH", + default_value="master", + ), + ) + + def artifact_entries( + self, + resolved_inputs: dict[str, BranchReference], + ) -> tuple[ArtifactEntry]: + return (text_entry("java-load-from-s3", resolved_inputs["library_branch"].sha),) + + +class Prod: + def artifact_inputs(self, env: dict[str, str]) -> tuple[GitHubLatestReleaseResolver]: + return (GitHubLatestReleaseResolver(name="release", repository="DataDog/dd-trace-java"),) + + def artifact_entries( + self, + resolved_inputs: dict[str, GitHubReleaseReference], + ) -> tuple[ArtifactEntry]: + return (text_entry("java-load-from-release", resolved_inputs["release"].tag_name),) diff --git a/utils/build/docker/java_otel/artifact.py b/utils/build/docker/java_otel/artifact.py new file mode 100644 index 00000000000..de80115f189 --- /dev/null +++ b/utils/build/docker/java_otel/artifact.py @@ -0,0 +1,33 @@ +from __future__ import annotations + + +from utils.target_artifacts.entry_helpers import text_entry +from utils.target_artifacts.models import ( + ArtifactEntry, + GitHubReleaseReference, +) +from utils.target_artifacts.resolvers import GitHubLatestReleaseResolver + +REPOSITORY = "open-telemetry/opentelemetry-java-instrumentation" + + +class Dev: + def artifact_inputs(self, env: dict[str, str]) -> tuple[GitHubLatestReleaseResolver]: + return (GitHubLatestReleaseResolver(name="release", repository=REPOSITORY),) + + def artifact_entries( + self, + resolved_inputs: dict[str, GitHubReleaseReference], + ) -> tuple[ArtifactEntry]: + return (text_entry("java-otel-load-from-release", resolved_inputs["release"].tag_name),) + + +class Prod: + def artifact_inputs(self, env: dict[str, str]) -> tuple[GitHubLatestReleaseResolver]: + return (GitHubLatestReleaseResolver(name="release", repository=REPOSITORY),) + + def artifact_entries( + self, + resolved_inputs: dict[str, GitHubReleaseReference], + ) -> tuple[ArtifactEntry]: + return (text_entry("java-otel-load-from-release", resolved_inputs["release"].tag_name),) diff --git a/utils/build/docker/java_otel/install_opentelemetry.sh b/utils/build/docker/java_otel/install_opentelemetry.sh index 47f9f36bd0f..80a33ae5e57 100755 --- a/utils/build/docker/java_otel/install_opentelemetry.sh +++ b/utils/build/docker/java_otel/install_opentelemetry.sh @@ -6,9 +6,14 @@ mkdir /otel-tracer # shellcheck disable=SC2012 if [ "$(ls /binaries/opentelemetry-javaagent*.jar | wc -l)" = 0 ]; then - BUILD_URL="https://github.com/open-telemetry/opentelemetry-java-instrumentation/releases/latest/download/opentelemetry-javaagent.jar" + if [ -f /binaries/java-otel-load-from-release ]; then + RELEASE_TAG=$(cat /binaries/java-otel-load-from-release) + BUILD_URL="https://github.com/open-telemetry/opentelemetry-java-instrumentation/releases/download/${RELEASE_TAG}/opentelemetry-javaagent.jar" + else + BUILD_URL="https://github.com/open-telemetry/opentelemetry-java-instrumentation/releases/latest/download/opentelemetry-javaagent.jar" + fi echo "install from Github release: $BUILD_URL" - curl -Lf -o /otel-tracer/opentelemetry-javaagent.jar $BUILD_URL + curl -Lf -o /otel-tracer/opentelemetry-javaagent.jar "$BUILD_URL" elif [ "$(ls /binaries/opentelemetry-javaagent*.jar | wc -l)" = 1 ]; then echo "Install local file $(ls /binaries/opentelemetry-javaagent*.jar)" @@ -22,4 +27,3 @@ fi java -jar /otel-tracer/opentelemetry-javaagent.jar > /binaries/SYSTEM_TESTS_LIBRARY_VERSION echo "opentelemetry-javaagent version: $(cat /binaries/SYSTEM_TESTS_LIBRARY_VERSION)" - diff --git a/utils/build/docker/nodejs/artifact.py b/utils/build/docker/nodejs/artifact.py new file mode 100644 index 00000000000..d40e0fdd3a8 --- /dev/null +++ b/utils/build/docker/nodejs/artifact.py @@ -0,0 +1,41 @@ +from __future__ import annotations + + +from utils.target_artifacts.entry_helpers import text_entry +from utils.target_artifacts.models import ( + ArtifactEntry, + BranchReference, + ModuleVersion, +) +from utils.target_artifacts.resolvers import GitHubBranchResolver, NpmLatestResolver + + +class Dev: + def artifact_inputs(self, env: dict[str, str]) -> tuple[GitHubBranchResolver]: + return ( + GitHubBranchResolver( + name="library_branch", + repository="DataDog/dd-trace-js", + variable_name="LIBRARY_TARGET_BRANCH", + default_value="master", + ), + ) + + def artifact_entries( + self, + resolved_inputs: dict[str, BranchReference], + ) -> tuple[ArtifactEntry]: + sha = resolved_inputs["library_branch"].sha + return (text_entry("nodejs-load-from-npm", f"DataDog/dd-trace-js#{sha}"),) + + +class Prod: + def artifact_inputs(self, env: dict[str, str]) -> tuple[NpmLatestResolver]: + return (NpmLatestResolver(name="dd-trace", package="dd-trace"),) + + def artifact_entries( + self, + resolved_inputs: dict[str, ModuleVersion], + ) -> tuple[ArtifactEntry]: + version = resolved_inputs["dd-trace"].version + return (text_entry("nodejs-load-from-npm", f"dd-trace@{version}"),) diff --git a/utils/build/docker/nodejs_lambda/artifact.py b/utils/build/docker/nodejs_lambda/artifact.py new file mode 100644 index 00000000000..58c431d5d1d --- /dev/null +++ b/utils/build/docker/nodejs_lambda/artifact.py @@ -0,0 +1,57 @@ +from __future__ import annotations + + +from utils.target_artifacts.entry_helpers import json_entry, text_entry +from utils.target_artifacts.models import ( + ArtifactEntry, + GitHubActionsArtifactReference, + GitHubReleaseReference, +) +from utils.target_artifacts.resolvers import GitHubActionsArtifactResolver, GitHubLatestReleaseResolver + + +class Dev: + def artifact_inputs(self, env: dict[str, str]) -> tuple[GitHubActionsArtifactResolver]: + return ( + GitHubActionsArtifactResolver( + name="workflow_artifact", + repository="DataDog/datadog-lambda-js", + workflow="build_layer.yml", + artifact_name="datadog_lambda_node18.12", + variable_name="LIBRARY_TARGET_BRANCH", + default_value="main", + ignore_failed_workflow=False, + ), + ) + + def artifact_entries( + self, + resolved_inputs: dict[str, GitHubActionsArtifactReference], + ) -> tuple[ArtifactEntry]: + artifact = resolved_inputs["workflow_artifact"] + return ( + json_entry( + "nodejs-lambda-github-actions-artifact.json", + { + "archive_download_url": artifact.archive_download_url, + "artifact_id": artifact.artifact_id, + "artifact_name": artifact.artifact_name, + "commit_sha": artifact.commit_sha, + "repository": artifact.repository, + "run_id": artifact.run_id, + "run_url": artifact.run_url, + "workflow": artifact.workflow, + }, + ), + ) + + +class Prod: + def artifact_inputs(self, env: dict[str, str]) -> tuple[GitHubLatestReleaseResolver]: + return (GitHubLatestReleaseResolver(name="release", repository="DataDog/datadog-lambda-js"),) + + def artifact_entries( + self, + resolved_inputs: dict[str, GitHubReleaseReference], + ) -> tuple[ArtifactEntry]: + return (text_entry("nodejs-lambda-load-from-release", resolved_inputs["release"].tag_name),) diff --git a/utils/build/docker/nodejs_lambda/install_datadog_lambda.sh b/utils/build/docker/nodejs_lambda/install_datadog_lambda.sh index 7ecc43ac725..a8317701c03 100755 --- a/utils/build/docker/nodejs_lambda/install_datadog_lambda.sh +++ b/utils/build/docker/nodejs_lambda/install_datadog_lambda.sh @@ -9,7 +9,6 @@ if [ "$(find . -maxdepth 1 -name "*.zip" | wc -l)" = "1" ]; then echo "Install datadog_lambda from ${path}" unzip "${path}" -d /opt else - echo "Fetching from latest GitHub release..." NODE_MAJOR=$(node -e "console.log(process.version.split('.')[0].slice(1))") # Map major version to the runtime version used by datadog-lambda-js release assets. # See https://github.com/DataDog/datadog-lambda-js/blob/main/.gitlab/datasources/runtimes.yaml @@ -22,15 +21,35 @@ else esac echo "Detected Node.js major: ${NODE_MAJOR}, using layer runtime version: ${NODE_VERSION}" - LATEST_TAG=$(curl -fsSL -H "Accept: application/vnd.github.v3+json" \ - https://api.github.com/repos/DataDog/datadog-lambda-js/releases/latest \ - | grep '"tag_name"' | head -1 | sed 's/.*"tag_name": *"//;s/".*//') - echo "Latest release tag: ${LATEST_TAG}" - ZIP_NAME="datadog_lambda_node${NODE_VERSION}.zip" - DOWNLOAD_URL="https://github.com/DataDog/datadog-lambda-js/releases/download/${LATEST_TAG}/${ZIP_NAME}" - echo "Downloading ${DOWNLOAD_URL}" - curl -fsSLO "${DOWNLOAD_URL}" + if [ -f nodejs-lambda-github-actions-artifact.json ]; then + echo "Fetching from staged GitHub Actions artifact metadata..." + ARCHIVE_URL=$(jq -r '.archive_download_url' nodejs-lambda-github-actions-artifact.json) + if [ -z "$ARCHIVE_URL" ] || [ "$ARCHIVE_URL" = "null" ]; then + echo "Staged GitHub Actions artifact metadata is missing archive_download_url" + exit 1 + fi + GITHUB_AUTH_HEADER=() + if [ -f /run/secrets/github_token ]; then + GITHUB_AUTH_HEADER=(-H "Authorization: Bearer $(cat /run/secrets/github_token)") + fi + curl -fsSL "${GITHUB_AUTH_HEADER[@]}" -o /tmp/nodejs-lambda-artifact.zip "$ARCHIVE_URL" + mkdir -p /tmp/nodejs-lambda-artifact + unzip -o /tmp/nodejs-lambda-artifact.zip -d /tmp/nodejs-lambda-artifact + cp "$(find /tmp/nodejs-lambda-artifact -name "$ZIP_NAME" | head -1)" . + else + if [ -f nodejs-lambda-load-from-release ]; then + LATEST_TAG=$(cat nodejs-lambda-load-from-release) + else + LATEST_TAG=$(curl -fsSL -H "Accept: application/vnd.github.v3+json" \ + https://api.github.com/repos/DataDog/datadog-lambda-js/releases/latest \ + | grep '"tag_name"' | head -1 | sed 's/.*"tag_name": *"//;s/".*//') + fi + echo "Release tag: ${LATEST_TAG}" + DOWNLOAD_URL="https://github.com/DataDog/datadog-lambda-js/releases/download/${LATEST_TAG}/${ZIP_NAME}" + echo "Downloading ${DOWNLOAD_URL}" + curl -fsSLO "${DOWNLOAD_URL}" + fi if [ ! -f "${ZIP_NAME}" ]; then echo "Failed to download ${ZIP_NAME}" diff --git a/utils/build/docker/nodejs_otel/artifact.py b/utils/build/docker/nodejs_otel/artifact.py new file mode 100644 index 00000000000..ebe214836ce --- /dev/null +++ b/utils/build/docker/nodejs_otel/artifact.py @@ -0,0 +1,35 @@ +from __future__ import annotations + + +from utils.target_artifacts.entry_helpers import text_entry +from utils.target_artifacts.models import ( + ArtifactEntry, + ModuleVersion, +) +from utils.target_artifacts.resolvers import NpmLatestResolver + +PACKAGE_NAME = "@opentelemetry/auto-instrumentations-node" + + +class Dev: + def artifact_inputs(self, env: dict[str, str]) -> tuple[NpmLatestResolver]: + return (NpmLatestResolver(name="otel_package", package=PACKAGE_NAME),) + + def artifact_entries( + self, + resolved_inputs: dict[str, ModuleVersion], + ) -> tuple[ArtifactEntry]: + version = resolved_inputs["otel_package"].version + return (text_entry("nodejs-otel-load-from-npm", f"{PACKAGE_NAME}@{version}"),) + + +class Prod: + def artifact_inputs(self, env: dict[str, str]) -> tuple[NpmLatestResolver]: + return (NpmLatestResolver(name="otel_package", package=PACKAGE_NAME),) + + def artifact_entries( + self, + resolved_inputs: dict[str, ModuleVersion], + ) -> tuple[ArtifactEntry]: + version = resolved_inputs["otel_package"].version + return (text_entry("nodejs-otel-load-from-npm", f"{PACKAGE_NAME}@{version}"),) diff --git a/utils/build/docker/nodejs_otel/express4-otel.Dockerfile b/utils/build/docker/nodejs_otel/express4-otel.Dockerfile index 304eb701d6e..ba16560312a 100644 --- a/utils/build/docker/nodejs_otel/express4-otel.Dockerfile +++ b/utils/build/docker/nodejs_otel/express4-otel.Dockerfile @@ -15,6 +15,8 @@ COPY utils/build/docker/nodejs/express /usr/app #overwrite app.js and package files COPY utils/build/docker/nodejs_otel/express4-otel /usr/app RUN npm ci || (sleep 30 && npm ci) +COPY binaries* /binaries/ +RUN if [ -f /binaries/nodejs-otel-load-from-npm ]; then npm install "$(cat /binaries/nodejs-otel-load-from-npm)"; fi EXPOSE 7777 diff --git a/utils/build/docker/otel_collector/artifact.py b/utils/build/docker/otel_collector/artifact.py new file mode 100644 index 00000000000..2ff28c59ed9 --- /dev/null +++ b/utils/build/docker/otel_collector/artifact.py @@ -0,0 +1,39 @@ +from __future__ import annotations + + +from utils.target_artifacts.entry_helpers import text_entry +from utils.target_artifacts.models import ( + ArtifactEntry, + OciImageReference, +) +from utils.target_artifacts.resolvers import OciDigestResolver + +DEFAULT_IMAGE = "otel/opentelemetry-collector-contrib:0.137.0" + + +class Dev: + def artifact_inputs(self, env: dict[str, str]) -> tuple[OciDigestResolver]: + return ( + OciDigestResolver( + name="collector_image", + image=DEFAULT_IMAGE, + variable_name="OTEL_COLLECTOR_IMAGE", + ), + ) + + def artifact_entries( + self, + resolved_inputs: dict[str, OciImageReference], + ) -> tuple[ArtifactEntry]: + return (text_entry("otel_collector-image", resolved_inputs["collector_image"].reference),) + + +class Prod: + def artifact_inputs(self, env: dict[str, str]) -> tuple[OciDigestResolver]: + return (OciDigestResolver(name="collector_image", image=DEFAULT_IMAGE),) + + def artifact_entries( + self, + resolved_inputs: dict[str, OciImageReference], + ) -> tuple[ArtifactEntry]: + return (text_entry("otel_collector-image", resolved_inputs["collector_image"].reference),) diff --git a/utils/build/docker/php/artifact.py b/utils/build/docker/php/artifact.py new file mode 100644 index 00000000000..42461c39685 --- /dev/null +++ b/utils/build/docker/php/artifact.py @@ -0,0 +1,55 @@ +from __future__ import annotations + +import re + +from utils.target_artifacts.entry_helpers import provider_fetch_entries, text_entry +from utils.target_artifacts.models import ( + ArtifactEntry, + BranchReference, + GitHubReleaseReference, +) +from utils.target_artifacts.resolvers import GitHubBranchResolver, GitHubLatestReleaseResolver + + +def _normalize_branch_for_image_tag(branch_name: str) -> str: + value = re.sub(r"[^a-z0-9]+", "-", branch_name.lower()) + value = re.sub(r"-+", "-", value).strip("-") + return value[:63].rstrip("-") + + +class Dev: + def artifact_inputs(self, env: dict[str, str]) -> tuple[GitHubBranchResolver]: + return ( + GitHubBranchResolver( + name="library_branch", + repository="DataDog/dd-trace-php", + variable_name="LIBRARY_TARGET_BRANCH", + default_value="master", + ), + ) + + def artifact_entries( + self, + resolved_inputs: dict[str, BranchReference], + ) -> tuple[ArtifactEntry, ArtifactEntry]: + resolved_branch = resolved_inputs["library_branch"] + fetch_selector = ( + f"ghcr.io/datadog/dd-trace-php/dd-library-php:{_normalize_branch_for_image_tag(resolved_branch.branch)}" + ) + return provider_fetch_entries( + fetch_filename="php-package-image", + fetch_selector=fetch_selector, + marker_filename="php-package-selection", + bounded_selector=resolved_branch.sha, + ) + + +class Prod: + def artifact_inputs(self, env: dict[str, str]) -> tuple[GitHubLatestReleaseResolver]: + return (GitHubLatestReleaseResolver(name="release", repository="DataDog/dd-trace-php"),) + + def artifact_entries( + self, + resolved_inputs: dict[str, GitHubReleaseReference], + ) -> tuple[ArtifactEntry]: + return (text_entry("php-load-from-release", resolved_inputs["release"].tag_name),) diff --git a/utils/build/docker/php/common/install_ddtrace.sh b/utils/build/docker/php/common/install_ddtrace.sh index a05b7411263..40ff25da7ec 100755 --- a/utils/build/docker/php/common/install_ddtrace.sh +++ b/utils/build/docker/php/common/install_ddtrace.sh @@ -52,8 +52,8 @@ if [ -d /opt/php/nts ]; then elif [[ $IS_APACHE -eq 0 ]]; then PHP_VERSION=$(php -r "echo PHP_MAJOR_VERSION.'.'.PHP_MINOR_VERSION;") INI_FILE=/etc/php/$PHP_VERSION/fpm/conf.d/98-ddtrace.ini - mkdir -p $(dirname $INI_FILE) - chmod 777 $(dirname $INI_FILE) + mkdir -p "$(dirname "$INI_FILE")" + chmod 777 "$(dirname "$INI_FILE")" fi # Always install from package first (to get recommended.json and other files) @@ -63,8 +63,13 @@ if [ "$PKG" != "" ] && [ ! -f "$SETUP" ]; then fi if [ "$PKG" == "" ]; then - #Download latest release - curl -LO https://github.com/DataDog/dd-trace-php/releases/latest/download/datadog-setup.php + if [ -f /binaries/php-load-from-release ]; then + RELEASE_TAG=$(cat /binaries/php-load-from-release) + curl -LO "https://github.com/DataDog/dd-trace-php/releases/download/${RELEASE_TAG}/datadog-setup.php" + else + # Download latest release for compatibility when artifact staging has not run. + curl -LO https://github.com/DataDog/dd-trace-php/releases/latest/download/datadog-setup.php + fi SETUP=datadog-setup.php unset PKG @@ -101,26 +106,26 @@ else fi # After package installation, override with custom ddtrace.so if present -if [ -f $DDTRACE_SO ]; then +if [ -f "$DDTRACE_SO" ]; then echo "Overriding package ddtrace.so with custom binary from $DDTRACE_SO" # Find and replace the installed ddtrace.so with custom one INSTALLED_DDTRACE=$(find /root /opt /usr/lib/php -name ddtrace.so 2>/dev/null | grep -v /binaries | head -1) if [ -n "$INSTALLED_DDTRACE" ]; then echo "Found installed ddtrace.so at $INSTALLED_DDTRACE, replacing with custom binary" - cp -f $DDTRACE_SO $INSTALLED_DDTRACE + cp -f "$DDTRACE_SO" "$INSTALLED_DDTRACE" else echo "Warning: Could not find installed ddtrace.so to replace" fi fi # After package installation, override with custom ddappsec.so and helper if present -if [ -f $DDAPPSEC_SO ] && [ -f $APPSEC_HELPER_SO ]; then +if [ -f "$DDAPPSEC_SO" ] && [ -f "$APPSEC_HELPER_SO" ]; then echo "Overriding package ddappsec.so and helper with custom binaries" # Find and replace the installed ddappsec.so INSTALLED_DDAPPSEC=$(find /root /opt /usr/lib/php -name ddappsec.so 2>/dev/null | grep -v /binaries | head -1) if [ -n "$INSTALLED_DDAPPSEC" ]; then echo "Found installed ddappsec.so at $INSTALLED_DDAPPSEC, replacing with custom binary" - cp -f $DDAPPSEC_SO $INSTALLED_DDAPPSEC + cp -f "$DDAPPSEC_SO" "$INSTALLED_DDAPPSEC" else echo "Warning: Could not find installed ddappsec.so to replace" fi @@ -129,44 +134,44 @@ if [ -f $DDAPPSEC_SO ] && [ -f $APPSEC_HELPER_SO ]; then INSTALLED_HELPER=$(find /root /opt -name libddappsec-helper.so 2>/dev/null | grep -v /binaries | head -1) if [ -n "$INSTALLED_HELPER" ]; then echo "Found installed helper at $INSTALLED_HELPER, replacing with custom binary" - cp -f $APPSEC_HELPER_SO $INSTALLED_HELPER + cp -f "$APPSEC_HELPER_SO" "$INSTALLED_HELPER" else echo "Warning: Could not find installed libddappsec-helper.so to replace" fi fi # Install the Rust helper alongside the C++ helper so DD_APPSEC_HELPER_RUST_REDIRECTION works -if [ -f $APPSEC_RUST_HELPER_SO ]; then +if [ -f "$APPSEC_RUST_HELPER_SO" ]; then INSTALLED_HELPER=$(find /root /opt -name libddappsec-helper.so 2>/dev/null | grep -v /binaries | head -1) if [ -n "$INSTALLED_HELPER" ]; then echo "Installing Rust helper at $(dirname "$INSTALLED_HELPER")/libddappsec-helper-rust.so" - cp -f $APPSEC_RUST_HELPER_SO "$(dirname "$INSTALLED_HELPER")/libddappsec-helper-rust.so" + cp -f "$APPSEC_RUST_HELPER_SO" "$(dirname "$INSTALLED_HELPER")/libddappsec-helper-rust.so" else echo "Warning: Could not find installed libddappsec-helper.so to install Rust helper alongside" fi fi -if [ -f $LIBDDWAF_SO ]; then +if [ -f "$LIBDDWAF_SO" ]; then echo "Copying libddwaf.so from /binaries" INSTALLED_HELPER=$(find /root /opt -name libddappsec-helper.so 2>/dev/null | grep -v /binaries | head -1) if [ -n "$INSTALLED_HELPER" ]; then echo "Found installed helper at $INSTALLED_HELPER, installing custom libddwaf.so alongside" - cp -v $LIBDDWAF_SO "$(dirname "$INSTALLED_HELPER")" + cp -v "$LIBDDWAF_SO" "$(dirname "$INSTALLED_HELPER")" else echo "Warning: Could not find installed libddappsec-helper.so" fi fi -if test -f $INI_FILE; then +if test -f "$INI_FILE"; then #There is a bug on 0.98.1 which disable explicitly appsec when it shouldnt. Delete this line when hotfix - sed -i "/datadog.appsec.enabled/s/^/;/g" $INI_FILE + sed -i "/datadog.appsec.enabled/s/^/;/g" "$INI_FILE" #Parametric tests don't need appsec - [ ! -z ${NO_EXTRACT_VERSION+x} ] && echo "datadog.appsec.enabled = Off" >> $INI_FILE + [ -n "${NO_EXTRACT_VERSION+x}" ] && echo "datadog.appsec.enabled = Off" >> "$INI_FILE" fi #Ensure parametric test compatibility -[ ! -z ${NO_EXTRACT_VERSION+x} ] && exit 0 +[ -n "${NO_EXTRACT_VERSION+x}" ] && exit 0 #Extract version info php -d error_reporting='' -d extension=ddtrace.so -d extension=ddappsec.so -r 'echo phpversion("ddtrace");' > \ diff --git a/utils/build/docker/python/artifact.py b/utils/build/docker/python/artifact.py new file mode 100644 index 00000000000..c2feaac9797 --- /dev/null +++ b/utils/build/docker/python/artifact.py @@ -0,0 +1,40 @@ +from __future__ import annotations + + +from utils.target_artifacts.entry_helpers import text_entry +from utils.target_artifacts.models import ( + ArtifactEntry, + BranchReference, + ModuleVersion, +) +from utils.target_artifacts.resolvers import GitHubBranchResolver, PypiLatestResolver + + +class Dev: + def artifact_inputs(self, env: dict[str, str]) -> tuple[GitHubBranchResolver]: + return ( + GitHubBranchResolver( + name="library_branch", + repository="DataDog/dd-trace-py", + variable_name="LIBRARY_TARGET_BRANCH", + default_value="main", + ), + ) + + def artifact_entries( + self, + resolved_inputs: dict[str, BranchReference], + ) -> tuple[ArtifactEntry]: + return (text_entry("python-load-from-s3", resolved_inputs["library_branch"].sha),) + + +class Prod: + def artifact_inputs(self, env: dict[str, str]) -> tuple[PypiLatestResolver]: + return (PypiLatestResolver(name="ddtrace", package="ddtrace"),) + + def artifact_entries( + self, + resolved_inputs: dict[str, ModuleVersion], + ) -> tuple[ArtifactEntry]: + version = resolved_inputs["ddtrace"].version + return (text_entry("python-load-from-pip", f"ddtrace=={version}"),) diff --git a/utils/build/docker/python_lambda/artifact.py b/utils/build/docker/python_lambda/artifact.py new file mode 100644 index 00000000000..c640906497c --- /dev/null +++ b/utils/build/docker/python_lambda/artifact.py @@ -0,0 +1,57 @@ +from __future__ import annotations + + +from utils.target_artifacts.entry_helpers import json_entry, text_entry +from utils.target_artifacts.models import ( + ArtifactEntry, + GitHubActionsArtifactReference, + GitHubReleaseReference, +) +from utils.target_artifacts.resolvers import GitHubActionsArtifactResolver, GitHubLatestReleaseResolver + + +class Dev: + def artifact_inputs(self, env: dict[str, str]) -> tuple[GitHubActionsArtifactResolver]: + return ( + GitHubActionsArtifactResolver( + name="workflow_artifact", + repository="DataDog/datadog-lambda-python", + workflow="build_layer.yml", + artifact_name="datadog-lambda-python-3.13-amd64", + variable_name="LIBRARY_TARGET_BRANCH", + default_value="main", + ignore_failed_workflow=False, + ), + ) + + def artifact_entries( + self, + resolved_inputs: dict[str, GitHubActionsArtifactReference], + ) -> tuple[ArtifactEntry]: + artifact = resolved_inputs["workflow_artifact"] + return ( + json_entry( + "python-lambda-github-actions-artifact.json", + { + "archive_download_url": artifact.archive_download_url, + "artifact_id": artifact.artifact_id, + "artifact_name": artifact.artifact_name, + "commit_sha": artifact.commit_sha, + "repository": artifact.repository, + "run_id": artifact.run_id, + "run_url": artifact.run_url, + "workflow": artifact.workflow, + }, + ), + ) + + +class Prod: + def artifact_inputs(self, env: dict[str, str]) -> tuple[GitHubLatestReleaseResolver]: + return (GitHubLatestReleaseResolver(name="release", repository="DataDog/datadog-lambda-python"),) + + def artifact_entries( + self, + resolved_inputs: dict[str, GitHubReleaseReference], + ) -> tuple[ArtifactEntry]: + return (text_entry("python-lambda-load-from-release", resolved_inputs["release"].tag_name),) diff --git a/utils/build/docker/python_lambda/install_datadog_lambda.sh b/utils/build/docker/python_lambda/install_datadog_lambda.sh index 45ba9172fca..6c1a3bbbe3f 100755 --- a/utils/build/docker/python_lambda/install_datadog_lambda.sh +++ b/utils/build/docker/python_lambda/install_datadog_lambda.sh @@ -9,14 +9,34 @@ if [ "$(find . -maxdepth 1 -name "*.zip" | wc -l)" = "1" ]; then echo "Install datadog_lambda from ${path}" unzip "${path}" -d /opt else - echo "Fetching from latest GitHub release..." ARCH=$(uname -m | sed 's/x86_64/amd64/' | sed 's/aarch64/arm64/') - echo https://github.com/DataDog/datadog-lambda-python/releases/latest/download/datadog_lambda_py-"$ARCH"-3.13.zip - curl -fsSLO https://github.com/DataDog/datadog-lambda-python/releases/latest/download/datadog_lambda_py-"$ARCH"-3.13.zip + ZIPFILE=datadog_lambda_py-"$ARCH"-3.13.zip + if [ -f python-lambda-github-actions-artifact.json ]; then + echo "Fetching from staged GitHub Actions artifact metadata..." + ARCHIVE_URL=$(jq -r '.archive_download_url' python-lambda-github-actions-artifact.json) + if [ -z "$ARCHIVE_URL" ] || [ "$ARCHIVE_URL" = "null" ]; then + echo "Staged GitHub Actions artifact metadata is missing archive_download_url" + exit 1 + fi + GITHUB_AUTH_HEADER=() + if [ -f /run/secrets/github_token ]; then + GITHUB_AUTH_HEADER=(-H "Authorization: Bearer $(cat /run/secrets/github_token)") + fi + curl -fsSL "${GITHUB_AUTH_HEADER[@]}" -o /tmp/python-lambda-artifact.zip "$ARCHIVE_URL" + mkdir -p /tmp/python-lambda-artifact + unzip -o /tmp/python-lambda-artifact.zip -d /tmp/python-lambda-artifact + cp "$(find /tmp/python-lambda-artifact -name "$ZIPFILE" | head -1)" . + elif [ -f python-lambda-load-from-release ]; then + RELEASE_TAG=$(cat python-lambda-load-from-release) + curl -fsSLO "https://github.com/DataDog/datadog-lambda-python/releases/download/${RELEASE_TAG}/${ZIPFILE}" + else + echo "Fetching from latest GitHub release..." + curl -fsSLO "https://github.com/DataDog/datadog-lambda-python/releases/latest/download/${ZIPFILE}" + fi unzip -o datadog_lambda_py-"$ARCH"-3.13.zip -d /opt - if [ ! -f datadog_lambda_py-"$ARCH"-3.13.zip ]; then - echo "Failed to download datadog_lambda_py-""$ARCH""-3.13.zip" + if [ ! -f "$ZIPFILE" ]; then + echo "Failed to download ${ZIPFILE}" exit 1 fi fi diff --git a/utils/build/docker/python_otel/artifact.py b/utils/build/docker/python_otel/artifact.py new file mode 100644 index 00000000000..d412eb1505c --- /dev/null +++ b/utils/build/docker/python_otel/artifact.py @@ -0,0 +1,35 @@ +from __future__ import annotations + + +from utils.target_artifacts.entry_helpers import text_entry +from utils.target_artifacts.models import ( + ArtifactEntry, + ModuleVersion, +) +from utils.target_artifacts.resolvers import PypiLatestResolver + +PACKAGE_NAME = "opentelemetry-distro" + + +class Dev: + def artifact_inputs(self, env: dict[str, str]) -> tuple[PypiLatestResolver]: + return (PypiLatestResolver(name="otel_package", package=PACKAGE_NAME),) + + def artifact_entries( + self, + resolved_inputs: dict[str, ModuleVersion], + ) -> tuple[ArtifactEntry]: + version = resolved_inputs["otel_package"].version + return (text_entry("python-otel-load-from-pip", f"{PACKAGE_NAME}[otlp]=={version}"),) + + +class Prod: + def artifact_inputs(self, env: dict[str, str]) -> tuple[PypiLatestResolver]: + return (PypiLatestResolver(name="otel_package", package=PACKAGE_NAME),) + + def artifact_entries( + self, + resolved_inputs: dict[str, ModuleVersion], + ) -> tuple[ArtifactEntry]: + version = resolved_inputs["otel_package"].version + return (text_entry("python-otel-load-from-pip", f"{PACKAGE_NAME}[otlp]=={version}"),) diff --git a/utils/build/docker/python_otel/flask-poc-otel.Dockerfile b/utils/build/docker/python_otel/flask-poc-otel.Dockerfile index e1ad5193d74..1de6e00bf47 100644 --- a/utils/build/docker/python_otel/flask-poc-otel.Dockerfile +++ b/utils/build/docker/python_otel/flask-poc-otel.Dockerfile @@ -7,10 +7,14 @@ RUN pip uninstall -y psycopg2-binary RUN pip install psycopg2 ############# -RUN pip install opentelemetry-distro[otlp]==0.49b0 +COPY binaries* /binaries/ +RUN if [ -f /binaries/python-otel-load-from-pip ]; then \ + pip install "$(cat /binaries/python-otel-load-from-pip)"; \ + else \ + pip install opentelemetry-distro[otlp]==0.49b0; \ + fi WORKDIR /app -COPY binaries* /binaries/ COPY utils/build/docker/python/flask /app COPY utils/build/docker/python_otel/flask-poc-otel/app.py /app @@ -22,4 +26,3 @@ RUN pip freeze | grep opentelemetry ENV OTEL_PYTHON_LOGGING_AUTO_INSTRUMENTATION_ENABLED=true ENV FLASK_APP=app.py CMD ./app.sh - diff --git a/utils/build/docker/ruby/artifact.py b/utils/build/docker/ruby/artifact.py new file mode 100644 index 00000000000..d4fff04e096 --- /dev/null +++ b/utils/build/docker/ruby/artifact.py @@ -0,0 +1,52 @@ +from __future__ import annotations + + +from utils.target_artifacts.entry_helpers import text_entry +from utils.target_artifacts.models import ( + ArtifactEntry, + BranchReference, + ModuleVersion, +) +from utils.target_artifacts.resolvers import GitHubBranchResolver, RubygemsLatestResolver + + +class Dev: + def artifact_inputs(self, env: dict[str, str]) -> tuple[GitHubBranchResolver]: + return ( + GitHubBranchResolver( + name="library_branch", + repository="DataDog/dd-trace-rb", + variable_name="LIBRARY_TARGET_BRANCH", + default_value="master", + ), + ) + + def artifact_entries( + self, + resolved_inputs: dict[str, BranchReference], + ) -> tuple[ArtifactEntry]: + sha = resolved_inputs["library_branch"].sha + return ( + text_entry( + "ruby-load-from-bundle-add", + "gem 'datadog', require: 'datadog/auto_instrument', " + f"git: 'https://github.com/DataDog/dd-trace-rb.git', ref: '{sha}'", + ), + ) + + +class Prod: + def artifact_inputs(self, env: dict[str, str]) -> tuple[RubygemsLatestResolver]: + return (RubygemsLatestResolver(name="datadog", package="datadog"),) + + def artifact_entries( + self, + resolved_inputs: dict[str, ModuleVersion], + ) -> tuple[ArtifactEntry]: + version = resolved_inputs["datadog"].version + return ( + text_entry( + "ruby-load-from-bundle-add", + f"gem 'datadog', '{version}', require: 'datadog/auto_instrument'", + ), + ) diff --git a/utils/build/docker/ruby_lambda/artifact.py b/utils/build/docker/ruby_lambda/artifact.py new file mode 100644 index 00000000000..18a0d868888 --- /dev/null +++ b/utils/build/docker/ruby_lambda/artifact.py @@ -0,0 +1,44 @@ +from __future__ import annotations + + +from utils.target_artifacts.entry_helpers import text_entry +from utils.target_artifacts.models import ( + ArtifactEntry, + BranchReference, + GitHubReleaseReference, +) +from utils.target_artifacts.resolvers import GitHubBranchResolver, GitHubLatestReleaseResolver + + +class Dev: + def artifact_inputs(self, env: dict[str, str]) -> tuple[GitHubBranchResolver]: + return ( + GitHubBranchResolver( + name="library_branch", + repository="DataDog/datadog-lambda-rb", + variable_name="LIBRARY_TARGET_BRANCH", + default_value="main", + ), + ) + + def artifact_entries( + self, + resolved_inputs: dict[str, BranchReference], + ) -> tuple[ArtifactEntry]: + return ( + text_entry( + "ruby-lambda-load-from-git", + f"https://github.com/DataDog/datadog-lambda-rb@{resolved_inputs['library_branch'].sha}", + ), + ) + + +class Prod: + def artifact_inputs(self, env: dict[str, str]) -> tuple[GitHubLatestReleaseResolver]: + return (GitHubLatestReleaseResolver(name="release", repository="DataDog/datadog-lambda-rb"),) + + def artifact_entries( + self, + resolved_inputs: dict[str, GitHubReleaseReference], + ) -> tuple[ArtifactEntry]: + return (text_entry("ruby-lambda-load-from-release", resolved_inputs["release"].tag_name),) diff --git a/utils/build/docker/ruby_lambda/install_datadog_lambda.sh b/utils/build/docker/ruby_lambda/install_datadog_lambda.sh index 8b1dfc0fe5f..7b0d755591c 100755 --- a/utils/build/docker/ruby_lambda/install_datadog_lambda.sh +++ b/utils/build/docker/ruby_lambda/install_datadog_lambda.sh @@ -28,15 +28,30 @@ elif [ "$(find . -maxdepth 1 -name '*.zip' | wc -l)" = "1" ]; then path=$(readlink -f "$(find . -maxdepth 1 -name '*.zip')") echo "Install datadog-lambda from ${path}" unzip "${path}" -d /opt +elif [ -f ruby-lambda-load-from-git ]; then + TARGET=$(cat ruby-lambda-load-from-git) + URL=$(echo "$TARGET" | cut -d "@" -f 1) + REF=$(echo "$TARGET" | cut -d "@" -f 2) + echo "Install datadog-lambda from ${TARGET}" + git clone "$URL" datadog-lambda-rb + git -C datadog-lambda-rb checkout "$REF" + cd datadog-lambda-rb + gem build datadog-lambda + gem install datadog-lambda-*.gem --install-dir "${GEM_DIR}" --no-document else - echo "Fetching from latest GitHub release..." ARCH=$(uname -m | sed 's/x86_64/amd64/' | sed 's/aarch64/arm64/') RUBY_MINOR=$(ruby -e 'puts RUBY_VERSION.split(".")[0..1].join(".")') # NOTE: Release assets are datadog-lambda_ruby--.zip # just one-dot (3.4, not 3.4.0). # The old datadog_lambda_rb-* name still resolved, but to a stale layer # with no AppSec - URL="https://github.com/DataDog/datadog-lambda-rb/releases/latest/download/datadog-lambda_ruby-${ARCH}-${RUBY_MINOR}.zip" + if [ -f ruby-lambda-load-from-release ]; then + RELEASE_TAG=$(cat ruby-lambda-load-from-release) + URL="https://github.com/DataDog/datadog-lambda-rb/releases/download/${RELEASE_TAG}/datadog-lambda_ruby-${ARCH}-${RUBY_MINOR}.zip" + else + echo "Fetching from latest GitHub release..." + URL="https://github.com/DataDog/datadog-lambda-rb/releases/latest/download/datadog-lambda_ruby-${ARCH}-${RUBY_MINOR}.zip" + fi echo "${URL}" curl -fsSLO "${URL}" ZIPFILE="datadog-lambda_ruby-${ARCH}-${RUBY_MINOR}.zip" diff --git a/utils/build/docker/rust/artifact.py b/utils/build/docker/rust/artifact.py new file mode 100644 index 00000000000..00b3f921ba0 --- /dev/null +++ b/utils/build/docker/rust/artifact.py @@ -0,0 +1,45 @@ +from __future__ import annotations + + +from utils.target_artifacts.entry_helpers import text_entry +from utils.target_artifacts.models import ( + ArtifactEntry, + BranchReference, + ModuleVersion, +) +from utils.target_artifacts.resolvers import CratesLatestResolver, GitHubBranchResolver + + +class Dev: + def artifact_inputs(self, env: dict[str, str]) -> tuple[GitHubBranchResolver]: + return ( + GitHubBranchResolver( + name="library_branch", + repository="DataDog/dd-trace-rs", + variable_name="LIBRARY_TARGET_BRANCH", + default_value="main", + ), + ) + + def artifact_entries( + self, + resolved_inputs: dict[str, BranchReference], + ) -> tuple[ArtifactEntry]: + return (text_entry("rust-load-from-git", resolved_inputs["library_branch"].sha),) + + +class Prod: + def artifact_inputs(self, env: dict[str, str]) -> tuple[CratesLatestResolver]: + return ( + CratesLatestResolver( + name="datadog_opentelemetry", + package="datadog-opentelemetry", + ), + ) + + def artifact_entries( + self, + resolved_inputs: dict[str, ModuleVersion], + ) -> tuple[ArtifactEntry]: + version = resolved_inputs["datadog_opentelemetry"].version + return (text_entry("rust-load-from-crates", version),) diff --git a/utils/build/docker/rust/install_ddtrace.sh b/utils/build/docker/rust/install_ddtrace.sh index b9c9f4267ed..94385bdf354 100755 --- a/utils/build/docker/rust/install_ddtrace.sh +++ b/utils/build/docker/rust/install_ddtrace.sh @@ -31,10 +31,13 @@ fail() { if [ -e /binaries/rust-load-from-git ]; then rev_or_branch=$(/dev/null 2>&1; then + echo "Clone $REPO_URL at $rev_or_branch into /binaries/dd-trace-rs" + if ! git clone "$REPO_URL" /binaries/dd-trace-rs >/dev/null 2>&1; then fail "could not clone dd-trace-rs ref '$rev_or_branch'. Check that the ref exists and is accessible." fi + if ! git -C /binaries/dd-trace-rs checkout "$rev_or_branch" >/dev/null 2>&1; then + fail "could not checkout dd-trace-rs ref '$rev_or_branch'. Check that the ref exists and is accessible." + fi fi if [ -e /binaries/dd-trace-rs ]; then @@ -74,7 +77,13 @@ else # remove previous dependency on datadog-opentelemetry and add the new one from crates.io cargo remove datadog-opentelemetry >/dev/null 2>&1 || true - if ! cargo add datadog-opentelemetry --features metrics-http,metrics-grpc,logs-http,logs-grpc >/dev/null 2>&1; then + if [ -e /binaries/rust-load-from-crates ]; then + crate_version=$(/dev/null 2>&1; then fail "could not install datadog-opentelemetry from crates.io. Check network access and the selected package version." fi fi diff --git a/utils/ci/gitlab/build_pipeline.py b/utils/ci/gitlab/build_pipeline.py index 4a7b9389959..d0771d2966c 100644 --- a/utils/ci/gitlab/build_pipeline.py +++ b/utils/ci/gitlab/build_pipeline.py @@ -100,6 +100,7 @@ def render_library( for scenario in job.get("scenarios", []) ] binaries_artifact = params["miscs"]["binaries_artifact"] + ci_environment = params["miscs"].get("ci_environment", "prod") parametric = params["parametric"] # Build the full list of artifact jobs for cross-pipeline downloads. # If binaries_artifacts is provided, use it; otherwise fall back to the single job. @@ -110,9 +111,10 @@ def render_library( else: binaries_artifacts_list = [] + is_default_branch = ci_commit_branch in {"main", "master", ci_default_branch} render_build = _generate_build_renderer( - push_main=True, - push_lib_main=(ci_project_name != "system-tests" and ci_commit_branch in {"main", "master", ci_default_branch}), + push_main=(ci_project_name == "system-tests" and is_default_branch), + push_lib_main=(ci_project_name != "system-tests" and is_default_branch), ) return _template.render( @@ -123,6 +125,7 @@ def render_library( binaries_artifact=binaries_artifact, binaries_artifacts_list=binaries_artifacts_list, binaries_artifact_path=binaries_artifact_path, + ci_environment=ci_environment, parametric=parametric, ci_image=ci_image, ref=ref, diff --git a/utils/ci/gitlab/system-tests.yml.j2 b/utils/ci/gitlab/system-tests.yml.j2 index 3268f4dc5dd..df786faa7d0 100644 --- a/utils/ci/gitlab/system-tests.yml.j2 +++ b/utils/ci/gitlab/system-tests.yml.j2 @@ -17,6 +17,11 @@ echo "SYSTEM_TESTS_GENERATED_PIPELINE_START_TIME not set or not numeric ('$SYSTEM_TESTS_GENERATED_PIPELINE_START_TIME'), skipping metric emission" fi {% endmacro %} +{% macro stage_target_artifacts() %} + - section_start "target_artifacts" "Staging target artifacts" + - python3 utils/scripts/stage-target-artifacts.py {{library}} {{ci_environment}} + - section_end "target_artifacts" +{% endmacro %} {% if not skip_header %} workflow: name: "System-tests end to end" @@ -90,6 +95,8 @@ system_tests_build_{{library}}_{{variant}}: script: {% if binaries_artifacts_list and binaries_artifact_path %} {{ copy_binaries(binaries_artifact_path) }} + {% elif not binaries_artifacts_list %} + {{ stage_target_artifacts() }} {% endif %} - section_start "build" "Building weblog" false {{ job_tag("build") }} @@ -178,6 +185,8 @@ system_tests_run_{{library}}_PARAMETRIC_{{job_index}}: {{ job_tag("run") }} {% if binaries_artifacts_list and binaries_artifact_path %} {{ copy_binaries(binaries_artifact_path) }} + {% elif not binaries_artifacts_list %} + {{ stage_target_artifacts() }} {% endif %} - {{ trace("run " ~ library ~ " PARAMETRIC " ~ job_index, "./run.sh PARAMETRIC -L " ~ library ~ " --splits=" ~ parametric.job_count ~ " --group=" ~ job_index, library, scenario="PARAMETRIC") }} - section_end "run" diff --git a/utils/scripts/compute_libraries_and_scenarios.py b/utils/scripts/compute_libraries_and_scenarios.py index eb5fe204a67..9d10b83de14 100644 --- a/utils/scripts/compute_libraries_and_scenarios.py +++ b/utils/scripts/compute_libraries_and_scenarios.py @@ -30,8 +30,8 @@ OTEL_LIBRARIES = COMPONENT_GROUPS.otel - {"nodejs_otel"} # nodejs_otel intentionally excluded ALL_LIBRARIES = LIBRARIES | OTEL_LIBRARIES GITHUB_EXCLUDED_LIBRARIES = {"c"} -GITLAB_PR_LIBRARIES = {"c", "python"} -GITLAB_MAIN = {} +GITLAB_PR_LIBRARIES = {"c"} +GITLAB_MAIN: set[str] = {"python"} def check_scenarios(scenarios: set[str]) -> bool: diff --git a/utils/scripts/docker_base_image.sh b/utils/scripts/docker_base_image.sh index 3a86ac8faf4..7c8e5849cc9 100755 --- a/utils/scripts/docker_base_image.sh +++ b/utils/scripts/docker_base_image.sh @@ -6,18 +6,18 @@ set -eu image="$1" target_dir="$2" -mkdir --parent $target_dir +mkdir --parent "$target_dir" echo "Extracting Docker base image $image to folder $target_dir" -docker pull $image -docker save -o $target_dir/image.tar $image -tar xf $target_dir/image.tar -C $target_dir -layers=$(jq -r '.[0].Layers[]' $target_dir/manifest.json) +docker pull "$image" +docker save -o "$target_dir/image.tar" "$image" +tar xf "$target_dir/image.tar" -C "$target_dir" +layers=$(jq -r '.[0].Layers[]' "$target_dir/manifest.json") for i in $layers; do - tar xf $target_dir/$i -C $target_dir + tar xf "$target_dir/$i" -C "$target_dir" done #Done! clean -rm -rf $target_dir/image.tar $target_dir/manifest.json $target_dir/oci-layout $target_dir/index.json -rm -rf $target_dir/blobs/ +rm -rf "$target_dir/image.tar" "$target_dir/manifest.json" "$target_dir/oci-layout" "$target_dir/index.json" +rm -rf "$target_dir/blobs/" diff --git a/utils/scripts/load-binary.sh b/utils/scripts/load-binary.sh index f063797e299..6140af107f7 100755 --- a/utils/scripts/load-binary.sh +++ b/utils/scripts/load-binary.sh @@ -4,405 +4,66 @@ # This product includes software developed at Datadog (https://www.datadoghq.com/). # Copyright 2021 Datadog, Inc. - -########################################################################################## -# The purpose of this script is to download the latest development version of a component. -# -# Binaries sources: -# -# * Agent: Docker hub datadog/agent-dev:master-py3 -# * cpp_httpd: Github action artifact -# * Golang: github.com/DataDog/dd-trace-go/v2@main -# * .NET: ghcr.io/datadog/dd-trace-dotnet -# * Java: S3 -# * Java Lambda: S3 (same binary as Java) -# * PHP: ghcr.io/datadog/dd-trace-php -# * Node.js: Direct from github source -# * Node.js Lambda: Fetch from GitHub Actions artifact -# * C++: Direct from github source -# * Python: S3 https://dd-trace-py-builds.s3.amazonaws.com//index.html -# * Ruby: Direct from github source -# * WAF: Direct from github source, but not working, as this repo is now private -# * Python Lambda: Fetch from GitHub Actions artifact -# * Ruby Lambda: Clone locally the github repo -# * Rust: Clone locally the github repo -########################################################################################## - set -eu -assert_version_is_dev() { - - if [ "$VERSION" = 'dev' ]; then - return 0 - fi - - echo "Don't know how to load version $VERSION for $TARGET" - - exit 1 -} - -assert_target_branch_is_not_set() { - - if [[ -z "${LIBRARY_TARGET_BRANCH:-}" ]]; then - return 0 - fi - - echo "It is not possible to specify the '$LIBRARY_TARGET_BRANCH' target branch for $TARGET library yet" - - exit 1 -} - -ghcr_login_if_token_set() { - if [ -n "$GITHUB_TOKEN" ]; then - echo "Log to GHCR with token" - echo "$GITHUB_TOKEN" | docker login ghcr.io --password-stdin -u "actor" # username is ignored - fi -} - -resolve_github_branch_sha() { - local repository="$1" - local branch="$2" - local encoded_branch - local response - local sha - - encoded_branch=$(jq -rn --arg value "$branch" '$value | @uri') - if ! response=$(curl --fail --location --silent --show-error \ - "${GITHUB_AUTH_HEADER[@]}" \ - "https://api.github.com/repos/${repository}/branches/${encoded_branch}"); then - echo "Unable to resolve branch '${branch}' in ${repository}" >&2 - exit 1 - fi - - sha=$(jq -r '.commit.sha // empty' <<< "$response") - if [[ ! "$sha" =~ ^[0-9a-f]{40}$ ]]; then - echo "Branch '${branch}' in ${repository} did not resolve to a commit SHA" >&2 - exit 1 - fi - - printf '%s' "$sha" -} - -validate_oci_image() { - local image="$1" - - if ! docker manifest inspect "$image" >/dev/null; then - echo "OCI package does not exist or is not accessible: ${image}" >&2 - exit 1 - fi -} - -get_github_action_artifact() { - rm -rf artifacts artifacts.zip - - SLUG=$1 - WORKFLOW=$2 - BRANCH=$3 - ARTIFACT_NAME=$4 - PATTERN=$5 - IGNORE_FAILED_WORKFLOW=${6:-true} # 6th arg, with default "true" - - # query filter seems not to be working ?? - WORKFLOWS=$(curl --silent --fail --show-error -H "Authorization: token $GITHUB_TOKEN" "https://api.github.com/repos/$SLUG/actions/workflows/$WORKFLOW/runs?per_page=100") - - if [ "$IGNORE_FAILED_WORKFLOW" = "true" ]; then - QUERY="[.workflow_runs[] | select(.conclusion != \"failure\" and .head_branch == \"$BRANCH\" and .status == \"completed\")][0]" - else - QUERY="[.workflow_runs[] | select(.head_branch == \"$BRANCH\" and .status == \"completed\")][0]" - fi - - # this wil fail if there are more than 100 artifacts - ARTIFACT_URL=$(echo "$WORKFLOWS" | jq -r "$QUERY | .artifacts_url") - ARTIFACT_URL="$ARTIFACT_URL?per_page=100" - - HTML_URL=$(echo "$WORKFLOWS" | jq -r "$QUERY | .html_url") - echo "Load artifacts for $HTML_URL" - ARTIFACTS=$(curl --silent -H "Authorization: token $GITHUB_TOKEN" "$ARTIFACT_URL") - ARCHIVE_URL=$(echo "$ARTIFACTS" | jq -r --arg ARTIFACT_NAME "$ARTIFACT_NAME" '.artifacts | map(select(.name | contains($ARTIFACT_NAME))) | .[0].archive_download_url') - echo "Load archive $ARCHIVE_URL" - - curl -H "Authorization: token $GITHUB_TOKEN" --output artifacts.zip -L "$ARCHIVE_URL" - - mkdir -p artifacts/ - unzip artifacts.zip -d artifacts/ - - find artifacts/ -type f -name "$PATTERN" -exec cp '{}' . ';' - - rm -rf artifacts artifacts.zip -} - -get_github_release_asset() { - SLUG=$1 - PATTERN=$2 - - release=$(curl --silent --fail --show-error -H "Authorization: token $GITHUB_TOKEN" "https://api.github.com/repos/$SLUG/releases/latest") - - name=$(echo "$release" | jq -r ".assets[].name | select(test(\"$PATTERN\"))") - url=$(echo "$release" | jq -r ".assets[].browser_download_url | select(test(\"$PATTERN\"))") - - echo "Load $url" - - curl -H "Authorization: token $GITHUB_TOKEN" --output "$name" -L "$url" -} - if test -f ".env"; then # shellcheck source=/dev/null source .env fi -TARGET=$1 -VERSION=${2:-'dev'} +TARGET=${1:-} +VERSION=${2:-dev} +BINARIES_DIR=${BINARIES_DIR:-binaries} +GITHUB_TOKEN=${GITHUB_TOKEN:-} -GITHUB_TOKEN="${GITHUB_TOKEN:-}" -GITHUB_AUTH_HEADER=() -if [ -n "$GITHUB_TOKEN" ]; then - GITHUB_AUTH_HEADER=(-H "Authorization: Bearer $GITHUB_TOKEN") +if [[ -z "$TARGET" ]]; then + echo "Usage: $0 [dev|prod|custom]" >&2 + exit 1 fi -echo "Load $VERSION binary for $TARGET" - -cd "${BINARIES_DIR:-binaries}/" - -if [ "$TARGET" = "c" ]; then - if [ "$VERSION" = "prod" ]; then - if [[ -n "${LIBRARY_TARGET_BRANCH:-}" || -n "${AUTO_INJECT_TARGET_BRANCH:-}" ]]; then - echo "Target branches can only be used with the development c packages" >&2 - exit 1 - fi - - C_LIBRARY_IMAGE="install.datadoghq.com/apm-library-c-package:latest" - C_INJECTOR_IMAGE="install.datadoghq.com/apm-inject-package:latest" - elif [ "$VERSION" = "dev" ]; then - if [[ -n "${LIBRARY_TARGET_BRANCH:-}" ]]; then - C_LIBRARY_SHA=$(resolve_github_branch_sha "DataDog/dd-trace-c" "$LIBRARY_TARGET_BRANCH") - C_LIBRARY_IMAGE="installtesting.datad0g.com/apm-library-c-package:${C_LIBRARY_SHA}" - else - C_LIBRARY_IMAGE="install.datadoghq.com/apm-library-c-package:latest" - fi - - if [[ -n "${AUTO_INJECT_TARGET_BRANCH:-}" ]]; then - C_INJECTOR_SHA=$(resolve_github_branch_sha "DataDog/auto_inject" "$AUTO_INJECT_TARGET_BRANCH") - C_INJECTOR_IMAGE="installtesting.datad0g.com/apm-inject-package:${C_INJECTOR_SHA}" - else - C_INJECTOR_IMAGE="install.datadoghq.com/apm-inject-package:latest" - fi - else - echo "Don't know how to load version $VERSION for $TARGET" >&2 - exit 1 +assert_version_is_dev() { + if [[ "$VERSION" == "dev" ]]; then + return 0 fi - validate_oci_image "$C_LIBRARY_IMAGE" - validate_oci_image "$C_INJECTOR_IMAGE" - - printf '%s\n' "$C_LIBRARY_IMAGE" > c-library-image - printf '%s\n' "$C_INJECTOR_IMAGE" > c-injector-image - echo "Using dd-trace-c package ${C_LIBRARY_IMAGE}" - echo "Using auto-inject package ${C_INJECTOR_IMAGE}" - -elif [ "$TARGET" = "java" ] || [ "$TARGET" = "java_lambda" ]; then - assert_version_is_dev - - LIBRARY_TARGET_BRANCH="${LIBRARY_TARGET_BRANCH:-master}" - - curl --fail --location --silent --show-error --output dd-java-agent.jar "https://s3.us-east-1.amazonaws.com/dd-trace-java-builds/${LIBRARY_TARGET_BRANCH}/dd-java-agent.jar" - -elif [ "$TARGET" = "dotnet" ]; then - assert_version_is_dev - - LIBRARY_TARGET_BRANCH="${LIBRARY_TARGET_BRANCH:-latest_snapshot}" - # Normalize branch name for image tag: replace '/' with '_' - NORMALIZED_BRANCH=$(echo "$LIBRARY_TARGET_BRANCH" | sed 's/\//_/g') - - rm -rf ./*.tar.gz - ghcr_login_if_token_set - - ../utils/scripts/docker_base_image.sh "ghcr.io/datadog/dd-trace-dotnet/dd-trace-dotnet:${NORMALIZED_BRANCH}" . - -elif [ "$TARGET" = "python" ]; then - assert_version_is_dev - - LIBRARY_TARGET_BRANCH="${LIBRARY_TARGET_BRANCH:-main}" - echo "Using $LIBRARY_TARGET_BRANCH in S3 for DataDog/dd-trace-py" - echo "$LIBRARY_TARGET_BRANCH" > python-load-from-s3 - -elif [ "$TARGET" = "ruby" ]; then - assert_version_is_dev - - LIBRARY_TARGET_BRANCH="${LIBRARY_TARGET_BRANCH:-master}" - echo "gem 'datadog', require: 'datadog/auto_instrument', git: 'https://github.com/Datadog/dd-trace-rb.git', branch: '$LIBRARY_TARGET_BRANCH'" > ruby-load-from-bundle-add - echo "Using $(cat ruby-load-from-bundle-add)" - -elif [ "$TARGET" = "php" ]; then - rm -rf ./*.tar.gz - mkdir -p temp - - if [ "${VERSION:-}" = 'prod' ]; then - ../utils/scripts/docker_base_image.sh ghcr.io/datadog/dd-trace-php/dd-library-php:latest ./temp - - elif [ -n "${LIBRARY_TARGET_BRANCH:-}" ]; then - # Match GitLab's CI_COMMIT_REF_SLUG: lowercase, non-alphanumeric → '-', collapse, truncate to 63 bytes and trim - NORMALIZED_BRANCH=$(echo "$LIBRARY_TARGET_BRANCH" | tr '[:upper:]' '[:lower:]' | sed 's/[^a-z0-9]/-/g;s/-\+/-/g;s/^-//;s/-$//' | cut -c1-63 | sed 's/-$//') - ghcr_login_if_token_set - ../utils/scripts/docker_base_image.sh \ - "ghcr.io/datadog/dd-trace-php/dd-library-php:${NORMALIZED_BRANCH}" \ - ./temp - - elif [ "${VERSION:-}" = 'dev' ]; then - URL="https://s3.us-east-1.amazonaws.com/dd-trace-php-builds/latest/datadog-setup.php" - echo "Downloading datadog-setup.php from: $URL" - curl --fail --location --silent --show-error --output ./temp/datadog-setup.php "$URL" - echo "datadog-setup.php downloaded" - - VERSION_HASH=$(grep "define('RELEASE_VERSION'" ./temp/datadog-setup.php | sed -E "s/.*urlencode\('([^']+)'\).*/\1/") - if [ -z "$VERSION_HASH" ]; then - echo "Failed to extract VERSION_HASH from datadog-setup.php" - exit 1 - fi - - VERSION_HASH_ENCODED=${VERSION_HASH//+/%2B} - URL="https://s3.us-east-1.amazonaws.com/dd-trace-php-builds/${VERSION_HASH_ENCODED}/dd-library-php-${VERSION_HASH_ENCODED}-$(arch)-linux-gnu.tar.gz" - echo "Downloading dd-library-php from: $URL" - curl --fail --location --silent --show-error --output "./temp/dd-library-php-${VERSION_HASH}-$(arch)-linux-gnu.tar.gz" "$URL" - echo "dd-library-php $(arch) downloaded" + echo "Don't know how to load version $VERSION for $TARGET" >&2 + exit 1 +} - else - echo "Don't know how to load version ${VERSION:-} for $TARGET" - exit 1 +assert_target_branch_is_not_set() { + if [[ -z "${LIBRARY_TARGET_BRANCH:-}" ]]; then + return 0 fi - mv ./temp/dd-library-php*.tar.gz . && mv ./temp/datadog-setup.php . && rm -rf ./temp - -elif [ "$TARGET" = "golang" ]; then - assert_version_is_dev - rm -rf golang-load-from-go-get - set -o pipefail - - LIBRARY_TARGET_BRANCH="${LIBRARY_TARGET_BRANCH:-main}" - echo "load last commit on $LIBRARY_TARGET_BRANCH for DataDog/dd-trace-go" - COMMIT_ID=$(curl -sS --fail "${GITHUB_AUTH_HEADER[@]}" "https://api.github.com/repos/DataDog/dd-trace-go/branches/$LIBRARY_TARGET_BRANCH" | jq -r .commit.sha) - - echo "Using github.com/DataDog/dd-trace-go/v2@$COMMIT_ID" - { - echo "github.com/DataDog/dd-trace-go/v2@$COMMIT_ID" - echo "github.com/DataDog/dd-trace-go/contrib/database/sql/v2@$COMMIT_ID" - echo "github.com/DataDog/dd-trace-go/contrib/net/http/v2@$COMMIT_ID" - echo "github.com/DataDog/dd-trace-go/contrib/google.golang.org/grpc/v2@$COMMIT_ID" - echo "github.com/DataDog/dd-trace-go/contrib/99designs/gqlgen/v2@$COMMIT_ID" - echo "github.com/DataDog/dd-trace-go/contrib/gin-gonic/gin/v2@$COMMIT_ID" - echo "github.com/DataDog/dd-trace-go/contrib/graphql-go/graphql/v2@$COMMIT_ID" - echo "github.com/DataDog/dd-trace-go/contrib/graph-gophers/graphql-go/v2@$COMMIT_ID" - echo "github.com/DataDog/dd-trace-go/contrib/go-chi/chi.v5/v2@$COMMIT_ID" - echo "github.com/DataDog/dd-trace-go/contrib/IBM/sarama/v2@$COMMIT_ID" - echo "github.com/DataDog/dd-trace-go/contrib/labstack/echo.v4/v2@$COMMIT_ID" - echo "github.com/DataDog/dd-trace-go/contrib/sirupsen/logrus/v2@$COMMIT_ID" - } > golang-load-from-go-get - - echo "Using github.com/DataDog/orchestrion@latest" - echo "github.com/DataDog/orchestrion@latest" > orchestrion-load-from-go-get - - # envoy integration - echo "Using ghcr.io/datadog/dd-trace-go/service-extensions-callout:dev" - echo "ghcr.io/datadog/dd-trace-go/service-extensions-callout:dev" > golang-service-extensions-callout-image - - # haproxy integration - echo "Using ghcr.io/datadog/dd-trace-go/haproxy-spoa:dev" - echo "ghcr.io/datadog/dd-trace-go/haproxy-spoa:dev" > golang-haproxy-spoa-image - -elif [ "$TARGET" = "cpp" ]; then - assert_version_is_dev - # PROFILER: The main version is stored in s3, though we can not access this in CI - # Not handled for now for system-tests. this handles artifact for parametric - LIBRARY_TARGET_BRANCH="${LIBRARY_TARGET_BRANCH:-main}" - echo "https://github.com/DataDog/dd-trace-cpp@$LIBRARY_TARGET_BRANCH" > cpp-load-from-git - echo "Using $(cat cpp-load-from-git)" - -elif [ "$TARGET" = "cpp_httpd" ]; then - assert_version_is_dev - get_github_action_artifact "DataDog/httpd-datadog" "dev.yml" "main" "mod_datadog_artifact" "mod_datadog.so" - -elif [ "$TARGET" = "cpp_kong" ]; then - assert_version_is_dev - LIBRARY_TARGET_BRANCH="${LIBRARY_TARGET_BRANCH:-main}" - echo "Cloning kong-plugin-ddtrace branch ${LIBRARY_TARGET_BRANCH}" - git clone --depth 1 --branch "$LIBRARY_TARGET_BRANCH" \ - https://github.com/DataDog/kong-plugin-ddtrace.git kong-plugin-ddtrace - echo "Using kong-plugin-ddtrace@$(git -C kong-plugin-ddtrace rev-parse --short HEAD)" - -elif [ "$TARGET" = "cpp_nginx" ]; then - assert_version_is_dev - get_github_action_artifact "DataDog/nginx-datadog" "system-tests.yml" "master" "binaries" "binaries.zip" "false" - -elif [ "$TARGET" = "agent" ]; then - assert_version_is_dev - AGENT_TARGET_BRANCH="${AGENT_TARGET_BRANCH:-master-py3}" - echo "datadog/agent-dev:$AGENT_TARGET_BRANCH" > agent-image - echo "Using $(cat agent-image) image" - -elif [ "$TARGET" = "nodejs" ]; then - assert_version_is_dev - - LIBRARY_TARGET_BRANCH="${LIBRARY_TARGET_BRANCH:-master}" - # NPM builds the package, so we put a trigger file that tells install script to get package from github#master - echo "DataDog/dd-trace-js#$LIBRARY_TARGET_BRANCH" > nodejs-load-from-npm - echo "Using $(cat nodejs-load-from-npm)" - -elif [ "$TARGET" = "rust" ]; then - assert_version_is_dev - - LIBRARY_TARGET_BRANCH="${LIBRARY_TARGET_BRANCH:-main}" - echo "$LIBRARY_TARGET_BRANCH" > rust-load-from-git - echo "Using $(cat rust-load-from-git)" - -elif [ "$TARGET" = "waf_rule_set_v1" ]; then + echo "It is not possible to specify the '$LIBRARY_TARGET_BRANCH' target branch for $TARGET library yet" >&2 exit 1 +} -elif [ "$TARGET" = "waf_rule_set_v2" ]; then - assert_version_is_dev - assert_target_branch_is_not_set - curl --silent \ - -H "Authorization: token $GITHUB_TOKEN" \ - -H "Accept: application/vnd.github.v3.raw" \ - --output "waf_rule_set.json" \ - https://api.github.com/repos/DataDog/appsec-event-rules/contents/build/recommended.json - -elif [ "$TARGET" = "waf_rule_set" ]; then - assert_version_is_dev - assert_target_branch_is_not_set - curl --fail --output "waf_rule_set.json" \ +load_waf_rule_set() { + mkdir -p "$BINARIES_DIR" + curl --fail --location --silent --show-error \ -H "Authorization: token $GITHUB_TOKEN" \ -H "Accept: application/vnd.github.v3.raw" \ + --output "$BINARIES_DIR/waf_rule_set.json" \ https://api.github.com/repos/DataDog/appsec-event-rules/contents/build/recommended.json +} -elif [ "$TARGET" = "python_lambda" ]; then - assert_version_is_dev - - LIBRARY_TARGET_BRANCH="${LIBRARY_TARGET_BRANCH:-main}" - get_github_action_artifact "DataDog/datadog-lambda-python" "build_layer.yml" "$LIBRARY_TARGET_BRANCH" "datadog-lambda-python-3.13-amd64" "datadog_lambda_py-amd64-3.13.zip" "false" - -elif [ "$TARGET" = "nodejs_lambda" ]; then - assert_version_is_dev - - LIBRARY_TARGET_BRANCH="${LIBRARY_TARGET_BRANCH:-main}" - get_github_action_artifact "DataDog/datadog-lambda-js" "build_layer.yml" "$LIBRARY_TARGET_BRANCH" "datadog_lambda_node18.12" "datadog_lambda_node18.12.zip" "false" - -elif [ "$TARGET" = "ruby_lambda" ]; then - assert_version_is_dev - - LIBRARY_TARGET_BRANCH="${LIBRARY_TARGET_BRANCH:-main}" - echo "Cloning datadog-lambda-rb branch ${LIBRARY_TARGET_BRANCH}" - rm -rf datadog-lambda-rb - git clone --depth 1 --branch "$LIBRARY_TARGET_BRANCH" \ - https://github.com/DataDog/datadog-lambda-rb.git datadog-lambda-rb - echo "Using datadog-lambda-rb@$(git -C datadog-lambda-rb rev-parse --short HEAD)" - -elif [ "$TARGET" = "otel_collector" ]; then - assert_version_is_dev - assert_target_branch_is_not_set - - echo "otel/opentelemetry-collector-contrib:nightly" > otel_collector-image - echo "Using $(cat otel_collector-image) image" +echo "Load $VERSION artifact entries for $TARGET" -else - echo "Unknown target: $1" - exit 1 -fi; +case "$TARGET" in + waf_rule_set_v1) + exit 1 + ;; + waf_rule_set|waf_rule_set_v2) + assert_version_is_dev + assert_target_branch_is_not_set + load_waf_rule_set + ;; + *) + python3 utils/scripts/stage-target-artifacts.py \ + "$TARGET" "$VERSION" \ + --binaries-dir "$BINARIES_DIR" \ + --repo-root . \ + --compatibility + ;; +esac diff --git a/utils/scripts/stage-target-artifacts.py b/utils/scripts/stage-target-artifacts.py new file mode 100755 index 00000000000..8f07aec28db --- /dev/null +++ b/utils/scripts/stage-target-artifacts.py @@ -0,0 +1,13 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +import sys +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(REPO_ROOT)) + +from utils.target_artifacts.cli import main # noqa: E402 + +raise SystemExit(main()) diff --git a/utils/target_artifacts/__init__.py b/utils/target_artifacts/__init__.py new file mode 100644 index 00000000000..11eb5f784ac --- /dev/null +++ b/utils/target_artifacts/__init__.py @@ -0,0 +1,32 @@ +from .models import ( + ArtifactEntry, + ArtifactResolver, + BranchReference, + GitHubActionsArtifactReference, + GitHubReleaseReference, + LiteralValue, + ModuleVersion, + OciImageReference, + ReleaseAsset, + ResolvedArtifactInput, + TargetArtifactEnvironment, + TargetArtifactError, +) +from .orchestrator import MANIFEST_FILENAME, stage_target + +__all__ = [ + "MANIFEST_FILENAME", + "ArtifactEntry", + "ArtifactResolver", + "BranchReference", + "GitHubActionsArtifactReference", + "GitHubReleaseReference", + "LiteralValue", + "ModuleVersion", + "OciImageReference", + "ReleaseAsset", + "ResolvedArtifactInput", + "TargetArtifactEnvironment", + "TargetArtifactError", + "stage_target", +] diff --git a/utils/target_artifacts/__main__.py b/utils/target_artifacts/__main__.py new file mode 100644 index 00000000000..eb53e2f31b2 --- /dev/null +++ b/utils/target_artifacts/__main__.py @@ -0,0 +1,3 @@ +from .cli import main + +raise SystemExit(main()) diff --git a/utils/target_artifacts/cli.py b/utils/target_artifacts/cli.py new file mode 100644 index 00000000000..ad7e4e765e7 --- /dev/null +++ b/utils/target_artifacts/cli.py @@ -0,0 +1,41 @@ +from __future__ import annotations + +import argparse +import os +import sys +from pathlib import Path + +from .compat import stage_legacy_dependency +from .models import TargetArtifactError +from .orchestrator import stage_target + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(prog="stage-target-artifacts") + parser.add_argument("target", help="Target artifact name, such as python, java, or custom") + parser.add_argument("environment", nargs="?", default="dev", help="dev, prod, or custom") + parser.add_argument("--binaries-dir", default=os.environ.get("BINARIES_DIR", "binaries")) + parser.add_argument("--repo-root", default=".") + parser.add_argument( + "--compatibility", + action="store_true", + help="Route legacy dependency or overlay targets through explicit compatibility handling", + ) + + args = parser.parse_args(argv) + repo_root = Path(args.repo_root) + binaries_dir = Path(args.binaries_dir) + + try: + if args.compatibility and stage_legacy_dependency( + args.target, + args.environment, + repo_root=repo_root, + binaries_dir=binaries_dir, + ): + return 0 + stage_target(args.target, args.environment, repo_root=repo_root, binaries_dir=binaries_dir) + except TargetArtifactError as exc: + sys.stderr.write(f"{exc}\n") + return 1 + return 0 diff --git a/utils/target_artifacts/compat.py b/utils/target_artifacts/compat.py new file mode 100644 index 00000000000..9388f3e4ea4 --- /dev/null +++ b/utils/target_artifacts/compat.py @@ -0,0 +1,36 @@ +from __future__ import annotations + +import os +from pathlib import Path + +from .entry_helpers import text_entry +from .models import TargetArtifactError +from .orchestrator import write_artifact_entries + + +def stage_legacy_dependency( + target: str, + environment: str, + *, + repo_root: Path | None = None, + binaries_dir: Path | None = None, + process_env: dict[str, str] | None = None, +) -> bool: + if target != "agent": + return False + if environment != "dev": + raise TargetArtifactError(f"Don't know how to load version {environment} for {target}") + + env = dict(os.environ if process_env is None else process_env) + output_dir = Path(env.get("BINARIES_DIR", "binaries")) if binaries_dir is None else binaries_dir + if not output_dir.is_absolute(): + output_dir = (Path.cwd() if repo_root is None else repo_root) / output_dir + + branch = env.get("AGENT_TARGET_BRANCH", "master-py3") + write_artifact_entries( + output_dir, + target, + "dependency", + (text_entry("agent-image", f"datadog/agent-dev:{branch}"),), + ) + return True diff --git a/utils/target_artifacts/entry_helpers.py b/utils/target_artifacts/entry_helpers.py new file mode 100644 index 00000000000..e159504d4c6 --- /dev/null +++ b/utils/target_artifacts/entry_helpers.py @@ -0,0 +1,38 @@ +from __future__ import annotations + +import json + +from .models import ( + ArtifactEntry, + TargetArtifactError, +) + + +def text_entry(filename: str, content: str) -> ArtifactEntry: + return ArtifactEntry(filename=filename, content=f"{content.rstrip()}\n") + + +def json_entry(filename: str, payload: dict[str, object]) -> ArtifactEntry: + if not filename.endswith(".json"): + raise TargetArtifactError(f"JSON artifact entry '{filename}' must use a .json extension") + return ArtifactEntry(filename=filename, content=f"{json.dumps(payload, sort_keys=True)}\n") + + +def provider_fetch_entries( + *, + fetch_filename: str, + fetch_selector: str, + marker_filename: str, + bounded_selector: str, +) -> tuple[ArtifactEntry, ArtifactEntry]: + """Create a provider fetch entry plus its bounded selection marker. + + Some providers require installer-facing fetch selectors that are not + themselves bounded, such as branch-derived package image tags. The fetch + entry is consumed by the build to retrieve the provider artifact, while the + marker entry records the bounded selector used for cache identity. + """ + return ( + text_entry(fetch_filename, fetch_selector), + text_entry(marker_filename, bounded_selector), + ) diff --git a/utils/target_artifacts/env.py b/utils/target_artifacts/env.py new file mode 100644 index 00000000000..e8f9b0d4c42 --- /dev/null +++ b/utils/target_artifacts/env.py @@ -0,0 +1,46 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from pathlib import Path + + +QUOTED_VALUE_MIN_LENGTH = 2 + + +def _parse_dotenv_line(line: str) -> tuple[str, str] | None: + stripped = line.strip() + if not stripped or stripped.startswith("#") or "=" not in stripped: + return None + + key, value = stripped.split("=", 1) + key = key.strip() + if key.startswith("export "): + key = key.removeprefix("export ").strip() + if not key: + return None + + value = value.strip() + if len(value) >= QUOTED_VALUE_MIN_LENGTH and value[0] == value[-1] and value[0] in {"'", '"'}: + value = value[1:-1] + return key, value + + +def read_dotenv(path: Path) -> dict[str, str]: + if not path.exists(): + return {} + + result: dict[str, str] = {} + for line in path.read_text(encoding="utf-8").splitlines(): + item = _parse_dotenv_line(line) + if item is not None: + key, value = item + result[key] = value + return result + + +def load_environment(repo_root: Path, process_env: dict[str, str]) -> dict[str, str]: + result = read_dotenv(repo_root / ".env") + result.update(process_env) + return result diff --git a/utils/target_artifacts/models.py b/utils/target_artifacts/models.py new file mode 100644 index 00000000000..5805d323ad9 --- /dev/null +++ b/utils/target_artifacts/models.py @@ -0,0 +1,109 @@ +from dataclasses import dataclass +from typing import Protocol, runtime_checkable + + +class TargetArtifactError(Exception): + """Expected target artifact configuration or resolution failure.""" + + +@dataclass(frozen=True) +class ArtifactEntry: + filename: str + content: str + + +@dataclass(frozen=True) +class LiteralValue: + name: str + value: str + + +@dataclass(frozen=True) +class BranchReference: + name: str + repository: str + branch: str + sha: str + + +@dataclass(frozen=True) +class ReleaseAsset: + name: str + browser_download_url: str + + +@dataclass(frozen=True) +class GitHubReleaseReference: + name: str + repository: str + tag_name: str + assets: tuple[ReleaseAsset, ...] = () + + +@dataclass(frozen=True) +class GitHubActionsArtifactReference: + name: str + repository: str + workflow: str + branch: str + commit_sha: str + run_id: int + run_url: str + artifact_id: int + artifact_name: str + archive_download_url: str + + +@dataclass(frozen=True) +class OciImageReference: + name: str + image: str + digest: str + reference: str + + +@dataclass(frozen=True) +class ModuleVersion: + name: str + module: str + version: str + + +type ResolvedArtifactInput = ( + LiteralValue + | BranchReference + | GitHubReleaseReference + | GitHubActionsArtifactReference + | OciImageReference + | ModuleVersion +) + + +class ArtifactResolver(Protocol): + """Resolver implementations document their resolved model type in their class docstring.""" + + @property + def name(self) -> str: + """Resolved input name.""" + ... + + def resolve(self, env: dict[str, str], /) -> ResolvedArtifactInput: + """Resolve one declared artifact input.""" + ... + + +@runtime_checkable +class TargetArtifactEnvironment(Protocol): + def artifact_inputs( + self, + env: dict[str, str], + ) -> tuple[ArtifactResolver, ...]: + """Declare the inputs needed to produce artifact entries.""" + ... + + def artifact_entries( + self, + resolved_inputs: dict[str, ResolvedArtifactInput], + ) -> tuple[ArtifactEntry, ...]: + """Return text artifact entries from resolved inputs.""" + ... diff --git a/utils/target_artifacts/orchestrator.py b/utils/target_artifacts/orchestrator.py new file mode 100644 index 00000000000..45a7bb6f373 --- /dev/null +++ b/utils/target_artifacts/orchestrator.py @@ -0,0 +1,169 @@ +from __future__ import annotations + +import hashlib +import importlib.util +import json +import os +from pathlib import Path +from typing import TYPE_CHECKING, Any + +from .env import load_environment +from .models import ( + ArtifactEntry, + TargetArtifactEnvironment, + TargetArtifactError, +) + +if TYPE_CHECKING: + from types import ModuleType + +MANIFEST_FILENAME = ".target-artifacts-manifest.json" +MANIFEST_VERSION = 1 + + +def stage_target( + target: str, + environment: str, + *, + repo_root: Path | None = None, + binaries_dir: Path | None = None, + process_env: dict[str, str] | None = None, + load_dotenv: bool = True, +) -> None: + root = Path.cwd() if repo_root is None else repo_root + output_dir = Path(os.environ.get("BINARIES_DIR", "binaries")) if binaries_dir is None else binaries_dir + output_dir = output_dir if output_dir.is_absolute() else root / output_dir + + env = dict(os.environ if process_env is None else process_env) + if load_dotenv: + env = load_environment(root, env) + + if environment == "custom": + return + if environment not in {"dev", "prod"}: + raise TargetArtifactError(f"Unknown target artifact environment: {environment}") + + target_environment = load_target_environment(root, target, environment) + resolved_inputs = { + artifact_resolver.name: artifact_resolver.resolve(env) + for artifact_resolver in target_environment.artifact_inputs(env) + } + entries = target_environment.artifact_entries(resolved_inputs) + write_artifact_entries(output_dir, target, environment, entries) + + +def load_target_environment(repo_root: Path, target: str, environment: str) -> TargetArtifactEnvironment: + module_path = repo_root / "utils" / "build" / "docker" / target / "artifact.py" + if not module_path.exists(): + raise TargetArtifactError(f"No target artifact module found for '{target}' at {module_path}") + + module = _load_module(module_path, f"system_tests_target_artifacts_{target}") + class_name = "Dev" if environment == "dev" else "Prod" + environment_class = getattr(module, class_name, None) + if environment_class is None: + raise TargetArtifactError(f"Target artifact module for '{target}' does not define {class_name}") + + instance = environment_class() + if not isinstance(instance, TargetArtifactEnvironment): + raise TargetArtifactError(f"{target}.{class_name} does not implement TargetArtifactEnvironment") + return instance + + +def write_artifact_entries( + binaries_dir: Path, + target: str, + environment: str, + entries: tuple[ArtifactEntry, ...], +) -> None: + manifest = _read_manifest(binaries_dir) + manifest_entries = _manifest_entries(manifest) + new_entries = _dedupe_entries(entries) + owner = {"target": target, "environment": environment} + + for filename in new_entries: + _validate_filename(filename) + existing_owner = manifest_entries.get(filename, {}).get("owner") + path = binaries_dir / filename + if existing_owner is not None and not _same_target(existing_owner, target): + owner_target = ( + existing_owner.get("target", "") if isinstance(existing_owner, dict) else "" + ) + raise TargetArtifactError(f"Artifact entry '{filename}' is already owned by target '{owner_target}'") + if path.exists() and existing_owner is None: + raise TargetArtifactError(f"Refusing to overwrite unowned artifact entry '{filename}'") + + for filename, metadata in list(manifest_entries.items()): + owner_data = metadata.get("owner") + if _same_target(owner_data, target) and filename not in new_entries: + path = binaries_dir / filename + if path.exists(): + path.unlink() + del manifest_entries[filename] + + binaries_dir.mkdir(parents=True, exist_ok=True) + for filename, entry in new_entries.items(): + path = binaries_dir / filename + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(entry.content, encoding="utf-8") + manifest_entries[filename] = { + "owner": owner, + "sha256": hashlib.sha256(entry.content.encode("utf-8")).hexdigest(), + } + + manifest["version"] = MANIFEST_VERSION + manifest["entries"] = dict(sorted(manifest_entries.items())) + manifest_content = f"{json.dumps(manifest, indent=2, sort_keys=True)}\n" + (binaries_dir / MANIFEST_FILENAME).write_text(manifest_content, encoding="utf-8") + + +def _load_module(module_path: Path, module_name: str) -> ModuleType: + spec = importlib.util.spec_from_file_location(module_name, module_path) + if spec is None or spec.loader is None: + raise TargetArtifactError(f"Unable to import target artifact module at {module_path}") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _read_manifest(binaries_dir: Path) -> dict[str, Any]: + path = binaries_dir / MANIFEST_FILENAME + if not path.exists(): + return {"version": MANIFEST_VERSION, "entries": {}} + payload = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(payload, dict): + raise TargetArtifactError(f"Artifact manifest {path} is not an object") + if payload.get("version") != MANIFEST_VERSION: + raise TargetArtifactError(f"Unsupported artifact manifest version in {path}") + return payload + + +def _manifest_entries(manifest: dict[str, Any]) -> dict[str, dict[str, Any]]: + entries = manifest.get("entries") + if not isinstance(entries, dict): + raise TargetArtifactError("Artifact manifest entries must be an object") + return {str(name): _metadata(metadata) for name, metadata in entries.items()} + + +def _metadata(value: object) -> dict[str, Any]: + if not isinstance(value, dict): + raise TargetArtifactError("Artifact manifest entry metadata must be an object") + return value + + +def _dedupe_entries(entries: tuple[ArtifactEntry, ...]) -> dict[str, ArtifactEntry]: + result: dict[str, ArtifactEntry] = {} + for entry in entries: + if entry.filename in result: + raise TargetArtifactError(f"Duplicate artifact entry '{entry.filename}'") + result[entry.filename] = entry + return result + + +def _validate_filename(filename: str) -> None: + path = Path(filename) + if path.is_absolute() or ".." in path.parts or filename == MANIFEST_FILENAME: + raise TargetArtifactError(f"Invalid artifact entry filename '{filename}'") + + +def _same_target(owner: object, target: str) -> bool: + return isinstance(owner, dict) and owner.get("target") == target diff --git a/utils/target_artifacts/resolvers.py b/utils/target_artifacts/resolvers.py new file mode 100644 index 00000000000..06d321dab43 --- /dev/null +++ b/utils/target_artifacts/resolvers.py @@ -0,0 +1,378 @@ +from __future__ import annotations + +import json +import re +import subprocess +from dataclasses import dataclass +from typing import Any +from urllib.parse import quote + +import requests + +from .models import ( + BranchReference, + GitHubActionsArtifactReference, + GitHubReleaseReference, + LiteralValue, + ModuleVersion, + OciImageReference, + ReleaseAsset, + TargetArtifactError, +) + +REQUEST_TIMEOUT_SECONDS = 30 +FULL_SHA_PATTERN = re.compile(r"^[0-9a-f]{40}$") +CRATES_IO_HEADERS = { + "Accept": "application/json", + "User-Agent": "system-tests-target-artifacts (https://github.com/DataDog/system-tests)", +} + + +@dataclass(frozen=True) +class EnvResolver: + """Resolve an environment variable to LiteralValue.""" + + name: str + variable_name: str = "" + default_value: str = "" + + def resolve(self, env: dict[str, str]) -> LiteralValue: + value = env.get(self.variable_name, self.default_value) + return LiteralValue(name=self.name, value=value) + + +class _GitHubResolver: + @staticmethod + def _github_headers(env: dict[str, str]) -> dict[str, str]: + headers = {"Accept": "application/vnd.github.v3+json"} + token = env.get("GITHUB_TOKEN", "") + if token: + headers["Authorization"] = f"Bearer {token}" + return headers + + def _github_get(self, url: str, env: dict[str, str]) -> dict[str, Any]: + return _get_json(url, self._github_headers(env)) + + +@dataclass(frozen=True) +class GitHubBranchResolver(_GitHubResolver): + """Resolve a GitHub branch or commit SHA to BranchReference.""" + + name: str + repository: str + variable_name: str = "" + default_value: str = "" + + def resolve(self, env: dict[str, str]) -> BranchReference: + branch = env.get(self.variable_name, self.default_value) + if not branch: + raise TargetArtifactError(f"Missing branch for input '{self.name}'") + if FULL_SHA_PATTERN.match(branch): + return BranchReference( + name=self.name, + repository=self.repository, + branch=branch, + sha=branch, + ) + + payload = self._github_get( + f"https://api.github.com/repos/{self.repository}/branches/{quote(branch, safe='')}", + env, + ) + commit = _mapping(payload.get("commit"), f"branch '{branch}' commit") + sha = commit.get("sha") + if not isinstance(sha, str) or FULL_SHA_PATTERN.match(sha) is None: + raise TargetArtifactError(f"Branch '{branch}' in {self.repository} did not resolve to a commit SHA") + return BranchReference( + name=self.name, + repository=self.repository, + branch=branch, + sha=sha, + ) + + +@dataclass(frozen=True) +class GitHubLatestReleaseResolver(_GitHubResolver): + """Resolve the latest GitHub release to GitHubReleaseReference.""" + + name: str + repository: str + include_assets: bool = False + + def resolve(self, env: dict[str, str]) -> GitHubReleaseReference: + payload = self._github_get(f"https://api.github.com/repos/{self.repository}/releases/latest", env) + tag_name = payload.get("tag_name") + if not isinstance(tag_name, str) or not tag_name: + raise TargetArtifactError(f"Latest release for {self.repository} did not include a tag") + + assets: tuple[ReleaseAsset, ...] = () + if self.include_assets: + raw_assets = payload.get("assets") + if not isinstance(raw_assets, list): + raise TargetArtifactError(f"Latest release for {self.repository} did not include assets") + assets = tuple(_release_asset(asset) for asset in raw_assets) + + return GitHubReleaseReference( + name=self.name, + repository=self.repository, + tag_name=tag_name, + assets=assets, + ) + + +@dataclass(frozen=True) +class GitHubActionsArtifactResolver(_GitHubResolver): + """Resolve a GitHub Actions workflow artifact to GitHubActionsArtifactReference.""" + + name: str + repository: str + workflow: str + artifact_name: str + variable_name: str = "" + default_value: str = "" + ignore_failed_workflow: bool = True + + def resolve(self, env: dict[str, str]) -> GitHubActionsArtifactReference: + branch = env.get(self.variable_name, self.default_value) + if not branch: + raise TargetArtifactError(f"Missing workflow branch for input '{self.name}'") + + runs_payload = self._github_get( + "https://api.github.com/repos/" + f"{self.repository}/actions/workflows/{self.workflow}/runs" + f"?branch={quote(branch, safe='')}&status=completed&per_page=100", + env, + ) + runs = runs_payload.get("workflow_runs") + if not isinstance(runs, list): + raise TargetArtifactError(f"Workflow runs were not returned for {self.repository}") + + selected_run: dict[str, Any] | None = None + for run in runs: + run_mapping = _mapping(run, "workflow run") + if self.ignore_failed_workflow and run_mapping.get("conclusion") == "failure": + continue + selected_run = run_mapping + break + + if selected_run is None: + raise TargetArtifactError(f"No completed workflow run found for {self.repository}@{branch}") + + artifacts_url = selected_run.get("artifacts_url") + if not isinstance(artifacts_url, str): + raise TargetArtifactError("Selected workflow run did not include artifacts_url") + artifacts_payload = self._github_get(f"{artifacts_url}?per_page=100", env) + artifacts = artifacts_payload.get("artifacts") + if not isinstance(artifacts, list): + raise TargetArtifactError("Workflow artifacts were not returned") + + selected_artifact: dict[str, Any] | None = None + for artifact in artifacts: + artifact_mapping = _mapping(artifact, "workflow artifact") + artifact_name = artifact_mapping.get("name") + if isinstance(artifact_name, str) and self.artifact_name in artifact_name: + selected_artifact = artifact_mapping + break + + if selected_artifact is None: + raise TargetArtifactError(f"No artifact containing '{self.artifact_name}' found for {self.repository}") + + return GitHubActionsArtifactReference( + name=self.name, + repository=self.repository, + workflow=self.workflow, + branch=branch, + commit_sha=_required_str(selected_run, "head_sha"), + run_id=_required_int(selected_run, "id"), + run_url=_required_str(selected_run, "html_url"), + artifact_id=_required_int(selected_artifact, "id"), + artifact_name=_required_str(selected_artifact, "name"), + archive_download_url=_required_str(selected_artifact, "archive_download_url"), + ) + + +@dataclass(frozen=True) +class OciDigestResolver: + """Resolve an OCI image tag to OciImageReference.""" + + name: str + image: str = "" + variable_name: str = "" + default_value: str = "" + + def resolve(self, env: dict[str, str]) -> OciImageReference: + image = env.get(self.variable_name, self.image or self.default_value) + if not image: + raise TargetArtifactError(f"Missing OCI image for input '{self.name}'") + if "@sha256:" in image: + digest = image.rsplit("@", 1)[1] + return OciImageReference( + name=self.name, + image=image, + digest=digest, + reference=image, + ) + + try: + result = subprocess.run( + ["docker", "buildx", "imagetools", "inspect", image], + capture_output=True, + check=False, + text=True, + ) + except FileNotFoundError as exc: + raise TargetArtifactError("Unable to resolve OCI digest: docker was not found") from exc + if result.returncode != 0: + raise TargetArtifactError(f"Unable to resolve OCI digest for {image}: {result.stderr.strip()}") + + digest = "" + for line in result.stdout.splitlines(): + stripped = line.strip() + if stripped.startswith("Digest:"): + digest = stripped.removeprefix("Digest:").strip() + break + + if not digest.startswith("sha256:"): + raise TargetArtifactError(f"Unable to find OCI digest for {image}") + + last_slash = image.rfind("/") + last_colon = image.rfind(":") + repository = image[:last_colon] if last_colon > last_slash else image + return OciImageReference( + name=self.name, + image=image, + digest=digest, + reference=f"{repository}@{digest}", + ) + + +@dataclass(frozen=True) +class NpmLatestResolver: + """Resolve the latest npm package version to ModuleVersion.""" + + name: str + package: str + + def resolve(self, _env: dict[str, str]) -> ModuleVersion: + payload = _get_json(f"https://registry.npmjs.org/{quote(self.package, safe='@/')}/latest", {}) + version = payload.get("version") + if not isinstance(version, str) or not version: + raise TargetArtifactError(f"NPM package {self.package} did not include a version") + return ModuleVersion(name=self.name, module=self.package, version=version) + + +@dataclass(frozen=True) +class PypiLatestResolver: + """Resolve the latest PyPI package version to ModuleVersion.""" + + name: str + package: str + + def resolve(self, _env: dict[str, str]) -> ModuleVersion: + payload = _get_json(f"https://pypi.org/pypi/{quote(self.package, safe='')}/json", {}) + info = _mapping(payload.get("info"), f"PyPI package {self.package} info") + version = info.get("version") + if not isinstance(version, str) or not version: + raise TargetArtifactError(f"PyPI package {self.package} did not include a version") + return ModuleVersion(name=self.name, module=self.package, version=version) + + +@dataclass(frozen=True) +class RubygemsLatestResolver: + """Resolve the latest RubyGems package version to ModuleVersion.""" + + name: str + package: str + + def resolve(self, _env: dict[str, str]) -> ModuleVersion: + payload = _get_json(f"https://rubygems.org/api/v1/gems/{quote(self.package, safe='')}.json", {}) + version = payload.get("version") + if not isinstance(version, str) or not version: + raise TargetArtifactError(f"RubyGems package {self.package} did not include a version") + return ModuleVersion(name=self.name, module=self.package, version=version) + + +@dataclass(frozen=True) +class CratesLatestResolver: + """Resolve the latest crates.io package version to ModuleVersion.""" + + name: str + package: str + + def resolve(self, _env: dict[str, str]) -> ModuleVersion: + payload = _get_json(f"https://crates.io/api/v1/crates/{quote(self.package, safe='')}", CRATES_IO_HEADERS) + crate = _mapping(payload.get("crate"), f"crate {self.package}") + version = crate.get("max_stable_version") or crate.get("max_version") + if not isinstance(version, str) or not version: + raise TargetArtifactError(f"crate {self.package} did not include a version") + return ModuleVersion(name=self.name, module=self.package, version=version) + + +@dataclass(frozen=True) +class GoModuleLatestResolver: + """Resolve the latest Go module version to ModuleVersion.""" + + name: str + module: str + + def resolve(self, _env: dict[str, str]) -> ModuleVersion: + try: + result = subprocess.run( + ["go", "list", "-m", "-json", f"{self.module}@latest"], + capture_output=True, + check=False, + text=True, + ) + except FileNotFoundError as exc: + raise TargetArtifactError("Unable to resolve Go module: go was not found") from exc + if result.returncode != 0: + raise TargetArtifactError(f"Unable to resolve Go module {self.module}: {result.stderr.strip()}") + try: + payload = json.loads(result.stdout) + except json.JSONDecodeError as exc: + raise TargetArtifactError(f"Unable to parse Go module metadata for {self.module}") from exc + version = payload.get("Version") + if not isinstance(version, str) or not version: + raise TargetArtifactError(f"Go module {self.module} did not include a version") + return ModuleVersion(name=self.name, module=self.module, version=version) + + +def _get_json(url: str, headers: dict[str, str]) -> dict[str, Any]: + try: + response = requests.get(url, headers=dict(headers), timeout=REQUEST_TIMEOUT_SECONDS) + response.raise_for_status() + except requests.RequestException as exc: + raise TargetArtifactError(f"Unable to resolve artifact metadata from {url}: {exc}") from exc + try: + payload = response.json() + except ValueError as exc: + raise TargetArtifactError(f"Unable to parse artifact metadata from {url}") from exc + return _mapping(payload, f"response from {url}") + + +def _mapping(value: object, description: str) -> dict[str, Any]: + if not isinstance(value, dict): + raise TargetArtifactError(f"Expected {description} to be an object") + return value + + +def _release_asset(value: object) -> ReleaseAsset: + item = _mapping(value, "release asset") + return ReleaseAsset( + name=_required_str(item, "name"), + browser_download_url=_required_str(item, "browser_download_url"), + ) + + +def _required_str(value: dict[str, Any], key: str) -> str: + result = value.get(key) + if not isinstance(result, str) or not result: + raise TargetArtifactError(f"Expected '{key}' to be a non-empty string") + return result + + +def _required_int(value: dict[str, Any], key: str) -> int: + result = value.get(key) + if not isinstance(result, int): + raise TargetArtifactError(f"Expected '{key}' to be an integer") + return result From 1dd6742f269a893824937edbeaba65a314d03355 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Tue, 11 Aug 2026 16:41:15 +0200 Subject: [PATCH 06/12] cleanup --- .../internals/target-artifact-staging-spec.md | 158 ------------------ 1 file changed, 158 deletions(-) delete mode 100644 docs/internals/target-artifact-staging-spec.md diff --git a/docs/internals/target-artifact-staging-spec.md b/docs/internals/target-artifact-staging-spec.md deleted file mode 100644 index 8c1126f117e..00000000000 --- a/docs/internals/target-artifact-staging-spec.md +++ /dev/null @@ -1,158 +0,0 @@ -# Target Artifact Staging Spec - -## Problem Statement - -System-tests currently has two different mechanisms for choosing the target artifact to test. -Development artifact selection is mostly centralized in the legacy binary-loading script, while production artifact selection often happens dynamically inside Dockerfiles or installer scripts. This makes ownership unclear, makes non-library test targets fit awkwardly into library-oriented workflows, and makes Docker layer cache behavior hard to reason about when production releases change. - -The user wants each test target to own its target artifact selection logic for both development and production. Artifact staging should happen before the Docker build or test run consumes the selected artifact entries. The Docker build should not dynamically discover the latest production release for the target artifact, because a mutable selector such as `latest` can match every version ever released and does not provide a bounded contract for cache invalidation. - -## Solution - -Introduce a Python-based artifact staging mechanism where every test target defines two top-level environment classes, one for development and one for production. Both classes implement a common Protocol. Each environment class declares the artifact inputs it needs, then maps resolved artifact inputs to generated artifact entries. - -The orchestrator owns side effects: loading local environment configuration, resolving declared artifact inputs through shared resolvers, writing artifact entries, and maintaining the artifact manifest. Target environment classes remain side-effect-free and only return text artifact entries. - -Generated artifact entries must represent bounded artifact selectors. They may use version numbers, commit SHAs, release tags, package versions, or image digests. They must not use unbounded rolling selectors such as `latest`. When a provider-specific fetch selector cannot be bounded, the target must also emit a visible selection marker containing the bounded selector used for cache identity. - -The legacy binary-loading command remains available as a compatibility wrapper. The canonical behavior becomes target artifact staging, but existing local and CI invocations keep working transparently. - -## User Stories - -1. As a system-tests user, I want each test target to define its own target artifact selection, so that target-specific behavior is easy to find and review. -2. As a system-tests user, I want development and production target artifact selection to live together, so that both environments follow the same model. -3. As a system-tests user, I want production artifact selection to happen before Docker builds dynamically install a tracer, so that Docker layer cache invalidation is easier to reason about. -4. As a system-tests user, I want production target artifact entries to avoid `latest`, so that a staged artifact selection has a bounded meaning. -5. As a system-tests user, I want version selectors to remain valid when they match multiple architecture-specific payloads, so that runtime-specific installers can still choose the correct artifact. -6. As a system-tests user, I want development branch inputs to resolve to bounded selectors when possible, so that testing a branch still has a stable cache identity. -7. As a system-tests user, I want providers that require branch-based fetching to write an additional selection marker, so that cache invalidation still tracks the resolved commit. -8. As a system-tests user, I want selection markers to be documented clearly, so that I understand when they are required and why. -9. As a system-tests user, I want generated artifact entries to be text files, so that the staging contract stays simple and inspectable. -10. As a system-tests user, I want local payload overrides to remain supported outside generated artifact staging, so that I can still test a local jar, wheel, archive, native library, or checkout. -11. As a system-tests user, I want generated artifact staging to avoid overwriting my manual payload overrides, so that local testing artifacts are not destroyed. -12. As a system-tests user, I want generated artifact staging to avoid overwriting my manual marker files, so that explicit local selections are not silently replaced. -13. As a system-tests user, I want a clear error when a generated artifact entry conflicts with an unowned file, so that I know which local file to remove. -14. As a system-tests user, I want generated artifact entries from previous runs to be refreshed safely, so that stale generated files do not keep affecting builds. -15. As a system-tests user, I want staging one environment for a target to replace the previous environment for that same target, so that development and production selections cannot be active for the same target at the same time. -16. As a system-tests user, I want staging one test target to leave other staged targets alone, so that dependency artifacts and multi-target workflows can coexist. -17. As a system-tests user, I want artifact staging to track generated ownership in a manifest, so that the orchestrator can distinguish generated files from manual files. -18. As a system-tests user, I want the artifact manifest to avoid duplicating artifact entry contents, so that the actual staged files remain the source of truth. -19. As a system-tests user, I want structured artifact entries to use JSON, so that multi-field references are not encoded with fragile delimiters. -20. As a system-tests user, I want JSON artifact entries to use a JSON file extension, so that format expectations are obvious. -21. As a target maintainer, I want my target's artifact inputs to be explicitly declared, so that repo names, defaults, and external providers are not hidden in orchestration code. -22. As a target maintainer, I want shared resolvers for common external metadata lookups, so that target files do not duplicate GitHub, registry, or environment parsing logic. -23. As a target maintainer, I want target environment classes to receive resolved inputs, so that I can unit test target mapping without network, filesystem, or environment side effects. -24. As a target maintainer, I want the target module to own ecosystem-specific selection semantics, so that Java, Go, PHP, C, Lambda, and native-web-server targets can express their different needs. -25. As a target maintainer, I want authenticated artifact entries to contain only non-secret metadata, so that uploaded artifact bundles do not leak credentials. -26. As a CI maintainer, I want GitLab build jobs to run artifact staging directly, so that small workloads do not pay the startup cost of a separate staging job. -27. As a CI maintainer, I want GitLab parametric jobs to run artifact staging directly, so that parametric runs have target artifacts even when no weblog build job exists. -28. As a CI maintainer, I want GitLab custom runs with upstream artifact bundles to skip artifact staging, so that external artifacts remain the source of truth. -29. As a CI maintainer, I want the staging command to accept custom as a no-op, so that templates can call a single command shape safely. -30. As a CI maintainer, I want GitHub workflows to keep working transparently during migration, so that the refactor does not require immediate GitHub production-flow changes. -31. As a CI maintainer, I want the legacy command name to keep working, so that existing workflows and local habits do not break during migration. -32. As a system-tests maintainer, I want every test target to define real development and production staging behavior, so that there are no placeholder loaders. -33. As a system-tests maintainer, I want dependency artifacts such as the agent to stay outside the test target protocol, so that test targets and dependencies remain separate domain concepts. -34. As a system-tests maintainer, I want WAF rule set loading to remain outside the test target protocol, so that overlay artifacts do not blur target ownership. -35. As a system-tests maintainer, I want the target artifact context to stay target-level, so that one staged artifact bundle can be reused across many weblog variants. -36. As a system-tests maintainer, I want architecture and runtime-specific payload selection to remain in installers when needed, so that the artifact staging phase does not need weblog-specific facts. -37. As a system-tests maintainer, I want remote lookups to fetch as little data as possible, so that staging resolves metadata or bounded references without downloading payloads unnecessarily. -38. As a system-tests maintainer, I want GitHub Actions artifact staging to return metadata rather than payloads, so that Docker builds fetch the selected artifact only once. -39. As a system-tests maintainer, I want OCI production image selectors to resolve to digests, so that image-based target artifacts do not use mutable tags. -40. As a system-tests maintainer, I want the bounded-selector rule documented as a contract rather than enforced by expensive runtime checks, so that the system remains practical. - -## Implementation Decisions - -- The canonical operation is artifact staging, not binary loading. The legacy command remains as a compatibility entrypoint. -- Every test target must provide real development and production artifact staging behavior. Checked-in placeholder behavior is not acceptable. -- A test target is the component selected as the CI matrix library value. Dependencies and overlays are not test targets. -- Dependency artifacts, including the agent, remain compatibility-only behavior in the orchestrator and are outside the per-test-target Protocol. -- WAF rule set loading remains compatibility-only behavior in the orchestrator and is outside the per-test-target Protocol. -- Each test target owns a target artifact module in its existing target-specific Docker area. -- Shared protocol, models, resolvers, and orchestration live in a normal importable Python package outside the Docker build asset tree. -- The orchestrator imports each target artifact module dynamically by file location instead of turning every Docker target directory into a Python package. -- Each target artifact module exposes two top-level classes, `Dev` and `Prod`. -- `Dev` and `Prod` explicitly inherit from the shared Protocol type supplied by the typing module. -- `Dev` and `Prod` have no constructor arguments. Runtime data is passed through context and resolved artifact inputs. -- The Protocol exposes one method for declaring artifact inputs and one method for returning artifact entries. -- Artifact input declarations are explicit for both development and production. The orchestrator does not infer repository names, default branches, production release sources, or ecosystem semantics. -- Resolved artifact inputs are accessed as a mapping keyed by input name. -- Resolved artifact input values are typed frozen dataclasses, not plain strings. -- Target artifact functions are side-effect-free. They do not read environment variables, read or write files, call subprocesses, or perform network requests. -- The orchestrator owns side effects: environment loading, remote metadata resolution, artifact entry writing, and manifest maintenance. -- Generated artifact entries are text-only. Payload bytes and local checkouts remain manual payload overrides outside this loader protocol. -- Single-value artifact entries use plain text. -- Multi-field artifact entries use JSON and their filenames indicate the JSON format. -- Artifact entries must use bounded artifact selectors by contract. The shared implementation should document this rule but should not try to prove arbitrary content is valid. -- Mutable development inputs such as branch names should resolve to bounded selectors before artifact entries are generated when the provider supports that. -- When a provider requires an unbounded or provider-specific fetch selector, the loader must also emit a selection marker that contains the bounded selector used for artifact selection identity. -- Selection markers are visible, documented artifact entries. They are required when the installer-facing entry cannot itself be bounded. -- A shared helper creates provider-fetch entries with the required selection marker, reducing the chance that a target forgets it. -- Production OCI image references resolve to digests. -- GitHub latest release inputs return minimal release metadata by default. Asset metadata is included only when a target requests it. -- GitHub Actions artifact inputs resolve to stable artifact metadata, not downloaded payloads. -- Artifact entries that require authenticated downloads contain only non-secret metadata. Credentials are supplied separately by the build environment. -- The command loads local environment configuration by default using the Python dotenv package. Process environment variables override dotenv values. -- The `custom` environment is an orchestrator-only no-op. It is not represented in target modules. -- The artifact manifest is a single generated manifest for all staged targets in the staging directory. -- The manifest stores versioned ownership metadata and content hashes, not duplicate artifact entry contents. -- The manifest forbids two owners from owning the same artifact entry filename. -- Staging a target in one environment removes previously owned entries for other environments of the same target. -- Staging a target does not remove generated entries owned by other targets. -- Staging removes stale previously owned entries for the same target when the new run no longer emits them. -- Staging refuses to overwrite unowned existing files. There is no force mode in the first implementation. -- There is no clean subcommand in the first implementation. -- GitHub integration is kept transparent for now. Existing development artifact preparation continues through the compatibility command, and new production outputs are ignored until GitHub workflows choose to consume them. -- GitLab generated build jobs run artifact staging directly using the job's CI environment. -- GitLab generated parametric jobs run artifact staging directly using the job's CI environment. -- GitLab custom jobs with upstream artifact bundles skip artifact staging because the upstream bundle is the selected artifact source of truth. -- GitLab accepts the small risk that per-job production resolution could differ if a release changes mid-pipeline. This race is considered extremely unlikely and preferable to adding job startup overhead. - -## Testing Decisions - -- The main test seam is the artifact staging CLI/orchestrator. Tests should execute the staging flow at the command boundary with fake or stubbed resolvers and inspect the staged artifact entries plus manifest behavior. -- Target environment classes should be tested through their public Protocol methods by passing fake resolved inputs and asserting returned artifact entries. Tests should not inspect target implementation internals. -- Manifest behavior should be tested through observable filesystem effects: safe overwrite of owned files, refusal to overwrite unowned files, cleanup of stale owned entries, replacement of a target's previous environment, preservation of other targets, and owner conflict failures. -- GitLab integration should be tested through the existing pipeline rendering seam. Generated jobs should include artifact staging in build and parametric jobs, skip it for custom upstream artifact bundles, and avoid adding a separate staging job. -- Compatibility behavior should be tested through the legacy command entrypoint, showing that existing invocations continue to route to the new staging behavior. -- Custom environment behavior should be tested at the CLI/orchestrator seam as a successful no-op that does not load target modules or write manifest changes. -- Expected user/configuration failures should raise the shared domain exception and produce clear CLI errors. -- Tests should not perform real network calls. GitHub release metadata, GitHub Actions artifact metadata, branch-to-SHA resolution, and OCI digest resolution should be exercised through resolver fakes. -- Tests should not assert private helper call sequences when the same behavior can be verified through generated artifact entries, manifest contents, and rendered CI commands. -- Prior art exists in current tests for the legacy binary-loading command and GitLab pipeline rendering. The new tests should reuse those high-level seams where possible rather than spreading assertions across many low-level helpers. - -## Out of Scope - -- Publishing this spec to an issue tracker. -- Adding GitHub production artifact staging to workflows immediately. -- Adding runtime purity enforcement, monkeypatch-based side-effect tests, or AST guards. -- Adding a force option. -- Adding a clean subcommand. -- Turning dependency artifacts into test target loaders. -- Turning WAF rule set loading into a test target loader. -- Generating payload artifact entries. -- Removing support for manual payload overrides. -- Making the target artifact context weblog-specific, runtime-specific, or architecture-specific. -- Proving bounded-selector validity for arbitrary strings at runtime. -- Adding broad generic downloader abstractions before a target needs them. - -## Further Notes - -- All test targets currently have a matching target-specific Docker directory. Extra Docker directories such as shared support, dependency, and proxy directories are not test targets. -- The current C target is a real test target even though it is exercised through GitLab rather than GitHub. -- The agent is a dependency artifact, not a test target. -- The accepted GitLab consistency tradeoff is deliberate: per-job staging may theoretically resolve different production selectors if a release changes mid-pipeline, but the risk is very low and avoids disproportionate job startup cost. -- Documentation should make selection markers highly visible because missing them breaks the cache identity contract when the installer-facing selector is not bounded. -- Teams can ask questions about system-tests behavior in `#apm-shared-testing`. - -## Maintainer Checklist - -When adding or changing a target's artifact staging behavior: - -1. Add or update `utils/build/docker//artifact.py`. -2. Define top-level `Dev` and `Prod` classes that implement `TargetArtifactEnvironment`. -3. Keep both classes side-effect-free: declare `ArtifactInput` values in `artifact_inputs`, and turn resolved values into text or JSON `ArtifactEntry` values in `artifact_entries`. -4. Put provider lookups in shared resolvers instead of target modules. -5. Emit bounded artifact selectors whenever possible. If an installer-facing entry must use a provider-specific fetch selector, emit a selection marker with `provider_fetch_entries`. -6. Keep local payload override handling in the installer script. Staging should write selectors and metadata, not jar, wheel, zip, tarball, or checkout payloads. -7. Use JSON entries for multi-field references, and give those files a `.json` extension. -8. Add or update `TEST_THE_TEST` coverage that exercises the target through the public Protocol methods with fake resolved inputs. From 58706f6701fcd0d0ca4bb0f4460b45d26bcbeb22 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Thu, 27 Aug 2026 11:57:23 +0200 Subject: [PATCH 07/12] Simplifications --- tests/test_the_test/test_load_binary.py | 4 +- tests/test_the_test/test_target_artifacts.py | 30 ------------- utils/build/docker/agent/artifact.py | 21 +++++++++ utils/build/docker/cpp_httpd/artifact.py | 19 +------- utils/build/docker/cpp_nginx/artifact.py | 16 +------ utils/build/docker/nodejs_lambda/artifact.py | 19 +------- utils/build/docker/python_lambda/artifact.py | 19 +------- utils/scripts/load-binary.sh | 3 +- utils/target_artifacts/cli.py | 13 ------ utils/target_artifacts/compat.py | 36 --------------- utils/target_artifacts/entry_helpers.py | 18 ++++++++ utils/target_artifacts/env.py | 46 -------------------- utils/target_artifacts/orchestrator.py | 4 -- 13 files changed, 50 insertions(+), 198 deletions(-) create mode 100644 utils/build/docker/agent/artifact.py delete mode 100644 utils/target_artifacts/compat.py delete mode 100644 utils/target_artifacts/env.py diff --git a/tests/test_the_test/test_load_binary.py b/tests/test_the_test/test_load_binary.py index 7b5e26927e6..793f2a61aa3 100644 --- a/tests/test_the_test/test_load_binary.py +++ b/tests/test_the_test/test_load_binary.py @@ -179,7 +179,7 @@ def test_missing_package_fails_with_clear_error(self, tmp_path: Path) -> None: assert result.returncode != 0 assert "Unable to resolve OCI digest" in result.stderr - def test_agent_dependency_uses_explicit_compatibility_path(self, tmp_path: Path) -> None: + def test_agent_dependency_uses_target_artifact_staging(self, tmp_path: Path) -> None: result = _run_loader( tmp_path, "dev", @@ -193,7 +193,7 @@ def test_agent_dependency_uses_explicit_compatibility_path(self, tmp_path: Path) manifest = json.loads((binaries_dir / MANIFEST_FILENAME).read_text(encoding="utf-8")) assert manifest["entries"]["agent-image"]["owner"] == { "target": "agent", - "environment": "dependency", + "environment": "dev", } def test_waf_rule_set_overlay_stays_outside_target_manifest(self, tmp_path: Path) -> None: diff --git a/tests/test_the_test/test_target_artifacts.py b/tests/test_the_test/test_target_artifacts.py index 367b77d714e..e39f987653f 100644 --- a/tests/test_the_test/test_target_artifacts.py +++ b/tests/test_the_test/test_target_artifacts.py @@ -175,36 +175,6 @@ def test_custom_environment_is_noop(self, tmp_path: Path) -> None: assert not binaries_dir.exists() - def test_dotenv_values_are_loaded_and_process_environment_wins(self, tmp_path: Path) -> None: - _write_target_module( - tmp_path, - """ -from utils.target_artifacts.entry_helpers import text_entry -from utils.target_artifacts.resolvers import EnvResolver - -class Dev: - def artifact_inputs(self, env): - return (EnvResolver(name="value", variable_name="STAGED_VALUE", default_value="default"),) - - def artifact_entries(self, resolved_inputs): - return (text_entry("value", resolved_inputs["value"].value),) - -class Prod(Dev): - pass -""", - ) - (tmp_path / ".env").write_text("STAGED_VALUE=dotenv\n", encoding="utf-8") - - stage_target( - "fake", - "dev", - repo_root=tmp_path, - binaries_dir=tmp_path / "binaries", - process_env={"STAGED_VALUE": "process"}, - ) - - assert (tmp_path / "binaries" / "value").read_text(encoding="utf-8") == "process\n" - def test_manifest_refreshes_owned_files_and_preserves_other_targets(self, tmp_path: Path) -> None: module_path = tmp_path / "utils" / "build" / "docker" / "fake" module_path.mkdir(parents=True) diff --git a/utils/build/docker/agent/artifact.py b/utils/build/docker/agent/artifact.py new file mode 100644 index 00000000000..fe6e5fee4fb --- /dev/null +++ b/utils/build/docker/agent/artifact.py @@ -0,0 +1,21 @@ +from __future__ import annotations + + +from utils.target_artifacts.entry_helpers import text_entry +from utils.target_artifacts.models import ArtifactEntry, LiteralValue +from utils.target_artifacts.resolvers import EnvResolver + + +class Dev: + def artifact_inputs(self, env: dict[str, str]) -> tuple[EnvResolver]: + return (EnvResolver(name="agent_branch", variable_name="AGENT_TARGET_BRANCH", default_value="master-py3"),) + + def artifact_entries( + self, + resolved_inputs: dict[str, LiteralValue], + ) -> tuple[ArtifactEntry]: + return (text_entry("agent-image", f"datadog/agent-dev:{resolved_inputs['agent_branch'].value}"),) + + +class Prod(Dev): + pass diff --git a/utils/build/docker/cpp_httpd/artifact.py b/utils/build/docker/cpp_httpd/artifact.py index e8b69d7eb21..9da0e13bbdc 100644 --- a/utils/build/docker/cpp_httpd/artifact.py +++ b/utils/build/docker/cpp_httpd/artifact.py @@ -1,7 +1,7 @@ from __future__ import annotations -from utils.target_artifacts.entry_helpers import json_entry, text_entry +from utils.target_artifacts.entry_helpers import gha_artifact_entry, text_entry from utils.target_artifacts.models import ( ArtifactEntry, GitHubActionsArtifactReference, @@ -27,22 +27,7 @@ def artifact_entries( self, resolved_inputs: dict[str, GitHubActionsArtifactReference], ) -> tuple[ArtifactEntry]: - artifact = resolved_inputs["workflow_artifact"] - return ( - json_entry( - "cpp-httpd-github-actions-artifact.json", - { - "archive_download_url": artifact.archive_download_url, - "artifact_id": artifact.artifact_id, - "artifact_name": artifact.artifact_name, - "commit_sha": artifact.commit_sha, - "repository": artifact.repository, - "run_id": artifact.run_id, - "run_url": artifact.run_url, - "workflow": artifact.workflow, - }, - ), - ) + return (gha_artifact_entry("cpp-httpd-github-actions-artifact.json", resolved_inputs["workflow_artifact"]),) class Prod: diff --git a/utils/build/docker/cpp_nginx/artifact.py b/utils/build/docker/cpp_nginx/artifact.py index b0dcdef45c5..7335a2cabac 100644 --- a/utils/build/docker/cpp_nginx/artifact.py +++ b/utils/build/docker/cpp_nginx/artifact.py @@ -2,7 +2,7 @@ from typing import cast -from utils.target_artifacts.entry_helpers import json_entry, text_entry +from utils.target_artifacts.entry_helpers import gha_artifact_entry, text_entry from utils.target_artifacts.models import ( ArtifactEntry, GitHubActionsArtifactReference, @@ -38,19 +38,7 @@ def artifact_entries( artifact = cast(GitHubActionsArtifactReference, resolved_inputs["workflow_artifact"]) ddprof_release = cast(GitHubReleaseReference, resolved_inputs["ddprof_release"]) return ( - json_entry( - "cpp-nginx-github-actions-artifact.json", - { - "archive_download_url": artifact.archive_download_url, - "artifact_id": artifact.artifact_id, - "artifact_name": artifact.artifact_name, - "commit_sha": artifact.commit_sha, - "repository": artifact.repository, - "run_id": artifact.run_id, - "run_url": artifact.run_url, - "workflow": artifact.workflow, - }, - ), + gha_artifact_entry("cpp-nginx-github-actions-artifact.json", artifact), text_entry("cpp-nginx-ddprof-load-from-release", ddprof_release.tag_name), ) diff --git a/utils/build/docker/nodejs_lambda/artifact.py b/utils/build/docker/nodejs_lambda/artifact.py index 58c431d5d1d..87746235fbc 100644 --- a/utils/build/docker/nodejs_lambda/artifact.py +++ b/utils/build/docker/nodejs_lambda/artifact.py @@ -1,7 +1,7 @@ from __future__ import annotations -from utils.target_artifacts.entry_helpers import json_entry, text_entry +from utils.target_artifacts.entry_helpers import gha_artifact_entry, text_entry from utils.target_artifacts.models import ( ArtifactEntry, GitHubActionsArtifactReference, @@ -28,22 +28,7 @@ def artifact_entries( self, resolved_inputs: dict[str, GitHubActionsArtifactReference], ) -> tuple[ArtifactEntry]: - artifact = resolved_inputs["workflow_artifact"] - return ( - json_entry( - "nodejs-lambda-github-actions-artifact.json", - { - "archive_download_url": artifact.archive_download_url, - "artifact_id": artifact.artifact_id, - "artifact_name": artifact.artifact_name, - "commit_sha": artifact.commit_sha, - "repository": artifact.repository, - "run_id": artifact.run_id, - "run_url": artifact.run_url, - "workflow": artifact.workflow, - }, - ), - ) + return (gha_artifact_entry("nodejs-lambda-github-actions-artifact.json", resolved_inputs["workflow_artifact"]),) class Prod: diff --git a/utils/build/docker/python_lambda/artifact.py b/utils/build/docker/python_lambda/artifact.py index c640906497c..d65785296c7 100644 --- a/utils/build/docker/python_lambda/artifact.py +++ b/utils/build/docker/python_lambda/artifact.py @@ -1,7 +1,7 @@ from __future__ import annotations -from utils.target_artifacts.entry_helpers import json_entry, text_entry +from utils.target_artifacts.entry_helpers import gha_artifact_entry, text_entry from utils.target_artifacts.models import ( ArtifactEntry, GitHubActionsArtifactReference, @@ -28,22 +28,7 @@ def artifact_entries( self, resolved_inputs: dict[str, GitHubActionsArtifactReference], ) -> tuple[ArtifactEntry]: - artifact = resolved_inputs["workflow_artifact"] - return ( - json_entry( - "python-lambda-github-actions-artifact.json", - { - "archive_download_url": artifact.archive_download_url, - "artifact_id": artifact.artifact_id, - "artifact_name": artifact.artifact_name, - "commit_sha": artifact.commit_sha, - "repository": artifact.repository, - "run_id": artifact.run_id, - "run_url": artifact.run_url, - "workflow": artifact.workflow, - }, - ), - ) + return (gha_artifact_entry("python-lambda-github-actions-artifact.json", resolved_inputs["workflow_artifact"]),) class Prod: diff --git a/utils/scripts/load-binary.sh b/utils/scripts/load-binary.sh index 6140af107f7..4439ac0b3c4 100755 --- a/utils/scripts/load-binary.sh +++ b/utils/scripts/load-binary.sh @@ -63,7 +63,6 @@ case "$TARGET" in python3 utils/scripts/stage-target-artifacts.py \ "$TARGET" "$VERSION" \ --binaries-dir "$BINARIES_DIR" \ - --repo-root . \ - --compatibility + --repo-root . ;; esac diff --git a/utils/target_artifacts/cli.py b/utils/target_artifacts/cli.py index ad7e4e765e7..eeb160d09a4 100644 --- a/utils/target_artifacts/cli.py +++ b/utils/target_artifacts/cli.py @@ -5,7 +5,6 @@ import sys from pathlib import Path -from .compat import stage_legacy_dependency from .models import TargetArtifactError from .orchestrator import stage_target @@ -16,24 +15,12 @@ def main(argv: list[str] | None = None) -> int: parser.add_argument("environment", nargs="?", default="dev", help="dev, prod, or custom") parser.add_argument("--binaries-dir", default=os.environ.get("BINARIES_DIR", "binaries")) parser.add_argument("--repo-root", default=".") - parser.add_argument( - "--compatibility", - action="store_true", - help="Route legacy dependency or overlay targets through explicit compatibility handling", - ) args = parser.parse_args(argv) repo_root = Path(args.repo_root) binaries_dir = Path(args.binaries_dir) try: - if args.compatibility and stage_legacy_dependency( - args.target, - args.environment, - repo_root=repo_root, - binaries_dir=binaries_dir, - ): - return 0 stage_target(args.target, args.environment, repo_root=repo_root, binaries_dir=binaries_dir) except TargetArtifactError as exc: sys.stderr.write(f"{exc}\n") diff --git a/utils/target_artifacts/compat.py b/utils/target_artifacts/compat.py deleted file mode 100644 index 9388f3e4ea4..00000000000 --- a/utils/target_artifacts/compat.py +++ /dev/null @@ -1,36 +0,0 @@ -from __future__ import annotations - -import os -from pathlib import Path - -from .entry_helpers import text_entry -from .models import TargetArtifactError -from .orchestrator import write_artifact_entries - - -def stage_legacy_dependency( - target: str, - environment: str, - *, - repo_root: Path | None = None, - binaries_dir: Path | None = None, - process_env: dict[str, str] | None = None, -) -> bool: - if target != "agent": - return False - if environment != "dev": - raise TargetArtifactError(f"Don't know how to load version {environment} for {target}") - - env = dict(os.environ if process_env is None else process_env) - output_dir = Path(env.get("BINARIES_DIR", "binaries")) if binaries_dir is None else binaries_dir - if not output_dir.is_absolute(): - output_dir = (Path.cwd() if repo_root is None else repo_root) / output_dir - - branch = env.get("AGENT_TARGET_BRANCH", "master-py3") - write_artifact_entries( - output_dir, - target, - "dependency", - (text_entry("agent-image", f"datadog/agent-dev:{branch}"),), - ) - return True diff --git a/utils/target_artifacts/entry_helpers.py b/utils/target_artifacts/entry_helpers.py index e159504d4c6..55fc6048668 100644 --- a/utils/target_artifacts/entry_helpers.py +++ b/utils/target_artifacts/entry_helpers.py @@ -4,6 +4,7 @@ from .models import ( ArtifactEntry, + GitHubActionsArtifactReference, TargetArtifactError, ) @@ -18,6 +19,23 @@ def json_entry(filename: str, payload: dict[str, object]) -> ArtifactEntry: return ArtifactEntry(filename=filename, content=f"{json.dumps(payload, sort_keys=True)}\n") +def gha_artifact_entry(filename: str, artifact: GitHubActionsArtifactReference) -> ArtifactEntry: + """Create a JSON artifact entry from a GitHub Actions artifact reference.""" + return json_entry( + filename, + { + "archive_download_url": artifact.archive_download_url, + "artifact_id": artifact.artifact_id, + "artifact_name": artifact.artifact_name, + "commit_sha": artifact.commit_sha, + "repository": artifact.repository, + "run_id": artifact.run_id, + "run_url": artifact.run_url, + "workflow": artifact.workflow, + }, + ) + + def provider_fetch_entries( *, fetch_filename: str, diff --git a/utils/target_artifacts/env.py b/utils/target_artifacts/env.py deleted file mode 100644 index e8f9b0d4c42..00000000000 --- a/utils/target_artifacts/env.py +++ /dev/null @@ -1,46 +0,0 @@ -from __future__ import annotations - -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from pathlib import Path - - -QUOTED_VALUE_MIN_LENGTH = 2 - - -def _parse_dotenv_line(line: str) -> tuple[str, str] | None: - stripped = line.strip() - if not stripped or stripped.startswith("#") or "=" not in stripped: - return None - - key, value = stripped.split("=", 1) - key = key.strip() - if key.startswith("export "): - key = key.removeprefix("export ").strip() - if not key: - return None - - value = value.strip() - if len(value) >= QUOTED_VALUE_MIN_LENGTH and value[0] == value[-1] and value[0] in {"'", '"'}: - value = value[1:-1] - return key, value - - -def read_dotenv(path: Path) -> dict[str, str]: - if not path.exists(): - return {} - - result: dict[str, str] = {} - for line in path.read_text(encoding="utf-8").splitlines(): - item = _parse_dotenv_line(line) - if item is not None: - key, value = item - result[key] = value - return result - - -def load_environment(repo_root: Path, process_env: dict[str, str]) -> dict[str, str]: - result = read_dotenv(repo_root / ".env") - result.update(process_env) - return result diff --git a/utils/target_artifacts/orchestrator.py b/utils/target_artifacts/orchestrator.py index 45a7bb6f373..fb157dca9b6 100644 --- a/utils/target_artifacts/orchestrator.py +++ b/utils/target_artifacts/orchestrator.py @@ -7,7 +7,6 @@ from pathlib import Path from typing import TYPE_CHECKING, Any -from .env import load_environment from .models import ( ArtifactEntry, TargetArtifactEnvironment, @@ -28,15 +27,12 @@ def stage_target( repo_root: Path | None = None, binaries_dir: Path | None = None, process_env: dict[str, str] | None = None, - load_dotenv: bool = True, ) -> None: root = Path.cwd() if repo_root is None else repo_root output_dir = Path(os.environ.get("BINARIES_DIR", "binaries")) if binaries_dir is None else binaries_dir output_dir = output_dir if output_dir.is_absolute() else root / output_dir env = dict(os.environ if process_env is None else process_env) - if load_dotenv: - env = load_environment(root, env) if environment == "custom": return From e65a73b8f90021f72e75b043c9c79e1be98b3382 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Thu, 27 Aug 2026 12:28:00 +0200 Subject: [PATCH 08/12] Simplification for simple targets --- utils/build/docker/agent/artifact.py | 14 ++-- utils/build/docker/cpp/artifact.py | 44 ++++-------- utils/build/docker/java/artifact.py | 44 ++++-------- utils/build/docker/java_lambda/artifact.py | 44 ++++-------- utils/build/docker/java_otel/artifact.py | 28 ++------ utils/build/docker/nodejs/artifact.py | 46 ++++-------- utils/build/docker/nodejs_otel/artifact.py | 30 ++------ utils/build/docker/otel_collector/artifact.py | 41 ++++------- utils/build/docker/python/artifact.py | 45 ++++-------- utils/build/docker/python_otel/artifact.py | 30 ++------ utils/build/docker/ruby/artifact.py | 72 +++++++------------ utils/build/docker/ruby_lambda/artifact.py | 49 ++++--------- utils/build/docker/rust/artifact.py | 59 ++++++--------- utils/target_artifacts/__init__.py | 2 + utils/target_artifacts/models.py | 22 ++++++ 15 files changed, 191 insertions(+), 379 deletions(-) diff --git a/utils/build/docker/agent/artifact.py b/utils/build/docker/agent/artifact.py index fe6e5fee4fb..12c83a2ac8e 100644 --- a/utils/build/docker/agent/artifact.py +++ b/utils/build/docker/agent/artifact.py @@ -2,19 +2,13 @@ from utils.target_artifacts.entry_helpers import text_entry -from utils.target_artifacts.models import ArtifactEntry, LiteralValue +from utils.target_artifacts.models import SimpleTarget from utils.target_artifacts.resolvers import EnvResolver -class Dev: - def artifact_inputs(self, env: dict[str, str]) -> tuple[EnvResolver]: - return (EnvResolver(name="agent_branch", variable_name="AGENT_TARGET_BRANCH", default_value="master-py3"),) - - def artifact_entries( - self, - resolved_inputs: dict[str, LiteralValue], - ) -> tuple[ArtifactEntry]: - return (text_entry("agent-image", f"datadog/agent-dev:{resolved_inputs['agent_branch'].value}"),) +class Dev(SimpleTarget): + inputs = (EnvResolver(name="agent_branch", variable_name="AGENT_TARGET_BRANCH", default_value="master-py3"),) + entries = (text_entry("agent-image", "datadog/agent-dev:{agent_branch.value}"),) class Prod(Dev): diff --git a/utils/build/docker/cpp/artifact.py b/utils/build/docker/cpp/artifact.py index e008184b4b8..2b830b713c3 100644 --- a/utils/build/docker/cpp/artifact.py +++ b/utils/build/docker/cpp/artifact.py @@ -2,41 +2,25 @@ from utils.target_artifacts.entry_helpers import text_entry -from utils.target_artifacts.models import ( - ArtifactEntry, - BranchReference, - GitHubReleaseReference, -) +from utils.target_artifacts.models import SimpleTarget from utils.target_artifacts.resolvers import GitHubBranchResolver, GitHubLatestReleaseResolver REPOSITORY = "DataDog/dd-trace-cpp" GIT_URL = "https://github.com/DataDog/dd-trace-cpp" -class Dev: - def artifact_inputs(self, env: dict[str, str]) -> tuple[GitHubBranchResolver]: - return ( - GitHubBranchResolver( - name="library_branch", - repository=REPOSITORY, - variable_name="LIBRARY_TARGET_BRANCH", - default_value="main", - ), - ) +class Dev(SimpleTarget): + inputs = ( + GitHubBranchResolver( + name="library_branch", + repository=REPOSITORY, + variable_name="LIBRARY_TARGET_BRANCH", + default_value="main", + ), + ) + entries = (text_entry("cpp-load-from-git", f"{GIT_URL}@{{library_branch.sha}}"),) - def artifact_entries( - self, - resolved_inputs: dict[str, BranchReference], - ) -> tuple[ArtifactEntry]: - return (text_entry("cpp-load-from-git", f"{GIT_URL}@{resolved_inputs['library_branch'].sha}"),) - -class Prod: - def artifact_inputs(self, env: dict[str, str]) -> tuple[GitHubLatestReleaseResolver]: - return (GitHubLatestReleaseResolver(name="release", repository=REPOSITORY),) - - def artifact_entries( - self, - resolved_inputs: dict[str, GitHubReleaseReference], - ) -> tuple[ArtifactEntry]: - return (text_entry("cpp-load-from-git", f"{GIT_URL}@{resolved_inputs['release'].tag_name}"),) +class Prod(SimpleTarget): + inputs = (GitHubLatestReleaseResolver(name="release", repository=REPOSITORY),) + entries = (text_entry("cpp-load-from-git", f"{GIT_URL}@{{release.tag_name}}"),) diff --git a/utils/build/docker/java/artifact.py b/utils/build/docker/java/artifact.py index 395a3809101..b9297c9f19f 100644 --- a/utils/build/docker/java/artifact.py +++ b/utils/build/docker/java/artifact.py @@ -2,38 +2,22 @@ from utils.target_artifacts.entry_helpers import text_entry -from utils.target_artifacts.models import ( - ArtifactEntry, - BranchReference, - GitHubReleaseReference, -) +from utils.target_artifacts.models import ArtifactEntry, SimpleTarget from utils.target_artifacts.resolvers import GitHubBranchResolver, GitHubLatestReleaseResolver -class Dev: - def artifact_inputs(self, env: dict[str, str]) -> tuple[GitHubBranchResolver]: - return ( - GitHubBranchResolver( - name="library_branch", - repository="DataDog/dd-trace-java", - variable_name="LIBRARY_TARGET_BRANCH", - default_value="master", - ), - ) +class Dev(SimpleTarget): + inputs = ( + GitHubBranchResolver( + name="library_branch", + repository="DataDog/dd-trace-java", + variable_name="LIBRARY_TARGET_BRANCH", + default_value="master", + ), + ) + entries = (text_entry("java-load-from-s3", "{library_branch.sha}"),) - def artifact_entries( - self, - resolved_inputs: dict[str, BranchReference], - ) -> tuple[ArtifactEntry]: - return (text_entry("java-load-from-s3", resolved_inputs["library_branch"].sha),) - -class Prod: - def artifact_inputs(self, env: dict[str, str]) -> tuple[GitHubLatestReleaseResolver]: - return (GitHubLatestReleaseResolver(name="release", repository="DataDog/dd-trace-java"),) - - def artifact_entries( - self, - resolved_inputs: dict[str, GitHubReleaseReference], - ) -> tuple[ArtifactEntry]: - return (text_entry("java-load-from-release", resolved_inputs["release"].tag_name),) +class Prod(SimpleTarget): + inputs = (GitHubLatestReleaseResolver(name="release", repository="DataDog/dd-trace-java"),) + entries = (text_entry("java-load-from-release", "{release.tag_name}"),) diff --git a/utils/build/docker/java_lambda/artifact.py b/utils/build/docker/java_lambda/artifact.py index 395a3809101..b9297c9f19f 100644 --- a/utils/build/docker/java_lambda/artifact.py +++ b/utils/build/docker/java_lambda/artifact.py @@ -2,38 +2,22 @@ from utils.target_artifacts.entry_helpers import text_entry -from utils.target_artifacts.models import ( - ArtifactEntry, - BranchReference, - GitHubReleaseReference, -) +from utils.target_artifacts.models import ArtifactEntry, SimpleTarget from utils.target_artifacts.resolvers import GitHubBranchResolver, GitHubLatestReleaseResolver -class Dev: - def artifact_inputs(self, env: dict[str, str]) -> tuple[GitHubBranchResolver]: - return ( - GitHubBranchResolver( - name="library_branch", - repository="DataDog/dd-trace-java", - variable_name="LIBRARY_TARGET_BRANCH", - default_value="master", - ), - ) +class Dev(SimpleTarget): + inputs = ( + GitHubBranchResolver( + name="library_branch", + repository="DataDog/dd-trace-java", + variable_name="LIBRARY_TARGET_BRANCH", + default_value="master", + ), + ) + entries = (text_entry("java-load-from-s3", "{library_branch.sha}"),) - def artifact_entries( - self, - resolved_inputs: dict[str, BranchReference], - ) -> tuple[ArtifactEntry]: - return (text_entry("java-load-from-s3", resolved_inputs["library_branch"].sha),) - -class Prod: - def artifact_inputs(self, env: dict[str, str]) -> tuple[GitHubLatestReleaseResolver]: - return (GitHubLatestReleaseResolver(name="release", repository="DataDog/dd-trace-java"),) - - def artifact_entries( - self, - resolved_inputs: dict[str, GitHubReleaseReference], - ) -> tuple[ArtifactEntry]: - return (text_entry("java-load-from-release", resolved_inputs["release"].tag_name),) +class Prod(SimpleTarget): + inputs = (GitHubLatestReleaseResolver(name="release", repository="DataDog/dd-trace-java"),) + entries = (text_entry("java-load-from-release", "{release.tag_name}"),) diff --git a/utils/build/docker/java_otel/artifact.py b/utils/build/docker/java_otel/artifact.py index de80115f189..c26eba19f2f 100644 --- a/utils/build/docker/java_otel/artifact.py +++ b/utils/build/docker/java_otel/artifact.py @@ -2,32 +2,16 @@ from utils.target_artifacts.entry_helpers import text_entry -from utils.target_artifacts.models import ( - ArtifactEntry, - GitHubReleaseReference, -) +from utils.target_artifacts.models import SimpleTarget from utils.target_artifacts.resolvers import GitHubLatestReleaseResolver REPOSITORY = "open-telemetry/opentelemetry-java-instrumentation" -class Dev: - def artifact_inputs(self, env: dict[str, str]) -> tuple[GitHubLatestReleaseResolver]: - return (GitHubLatestReleaseResolver(name="release", repository=REPOSITORY),) +class Dev(SimpleTarget): + inputs = (GitHubLatestReleaseResolver(name="release", repository=REPOSITORY),) + entries = (text_entry("java-otel-load-from-release", "{release.tag_name}"),) - def artifact_entries( - self, - resolved_inputs: dict[str, GitHubReleaseReference], - ) -> tuple[ArtifactEntry]: - return (text_entry("java-otel-load-from-release", resolved_inputs["release"].tag_name),) - -class Prod: - def artifact_inputs(self, env: dict[str, str]) -> tuple[GitHubLatestReleaseResolver]: - return (GitHubLatestReleaseResolver(name="release", repository=REPOSITORY),) - - def artifact_entries( - self, - resolved_inputs: dict[str, GitHubReleaseReference], - ) -> tuple[ArtifactEntry]: - return (text_entry("java-otel-load-from-release", resolved_inputs["release"].tag_name),) +class Prod(Dev): + pass diff --git a/utils/build/docker/nodejs/artifact.py b/utils/build/docker/nodejs/artifact.py index d40e0fdd3a8..e54cf2aded6 100644 --- a/utils/build/docker/nodejs/artifact.py +++ b/utils/build/docker/nodejs/artifact.py @@ -2,40 +2,22 @@ from utils.target_artifacts.entry_helpers import text_entry -from utils.target_artifacts.models import ( - ArtifactEntry, - BranchReference, - ModuleVersion, -) +from utils.target_artifacts.models import SimpleTarget from utils.target_artifacts.resolvers import GitHubBranchResolver, NpmLatestResolver -class Dev: - def artifact_inputs(self, env: dict[str, str]) -> tuple[GitHubBranchResolver]: - return ( - GitHubBranchResolver( - name="library_branch", - repository="DataDog/dd-trace-js", - variable_name="LIBRARY_TARGET_BRANCH", - default_value="master", - ), - ) +class Dev(SimpleTarget): + inputs = ( + GitHubBranchResolver( + name="library_branch", + repository="DataDog/dd-trace-js", + variable_name="LIBRARY_TARGET_BRANCH", + default_value="master", + ), + ) + entries = (text_entry("nodejs-load-from-npm", "DataDog/dd-trace-js#{library_branch.sha}"),) - def artifact_entries( - self, - resolved_inputs: dict[str, BranchReference], - ) -> tuple[ArtifactEntry]: - sha = resolved_inputs["library_branch"].sha - return (text_entry("nodejs-load-from-npm", f"DataDog/dd-trace-js#{sha}"),) - -class Prod: - def artifact_inputs(self, env: dict[str, str]) -> tuple[NpmLatestResolver]: - return (NpmLatestResolver(name="dd-trace", package="dd-trace"),) - - def artifact_entries( - self, - resolved_inputs: dict[str, ModuleVersion], - ) -> tuple[ArtifactEntry]: - version = resolved_inputs["dd-trace"].version - return (text_entry("nodejs-load-from-npm", f"dd-trace@{version}"),) +class Prod(SimpleTarget): + inputs = (NpmLatestResolver(name="dd-trace", package="dd-trace"),) + entries = (text_entry("nodejs-load-from-npm", "dd-trace@{dd-trace.version}"),) diff --git a/utils/build/docker/nodejs_otel/artifact.py b/utils/build/docker/nodejs_otel/artifact.py index ebe214836ce..d0e33780ba2 100644 --- a/utils/build/docker/nodejs_otel/artifact.py +++ b/utils/build/docker/nodejs_otel/artifact.py @@ -2,34 +2,16 @@ from utils.target_artifacts.entry_helpers import text_entry -from utils.target_artifacts.models import ( - ArtifactEntry, - ModuleVersion, -) +from utils.target_artifacts.models import SimpleTarget from utils.target_artifacts.resolvers import NpmLatestResolver PACKAGE_NAME = "@opentelemetry/auto-instrumentations-node" -class Dev: - def artifact_inputs(self, env: dict[str, str]) -> tuple[NpmLatestResolver]: - return (NpmLatestResolver(name="otel_package", package=PACKAGE_NAME),) +class Dev(SimpleTarget): + inputs = (NpmLatestResolver(name="otel_package", package=PACKAGE_NAME),) + entries = (text_entry("nodejs-otel-load-from-npm", f"{PACKAGE_NAME}@{{otel_package.version}}"),) - def artifact_entries( - self, - resolved_inputs: dict[str, ModuleVersion], - ) -> tuple[ArtifactEntry]: - version = resolved_inputs["otel_package"].version - return (text_entry("nodejs-otel-load-from-npm", f"{PACKAGE_NAME}@{version}"),) - -class Prod: - def artifact_inputs(self, env: dict[str, str]) -> tuple[NpmLatestResolver]: - return (NpmLatestResolver(name="otel_package", package=PACKAGE_NAME),) - - def artifact_entries( - self, - resolved_inputs: dict[str, ModuleVersion], - ) -> tuple[ArtifactEntry]: - version = resolved_inputs["otel_package"].version - return (text_entry("nodejs-otel-load-from-npm", f"{PACKAGE_NAME}@{version}"),) +class Prod(Dev): + pass diff --git a/utils/build/docker/otel_collector/artifact.py b/utils/build/docker/otel_collector/artifact.py index 2ff28c59ed9..36d4347c96d 100644 --- a/utils/build/docker/otel_collector/artifact.py +++ b/utils/build/docker/otel_collector/artifact.py @@ -2,38 +2,23 @@ from utils.target_artifacts.entry_helpers import text_entry -from utils.target_artifacts.models import ( - ArtifactEntry, - OciImageReference, -) +from utils.target_artifacts.models import SimpleTarget from utils.target_artifacts.resolvers import OciDigestResolver DEFAULT_IMAGE = "otel/opentelemetry-collector-contrib:0.137.0" -class Dev: - def artifact_inputs(self, env: dict[str, str]) -> tuple[OciDigestResolver]: - return ( - OciDigestResolver( - name="collector_image", - image=DEFAULT_IMAGE, - variable_name="OTEL_COLLECTOR_IMAGE", - ), - ) +class Dev(SimpleTarget): + inputs = ( + OciDigestResolver( + name="collector_image", + image=DEFAULT_IMAGE, + variable_name="OTEL_COLLECTOR_IMAGE", + ), + ) + entries = (text_entry("otel_collector-image", "{collector_image.reference}"),) - def artifact_entries( - self, - resolved_inputs: dict[str, OciImageReference], - ) -> tuple[ArtifactEntry]: - return (text_entry("otel_collector-image", resolved_inputs["collector_image"].reference),) - -class Prod: - def artifact_inputs(self, env: dict[str, str]) -> tuple[OciDigestResolver]: - return (OciDigestResolver(name="collector_image", image=DEFAULT_IMAGE),) - - def artifact_entries( - self, - resolved_inputs: dict[str, OciImageReference], - ) -> tuple[ArtifactEntry]: - return (text_entry("otel_collector-image", resolved_inputs["collector_image"].reference),) +class Prod(SimpleTarget): + inputs = (OciDigestResolver(name="collector_image", image=DEFAULT_IMAGE),) + entries = (text_entry("otel_collector-image", "{collector_image.reference}"),) diff --git a/utils/build/docker/python/artifact.py b/utils/build/docker/python/artifact.py index c2feaac9797..9af29c5006f 100644 --- a/utils/build/docker/python/artifact.py +++ b/utils/build/docker/python/artifact.py @@ -2,39 +2,22 @@ from utils.target_artifacts.entry_helpers import text_entry -from utils.target_artifacts.models import ( - ArtifactEntry, - BranchReference, - ModuleVersion, -) +from utils.target_artifacts.models import SimpleTarget from utils.target_artifacts.resolvers import GitHubBranchResolver, PypiLatestResolver -class Dev: - def artifact_inputs(self, env: dict[str, str]) -> tuple[GitHubBranchResolver]: - return ( - GitHubBranchResolver( - name="library_branch", - repository="DataDog/dd-trace-py", - variable_name="LIBRARY_TARGET_BRANCH", - default_value="main", - ), - ) +class Dev(SimpleTarget): + inputs = ( + GitHubBranchResolver( + name="library_branch", + repository="DataDog/dd-trace-py", + variable_name="LIBRARY_TARGET_BRANCH", + default_value="main", + ), + ) + entries = (text_entry("python-load-from-s3", "{library_branch.sha}"),) - def artifact_entries( - self, - resolved_inputs: dict[str, BranchReference], - ) -> tuple[ArtifactEntry]: - return (text_entry("python-load-from-s3", resolved_inputs["library_branch"].sha),) - -class Prod: - def artifact_inputs(self, env: dict[str, str]) -> tuple[PypiLatestResolver]: - return (PypiLatestResolver(name="ddtrace", package="ddtrace"),) - - def artifact_entries( - self, - resolved_inputs: dict[str, ModuleVersion], - ) -> tuple[ArtifactEntry]: - version = resolved_inputs["ddtrace"].version - return (text_entry("python-load-from-pip", f"ddtrace=={version}"),) +class Prod(SimpleTarget): + inputs = (PypiLatestResolver(name="ddtrace", package="ddtrace"),) + entries = (text_entry("python-load-from-pip", "ddtrace=={ddtrace.version}"),) diff --git a/utils/build/docker/python_otel/artifact.py b/utils/build/docker/python_otel/artifact.py index d412eb1505c..302f746060f 100644 --- a/utils/build/docker/python_otel/artifact.py +++ b/utils/build/docker/python_otel/artifact.py @@ -2,34 +2,16 @@ from utils.target_artifacts.entry_helpers import text_entry -from utils.target_artifacts.models import ( - ArtifactEntry, - ModuleVersion, -) +from utils.target_artifacts.models import SimpleTarget from utils.target_artifacts.resolvers import PypiLatestResolver PACKAGE_NAME = "opentelemetry-distro" -class Dev: - def artifact_inputs(self, env: dict[str, str]) -> tuple[PypiLatestResolver]: - return (PypiLatestResolver(name="otel_package", package=PACKAGE_NAME),) +class Dev(SimpleTarget): + inputs = (PypiLatestResolver(name="otel_package", package=PACKAGE_NAME),) + entries = (text_entry("python-otel-load-from-pip", f"{PACKAGE_NAME}[otlp]=={{otel_package.version}}"),) - def artifact_entries( - self, - resolved_inputs: dict[str, ModuleVersion], - ) -> tuple[ArtifactEntry]: - version = resolved_inputs["otel_package"].version - return (text_entry("python-otel-load-from-pip", f"{PACKAGE_NAME}[otlp]=={version}"),) - -class Prod: - def artifact_inputs(self, env: dict[str, str]) -> tuple[PypiLatestResolver]: - return (PypiLatestResolver(name="otel_package", package=PACKAGE_NAME),) - - def artifact_entries( - self, - resolved_inputs: dict[str, ModuleVersion], - ) -> tuple[ArtifactEntry]: - version = resolved_inputs["otel_package"].version - return (text_entry("python-otel-load-from-pip", f"{PACKAGE_NAME}[otlp]=={version}"),) +class Prod(Dev): + pass diff --git a/utils/build/docker/ruby/artifact.py b/utils/build/docker/ruby/artifact.py index d4fff04e096..f49c5534225 100644 --- a/utils/build/docker/ruby/artifact.py +++ b/utils/build/docker/ruby/artifact.py @@ -2,51 +2,33 @@ from utils.target_artifacts.entry_helpers import text_entry -from utils.target_artifacts.models import ( - ArtifactEntry, - BranchReference, - ModuleVersion, -) +from utils.target_artifacts.models import SimpleTarget from utils.target_artifacts.resolvers import GitHubBranchResolver, RubygemsLatestResolver -class Dev: - def artifact_inputs(self, env: dict[str, str]) -> tuple[GitHubBranchResolver]: - return ( - GitHubBranchResolver( - name="library_branch", - repository="DataDog/dd-trace-rb", - variable_name="LIBRARY_TARGET_BRANCH", - default_value="master", - ), - ) - - def artifact_entries( - self, - resolved_inputs: dict[str, BranchReference], - ) -> tuple[ArtifactEntry]: - sha = resolved_inputs["library_branch"].sha - return ( - text_entry( - "ruby-load-from-bundle-add", - "gem 'datadog', require: 'datadog/auto_instrument', " - f"git: 'https://github.com/DataDog/dd-trace-rb.git', ref: '{sha}'", - ), - ) - - -class Prod: - def artifact_inputs(self, env: dict[str, str]) -> tuple[RubygemsLatestResolver]: - return (RubygemsLatestResolver(name="datadog", package="datadog"),) - - def artifact_entries( - self, - resolved_inputs: dict[str, ModuleVersion], - ) -> tuple[ArtifactEntry]: - version = resolved_inputs["datadog"].version - return ( - text_entry( - "ruby-load-from-bundle-add", - f"gem 'datadog', '{version}', require: 'datadog/auto_instrument'", - ), - ) +class Dev(SimpleTarget): + inputs = ( + GitHubBranchResolver( + name="library_branch", + repository="DataDog/dd-trace-rb", + variable_name="LIBRARY_TARGET_BRANCH", + default_value="master", + ), + ) + entries = ( + text_entry( + "ruby-load-from-bundle-add", + "gem 'datadog', require: 'datadog/auto_instrument', " + "git: 'https://github.com/DataDog/dd-trace-rb.git', ref: '{library_branch.sha}'", + ), + ) + + +class Prod(SimpleTarget): + inputs = (RubygemsLatestResolver(name="datadog", package="datadog"),) + entries = ( + text_entry( + "ruby-load-from-bundle-add", + "gem 'datadog', '{datadog.version}', require: 'datadog/auto_instrument'", + ), + ) diff --git a/utils/build/docker/ruby_lambda/artifact.py b/utils/build/docker/ruby_lambda/artifact.py index 18a0d868888..f1540113f75 100644 --- a/utils/build/docker/ruby_lambda/artifact.py +++ b/utils/build/docker/ruby_lambda/artifact.py @@ -2,43 +2,22 @@ from utils.target_artifacts.entry_helpers import text_entry -from utils.target_artifacts.models import ( - ArtifactEntry, - BranchReference, - GitHubReleaseReference, -) +from utils.target_artifacts.models import SimpleTarget from utils.target_artifacts.resolvers import GitHubBranchResolver, GitHubLatestReleaseResolver -class Dev: - def artifact_inputs(self, env: dict[str, str]) -> tuple[GitHubBranchResolver]: - return ( - GitHubBranchResolver( - name="library_branch", - repository="DataDog/datadog-lambda-rb", - variable_name="LIBRARY_TARGET_BRANCH", - default_value="main", - ), - ) +class Dev(SimpleTarget): + inputs = ( + GitHubBranchResolver( + name="library_branch", + repository="DataDog/datadog-lambda-rb", + variable_name="LIBRARY_TARGET_BRANCH", + default_value="main", + ), + ) + entries = (text_entry("ruby-lambda-load-from-git", "https://github.com/DataDog/datadog-lambda-rb@{library_branch.sha}"),) - def artifact_entries( - self, - resolved_inputs: dict[str, BranchReference], - ) -> tuple[ArtifactEntry]: - return ( - text_entry( - "ruby-lambda-load-from-git", - f"https://github.com/DataDog/datadog-lambda-rb@{resolved_inputs['library_branch'].sha}", - ), - ) - -class Prod: - def artifact_inputs(self, env: dict[str, str]) -> tuple[GitHubLatestReleaseResolver]: - return (GitHubLatestReleaseResolver(name="release", repository="DataDog/datadog-lambda-rb"),) - - def artifact_entries( - self, - resolved_inputs: dict[str, GitHubReleaseReference], - ) -> tuple[ArtifactEntry]: - return (text_entry("ruby-lambda-load-from-release", resolved_inputs["release"].tag_name),) +class Prod(SimpleTarget): + inputs = (GitHubLatestReleaseResolver(name="release", repository="DataDog/datadog-lambda-rb"),) + entries = (text_entry("ruby-lambda-load-from-release", "{release.tag_name}"),) diff --git a/utils/build/docker/rust/artifact.py b/utils/build/docker/rust/artifact.py index 00b3f921ba0..8e3210cf240 100644 --- a/utils/build/docker/rust/artifact.py +++ b/utils/build/docker/rust/artifact.py @@ -2,44 +2,27 @@ from utils.target_artifacts.entry_helpers import text_entry -from utils.target_artifacts.models import ( - ArtifactEntry, - BranchReference, - ModuleVersion, -) +from utils.target_artifacts.models import SimpleTarget from utils.target_artifacts.resolvers import CratesLatestResolver, GitHubBranchResolver -class Dev: - def artifact_inputs(self, env: dict[str, str]) -> tuple[GitHubBranchResolver]: - return ( - GitHubBranchResolver( - name="library_branch", - repository="DataDog/dd-trace-rs", - variable_name="LIBRARY_TARGET_BRANCH", - default_value="main", - ), - ) - - def artifact_entries( - self, - resolved_inputs: dict[str, BranchReference], - ) -> tuple[ArtifactEntry]: - return (text_entry("rust-load-from-git", resolved_inputs["library_branch"].sha),) - - -class Prod: - def artifact_inputs(self, env: dict[str, str]) -> tuple[CratesLatestResolver]: - return ( - CratesLatestResolver( - name="datadog_opentelemetry", - package="datadog-opentelemetry", - ), - ) - - def artifact_entries( - self, - resolved_inputs: dict[str, ModuleVersion], - ) -> tuple[ArtifactEntry]: - version = resolved_inputs["datadog_opentelemetry"].version - return (text_entry("rust-load-from-crates", version),) +class Dev(SimpleTarget): + inputs = ( + GitHubBranchResolver( + name="library_branch", + repository="DataDog/dd-trace-rs", + variable_name="LIBRARY_TARGET_BRANCH", + default_value="main", + ), + ) + entries = (text_entry("rust-load-from-git", "{library_branch.sha}"),) + + +class Prod(SimpleTarget): + inputs = ( + CratesLatestResolver( + name="datadog_opentelemetry", + package="datadog-opentelemetry", + ), + ) + entries = (text_entry("rust-load-from-crates", "{datadog_opentelemetry.version}"),) diff --git a/utils/target_artifacts/__init__.py b/utils/target_artifacts/__init__.py index 11eb5f784ac..38f0b1a076d 100644 --- a/utils/target_artifacts/__init__.py +++ b/utils/target_artifacts/__init__.py @@ -9,6 +9,7 @@ OciImageReference, ReleaseAsset, ResolvedArtifactInput, + SimpleTarget, TargetArtifactEnvironment, TargetArtifactError, ) @@ -26,6 +27,7 @@ "OciImageReference", "ReleaseAsset", "ResolvedArtifactInput", + "SimpleTarget", "TargetArtifactEnvironment", "TargetArtifactError", "stage_target", diff --git a/utils/target_artifacts/models.py b/utils/target_artifacts/models.py index 5805d323ad9..5c227265936 100644 --- a/utils/target_artifacts/models.py +++ b/utils/target_artifacts/models.py @@ -107,3 +107,25 @@ def artifact_entries( ) -> tuple[ArtifactEntry, ...]: """Return text artifact entries from resolved inputs.""" ... + + +class SimpleTarget: + """Declarative base for targets with static inputs and template entries. + + Subclasses set ``inputs`` (a tuple of resolvers) and ``entries`` (a tuple + of ``ArtifactEntry`` whose ``content`` uses ``{resolver_name.field}`` + format placeholders). The orchestrator resolves the inputs and formats + each entry's content with the resolved values. + """ + + inputs: tuple[ArtifactResolver, ...] = () + entries: tuple[ArtifactEntry, ...] = () + + def artifact_inputs(self, _env: dict[str, str]) -> tuple[ArtifactResolver, ...]: + return self.inputs + + def artifact_entries(self, resolved_inputs: dict[str, ResolvedArtifactInput]) -> tuple[ArtifactEntry, ...]: + return tuple( + ArtifactEntry(filename=entry.filename, content=entry.content.format(**resolved_inputs)) + for entry in self.entries + ) From 55886cf0f8a64383100f908852476096891ed269 Mon Sep 17 00:00:00 2001 From: datadog-bits <263423550+datadog-bits@users.noreply.github.com> Date: Mon, 14 Sep 2026 14:47:32 +0000 Subject: [PATCH 09/12] Split staging framework and Python migration Co-authored-by: nccatoni <222672590+nccatoni@users.noreply.github.com> --- docs/CI/README.md | 6 - docs/CI/system-tests-ci.md | 12 +- docs/execute/binaries.md | 21 +- docs/execute/build.md | 9 +- .../internals/target-artifact-staging-spec.md | 60 +++ tests/test_the_test/test_build_pipeline.py | 181 -------- tests/test_the_test/test_load_binary.py | 134 +++--- tests/test_the_test/test_target_artifacts.py | 226 +++------- utils/build/build.sh | 59 --- utils/build/docker/agent/artifact.py | 15 - utils/build/docker/c/artifact.py | 89 ---- .../docker/c/perl-mojolicious.Dockerfile | 6 +- utils/build/docker/cpp/artifact.py | 26 -- utils/build/docker/cpp_httpd/artifact.py | 41 -- .../build/docker/cpp_httpd/install_ddtrace.sh | 19 +- utils/build/docker/cpp_kong/artifact.py | 63 --- .../build/docker/cpp_kong/install_ddtrace.sh | 18 +- utils/build/docker/cpp_nginx/artifact.py | 60 --- .../build/docker/cpp_nginx/install_ddprof.sh | 29 +- .../build/docker/cpp_nginx/install_ddtrace.sh | 93 ++-- utils/build/docker/dotnet/artifact.py | 49 -- utils/build/docker/dotnet/install_ddtrace.sh | 22 +- utils/build/docker/golang/artifact.py | 114 ----- utils/build/docker/java/artifact.py | 23 - utils/build/docker/java/install_ddtrace.sh | 33 +- .../docker/java/parametric/install_ddtrace.sh | 13 +- utils/build/docker/java_lambda/artifact.py | 23 - utils/build/docker/java_otel/artifact.py | 17 - .../docker/java_otel/install_opentelemetry.sh | 10 +- utils/build/docker/nodejs/artifact.py | 23 - utils/build/docker/nodejs_lambda/artifact.py | 42 -- .../nodejs_lambda/install_datadog_lambda.sh | 37 +- utils/build/docker/nodejs_otel/artifact.py | 17 - .../nodejs_otel/express4-otel.Dockerfile | 2 - utils/build/docker/otel_collector/artifact.py | 24 - utils/build/docker/php/artifact.py | 55 --- .../docker/php/common/install_ddtrace.sh | 39 +- utils/build/docker/python_lambda/artifact.py | 42 -- .../python_lambda/install_datadog_lambda.sh | 30 +- utils/build/docker/python_otel/artifact.py | 17 - .../python_otel/flask-poc-otel.Dockerfile | 9 +- utils/build/docker/ruby/artifact.py | 34 -- utils/build/docker/ruby_lambda/artifact.py | 23 - .../ruby_lambda/install_datadog_lambda.sh | 19 +- utils/build/docker/rust/artifact.py | 28 -- utils/build/docker/rust/install_ddtrace.sh | 15 +- utils/ci/gitlab/build_pipeline.py | 77 +--- utils/ci/gitlab/main.yml | 6 +- utils/ci/gitlab/system-tests.yml.j2 | 33 +- .../compute_libraries_and_scenarios.py | 2 +- utils/scripts/docker_base_image.sh | 16 +- utils/scripts/load-binary.sh | 424 ++++++++++++++++-- utils/target_artifacts/orchestrator.py | 10 +- 53 files changed, 753 insertions(+), 1742 deletions(-) create mode 100644 docs/internals/target-artifact-staging-spec.md delete mode 100644 utils/build/docker/agent/artifact.py delete mode 100644 utils/build/docker/c/artifact.py delete mode 100644 utils/build/docker/cpp/artifact.py delete mode 100644 utils/build/docker/cpp_httpd/artifact.py delete mode 100644 utils/build/docker/cpp_kong/artifact.py delete mode 100644 utils/build/docker/cpp_nginx/artifact.py delete mode 100644 utils/build/docker/dotnet/artifact.py delete mode 100644 utils/build/docker/golang/artifact.py delete mode 100644 utils/build/docker/java/artifact.py delete mode 100644 utils/build/docker/java_lambda/artifact.py delete mode 100644 utils/build/docker/java_otel/artifact.py delete mode 100644 utils/build/docker/nodejs/artifact.py delete mode 100644 utils/build/docker/nodejs_lambda/artifact.py delete mode 100644 utils/build/docker/nodejs_otel/artifact.py delete mode 100644 utils/build/docker/otel_collector/artifact.py delete mode 100644 utils/build/docker/php/artifact.py delete mode 100644 utils/build/docker/python_lambda/artifact.py delete mode 100644 utils/build/docker/python_otel/artifact.py delete mode 100644 utils/build/docker/ruby/artifact.py delete mode 100644 utils/build/docker/ruby_lambda/artifact.py delete mode 100644 utils/build/docker/rust/artifact.py diff --git a/docs/CI/README.md b/docs/CI/README.md index 635290c00df..6ece16c0386 100644 --- a/docs/CI/README.md +++ b/docs/CI/README.md @@ -19,12 +19,6 @@ For the system-tests own CI, see also: * [CI test selection](./ci-test-selection.md): how the CI decides which libraries and scenarios to run based on modified files -### Target artifact staging in generated GitLab jobs - -Generated GitLab child-pipeline jobs run target artifact staging directly where the artifact entries are consumed. Build jobs stage before building a weblog when no upstream binaries bundle takes precedence. Parametric jobs stage before `./run.sh PARAMETRIC` when no build artifact bundle exists. Custom jobs that receive an upstream binaries bundle skip staging because that bundle is the selected source of truth. - -GitHub workflows keep their existing compatibility path during the migration. The `load-binary.sh` command remains available for local and workflow compatibility, but test target selection is routed through the Python staging model. - ### GitLab CI secrets setup 1. Install aws-cli diff --git a/docs/CI/system-tests-ci.md b/docs/CI/system-tests-ci.md index ee9b6fd0df5..7f9be6d21af 100644 --- a/docs/CI/system-tests-ci.md +++ b/docs/CI/system-tests-ci.md @@ -26,20 +26,10 @@ Each library in the CI matrix will use its own specified branch. Libraries witho As a security measure, the "Fail if target branch is specified" job always fails if a target branch is selected. -### Target artifact staging - -GitLab generated jobs stage target artifacts at the point where the generated `binaries/` entries are needed: - -- Build jobs run `python3 utils/scripts/stage-target-artifacts.py ` before building a weblog when no upstream binaries bundle takes precedence. -- Parametric jobs run the same command before `./run.sh PARAMETRIC` when no build artifact bundle exists. -- Run jobs that consume a build job's artifact bundle do not repeat staging. -- Custom jobs with upstream binaries bundles skip staging because the upstream bundle is the selected artifact source of truth. - -The command accepts `custom` as a no-op, so generated templates can use one command shape safely. GitHub workflows keep using existing compatibility behavior until they choose to consume the new production artifact entries directly. - ### Scenario detection in CI When a modification is made in system tests, the CI tries to detect which scenario to run: 1. based on modified files in `tests/`, by extracting scenarios targeted by those files 2. based on any modification in a `tests/**/utils.py`, and applying the logic 1. on any sub file in `tests/**` + diff --git a/docs/execute/binaries.md b/docs/execute/binaries.md index 26a656f5f16..b4cdaf330f5 100644 --- a/docs/execute/binaries.md +++ b/docs/execute/binaries.md @@ -4,29 +4,28 @@ But we often want to run system tests against unmerged changes. The general appr ## Target artifact staging -Target artifact staging is the preferred way to generate the text files consumed from `binaries/`. -Run it with: +Python is the first target using the target artifact staging framework. The existing +compatibility command continues to work: ```bash -python3 utils/scripts/stage-target-artifacts.py +./utils/scripts/load-binary.sh python ``` -The legacy compatibility command still works and delegates test targets to the same staging model: +The equivalent direct command is: ```bash -./utils/scripts/load-binary.sh +python3 utils/scripts/stage-target-artifacts.py python ``` -Generated artifact entries are text-only. They select an artifact by a bounded artifact selector, such as a commit SHA, release tag, package version, or OCI digest. Generated entries must not use unbounded rolling selectors such as `latest`. +Staging writes bounded text selectors and records generated-file ownership in +`binaries/.target-artifacts-manifest.json`. It refuses to overwrite manual files in +`binaries/`. Other targets continue to use their existing loading behavior until +they are migrated separately. -Manual payload overrides remain supported. If you put a jar, wheel, archive, native module, local checkout, or explicit marker file in `binaries/`, the installer behavior documented below still applies. Staging refuses to overwrite unowned files so local payload overrides are not silently replaced. - -Some providers require an installer-facing fetch selector that is not itself bounded. In that case staging also writes a selection marker containing the bounded selector used for cache identity. The generated `binaries/.target-artifacts-manifest.json` file records which target owns generated entries and lets later staging refresh stale entries safely. ## Agent * Add a file `agent-image` in `binaries/`. The content must be a valid docker image name containing the datadog agent, like `datadog/agent` or `datadog/agent-dev:master-py3`. -* Compatibility command: `./utils/scripts/load-binary.sh agent dev` ### Building an agent image from a local datadog-agent branch @@ -84,7 +83,6 @@ There are three ways to run system-tests with a custom Kong plugin: ```bash ./utils/scripts/load-binary.sh cpp_kong ``` - The command now stages bounded references and metadata; the Docker build fetches the selected payload when needed. To test with a custom dd-trace-cpp C binding, you can additionally: * Create a file `cpp-load-from-git` in `binaries/` (e.g. `https://github.com/DataDog/dd-trace-cpp@main`) @@ -294,7 +292,6 @@ You can also use `utils/scripts/watch.sh` script to sync your local `dd-trace-rs ## WAF rule set * copy a file `waf_rule_set` in `binaries/` -* Compatibility command: `./utils/scripts/load-binary.sh waf_rule_set dev` #### After Testing with a Custom Tracer: Most of the ways to run system-tests with a custom tracer version involve modifying the binaries directory. Modifying the binaries will alter the tracer version used across your local computer. Once you're done testing with the custom tracer, ensure you **remove** it. For example for Python: diff --git a/docs/execute/build.md b/docs/execute/build.md index f521666f8f5..a4ae354c60a 100644 --- a/docs/execute/build.md +++ b/docs/execute/build.md @@ -59,15 +59,14 @@ Build the native C tracer workload with: ./build.sh c -w perl-mojolicious ``` -The production build starts from the published `apm-library-c-package:latest` -and `apm-inject-package:latest` images from `install.datadoghq.com`, then -records immutable digest references in `binaries/`. Run +The production build uses the published `apm-library-c-package:latest` and +`apm-inject-package:latest` images from `install.datadoghq.com`. Run `./utils/scripts/load-binary.sh c` to validate and record both production image -references. For a development build, set +references in `binaries/`. For a development build, set `LIBRARY_TARGET_BRANCH`, `AUTO_INJECT_TARGET_BRANCH`, or both before running the loader. Each branch override is resolved to an immutable commit-SHA tag from `installtesting.datad0g.com` (with a zero in `datad0g`); components without a -branch override continue to use production image digest references. +branch override continue to use the production `latest` image. The `perl-mojolicious` workload supports `DEFAULT`, `SAMPLING`, and `IPV6`. It uses Perl and Mojolicious without a Datadog Perl tracer; all tracing comes from diff --git a/docs/internals/target-artifact-staging-spec.md b/docs/internals/target-artifact-staging-spec.md new file mode 100644 index 00000000000..aaf85cb13cb --- /dev/null +++ b/docs/internals/target-artifact-staging-spec.md @@ -0,0 +1,60 @@ +# Target artifact staging + +Target artifact staging resolves a test target to bounded, inspectable entries in +`binaries/` before a build consumes them. The first migration covers Python; other +targets continue to use `utils/scripts/load-binary.sh` until migrated separately. + +## Contract + +Each migrated target provides `utils/build/docker//artifact.py` with `Dev` +and `Prod` implementations. They declare resolver inputs and map the resolved values +to text entries without performing network or filesystem side effects themselves. + +The shared orchestrator owns external lookups and writes the generated entries. It +also maintains `binaries/.target-artifacts-manifest.json`, which records the owner +and content hash of every generated file. Staging: + +- refreshes entries previously owned by the same target; +- removes stale entries owned by that target; +- preserves entries owned by other targets; and +- refuses to overwrite unowned files or entries owned by another target. + +Selectors should be bounded, such as a commit SHA, release tag, package version, or +OCI digest. If an installer must consume a mutable provider selector, the target must +also emit a bounded selection marker with `provider_fetch_entries`. + +The `custom` environment is a no-op because an upstream or local artifact bundle is +already the source of truth. + +## Commands + +The canonical entry point is: + +```bash +python3 utils/scripts/stage-target-artifacts.py +``` + +During migration, the existing compatibility command delegates migrated targets to +the same implementation: + +```bash +./utils/scripts/load-binary.sh +``` + +## Python demonstration + +For `python dev`, the configured `LIBRARY_TARGET_BRANCH` (default: `main`) resolves +to a commit SHA and produces `python-load-from-s3`. For `python prod`, the latest +published `ddtrace` package version produces `python-load-from-pip`. Existing Python +installer behavior consumes both files, so no installer change is needed. + +## Adding a target + +1. Add the target's `artifact.py` with `Dev` and `Prod` implementations. +2. Reuse shared resolvers, or add a resolver with isolated unit coverage. +3. Emit text entries only; keep payload downloads in existing build/install steps. +4. Preserve local payload overrides and add public-contract tests for the target. +5. Route only that target through the compatibility loader. + +GitLab job integration and Buildx remote caching are intentionally handled in later +changes after target migrations are reviewed. diff --git a/tests/test_the_test/test_build_pipeline.py b/tests/test_the_test/test_build_pipeline.py index 36ea8d80b33..bf8571f48ab 100644 --- a/tests/test_the_test/test_build_pipeline.py +++ b/tests/test_the_test/test_build_pipeline.py @@ -134,184 +134,3 @@ def test_c_pipeline_renders_three_scenarios_and_package_artifact(self, tmp_path: for job_name in expected_run_jobs: assert ".system_tests_base" in pipeline[job_name]["extends"] - - def test_build_job_stages_target_artifacts_without_upstream_bundle(self, tmp_path: Path) -> None: - params = { - "endtoend_defs": { - "parallel_weblogs": [{"name": "flask"}], - "parallel_jobs": [{"weblog": "flask", "scenarios": ["DEFAULT"], "weblog_build_required": True}], - }, - "miscs": {"binaries_artifact": "", "ci_environment": "prod"}, - "parametric": {"enable": False, "parallel_jobs": []}, - } - (tmp_path / "params_python.json").write_text(json.dumps(params)) - out = tmp_path / "out" - - build(["python"], tmp_path, out, stage="e2e", ci_image="myimage", chunks=1) - - pipeline = yaml.safe_load((out / "generated-pipeline-chunk-0.yml").read_text()) - build_script = pipeline["system_tests_build_python_flask"]["script"] - assert "python3 utils/scripts/stage-target-artifacts.py python prod" in build_script - assert not any(job_name.startswith("system_tests_stage") for job_name in pipeline) - - def test_parametric_job_stages_target_artifacts_without_upstream_bundle(self, tmp_path: Path) -> None: - params = { - "endtoend_defs": {"parallel_weblogs": [], "parallel_jobs": []}, - "miscs": {"binaries_artifact": "", "ci_environment": "dev"}, - "parametric": {"enable": True, "job_count": 1, "job_matrix": [1]}, - } - (tmp_path / "params_nodejs.json").write_text(json.dumps(params)) - out = tmp_path / "out" - - build(["nodejs"], tmp_path, out, stage="e2e", ci_image="myimage", chunks=1) - - pipeline = yaml.safe_load((out / "generated-pipeline-chunk-0.yml").read_text()) - run_script = pipeline["system_tests_run_nodejs_PARAMETRIC_1"]["script"] - assert "python3 utils/scripts/stage-target-artifacts.py nodejs dev" in run_script - - def test_upstream_artifact_bundle_skips_target_artifact_staging(self, tmp_path: Path) -> None: - params = { - "endtoend_defs": { - "parallel_weblogs": [{"name": "flask"}], - "parallel_jobs": [{"weblog": "flask", "scenarios": ["DEFAULT"], "weblog_build_required": True}], - }, - "miscs": {"binaries_artifact": "", "ci_environment": "custom"}, - "parametric": {"enable": True, "job_count": 1, "job_matrix": [1]}, - } - (tmp_path / "params_python.json").write_text(json.dumps(params)) - out = tmp_path / "out" - - build( - ["python"], - tmp_path, - out, - stage="e2e", - ci_image="myimage", - chunks=1, - binaries_artifacts="upstream-binaries", - binaries_artifact_path="system-tests-binaries", - ) - - text = (out / "generated-pipeline-chunk-0.yml").read_text() - assert "stage-target-artifacts.py" not in text - - def test_buildx_cache_updates_system_tests_main(self, tmp_path: Path) -> None: - params = { - "endtoend_defs": { - "parallel_weblogs": [{"name": "perl-mojolicious"}], - "parallel_jobs": [ - { - "weblog": "perl-mojolicious", - "scenarios": ["DEFAULT", "SAMPLING", "IPV6"], - "weblog_build_required": True, - } - ], - }, - "miscs": {"binaries_artifact": ""}, - "parametric": {"enable": False, "parallel_jobs": []}, - } - (tmp_path / "params_c.json").write_text(json.dumps(params)) - out = tmp_path / "out" - - build( - ["c"], - tmp_path, - out, - stage="e2e", - ci_image="myimage", - chunks=1, - binaries_artifacts="system_tests_package_refs", - binaries_artifact_path="system-tests-binaries", - ref="main", - ci_project_name="system-tests", - ci_commit_branch="main", - ci_default_branch="main", - ) - - text = (out / "generated-pipeline-chunk-0.yml").read_text() - assert ( - "--cache-to=type=registry,ref=registry.ddbuild.io/system-tests/cache/c/perl-mojolicious:main,mode=max" - in text - ) - assert ( - "--cache-to=type=registry,ref=registry.ddbuild.io/system-tests/cache/c/perl-mojolicious:lib_main,mode=max" - not in text - ) - - def test_buildx_cache_updates_library_default_branch(self, tmp_path: Path) -> None: - params = { - "endtoend_defs": { - "parallel_weblogs": [{"name": "perl-mojolicious"}], - "parallel_jobs": [ - { - "weblog": "perl-mojolicious", - "scenarios": ["DEFAULT", "SAMPLING", "IPV6"], - "weblog_build_required": True, - } - ], - }, - "miscs": {"binaries_artifact": ""}, - "parametric": {"enable": False, "parallel_jobs": []}, - } - (tmp_path / "params_c.json").write_text(json.dumps(params)) - out = tmp_path / "out" - - build( - ["c"], - tmp_path, - out, - stage="e2e", - ci_image="myimage", - chunks=1, - binaries_artifacts="system_tests_package_refs", - binaries_artifact_path="system-tests-binaries", - ref="main", - ci_project_name="dd-trace-rb", - ci_commit_branch="master", - ci_default_branch="master", - ) - - text = (out / "generated-pipeline-chunk-0.yml").read_text() - assert ( - "--cache-to=type=registry,ref=registry.ddbuild.io/system-tests/cache/c/perl-mojolicious:lib_main,mode=max" - in text - ) - assert ( - "--cache-to=type=registry,ref=registry.ddbuild.io/system-tests/cache/c/perl-mojolicious:main,mode=max" - not in text - ) - - def test_buildx_cache_does_not_update_from_not_main(self, tmp_path: Path) -> None: - params = { - "endtoend_defs": { - "parallel_weblogs": [{"name": "perl-mojolicious"}], - "parallel_jobs": [ - { - "weblog": "perl-mojolicious", - "scenarios": ["DEFAULT", "SAMPLING", "IPV6"], - "weblog_build_required": True, - } - ], - }, - "miscs": {"binaries_artifact": ""}, - "parametric": {"enable": False, "parallel_jobs": []}, - } - (tmp_path / "params_c.json").write_text(json.dumps(params)) - out = tmp_path / "out" - - build( - ["c"], - tmp_path, - out, - stage="e2e", - ci_image="myimage", - chunks=1, - binaries_artifacts="system_tests_package_refs", - binaries_artifact_path="system-tests-binaries", - ref="some-branch", - ci_project_name="system-tests", - ci_commit_branch="some-branch", - ci_default_branch="main", - ) - - assert not re.search("--cache-to=type=registry,ref=", (out / "generated-pipeline-chunk-0.yml").read_text()) diff --git a/tests/test_the_test/test_load_binary.py b/tests/test_the_test/test_load_binary.py index 793f2a61aa3..f5ff335d1cc 100644 --- a/tests/test_the_test/test_load_binary.py +++ b/tests/test_the_test/test_load_binary.py @@ -1,6 +1,5 @@ from __future__ import annotations -import json import os from pathlib import Path import subprocess @@ -10,12 +9,11 @@ SCRIPT = Path("utils/scripts/load-binary.sh") -C_LIBRARY_DIGEST = "sha256:" + ("a" * 64) -C_INJECTOR_DIGEST = "sha256:" + ("b" * 64) -C_LIBRARY_PROD_IMAGE = f"install.datadoghq.com/apm-library-c-package@{C_LIBRARY_DIGEST}" -C_INJECTOR_PROD_IMAGE = f"install.datadoghq.com/apm-inject-package@{C_INJECTOR_DIGEST}" +C_LIBRARY_PROD_IMAGE = "install.datadoghq.com/apm-library-c-package:latest" +C_INJECTOR_PROD_IMAGE = "install.datadoghq.com/apm-inject-package:latest" C_LIBRARY_SHA = "1" * 40 C_INJECTOR_SHA = "2" * 40 +PYTHON_SHA = "3" * 40 def _write_executable(path: Path, contents: str) -> None: @@ -27,7 +25,6 @@ def _run_loader( tmp_path: Path, version: str, *, - target: str = "c", extra_env: dict[str, str] | None = None, ) -> subprocess.CompletedProcess[str]: bin_dir = tmp_path / "bin" @@ -36,33 +33,29 @@ def _run_loader( binaries_dir.mkdir() _write_executable( - bin_dir / "docker", + bin_dir / "curl", f"""#!/usr/bin/env bash set -eu -printf '%s\\n' "$*" >> "$DOCKER_CALLS" -if [[ "${{FAIL_IMAGE:-}}" != "" && "$*" == *"$FAIL_IMAGE"* ]]; then - exit 1 +url="${{!#}}" +printf '%s\\n' "$url" >> "$CURL_CALLS" +if [[ "${{MISSING_BRANCH:-}}" != "" && "$url" == *"${{MISSING_BRANCH}}"* ]]; then + exit 22 fi -if [[ "$*" == *"apm-library-c-package"* ]]; then - printf 'Name: apm-library-c-package\\nDigest: {C_LIBRARY_DIGEST}\\n' +if [[ "$url" == *"DataDog/dd-trace-c"* ]]; then + printf '%s\\n' '{{"commit":{{"sha":"{C_LIBRARY_SHA}"}}}}' else - printf 'Name: apm-inject-package\\nDigest: {C_INJECTOR_DIGEST}\\n' + printf '%s\\n' '{{"commit":{{"sha":"{C_INJECTOR_SHA}"}}}}' fi """, ) _write_executable( - bin_dir / "curl", + bin_dir / "docker", """#!/usr/bin/env bash set -eu -output="" -while [ "$#" -gt 0 ]; do - if [ "$1" = "--output" ]; then - shift - output="$1" - fi - shift -done -printf '{"rules":[]}\\n' > "$output" +printf '%s\\n' "$*" >> "$DOCKER_CALLS" +if [[ "${FAIL_IMAGE:-}" != "" && "$*" == *"$FAIL_IMAGE"* ]]; then + exit 1 +fi """, ) @@ -70,13 +63,14 @@ def _run_loader( **os.environ, "PATH": f"{bin_dir}:{os.environ['PATH']}", "BINARIES_DIR": str(binaries_dir), + "CURL_CALLS": str(tmp_path / "curl-calls"), "DOCKER_CALLS": str(tmp_path / "docker-calls"), } env.pop("LIBRARY_TARGET_BRANCH", None) env.pop("AUTO_INJECT_TARGET_BRANCH", None) env.update(extra_env or {}) return subprocess.run( - ["bash", str(SCRIPT), target, version], + ["bash", str(SCRIPT), "c", version], check=False, capture_output=True, text=True, @@ -86,21 +80,6 @@ def _run_loader( @scenarios.test_the_test class Test_LoadBinaryC: - def test_stage_target_artifacts_entrypoint_imports_without_pythonpath(self) -> None: - env = dict(os.environ) - env.pop("PYTHONPATH", None) - - result = subprocess.run( - ["python3", "utils/scripts/stage-target-artifacts.py", "--help"], - check=False, - capture_output=True, - text=True, - env=env, - ) - - assert result.returncode == 0, result.stderr - assert "usage: stage-target-artifacts" in result.stdout - def test_native_library_is_loaded_by_auto_inject(self) -> None: dockerfile = Path("utils/build/docker/c/perl-mojolicious.Dockerfile").read_text(encoding="utf-8") launcher = Path("utils/build/docker/c/perl-mojolicious/app.sh").read_text(encoding="utf-8") @@ -117,21 +96,22 @@ def test_production_package_defaults(self, tmp_path: Path) -> None: assert (tmp_path / "binaries/c-library-image").read_text(encoding="utf-8").strip() == C_LIBRARY_PROD_IMAGE assert (tmp_path / "binaries/c-injector-image").read_text(encoding="utf-8").strip() == C_INJECTOR_PROD_IMAGE docker_calls = (tmp_path / "docker-calls").read_text(encoding="utf-8") - assert "install.datadoghq.com/apm-library-c-package:latest" in docker_calls - assert "install.datadoghq.com/apm-inject-package:latest" in docker_calls + assert C_LIBRARY_PROD_IMAGE in docker_calls + assert C_INJECTOR_PROD_IMAGE in docker_calls def test_development_package_defaults_to_production_without_overrides(self, tmp_path: Path) -> None: result = _run_loader(tmp_path, "dev") assert result.returncode == 0, result.stderr + assert not (tmp_path / "curl-calls").exists() assert (tmp_path / "binaries/c-library-image").read_text(encoding="utf-8").strip() == C_LIBRARY_PROD_IMAGE assert (tmp_path / "binaries/c-injector-image").read_text(encoding="utf-8").strip() == C_INJECTOR_PROD_IMAGE docker_calls = (tmp_path / "docker-calls").read_text(encoding="utf-8") - assert "install.datadoghq.com/apm-library-c-package:latest" in docker_calls - assert "install.datadoghq.com/apm-inject-package:latest" in docker_calls + assert C_LIBRARY_PROD_IMAGE in docker_calls + assert C_INJECTOR_PROD_IMAGE in docker_calls def test_single_branch_override_keeps_other_component_on_production(self, tmp_path: Path) -> None: - result = _run_loader(tmp_path, "dev", extra_env={"LIBRARY_TARGET_BRANCH": C_LIBRARY_SHA}) + result = _run_loader(tmp_path, "dev", extra_env={"LIBRARY_TARGET_BRANCH": "feature/c-client"}) assert result.returncode == 0, result.stderr assert (tmp_path / "binaries/c-library-image").read_text(encoding="utf-8").strip() == ( @@ -144,13 +124,15 @@ def test_independent_branch_overrides_resolve_to_sha_tags(self, tmp_path: Path) tmp_path, "dev", extra_env={ - "LIBRARY_TARGET_BRANCH": C_LIBRARY_SHA, - "AUTO_INJECT_TARGET_BRANCH": C_INJECTOR_SHA, + "LIBRARY_TARGET_BRANCH": "feature/c-client", + "AUTO_INJECT_TARGET_BRANCH": "feature/injector", }, ) assert result.returncode == 0, result.stderr - assert not (tmp_path / "docker-calls").exists() + curl_calls = (tmp_path / "curl-calls").read_text(encoding="utf-8") + assert "feature%2Fc-client" in curl_calls + assert "feature%2Finjector" in curl_calls assert (tmp_path / "binaries/c-library-image").read_text(encoding="utf-8").strip() == ( f"installtesting.datad0g.com/apm-library-c-package:{C_LIBRARY_SHA}" ) @@ -158,15 +140,15 @@ def test_independent_branch_overrides_resolve_to_sha_tags(self, tmp_path: Path) f"installtesting.datad0g.com/apm-inject-package:{C_INJECTOR_SHA}" ) - def test_production_rejects_branch_overrides_before_package_validation(self, tmp_path: Path) -> None: + def test_missing_branch_fails_before_package_validation(self, tmp_path: Path) -> None: result = _run_loader( tmp_path, - "prod", - extra_env={"LIBRARY_TARGET_BRANCH": C_LIBRARY_SHA}, + "dev", + extra_env={"LIBRARY_TARGET_BRANCH": "missing", "MISSING_BRANCH": "missing"}, ) assert result.returncode != 0 - assert "Target branches can only be used with the development c packages" in result.stderr + assert "Unable to resolve branch 'missing' in DataDog/dd-trace-c" in result.stderr assert not (tmp_path / "docker-calls").exists() def test_missing_package_fails_with_clear_error(self, tmp_path: Path) -> None: @@ -177,29 +159,45 @@ def test_missing_package_fails_with_clear_error(self, tmp_path: Path) -> None: ) assert result.returncode != 0 - assert "Unable to resolve OCI digest" in result.stderr + assert "OCI package does not exist or is not accessible" in result.stderr - def test_agent_dependency_uses_target_artifact_staging(self, tmp_path: Path) -> None: - result = _run_loader( - tmp_path, - "dev", - target="agent", - extra_env={"AGENT_TARGET_BRANCH": "feature-agent"}, - ) - assert result.returncode == 0, result.stderr +@scenarios.test_the_test +class Test_LoadBinaryPython: + def test_development_branch_uses_target_artifact_staging(self, tmp_path: Path) -> None: binaries_dir = tmp_path / "binaries" - assert (binaries_dir / "agent-image").read_text(encoding="utf-8").strip() == ("datadog/agent-dev:feature-agent") - manifest = json.loads((binaries_dir / MANIFEST_FILENAME).read_text(encoding="utf-8")) - assert manifest["entries"]["agent-image"]["owner"] == { - "target": "agent", - "environment": "dev", + env = { + **os.environ, + "BINARIES_DIR": str(binaries_dir), + "LIBRARY_TARGET_BRANCH": PYTHON_SHA, } - def test_waf_rule_set_overlay_stays_outside_target_manifest(self, tmp_path: Path) -> None: - result = _run_loader(tmp_path, "dev", target="waf_rule_set") + result = subprocess.run( + ["bash", str(SCRIPT), "python", "dev"], + check=False, + capture_output=True, + text=True, + env=env, + ) assert result.returncode == 0, result.stderr + assert (binaries_dir / "python-load-from-s3").read_text(encoding="utf-8") == f"{PYTHON_SHA}\n" + assert (binaries_dir / MANIFEST_FILENAME).exists() + + def test_custom_environment_preserves_manual_python_artifacts(self, tmp_path: Path) -> None: binaries_dir = tmp_path / "binaries" - assert json.loads((binaries_dir / "waf_rule_set.json").read_text(encoding="utf-8")) == {"rules": []} + binaries_dir.mkdir() + manual_artifact = binaries_dir / "python-load-from-s3" + manual_artifact.write_text("manual\n", encoding="utf-8") + + result = subprocess.run( + ["bash", str(SCRIPT), "python", "custom"], + check=False, + capture_output=True, + text=True, + env={**os.environ, "BINARIES_DIR": str(binaries_dir)}, + ) + + assert result.returncode == 0, result.stderr + assert manual_artifact.read_text(encoding="utf-8") == "manual\n" assert not (binaries_dir / MANIFEST_FILENAME).exists() diff --git a/tests/test_the_test/test_target_artifacts.py b/tests/test_the_test/test_target_artifacts.py index e39f987653f..3c26ea29328 100644 --- a/tests/test_the_test/test_target_artifacts.py +++ b/tests/test_the_test/test_target_artifacts.py @@ -9,7 +9,6 @@ import requests from utils import scenarios -from utils.const import COMPONENT_GROUPS from utils.target_artifacts.models import ( ArtifactResolver, BranchReference, @@ -254,6 +253,58 @@ class Prod(Dev): assert (binaries_dir / "manual").read_text(encoding="utf-8") == "user\n" + @pytest.mark.parametrize("filename", ["", "../outside", "nested/entry", MANIFEST_FILENAME]) + def test_invalid_entry_filename_is_rejected(self, tmp_path: Path, filename: str) -> None: + _write_target_module( + tmp_path, + f""" +from utils.target_artifacts.entry_helpers import text_entry + +class Dev: + def artifact_inputs(self, env): + return () + + def artifact_entries(self, resolved_inputs): + return (text_entry({filename!r}, "generated"),) + +class Prod(Dev): + pass +""", + ) + + with pytest.raises(TargetArtifactError, match="Invalid artifact entry filename"): + stage_target("fake", "dev", repo_root=tmp_path, binaries_dir=tmp_path / "binaries") + + def test_manifest_cannot_delete_files_outside_binaries(self, tmp_path: Path) -> None: + _write_target_module( + tmp_path, + """ +class Dev: + def artifact_inputs(self, env): + return () + + def artifact_entries(self, resolved_inputs): + return () + +class Prod(Dev): + pass +""", + ) + outside = tmp_path / "outside" + outside.write_text("keep\n", encoding="utf-8") + binaries_dir = tmp_path / "binaries" + binaries_dir.mkdir() + manifest = { + "version": 1, + "entries": {"../outside": {"owner": {"target": "fake", "environment": "dev"}}}, + } + (binaries_dir / MANIFEST_FILENAME).write_text(json.dumps(manifest), encoding="utf-8") + + with pytest.raises(TargetArtifactError, match="Invalid artifact entry filename"): + stage_target("fake", "dev", repo_root=tmp_path, binaries_dir=binaries_dir) + + assert outside.read_text(encoding="utf-8") == "keep\n" + def test_github_release_resolver_wraps_request_failures(self, monkeypatch: pytest.MonkeyPatch) -> None: def fail_get(*_args: object, **_kwargs: object) -> object: raise requests.ConnectionError("network unavailable") @@ -873,91 +924,12 @@ def fake_run( resolver.resolve({}) -@scenarios.test_the_test -class Test_TargetArtifactExternalContracts: - def test_public_github_branch_contract(self) -> None: - resolved = GitHubBranchResolver( - name="library_branch", - repository="DataDog/dd-trace-py", - default_value="main", - ).resolve({}) - - assert resolved.branch == "main" - assert resolved.repository == "DataDog/dd-trace-py" - assert len(resolved.sha) == 40 - assert all(character in "0123456789abcdef" for character in resolved.sha) - - def test_public_github_latest_release_contract_includes_assets(self) -> None: - resolved = GitHubLatestReleaseResolver( - name="release", - repository="DataDog/datadog-lambda-python", - include_assets=True, - ).resolve({}) - - assert resolved.repository == "DataDog/datadog-lambda-python" - assert resolved.tag_name.startswith("v") - assert resolved.assets - assert all(asset.name for asset in resolved.assets) - assert all( - asset.browser_download_url.startswith( - "https://github.com/DataDog/datadog-lambda-python/releases/download/", - ) - for asset in resolved.assets - ) - - def test_public_github_actions_artifact_contract_uses_unauthenticated_request(self) -> None: - resolved = GitHubActionsArtifactResolver( - name="workflow_artifact", - repository="DataDog/httpd-datadog", - workflow="dev.yml", - artifact_name="mod_datadog_artifact", - default_value="main", - ).resolve({}) - - assert resolved.repository == "DataDog/httpd-datadog" - assert resolved.workflow == "dev.yml" - assert resolved.branch == "main" - assert len(resolved.commit_sha) == 40 - assert all(character in "0123456789abcdef" for character in resolved.commit_sha) - assert resolved.run_url.startswith("https://github.com/DataDog/httpd-datadog/actions/runs/") - assert "mod_datadog_artifact" in resolved.artifact_name - assert resolved.archive_download_url.startswith( - "https://api.github.com/repos/DataDog/httpd-datadog/actions/artifacts/", - ) - - @pytest.mark.parametrize( - "resolver", - [ - NpmLatestResolver(name="package", package="dd-trace"), - PypiLatestResolver(name="package", package="ddtrace"), - RubygemsLatestResolver(name="package", package="datadog"), - CratesLatestResolver(name="package", package="datadog-opentelemetry"), - ], - ) - def test_public_package_registry_contracts_return_versions( - self, - resolver: NpmLatestResolver | PypiLatestResolver | RubygemsLatestResolver | CratesLatestResolver, - ) -> None: - resolved = resolver.resolve({}) - - assert resolved.module - assert resolved.version - assert any(character.isdigit() for character in resolved.version) - - @scenarios.test_the_test class Test_TargetArtifactModules: - @pytest.mark.parametrize("target", sorted(COMPONENT_GROUPS.all)) @pytest.mark.parametrize("environment", ["dev", "prod"]) - def test_every_target_has_real_staging_behavior(self, target: str, environment: str) -> None: - target_environment = load_target_environment(Path.cwd(), target, environment) - env = {} - if environment == "dev": - env = { - "AUTO_INJECT_TARGET_BRANCH": "auto-inject-branch", - "LIBRARY_TARGET_BRANCH": "library-branch", - "ORCHESTRION_TARGET_BRANCH": "orchestrion-branch", - } + def test_python_staging_emits_a_bounded_selector(self, environment: str) -> None: + target_environment = load_target_environment(Path.cwd(), "python", environment) + env = {"LIBRARY_TARGET_BRANCH": "feature-branch"} if environment == "dev" else {} resolver = FakeResolver() resolved = { artifact_resolver.name: resolver.resolve(artifact_resolver, env) @@ -966,84 +938,10 @@ def test_every_target_has_real_staging_behavior(self, target: str, environment: entries = target_environment.artifact_entries(resolved) - assert entries, f"{target} {environment} did not emit artifact entries" - assert all(entry.content.endswith("\n") for entry in entries) - assert all("placeholder" not in entry.content.lower() for entry in entries) - if environment == "prod": - assert all(":latest" not in entry.content for entry in entries) - assert all("@latest" not in entry.content for entry in entries) - - def test_c_dev_supports_independent_branch_overrides(self) -> None: - target_environment = load_target_environment(Path.cwd(), "c", "dev") - env = { - "AUTO_INJECT_TARGET_BRANCH": "auto-inject-branch", - "LIBRARY_TARGET_BRANCH": "library-branch", - } - resolver = FakeResolver() - resolved = { - artifact_resolver.name: resolver.resolve(artifact_resolver, env) - for artifact_resolver in target_environment.artifact_inputs(env) - } - - entries = {entry.filename: entry.content.strip() for entry in target_environment.artifact_entries(resolved)} - - assert entries == { - "c-injector-image": f"installtesting.datad0g.com/apm-inject-package:{SHA}", - "c-library-image": f"installtesting.datad0g.com/apm-library-c-package:{SHA}", - } - - def test_workflow_artifact_entries_are_credential_free_json(self) -> None: - target_environment = load_target_environment(Path.cwd(), "python_lambda", "dev") - env: dict[str, str] = {} - resolver = FakeResolver() - resolved = { - artifact_resolver.name: resolver.resolve(artifact_resolver, env) - for artifact_resolver in target_environment.artifact_inputs(env) - } - - entry = target_environment.artifact_entries(resolved)[0] - payload = json.loads(entry.content) - - assert entry.filename.endswith(".json") - assert payload["commit_sha"] == SHA - assert "token" not in entry.content.lower() - - def test_provider_package_selectors_have_build_consumers(self) -> None: - build_script = Path("utils/build/build.sh").read_text(encoding="utf-8") - - assert "binaries/dotnet-package-image" in build_script - assert "datadog-dotnet-apm*.tar.gz" in build_script - assert "binaries/php-package-image" in build_script - assert "dd-library-php-*-linux-gnu.tar.gz" in build_script - assert "datadog-setup.php" in build_script - - def test_staged_java_otel_selector_has_installer_consumer(self) -> None: - target_environment = load_target_environment(Path.cwd(), "java_otel", "dev") - env = {"LIBRARY_TARGET_BRANCH": "ignored"} - resolver = FakeResolver() - resolved = { - artifact_resolver.name: resolver.resolve(artifact_resolver, env) - for artifact_resolver in target_environment.artifact_inputs(env) - } - - entries = target_environment.artifact_entries(resolved) - installer = Path("utils/build/docker/java_otel/install_opentelemetry.sh").read_text(encoding="utf-8") - - assert {entry.filename for entry in entries} == {"java-otel-load-from-release"} - assert "java-otel-load-from-release" in installer - - def test_lambda_workflow_metadata_is_parsed_with_jq(self) -> None: - for installer_path, metadata_filename in ( - ( - Path("utils/build/docker/python_lambda/install_datadog_lambda.sh"), - "python-lambda-github-actions-artifact.json", - ), - ( - Path("utils/build/docker/nodejs_lambda/install_datadog_lambda.sh"), - "nodejs-lambda-github-actions-artifact.json", - ), - ): - installer = installer_path.read_text(encoding="utf-8") - - assert metadata_filename in installer - assert "jq -r '.archive_download_url'" in installer + assert len(entries) == 1 + if environment == "dev": + assert entries[0].filename == "python-load-from-s3" + assert entries[0].content == f"{SHA}\n" + else: + assert entries[0].filename == "python-load-from-pip" + assert entries[0].content == "ddtrace==1.2.3\n" diff --git a/utils/build/build.sh b/utils/build/build.sh index 20716402921..e022bf278c2 100755 --- a/utils/build/build.sh +++ b/utils/build/build.sh @@ -143,63 +143,6 @@ run_build_command() { return "${exit_code}" } -copy_staged_package_files() { - local source_dir=$1 - local package_pattern=$2 - local setup_pattern=${3:-} - local package_count - local setup_count=1 - - package_count=$(find "$source_dir" -type f -name "$package_pattern" | wc -l) - if [[ "$package_count" -eq 0 ]]; then - echo "ERROR: extracted staged package image did not contain $package_pattern" >&2 - exit 1 - fi - - if [[ -n "$setup_pattern" ]]; then - setup_count=$(find "$source_dir" -type f -name "$setup_pattern" | wc -l) - if [[ "$setup_count" -eq 0 ]]; then - echo "ERROR: extracted staged package image did not contain $setup_pattern" >&2 - exit 1 - fi - find "$source_dir" -type f -name "$setup_pattern" -exec cp {} binaries/ \; - fi - - find "$source_dir" -type f -name "$package_pattern" -exec cp {} binaries/ \; -} - -materialize_staged_package_image() { - local selector_file=$1 - local package_pattern=$2 - local setup_pattern=${3:-} - local temp_dir - local image - - image=$(<"$selector_file") - temp_dir=$(mktemp -d "${TMPDIR:-/tmp}/system-tests-staged-package.XXXXXX") - run_build_command utils/scripts/docker_base_image.sh "$image" "$temp_dir" - copy_staged_package_files "$temp_dir" "$package_pattern" "$setup_pattern" - rm -rf "$temp_dir" -} - -materialize_staged_provider_artifacts() { - if [[ $TEST_LIBRARY == dotnet ]] && [[ -f binaries/dotnet-package-image ]]; then - if [[ "$(find binaries -maxdepth 1 \( -name 'datadog-dotnet-apm*.tar.gz' -o -name 'Datadog.Trace.ClrProfiler.Native.so' \) | wc -l)" -gt 0 ]]; then - echo "Skipping staged .NET package image because local .NET artifacts already exist in binaries/" - else - materialize_staged_package_image binaries/dotnet-package-image 'datadog-dotnet-apm*.tar.gz' - fi - fi - - if [[ $TEST_LIBRARY == php ]] && [[ -f binaries/php-package-image ]]; then - if [[ "$(find binaries -maxdepth 1 \( -name 'dd-library-php-*-linux-gnu.tar.gz' -o -name 'datadog-setup.php' \) | wc -l)" -gt 0 ]]; then - echo "Skipping staged PHP package image because local PHP artifacts already exist in binaries/" - else - materialize_staged_package_image binaries/php-package-image 'dd-library-php-*-linux-gnu.tar.gz' 'datadog-setup.php' - fi - fi -} - build() { echo "==================================" @@ -359,8 +302,6 @@ build() { run_build_command docker run ${DOCKER_PLATFORM_ARGS} -v ./binaries/:/app -w /app ghcr.io/datadog/dd-trace-py/testrunner bash -c "pyenv global $PYTHON_VERSION; pip wheel --no-deps -w . /app/dd-trace-py" fi - materialize_staged_provider_artifacts - DOCKERFILE=utils/build/docker/${TEST_LIBRARY}/${WEBLOG_VARIANT}.Dockerfile BASE_IMAGE_CONTEXT_ARGS=() BASE_IMAGE_CONTEXTS=$(python3 utils/scripts/resolve-base-images-contexts.py --build-contexts "$DOCKERFILE") diff --git a/utils/build/docker/agent/artifact.py b/utils/build/docker/agent/artifact.py deleted file mode 100644 index 12c83a2ac8e..00000000000 --- a/utils/build/docker/agent/artifact.py +++ /dev/null @@ -1,15 +0,0 @@ -from __future__ import annotations - - -from utils.target_artifacts.entry_helpers import text_entry -from utils.target_artifacts.models import SimpleTarget -from utils.target_artifacts.resolvers import EnvResolver - - -class Dev(SimpleTarget): - inputs = (EnvResolver(name="agent_branch", variable_name="AGENT_TARGET_BRANCH", default_value="master-py3"),) - entries = (text_entry("agent-image", "datadog/agent-dev:{agent_branch.value}"),) - - -class Prod(Dev): - pass diff --git a/utils/build/docker/c/artifact.py b/utils/build/docker/c/artifact.py deleted file mode 100644 index 255b229c309..00000000000 --- a/utils/build/docker/c/artifact.py +++ /dev/null @@ -1,89 +0,0 @@ -from __future__ import annotations - -from typing import cast - -from utils.target_artifacts.entry_helpers import text_entry -from utils.target_artifacts.models import ( - ArtifactEntry, - BranchReference, - OciImageReference, - TargetArtifactError, -) -from utils.target_artifacts.resolvers import GitHubBranchResolver, OciDigestResolver - -type ArtifactInputResolver = GitHubBranchResolver | OciDigestResolver -type ResolvedCInput = BranchReference | OciImageReference - -PROD_LIBRARY_IMAGE = "install.datadoghq.com/apm-library-c-package:latest" -PROD_INJECTOR_IMAGE = "install.datadoghq.com/apm-inject-package:latest" -DEV_LIBRARY_IMAGE = "installtesting.datad0g.com/apm-library-c-package" -DEV_INJECTOR_IMAGE = "installtesting.datad0g.com/apm-inject-package" - - -class Dev: - def artifact_inputs(self, env: dict[str, str]) -> tuple[ArtifactInputResolver, ...]: - inputs: list[ArtifactInputResolver] = [] - if env.get("LIBRARY_TARGET_BRANCH"): - inputs.append( - GitHubBranchResolver( - name="library_branch", - repository="DataDog/dd-trace-c", - variable_name="LIBRARY_TARGET_BRANCH", - ) - ) - else: - inputs.append(OciDigestResolver(name="library_image", image=PROD_LIBRARY_IMAGE)) - - if env.get("AUTO_INJECT_TARGET_BRANCH"): - inputs.append( - GitHubBranchResolver( - name="injector_branch", - repository="DataDog/auto_inject", - variable_name="AUTO_INJECT_TARGET_BRANCH", - ) - ) - else: - inputs.append(OciDigestResolver(name="injector_image", image=PROD_INJECTOR_IMAGE)) - return tuple(inputs) - - def artifact_entries( - self, - resolved_inputs: dict[str, ResolvedCInput], - ) -> tuple[ArtifactEntry, ArtifactEntry]: - if "library_branch" in resolved_inputs: - library_branch = cast(BranchReference, resolved_inputs["library_branch"]) - library_ref = f"{DEV_LIBRARY_IMAGE}:{library_branch.sha}" - else: - library_image = cast(OciImageReference, resolved_inputs["library_image"]) - library_ref = library_image.reference - - if "injector_branch" in resolved_inputs: - injector_branch = cast(BranchReference, resolved_inputs["injector_branch"]) - injector_ref = f"{DEV_INJECTOR_IMAGE}:{injector_branch.sha}" - else: - injector_image = cast(OciImageReference, resolved_inputs["injector_image"]) - injector_ref = injector_image.reference - - return ( - text_entry("c-library-image", library_ref), - text_entry("c-injector-image", injector_ref), - ) - - -class Prod: - def artifact_inputs(self, env: dict[str, str]) -> tuple[OciDigestResolver, OciDigestResolver]: - if env.get("LIBRARY_TARGET_BRANCH") or env.get("AUTO_INJECT_TARGET_BRANCH"): - raise TargetArtifactError("Target branches can only be used with the development c packages") - return ( - OciDigestResolver(name="library_image", image=PROD_LIBRARY_IMAGE), - OciDigestResolver(name="injector_image", image=PROD_INJECTOR_IMAGE), - ) - - def artifact_entries( - self, - resolved_inputs: dict[str, OciImageReference], - ) -> tuple[ArtifactEntry, ArtifactEntry]: - return ( - text_entry("c-library-image", resolved_inputs["library_image"].reference), - text_entry("c-injector-image", resolved_inputs["injector_image"].reference), - ) diff --git a/utils/build/docker/c/perl-mojolicious.Dockerfile b/utils/build/docker/c/perl-mojolicious.Dockerfile index 1e36a65e96e..c784ded3313 100644 --- a/utils/build/docker/c/perl-mojolicious.Dockerfile +++ b/utils/build/docker/c/perl-mojolicious.Dockerfile @@ -17,11 +17,7 @@ RUN apk add --no-cache jq zstd \ output="$2"; \ manifest="$(oras manifest fetch --platform "linux/${TARGETARCH}" "$reference")"; \ digest="$(printf '%s' "$manifest" | jq -er '.layers[0].digest')"; \ - if printf '%s' "$reference" | grep -q '@'; then \ - repository="${reference%%@*}"; \ - else \ - repository="${reference%:*}"; \ - fi; \ + repository="${reference%:*}"; \ mkdir -p "$output"; \ oras blob fetch --output /tmp/package.tar.zst "${repository}@${digest}"; \ zstd --decompress --stdout /tmp/package.tar.zst | tar -x -C "$output"; \ diff --git a/utils/build/docker/cpp/artifact.py b/utils/build/docker/cpp/artifact.py deleted file mode 100644 index 2b830b713c3..00000000000 --- a/utils/build/docker/cpp/artifact.py +++ /dev/null @@ -1,26 +0,0 @@ -from __future__ import annotations - - -from utils.target_artifacts.entry_helpers import text_entry -from utils.target_artifacts.models import SimpleTarget -from utils.target_artifacts.resolvers import GitHubBranchResolver, GitHubLatestReleaseResolver - -REPOSITORY = "DataDog/dd-trace-cpp" -GIT_URL = "https://github.com/DataDog/dd-trace-cpp" - - -class Dev(SimpleTarget): - inputs = ( - GitHubBranchResolver( - name="library_branch", - repository=REPOSITORY, - variable_name="LIBRARY_TARGET_BRANCH", - default_value="main", - ), - ) - entries = (text_entry("cpp-load-from-git", f"{GIT_URL}@{{library_branch.sha}}"),) - - -class Prod(SimpleTarget): - inputs = (GitHubLatestReleaseResolver(name="release", repository=REPOSITORY),) - entries = (text_entry("cpp-load-from-git", f"{GIT_URL}@{{release.tag_name}}"),) diff --git a/utils/build/docker/cpp_httpd/artifact.py b/utils/build/docker/cpp_httpd/artifact.py deleted file mode 100644 index 9da0e13bbdc..00000000000 --- a/utils/build/docker/cpp_httpd/artifact.py +++ /dev/null @@ -1,41 +0,0 @@ -from __future__ import annotations - - -from utils.target_artifacts.entry_helpers import gha_artifact_entry, text_entry -from utils.target_artifacts.models import ( - ArtifactEntry, - GitHubActionsArtifactReference, - GitHubReleaseReference, -) -from utils.target_artifacts.resolvers import GitHubActionsArtifactResolver, GitHubLatestReleaseResolver - - -class Dev: - def artifact_inputs(self, env: dict[str, str]) -> tuple[GitHubActionsArtifactResolver]: - return ( - GitHubActionsArtifactResolver( - name="workflow_artifact", - repository="DataDog/httpd-datadog", - workflow="dev.yml", - artifact_name="mod_datadog_artifact", - variable_name="LIBRARY_TARGET_BRANCH", - default_value="main", - ), - ) - - def artifact_entries( - self, - resolved_inputs: dict[str, GitHubActionsArtifactReference], - ) -> tuple[ArtifactEntry]: - return (gha_artifact_entry("cpp-httpd-github-actions-artifact.json", resolved_inputs["workflow_artifact"]),) - - -class Prod: - def artifact_inputs(self, env: dict[str, str]) -> tuple[GitHubLatestReleaseResolver]: - return (GitHubLatestReleaseResolver(name="release", repository="DataDog/httpd-datadog"),) - - def artifact_entries( - self, - resolved_inputs: dict[str, GitHubReleaseReference], - ) -> tuple[ArtifactEntry]: - return (text_entry("cpp-httpd-load-from-release", resolved_inputs["release"].tag_name),) diff --git a/utils/build/docker/cpp_httpd/install_ddtrace.sh b/utils/build/docker/cpp_httpd/install_ddtrace.sh index 3dcbe0c5619..dece759f857 100755 --- a/utils/build/docker/cpp_httpd/install_ddtrace.sh +++ b/utils/build/docker/cpp_httpd/install_ddtrace.sh @@ -12,23 +12,9 @@ cd /binaries if [ -f "$FILENAME" ]; then echo "Install HTTPD plugin from binaries/$FILENAME" HTTPD_DATADOG_VERSION="v99.99.99" # TODO: get version from the binary. Right now, use the "big-version" trick - cp "$FILENAME" "$DEST_FOLDER/$FILENAME" -elif [ -f cpp-httpd-github-actions-artifact.json ]; then - echo "Install HTTPD plugin from staged GitHub Actions artifact metadata" - auth_header=$(get_authentication_header) - ARCHIVE_URL=$(jq -r '.archive_download_url' cpp-httpd-github-actions-artifact.json) - curl_cmd="curl -Lf $auth_header -o mod_datadog_artifact.zip ${ARCHIVE_URL}" - eval "$curl_cmd" - mkdir -p /tmp/mod-datadog-artifact - unzip -o mod_datadog_artifact.zip -d /tmp/mod-datadog-artifact - cp "$(find /tmp/mod-datadog-artifact -name "$FILENAME" | head -1)" "$DEST_FOLDER/$FILENAME" - HTTPD_DATADOG_VERSION="$(jq -r '.commit_sha' cpp-httpd-github-actions-artifact.json | cut -c1-12)" + cp $FILENAME "$DEST_FOLDER/$FILENAME" else - if [ -f cpp-httpd-load-from-release ]; then - HTTPD_DATADOG_VERSION=$(cat cpp-httpd-load-from-release) - else - HTTPD_DATADOG_VERSION="$(get_latest_release DataDog/httpd-datadog)" - fi + HTTPD_DATADOG_VERSION="$(get_latest_release DataDog/httpd-datadog)" TARBALL="mod_datadog_artifact.zip" URL="https://github.com/DataDog/httpd-datadog/releases/download/${HTTPD_DATADOG_VERSION}/${TARBALL}" echo "Get APACHE plugin from $URL" @@ -40,3 +26,4 @@ fi echo '{"status": "ok", "library": {"name": "cpp_httpd", "version": "'"$HTTPD_DATADOG_VERSION"'"}}' > /app/healthcheck.json echo "$HTTPD_DATADOG_VERSION" > SYSTEM_TESTS_LIBRARY_VERSION cat /app/healthcheck.json + diff --git a/utils/build/docker/cpp_kong/artifact.py b/utils/build/docker/cpp_kong/artifact.py deleted file mode 100644 index 7335571c91c..00000000000 --- a/utils/build/docker/cpp_kong/artifact.py +++ /dev/null @@ -1,63 +0,0 @@ -from __future__ import annotations - - -from utils.target_artifacts.entry_helpers import text_entry -from utils.target_artifacts.models import ( - ArtifactEntry, - BranchReference, - GitHubReleaseReference, -) -from utils.target_artifacts.resolvers import GitHubBranchResolver, GitHubLatestReleaseResolver - - -class Dev: - def artifact_inputs(self, env: dict[str, str]) -> tuple[GitHubBranchResolver, GitHubBranchResolver]: - return ( - GitHubBranchResolver( - name="cpp_branch", - repository="DataDog/dd-trace-cpp", - variable_name="DD_TRACE_CPP_TARGET_BRANCH", - default_value="main", - ), - GitHubBranchResolver( - name="plugin_branch", - repository="DataDog/kong-plugin-ddtrace", - variable_name="LIBRARY_TARGET_BRANCH", - default_value="main", - ), - ) - - def artifact_entries( - self, - resolved_inputs: dict[str, BranchReference], - ) -> tuple[ArtifactEntry, ArtifactEntry]: - return ( - text_entry( - "cpp-load-from-git", - f"https://github.com/DataDog/dd-trace-cpp@{resolved_inputs['cpp_branch'].sha}", - ), - text_entry( - "cpp-kong-plugin-git", - f"https://github.com/DataDog/kong-plugin-ddtrace@{resolved_inputs['plugin_branch'].sha}", - ), - ) - - -class Prod: - def artifact_inputs(self, env: dict[str, str]) -> tuple[GitHubLatestReleaseResolver, GitHubLatestReleaseResolver]: - return ( - GitHubLatestReleaseResolver(name="cpp_release", repository="DataDog/dd-trace-cpp"), - GitHubLatestReleaseResolver(name="plugin_release", repository="DataDog/kong-plugin-ddtrace"), - ) - - def artifact_entries( - self, - resolved_inputs: dict[str, GitHubReleaseReference], - ) -> tuple[ArtifactEntry, ArtifactEntry]: - return ( - text_entry( - "cpp-load-from-git", - f"https://github.com/DataDog/dd-trace-cpp@{resolved_inputs['cpp_release'].tag_name}", - ), - text_entry("cpp-kong-load-from-release", resolved_inputs["plugin_release"].tag_name), - ) diff --git a/utils/build/docker/cpp_kong/install_ddtrace.sh b/utils/build/docker/cpp_kong/install_ddtrace.sh index 3c448074b74..11ef895c154 100755 --- a/utils/build/docker/cpp_kong/install_ddtrace.sh +++ b/utils/build/docker/cpp_kong/install_ddtrace.sh @@ -31,9 +31,8 @@ elif [ -f cpp-load-from-git ]; then echo "Build libdd_trace_c.so from cpp-load-from-git" TARGET=$(cat cpp-load-from-git) URL=$(echo "$TARGET" | cut -d "@" -f 1) - REF=$(echo "$TARGET" | cut -d "@" -f 2) - git clone "$URL" dd-trace-cpp - git -C dd-trace-cpp checkout "$REF" + BRANCH=$(echo "$TARGET" | cut -d "@" -f 2) + git clone --depth 1 --branch "$BRANCH" "$URL" dd-trace-cpp cd dd-trace-cpp cmake -S . -B build \ -DDD_TRACE_BUILD_C_BINDING=ON \ @@ -97,19 +96,8 @@ if [ -n "$rock_file" ]; then elif [ -d kong-plugin-ddtrace ]; then echo "Using Kong plugin from binaries/kong-plugin-ddtrace" -elif [ -f cpp-kong-plugin-git ]; then - TARGET=$(cat cpp-kong-plugin-git) - URL=$(echo "$TARGET" | cut -d "@" -f 1) - REF=$(echo "$TARGET" | cut -d "@" -f 2) - git clone "$URL" kong-plugin-ddtrace - git -C kong-plugin-ddtrace checkout "$REF" - else - if [ -f cpp-kong-load-from-release ]; then - TAG=$(cat cpp-kong-load-from-release) - else - TAG=$(get_latest_release "DataDog/kong-plugin-ddtrace") - fi + TAG=$(get_latest_release "DataDog/kong-plugin-ddtrace") echo "Installing kong-plugin-ddtrace from latest release ${TAG}" curl -sL "https://github.com/DataDog/kong-plugin-ddtrace/archive/refs/tags/${TAG}.tar.gz" \ | tar -xz diff --git a/utils/build/docker/cpp_nginx/artifact.py b/utils/build/docker/cpp_nginx/artifact.py deleted file mode 100644 index 7335a2cabac..00000000000 --- a/utils/build/docker/cpp_nginx/artifact.py +++ /dev/null @@ -1,60 +0,0 @@ -from __future__ import annotations - -from typing import cast - -from utils.target_artifacts.entry_helpers import gha_artifact_entry, text_entry -from utils.target_artifacts.models import ( - ArtifactEntry, - GitHubActionsArtifactReference, - GitHubReleaseReference, -) -from utils.target_artifacts.resolvers import GitHubActionsArtifactResolver, GitHubLatestReleaseResolver - -type ResolvedNginxInput = GitHubActionsArtifactReference | GitHubReleaseReference - - -class Dev: - def artifact_inputs( - self, - env: dict[str, str], - ) -> tuple[GitHubActionsArtifactResolver, GitHubLatestReleaseResolver]: - return ( - GitHubActionsArtifactResolver( - name="workflow_artifact", - repository="DataDog/nginx-datadog", - workflow="system-tests.yml", - artifact_name="binaries", - variable_name="LIBRARY_TARGET_BRANCH", - default_value="master", - ignore_failed_workflow=False, - ), - GitHubLatestReleaseResolver(name="ddprof_release", repository="DataDog/ddprof"), - ) - - def artifact_entries( - self, - resolved_inputs: dict[str, ResolvedNginxInput], - ) -> tuple[ArtifactEntry, ArtifactEntry]: - artifact = cast(GitHubActionsArtifactReference, resolved_inputs["workflow_artifact"]) - ddprof_release = cast(GitHubReleaseReference, resolved_inputs["ddprof_release"]) - return ( - gha_artifact_entry("cpp-nginx-github-actions-artifact.json", artifact), - text_entry("cpp-nginx-ddprof-load-from-release", ddprof_release.tag_name), - ) - - -class Prod: - def artifact_inputs(self, env: dict[str, str]) -> tuple[GitHubLatestReleaseResolver, GitHubLatestReleaseResolver]: - return ( - GitHubLatestReleaseResolver(name="release", repository="DataDog/nginx-datadog"), - GitHubLatestReleaseResolver(name="ddprof_release", repository="DataDog/ddprof"), - ) - - def artifact_entries( - self, - resolved_inputs: dict[str, GitHubReleaseReference], - ) -> tuple[ArtifactEntry, ArtifactEntry]: - return ( - text_entry("cpp-nginx-load-from-release", resolved_inputs["release"].tag_name), - text_entry("cpp-nginx-ddprof-load-from-release", resolved_inputs["ddprof_release"].tag_name), - ) diff --git a/utils/build/docker/cpp_nginx/install_ddprof.sh b/utils/build/docker/cpp_nginx/install_ddprof.sh index 9f4063cbf01..15e51085ef8 100755 --- a/utils/build/docker/cpp_nginx/install_ddprof.sh +++ b/utils/build/docker/cpp_nginx/install_ddprof.sh @@ -4,44 +4,39 @@ set -euo pipefail # Checks in binary folder otherwise download from github ddprof_name=$(ls -1 ddprof*.xz 2> /dev/null || true) -if [ "$(echo "$ddprof_name" | wc -l)" -ge "2" ]; then +if [ "$(echo $ddprof_name | wc -l)" -ge "2" ]; then echo "Clean up the folder in ${PWD}" exit 1 fi -curl_install=$(command -v curl 2> /dev/null || true) +curl_install=$(which curl 2> /dev/null || true) -if [ -z "$curl_install" ]; then +if [ -z $curl_install ]; then echo "please install curl" exit 1 fi if [ -z "${ddprof_name}" ] || [ ! -e "${ddprof_name}" ]; then - if [ -f /binaries/cpp-nginx-ddprof-load-from-release ]; then - tag_name=$(cut -c2- /binaries/cpp-nginx-ddprof-load-from-release) - else - url_releases="https://api.github.com/repos/DataDog/ddprof/releases/latest" - echo "Could not find a version of ddprof in ${PWD}, get last release in ${url_releases}" - tag_name=$(curl -s --retry 3 "${url_releases}" | jq -r '.tag_name' | cut -c2-) - fi + url_releases="https://api.github.com/repos/DataDog/ddprof/releases/latest" + echo "Could not find a version of ddprof in ${PWD}, get last release in ${url_releases}" + tag_name=$(curl -s --retry 3 "${url_releases}" | jq -r '.tag_name' | cut -c2-) url_release="https://github.com/DataDog/ddprof/releases/download/v${tag_name}/ddprof-${tag_name}-amd64-linux.tar.xz" echo "Using $url_release" - curl -L -O -s --retry 3 "${url_release}" + curl -L -O -s --retry 3 ${url_release} ddprof_name=$(ls ddprof*.xz) else echo "using existing ddprof ${ddprof_name}" fi ddprof_install_path=${1-""} -if [ -z "$ddprof_install_path" ]; then +if [ -z ${ddprof_install_path-:""} ]; then echo "Specify install path" ddprof_install_path="/usr/local/bin/" echo "Override install path to: ${ddprof_install_path}" fi -ddprof_binary="${ddprof_install_path%/}/ddprof" -tar xvf "${ddprof_name}" ddprof/bin/ddprof -O > "$ddprof_binary" -chmod +x "$ddprof_binary" +tar xvf ${ddprof_name} ddprof/bin/ddprof -O > ${ddprof_install_path}/ddprof +chmod +x ${ddprof_install_path}/ddprof -SYSTEM_TESTS_PROFILER_VERSION=$("$ddprof_binary" --version) -echo "Profiler version: ${SYSTEM_TESTS_PROFILER_VERSION}" +SYSTEM_TESTS_PROFILER_VERSION=$(${ddprof_install_path}/ddprof --version) +echo "Profiler version: $(echo ${SYSTEM_TESTS_PROFILER_VERSION})" diff --git a/utils/build/docker/cpp_nginx/install_ddtrace.sh b/utils/build/docker/cpp_nginx/install_ddtrace.sh index b70a89e56d2..078afb5c30a 100755 --- a/utils/build/docker/cpp_nginx/install_ddtrace.sh +++ b/utils/build/docker/cpp_nginx/install_ddtrace.sh @@ -41,10 +41,12 @@ function epilogue { } version_first_is_greater() { - local v1=() - local v2=() - IFS='.' read -r -a v1 <<< "${1#v}" - IFS='.' read -r -a v2 <<< "${2#v}" + local v1=(${1//./ }) + local v2=(${2//./ }) + + # Remove the 'v' prefix from the version numbers + v1[0]=${v1[0]//v/} + v2[0]=${v2[0]//v/} # Compare the major, minor, and patch numbers for i in {0..2}; do @@ -59,69 +61,43 @@ version_first_is_greater() { return 1 } -function install_staged_binaries { - if [[ $(find /binaries -name 'ngx_http_datadog_module-*.so.tgz' | wc -l) -gt 0 ]]; then - echo "Found module in /binaries" - - if [[ $(find /binaries -name 'ngx_http_datadog_module-*.so.tgz' | wc -l) -gt 1 ]]; then - echo "ERROR: Found several ngx_http_datadog_module-*.so.tgz files in binaries/, abort." - exit 1 - fi - - NGINX_VERSION_OF_MODULE=$(find /binaries -name 'ngx_http_datadog_module-*.so.tgz' | grep -Po '(\d+\.\d+\.\d+)') - if [[ $NGINX_VERSION_OF_MODULE != "$NGINX_VERSION" ]]; then - echo "ERROR: nginx mismatch: module for $NGINX_VERSION_OF_MODULE, but base image of $NGINX_VERSION" - exit 1 - fi +if [[ $(find /binaries -name 'ngx_http_datadog_module-*.so.tgz' | wc -l) -gt 0 ]]; then + echo "Found module in /binaries" - MAIN_TARBALL=$(find /binaries -name 'ngx_http_datadog_module-*.so.tgz') - tar -xzvf "$MAIN_TARBALL" -C /usr/lib/nginx/modules - if [[ $(find /binaries -name 'ngx_http_datadog_module-*.so.debug.tgz' | wc -l) -eq 1 ]]; then - tar -xzvf /binaries/ngx_http_datadog_module-*.so.debug.tgz -C /usr/lib/nginx/modules - fi - - epilogue unknown_mod_version - exit 0 + if [[ $(find /binaries -name 'ngx_http_datadog_module-*.so.tgz' | wc -l) -gt 1 ]]; then + echo "ERROR: Found several ngx_http_datadog_module-*.so.tgz files in binaries/, abort." + exit 1 fi - if [[ -f /binaries/ngx_http_datadog_module.so ]]; then - cp -v /binaries/ngx_http_datadog_module.so /usr/lib/nginx/modules - if [[ -f /binaries/ngx_http_datadog_module.so.debug ]]; then - cp -v /binaries/ngx_http_datadog_module.so.debug /usr/lib/nginx/modules - fi + NGINX_VERSION_OF_MODULE=$(find /binaries -name 'ngx_http_datadog_module-*.so.tgz' | grep -Po '(\d+\.\d+\.\d+)') + if [[ $NGINX_VERSION_OF_MODULE != $NGINX_VERSION ]]; then + echo "ERROR: nginx mismatch: module for $NGINX_VERSION_OF_MODULE, but base image of $NGINX_VERSION" + exit 1 + fi - epilogue unknown_mod_version - exit 0 + MAIN_TARBALL=$(find /binaries -name 'ngx_http_datadog_module-*.so.tgz') + tar -xzvf "$MAIN_TARBALL" -C /usr/lib/nginx/modules + if [[ $(find /binaries -name 'ngx_http_datadog_module-*.so.debug.tgz' | wc -l) -eq 1 ]]; then + tar -xzvf /binaries/ngx_http_datadog_module-*.so.debug.tgz -C /usr/lib/nginx/modules fi -} -install_staged_binaries + epilogue unknown_mod_version + exit 0 +fi -if [[ -f /binaries/cpp-nginx-github-actions-artifact.json ]]; then - echo "Install NGINX plugin from staged GitHub Actions artifact metadata" - ARCHIVE_URL=$(jq -r '.archive_download_url' /binaries/cpp-nginx-github-actions-artifact.json) - AUTH_HEADER=() - if [[ -f /run/secrets/github_token ]]; then - AUTH_HEADER=(-H "Authorization: Bearer $(cat /run/secrets/github_token)") +if [[ -f /binaries/ngx_http_datadog_module.so ]]; then + cp -v /binaries/ngx_http_datadog_module.so /usr/lib/nginx/modules + if [[ -f /binaries/ngx_http_datadog_module.so.debug ]]; then + cp -v /binaries/ngx_http_datadog_module.so.debug /usr/lib/nginx/modules fi - curl -Lf "${AUTH_HEADER[@]}" -o /tmp/nginx-datadog-artifact.zip "$ARCHIVE_URL" - mkdir -p /tmp/nginx-datadog-artifact - unzip -o /tmp/nginx-datadog-artifact.zip -d /tmp/nginx-datadog-artifact - if [[ -f /tmp/nginx-datadog-artifact/binaries.zip ]]; then - unzip -o /tmp/nginx-datadog-artifact/binaries.zip -d /binaries - else - find /tmp/nginx-datadog-artifact -type f -name 'ngx_http_datadog_module*' -exec cp '{}' /binaries/ ';' - fi - install_staged_binaries + + epilogue unknown_mod_version + exit 0 fi get_latest_release() { - if [[ -f /binaries/cpp-nginx-load-from-release ]]; then - cat /binaries/cpp-nginx-load-from-release - else wget -qO- "https://api.github.com/repos/DataDog/nginx-datadog/releases/latest" \ | jq -r '.tag_name' - fi } get_architecture() { @@ -129,13 +105,12 @@ get_architecture() { } -if [[ -z ${NGINX_VERSION:-} ]]; then +if [ NGINX_VERSION == "" ]; then echo 1>&2 "ERROR: Missing NGINX_VERSION." exit 1 fi -ARCH=$(get_architecture) -readonly ARCH +readonly ARCH=$(get_architecture) if [[ $ARCH != "amd64" && $ARCH != "arm64" ]]; then echo 1>&2 "ERROR: Architecture ${ARCH} is not supported." @@ -146,10 +121,10 @@ FILENAME=ngx_http_datadog_module-appsec-$ARCH-$NGINX_VERSION.so if [ -f "$FILENAME" ]; then echo "Install NGINX plugin from binaries/$FILENAME" - cp "$FILENAME" /usr/lib/nginx/modules/ngx_http_datadog_module.so + cp $FILENAME /usr/lib/nginx/modules/ngx_http_datadog_module.so NGINX_DATADOG_VERSION="v99.99.99" # TODO: get version from the binary. Right now, use the "big-version" trick else - NGINX_DATADOG_VERSION="$(get_latest_release)" + readonly NGINX_DATADOG_VERSION="$(get_latest_release)" if version_first_is_greater "$NGINX_DATADOG_VERSION" "v1.1.0"; then TARBALLS=( diff --git a/utils/build/docker/dotnet/artifact.py b/utils/build/docker/dotnet/artifact.py deleted file mode 100644 index 9e94eac4269..00000000000 --- a/utils/build/docker/dotnet/artifact.py +++ /dev/null @@ -1,49 +0,0 @@ -from __future__ import annotations - - -from utils.target_artifacts.entry_helpers import provider_fetch_entries, text_entry -from utils.target_artifacts.models import ( - ArtifactEntry, - BranchReference, - GitHubReleaseReference, -) -from utils.target_artifacts.resolvers import GitHubBranchResolver, GitHubLatestReleaseResolver - - -def _normalize_branch_for_image_tag(branch_name: str) -> str: - return branch_name.replace("/", "_") - - -class Dev: - def artifact_inputs(self, env: dict[str, str]) -> tuple[GitHubBranchResolver]: - return ( - GitHubBranchResolver( - name="library_branch", - repository="DataDog/dd-trace-dotnet", - variable_name="LIBRARY_TARGET_BRANCH", - default_value="master", - ), - ) - - def artifact_entries(self, resolved_inputs: dict[str, BranchReference]) -> tuple[ArtifactEntry, ArtifactEntry]: - resolved_branch = resolved_inputs["library_branch"] - fetch_selector = ( - f"ghcr.io/datadog/dd-trace-dotnet/dd-trace-dotnet:{_normalize_branch_for_image_tag(resolved_branch.branch)}" - ) - return provider_fetch_entries( - fetch_filename="dotnet-package-image", - fetch_selector=fetch_selector, - marker_filename="dotnet-package-selection", - bounded_selector=resolved_branch.sha, - ) - - -class Prod: - def artifact_inputs(self, env: dict[str, str]) -> tuple[GitHubLatestReleaseResolver]: - return (GitHubLatestReleaseResolver(name="release", repository="DataDog/dd-trace-dotnet"),) - - def artifact_entries( - self, - resolved_inputs: dict[str, GitHubReleaseReference], - ) -> tuple[ArtifactEntry]: - return (text_entry("dotnet-load-from-release", resolved_inputs["release"].tag_name),) diff --git a/utils/build/docker/dotnet/install_ddtrace.sh b/utils/build/docker/dotnet/install_ddtrace.sh index e8943d6470c..f785cf445cd 100755 --- a/utils/build/docker/dotnet/install_ddtrace.sh +++ b/utils/build/docker/dotnet/install_ddtrace.sh @@ -17,39 +17,33 @@ get_latest_release() { echo "Failed to get latest release" exit 1 fi - echo "$releases" | grep '"tag_name":' | sed -E 's/.*"v([^"]+)".*/\1/'; + echo $releases | grep '"tag_name":' | sed -E 's/.*"v([^"]+)".*/\1/'; } mkdir -p /opt/datadog -native_file_count=$(find /binaries -maxdepth 1 -name 'Datadog.Trace.ClrProfiler.Native.so' | wc -l) -if [ "$native_file_count" = 1 ]; then +if [ $(ls /binaries/Datadog.Trace.ClrProfiler.Native.so | wc -l) = 1 ]; then echo "Install ddtrace from local folder" cp -r /binaries/* /opt/datadog/ else - tarball_count=$(find /binaries -maxdepth 1 -name 'datadog-dotnet-apm*.tar.gz' | wc -l) - if [ "$tarball_count" = 1 ]; then - echo "Install ddtrace from $(find /binaries -maxdepth 1 -name 'datadog-dotnet-apm*.tar.gz')" + if [ $(ls datadog-dotnet-apm*.tar.gz | wc -l) = 1 ]; then + echo "Install ddtrace from $(ls datadog-dotnet-apm*.tar.gz)" else echo "Install ddtrace from github releases" - if [ -f /binaries/dotnet-load-from-release ]; then - DDTRACE_VERSION="$(cat /binaries/dotnet-load-from-release)" - DDTRACE_VERSION="${DDTRACE_VERSION#v}" - elif ! DDTRACE_VERSION="$(get_latest_release DataDog/dd-trace-dotnet)"; then + if ! DDTRACE_VERSION="$(get_latest_release DataDog/dd-trace-dotnet)"; then echo "Failed to get latest release version" exit 1 fi - if [ "$(uname -m)" = "aarch64" ]; then + if [ $(uname -m) = "aarch64" ]; then artifact=datadog-dotnet-apm-${DDTRACE_VERSION}.arm64.tar.gz else artifact=datadog-dotnet-apm-${DDTRACE_VERSION}.tar.gz fi echo "Using artifact ${artifact}" - curl -L --fail "${GITHUB_AUTH_HEADER[@]}" "https://github.com/DataDog/dd-trace-dotnet/releases/download/v${DDTRACE_VERSION}/${artifact}" --output "${artifact}" + curl -L --fail "${GITHUB_AUTH_HEADER[@]}" https://github.com/DataDog/dd-trace-dotnet/releases/download/v${DDTRACE_VERSION}/${artifact} --output ${artifact} fi - tarball=$(find /binaries -maxdepth 1 -name 'datadog-dotnet-apm*.tar.gz') - tar xzf "$tarball" -C /opt/datadog + tar xzf $(ls datadog-dotnet-apm*.tar.gz) -C /opt/datadog fi diff --git a/utils/build/docker/golang/artifact.py b/utils/build/docker/golang/artifact.py deleted file mode 100644 index 9ef119ebdf3..00000000000 --- a/utils/build/docker/golang/artifact.py +++ /dev/null @@ -1,114 +0,0 @@ -from __future__ import annotations - -from typing import cast - -from utils.target_artifacts.entry_helpers import text_entry -from utils.target_artifacts.models import ( - ArtifactEntry, - BranchReference, - ModuleVersion, - OciImageReference, -) -from utils.target_artifacts.resolvers import GitHubBranchResolver, GoModuleLatestResolver, OciDigestResolver - -type ArtifactInputResolver = GitHubBranchResolver | GoModuleLatestResolver | OciDigestResolver -type ResolvedGoInput = BranchReference | ModuleVersion | OciImageReference - -GO_MODULES = ( - "github.com/DataDog/dd-trace-go/v2", - "github.com/DataDog/dd-trace-go/contrib/database/sql/v2", - "github.com/DataDog/dd-trace-go/contrib/net/http/v2", - "github.com/DataDog/dd-trace-go/contrib/google.golang.org/grpc/v2", - "github.com/DataDog/dd-trace-go/contrib/99designs/gqlgen/v2", - "github.com/DataDog/dd-trace-go/contrib/gin-gonic/gin/v2", - "github.com/DataDog/dd-trace-go/contrib/graphql-go/graphql/v2", - "github.com/DataDog/dd-trace-go/contrib/graph-gophers/graphql-go/v2", - "github.com/DataDog/dd-trace-go/contrib/go-chi/chi.v5/v2", - "github.com/DataDog/dd-trace-go/contrib/IBM/sarama/v2", - "github.com/DataDog/dd-trace-go/contrib/labstack/echo.v4/v2", - "github.com/DataDog/dd-trace-go/contrib/sirupsen/logrus/v2", -) - -DEV_SERVICE_EXTENSIONS_IMAGE = "ghcr.io/datadog/dd-trace-go/service-extensions-callout:dev" -DEV_HAPROXY_SPOA_IMAGE = "ghcr.io/datadog/dd-trace-go/haproxy-spoa:dev" -PROD_SERVICE_EXTENSIONS_IMAGE = "ghcr.io/datadog/dd-trace-go/service-extensions-callout:latest" -PROD_HAPROXY_SPOA_IMAGE = "ghcr.io/datadog/dd-trace-go/haproxy-spoa:latest" - - -class Dev: - def artifact_inputs(self, env: dict[str, str]) -> tuple[ArtifactInputResolver, ...]: - inputs: list[ArtifactInputResolver] = [ - GitHubBranchResolver( - name="library_branch", - repository="DataDog/dd-trace-go", - variable_name="LIBRARY_TARGET_BRANCH", - default_value="main", - ), - OciDigestResolver(name="service_extensions_image", image=DEV_SERVICE_EXTENSIONS_IMAGE), - OciDigestResolver(name="haproxy_spoa_image", image=DEV_HAPROXY_SPOA_IMAGE), - ] - if env.get("ORCHESTRION_TARGET_BRANCH"): - inputs.append( - GitHubBranchResolver( - name="orchestrion_branch", - repository="DataDog/orchestrion", - variable_name="ORCHESTRION_TARGET_BRANCH", - ) - ) - else: - inputs.append(GoModuleLatestResolver(name="orchestrion_version", module="github.com/DataDog/orchestrion")) - return tuple(inputs) - - def artifact_entries( - self, - resolved_inputs: dict[str, ResolvedGoInput], - ) -> tuple[ArtifactEntry, ArtifactEntry, ArtifactEntry, ArtifactEntry]: - library_branch = cast(BranchReference, resolved_inputs["library_branch"]) - sha = library_branch.sha - return _entries_for_go_ref(resolved_inputs, sha) - - -class Prod: - def artifact_inputs( - self, - env: dict[str, str], - ) -> tuple[GoModuleLatestResolver, GoModuleLatestResolver, OciDigestResolver, OciDigestResolver]: - return ( - GoModuleLatestResolver(name="library_version", module="github.com/DataDog/dd-trace-go/v2"), - GoModuleLatestResolver(name="orchestrion_version", module="github.com/DataDog/orchestrion"), - OciDigestResolver(name="service_extensions_image", image=PROD_SERVICE_EXTENSIONS_IMAGE), - OciDigestResolver(name="haproxy_spoa_image", image=PROD_HAPROXY_SPOA_IMAGE), - ) - - def artifact_entries( - self, - resolved_inputs: dict[str, ResolvedGoInput], - ) -> tuple[ArtifactEntry, ArtifactEntry, ArtifactEntry, ArtifactEntry]: - library_version = cast(ModuleVersion, resolved_inputs["library_version"]) - version = library_version.version - return _entries_for_go_ref(resolved_inputs, version) - - -def _entries_for_go_ref( - resolved_inputs: dict[str, ResolvedGoInput], - go_ref: str, -) -> tuple[ArtifactEntry, ArtifactEntry, ArtifactEntry, ArtifactEntry]: - if "orchestrion_branch" in resolved_inputs: - orchestrion_branch = cast(BranchReference, resolved_inputs["orchestrion_branch"]) - orchestrion_ref = orchestrion_branch.sha - else: - orchestrion_version = cast(ModuleVersion, resolved_inputs["orchestrion_version"]) - orchestrion_ref = orchestrion_version.version - - service_extensions_image = cast(OciImageReference, resolved_inputs["service_extensions_image"]) - haproxy_spoa_image = cast(OciImageReference, resolved_inputs["haproxy_spoa_image"]) - - return ( - text_entry("golang-load-from-go-get", "\n".join(f"{module}@{go_ref}" for module in GO_MODULES)), - text_entry("orchestrion-load-from-go-get", f"github.com/DataDog/orchestrion@{orchestrion_ref}"), - text_entry( - "golang-service-extensions-callout-image", - service_extensions_image.reference, - ), - text_entry("golang-haproxy-spoa-image", haproxy_spoa_image.reference), - ) diff --git a/utils/build/docker/java/artifact.py b/utils/build/docker/java/artifact.py deleted file mode 100644 index b9297c9f19f..00000000000 --- a/utils/build/docker/java/artifact.py +++ /dev/null @@ -1,23 +0,0 @@ -from __future__ import annotations - - -from utils.target_artifacts.entry_helpers import text_entry -from utils.target_artifacts.models import ArtifactEntry, SimpleTarget -from utils.target_artifacts.resolvers import GitHubBranchResolver, GitHubLatestReleaseResolver - - -class Dev(SimpleTarget): - inputs = ( - GitHubBranchResolver( - name="library_branch", - repository="DataDog/dd-trace-java", - variable_name="LIBRARY_TARGET_BRANCH", - default_value="master", - ), - ) - entries = (text_entry("java-load-from-s3", "{library_branch.sha}"),) - - -class Prod(SimpleTarget): - inputs = (GitHubLatestReleaseResolver(name="release", repository="DataDog/dd-trace-java"),) - entries = (text_entry("java-load-from-release", "{release.tag_name}"),) diff --git a/utils/build/docker/java/install_ddtrace.sh b/utils/build/docker/java/install_ddtrace.sh index 13eb98509ca..20d67c955ac 100755 --- a/utils/build/docker/java/install_ddtrace.sh +++ b/utils/build/docker/java/install_ddtrace.sh @@ -14,15 +14,10 @@ install_custom_jar() { echo "Using default $artifact_id" elif [ "$jar_count" = 1 ]; then [[ "$#" -lt 3 ]] && MVN_OPTS= || MVN_OPTS="$3" - local mvn_args=() - if [[ -n "$MVN_OPTS" ]]; then - read -r -a mvn_args <<< "$MVN_OPTS" - fi local custom_jar custom_jar=$(find /binaries/ -name "${jar_pattern}") echo "Using custom $artifact_id: ${custom_jar}" - mvn -Dfile="$custom_jar" -DgroupId=com.datadoghq -DartifactId="$artifact_id" \ - -Dversion=9999 -Dpackaging=jar "${mvn_args[@]}" install:install-file + mvn -Dfile="$custom_jar" -DgroupId=com.datadoghq -DartifactId="$artifact_id" -Dversion=9999 -Dpackaging=jar $MVN_OPTS install:install-file else echo "Too many $artifact_id within binaries folder" exit 1 @@ -38,23 +33,14 @@ install_custom_jar "dd-trace-api*.jar" "dd-trace-api" "$MVN_OPTS" install_custom_jar "dd-openfeature*.jar" "dd-openfeature" "$MVN_OPTS" # Look for custom dd-trace-java jar in custom binaries folder -if [ "$(find /binaries -maxdepth 1 -name 'dd-java-agent*.jar' | wc -l)" = 0 ]; then - if [ -f /binaries/java-load-from-s3 ]; then - GIT_REF=$(cat /binaries/java-load-from-s3) - BUILD_URL="https://s3.us-east-1.amazonaws.com/dd-trace-java-builds/${GIT_REF}/dd-java-agent.jar" - elif [ -f /binaries/java-load-from-release ]; then - RELEASE_TAG=$(cat /binaries/java-load-from-release) - BUILD_URL="https://github.com/DataDog/dd-trace-java/releases/download/${RELEASE_TAG}/dd-java-agent.jar" - else - BUILD_URL="https://github.com/DataDog/dd-trace-java/releases/latest/download/dd-java-agent.jar" - fi - echo "install from reference: $BUILD_URL" - curl -Lf -o /dd-tracer/dd-java-agent.jar "$BUILD_URL" +if [ $(ls /binaries/dd-java-agent*.jar | wc -l) = 0 ]; then + BUILD_URL="https://github.com/DataDog/dd-trace-java/releases/latest/download/dd-java-agent.jar" + echo "install from Github release: $BUILD_URL" + curl -Lf -o /dd-tracer/dd-java-agent.jar $BUILD_URL -elif [ "$(find /binaries -maxdepth 1 -name 'dd-java-agent*.jar' | wc -l)" = 1 ]; then - CUSTOM_JAR=$(find /binaries -maxdepth 1 -name 'dd-java-agent*.jar') - echo "Install local file $CUSTOM_JAR" - cp "$CUSTOM_JAR" /dd-tracer/dd-java-agent.jar +elif [ $(ls /binaries/dd-java-agent*.jar | wc -l) = 1 ]; then + echo "Install local file $(ls /binaries/dd-java-agent*.jar)" + cp $(ls /binaries/dd-java-agent*.jar) /dd-tracer/dd-java-agent.jar else echo "Too many jar files in binaries" @@ -65,4 +51,7 @@ java -jar /dd-tracer/dd-java-agent.jar > /binaries/SYSTEM_TESTS_LIBRARY_VERSION echo "Installed $(cat /binaries/SYSTEM_TESTS_LIBRARY_VERSION) java library" +SYSTEM_TESTS_LIBRARY_VERSION=$(cat /binaries/SYSTEM_TESTS_LIBRARY_VERSION) + echo "dd-trace version: $(cat /binaries/SYSTEM_TESTS_LIBRARY_VERSION)" + diff --git a/utils/build/docker/java/parametric/install_ddtrace.sh b/utils/build/docker/java/parametric/install_ddtrace.sh index a40c0229634..580bfc7c6e4 100755 --- a/utils/build/docker/java/parametric/install_ddtrace.sh +++ b/utils/build/docker/java/parametric/install_ddtrace.sh @@ -31,17 +31,8 @@ configure_custom_jar "dd-openfeature*.jar" "dd-openfeature" "customDdOpenfeature # Look for custom dd-java-agent jar in custom binaries folder CUSTOM_DD_JAVA_AGENT_COUNT=$(find /binaries/dd-java-agent*.jar 2>/dev/null | wc -l) if [ "$CUSTOM_DD_JAVA_AGENT_COUNT" = 0 ]; then - if [ -f /binaries/java-load-from-s3 ]; then - GIT_REF=$(cat /binaries/java-load-from-s3) - BUILD_URL="https://s3.us-east-1.amazonaws.com/dd-trace-java-builds/${GIT_REF}/dd-java-agent.jar" - elif [ -f /binaries/java-load-from-release ]; then - RELEASE_TAG=$(cat /binaries/java-load-from-release) - BUILD_URL="https://github.com/DataDog/dd-trace-java/releases/download/${RELEASE_TAG}/dd-java-agent.jar" - else - BUILD_URL="https://github.com/DataDog/dd-trace-java/releases/latest/download/dd-java-agent.jar" - fi - echo "Using dd-java-agent from $BUILD_URL" - wget -O /client/tracer/dd-java-agent.jar --no-cache "$BUILD_URL" + echo "Using latest dd-java-agent" + wget -O /client/tracer/dd-java-agent.jar --no-cache https://github.com/DataDog/dd-trace-java/releases/latest/download/dd-java-agent.jar elif [ "$CUSTOM_DD_JAVA_AGENT_COUNT" = 1 ]; then CUSTOM_DD_JAVA_AGENT=$(find /binaries/dd-java-agent*.jar) echo "Using custom dd-java-agent: ${CUSTOM_DD_JAVA_AGENT}" diff --git a/utils/build/docker/java_lambda/artifact.py b/utils/build/docker/java_lambda/artifact.py deleted file mode 100644 index b9297c9f19f..00000000000 --- a/utils/build/docker/java_lambda/artifact.py +++ /dev/null @@ -1,23 +0,0 @@ -from __future__ import annotations - - -from utils.target_artifacts.entry_helpers import text_entry -from utils.target_artifacts.models import ArtifactEntry, SimpleTarget -from utils.target_artifacts.resolvers import GitHubBranchResolver, GitHubLatestReleaseResolver - - -class Dev(SimpleTarget): - inputs = ( - GitHubBranchResolver( - name="library_branch", - repository="DataDog/dd-trace-java", - variable_name="LIBRARY_TARGET_BRANCH", - default_value="master", - ), - ) - entries = (text_entry("java-load-from-s3", "{library_branch.sha}"),) - - -class Prod(SimpleTarget): - inputs = (GitHubLatestReleaseResolver(name="release", repository="DataDog/dd-trace-java"),) - entries = (text_entry("java-load-from-release", "{release.tag_name}"),) diff --git a/utils/build/docker/java_otel/artifact.py b/utils/build/docker/java_otel/artifact.py deleted file mode 100644 index c26eba19f2f..00000000000 --- a/utils/build/docker/java_otel/artifact.py +++ /dev/null @@ -1,17 +0,0 @@ -from __future__ import annotations - - -from utils.target_artifacts.entry_helpers import text_entry -from utils.target_artifacts.models import SimpleTarget -from utils.target_artifacts.resolvers import GitHubLatestReleaseResolver - -REPOSITORY = "open-telemetry/opentelemetry-java-instrumentation" - - -class Dev(SimpleTarget): - inputs = (GitHubLatestReleaseResolver(name="release", repository=REPOSITORY),) - entries = (text_entry("java-otel-load-from-release", "{release.tag_name}"),) - - -class Prod(Dev): - pass diff --git a/utils/build/docker/java_otel/install_opentelemetry.sh b/utils/build/docker/java_otel/install_opentelemetry.sh index 80a33ae5e57..47f9f36bd0f 100755 --- a/utils/build/docker/java_otel/install_opentelemetry.sh +++ b/utils/build/docker/java_otel/install_opentelemetry.sh @@ -6,14 +6,9 @@ mkdir /otel-tracer # shellcheck disable=SC2012 if [ "$(ls /binaries/opentelemetry-javaagent*.jar | wc -l)" = 0 ]; then - if [ -f /binaries/java-otel-load-from-release ]; then - RELEASE_TAG=$(cat /binaries/java-otel-load-from-release) - BUILD_URL="https://github.com/open-telemetry/opentelemetry-java-instrumentation/releases/download/${RELEASE_TAG}/opentelemetry-javaagent.jar" - else - BUILD_URL="https://github.com/open-telemetry/opentelemetry-java-instrumentation/releases/latest/download/opentelemetry-javaagent.jar" - fi + BUILD_URL="https://github.com/open-telemetry/opentelemetry-java-instrumentation/releases/latest/download/opentelemetry-javaagent.jar" echo "install from Github release: $BUILD_URL" - curl -Lf -o /otel-tracer/opentelemetry-javaagent.jar "$BUILD_URL" + curl -Lf -o /otel-tracer/opentelemetry-javaagent.jar $BUILD_URL elif [ "$(ls /binaries/opentelemetry-javaagent*.jar | wc -l)" = 1 ]; then echo "Install local file $(ls /binaries/opentelemetry-javaagent*.jar)" @@ -27,3 +22,4 @@ fi java -jar /otel-tracer/opentelemetry-javaagent.jar > /binaries/SYSTEM_TESTS_LIBRARY_VERSION echo "opentelemetry-javaagent version: $(cat /binaries/SYSTEM_TESTS_LIBRARY_VERSION)" + diff --git a/utils/build/docker/nodejs/artifact.py b/utils/build/docker/nodejs/artifact.py deleted file mode 100644 index e54cf2aded6..00000000000 --- a/utils/build/docker/nodejs/artifact.py +++ /dev/null @@ -1,23 +0,0 @@ -from __future__ import annotations - - -from utils.target_artifacts.entry_helpers import text_entry -from utils.target_artifacts.models import SimpleTarget -from utils.target_artifacts.resolvers import GitHubBranchResolver, NpmLatestResolver - - -class Dev(SimpleTarget): - inputs = ( - GitHubBranchResolver( - name="library_branch", - repository="DataDog/dd-trace-js", - variable_name="LIBRARY_TARGET_BRANCH", - default_value="master", - ), - ) - entries = (text_entry("nodejs-load-from-npm", "DataDog/dd-trace-js#{library_branch.sha}"),) - - -class Prod(SimpleTarget): - inputs = (NpmLatestResolver(name="dd-trace", package="dd-trace"),) - entries = (text_entry("nodejs-load-from-npm", "dd-trace@{dd-trace.version}"),) diff --git a/utils/build/docker/nodejs_lambda/artifact.py b/utils/build/docker/nodejs_lambda/artifact.py deleted file mode 100644 index 87746235fbc..00000000000 --- a/utils/build/docker/nodejs_lambda/artifact.py +++ /dev/null @@ -1,42 +0,0 @@ -from __future__ import annotations - - -from utils.target_artifacts.entry_helpers import gha_artifact_entry, text_entry -from utils.target_artifacts.models import ( - ArtifactEntry, - GitHubActionsArtifactReference, - GitHubReleaseReference, -) -from utils.target_artifacts.resolvers import GitHubActionsArtifactResolver, GitHubLatestReleaseResolver - - -class Dev: - def artifact_inputs(self, env: dict[str, str]) -> tuple[GitHubActionsArtifactResolver]: - return ( - GitHubActionsArtifactResolver( - name="workflow_artifact", - repository="DataDog/datadog-lambda-js", - workflow="build_layer.yml", - artifact_name="datadog_lambda_node18.12", - variable_name="LIBRARY_TARGET_BRANCH", - default_value="main", - ignore_failed_workflow=False, - ), - ) - - def artifact_entries( - self, - resolved_inputs: dict[str, GitHubActionsArtifactReference], - ) -> tuple[ArtifactEntry]: - return (gha_artifact_entry("nodejs-lambda-github-actions-artifact.json", resolved_inputs["workflow_artifact"]),) - - -class Prod: - def artifact_inputs(self, env: dict[str, str]) -> tuple[GitHubLatestReleaseResolver]: - return (GitHubLatestReleaseResolver(name="release", repository="DataDog/datadog-lambda-js"),) - - def artifact_entries( - self, - resolved_inputs: dict[str, GitHubReleaseReference], - ) -> tuple[ArtifactEntry]: - return (text_entry("nodejs-lambda-load-from-release", resolved_inputs["release"].tag_name),) diff --git a/utils/build/docker/nodejs_lambda/install_datadog_lambda.sh b/utils/build/docker/nodejs_lambda/install_datadog_lambda.sh index a8317701c03..7ecc43ac725 100755 --- a/utils/build/docker/nodejs_lambda/install_datadog_lambda.sh +++ b/utils/build/docker/nodejs_lambda/install_datadog_lambda.sh @@ -9,6 +9,7 @@ if [ "$(find . -maxdepth 1 -name "*.zip" | wc -l)" = "1" ]; then echo "Install datadog_lambda from ${path}" unzip "${path}" -d /opt else + echo "Fetching from latest GitHub release..." NODE_MAJOR=$(node -e "console.log(process.version.split('.')[0].slice(1))") # Map major version to the runtime version used by datadog-lambda-js release assets. # See https://github.com/DataDog/datadog-lambda-js/blob/main/.gitlab/datasources/runtimes.yaml @@ -21,35 +22,15 @@ else esac echo "Detected Node.js major: ${NODE_MAJOR}, using layer runtime version: ${NODE_VERSION}" + LATEST_TAG=$(curl -fsSL -H "Accept: application/vnd.github.v3+json" \ + https://api.github.com/repos/DataDog/datadog-lambda-js/releases/latest \ + | grep '"tag_name"' | head -1 | sed 's/.*"tag_name": *"//;s/".*//') + echo "Latest release tag: ${LATEST_TAG}" + ZIP_NAME="datadog_lambda_node${NODE_VERSION}.zip" - if [ -f nodejs-lambda-github-actions-artifact.json ]; then - echo "Fetching from staged GitHub Actions artifact metadata..." - ARCHIVE_URL=$(jq -r '.archive_download_url' nodejs-lambda-github-actions-artifact.json) - if [ -z "$ARCHIVE_URL" ] || [ "$ARCHIVE_URL" = "null" ]; then - echo "Staged GitHub Actions artifact metadata is missing archive_download_url" - exit 1 - fi - GITHUB_AUTH_HEADER=() - if [ -f /run/secrets/github_token ]; then - GITHUB_AUTH_HEADER=(-H "Authorization: Bearer $(cat /run/secrets/github_token)") - fi - curl -fsSL "${GITHUB_AUTH_HEADER[@]}" -o /tmp/nodejs-lambda-artifact.zip "$ARCHIVE_URL" - mkdir -p /tmp/nodejs-lambda-artifact - unzip -o /tmp/nodejs-lambda-artifact.zip -d /tmp/nodejs-lambda-artifact - cp "$(find /tmp/nodejs-lambda-artifact -name "$ZIP_NAME" | head -1)" . - else - if [ -f nodejs-lambda-load-from-release ]; then - LATEST_TAG=$(cat nodejs-lambda-load-from-release) - else - LATEST_TAG=$(curl -fsSL -H "Accept: application/vnd.github.v3+json" \ - https://api.github.com/repos/DataDog/datadog-lambda-js/releases/latest \ - | grep '"tag_name"' | head -1 | sed 's/.*"tag_name": *"//;s/".*//') - fi - echo "Release tag: ${LATEST_TAG}" - DOWNLOAD_URL="https://github.com/DataDog/datadog-lambda-js/releases/download/${LATEST_TAG}/${ZIP_NAME}" - echo "Downloading ${DOWNLOAD_URL}" - curl -fsSLO "${DOWNLOAD_URL}" - fi + DOWNLOAD_URL="https://github.com/DataDog/datadog-lambda-js/releases/download/${LATEST_TAG}/${ZIP_NAME}" + echo "Downloading ${DOWNLOAD_URL}" + curl -fsSLO "${DOWNLOAD_URL}" if [ ! -f "${ZIP_NAME}" ]; then echo "Failed to download ${ZIP_NAME}" diff --git a/utils/build/docker/nodejs_otel/artifact.py b/utils/build/docker/nodejs_otel/artifact.py deleted file mode 100644 index d0e33780ba2..00000000000 --- a/utils/build/docker/nodejs_otel/artifact.py +++ /dev/null @@ -1,17 +0,0 @@ -from __future__ import annotations - - -from utils.target_artifacts.entry_helpers import text_entry -from utils.target_artifacts.models import SimpleTarget -from utils.target_artifacts.resolvers import NpmLatestResolver - -PACKAGE_NAME = "@opentelemetry/auto-instrumentations-node" - - -class Dev(SimpleTarget): - inputs = (NpmLatestResolver(name="otel_package", package=PACKAGE_NAME),) - entries = (text_entry("nodejs-otel-load-from-npm", f"{PACKAGE_NAME}@{{otel_package.version}}"),) - - -class Prod(Dev): - pass diff --git a/utils/build/docker/nodejs_otel/express4-otel.Dockerfile b/utils/build/docker/nodejs_otel/express4-otel.Dockerfile index ba16560312a..304eb701d6e 100644 --- a/utils/build/docker/nodejs_otel/express4-otel.Dockerfile +++ b/utils/build/docker/nodejs_otel/express4-otel.Dockerfile @@ -15,8 +15,6 @@ COPY utils/build/docker/nodejs/express /usr/app #overwrite app.js and package files COPY utils/build/docker/nodejs_otel/express4-otel /usr/app RUN npm ci || (sleep 30 && npm ci) -COPY binaries* /binaries/ -RUN if [ -f /binaries/nodejs-otel-load-from-npm ]; then npm install "$(cat /binaries/nodejs-otel-load-from-npm)"; fi EXPOSE 7777 diff --git a/utils/build/docker/otel_collector/artifact.py b/utils/build/docker/otel_collector/artifact.py deleted file mode 100644 index 36d4347c96d..00000000000 --- a/utils/build/docker/otel_collector/artifact.py +++ /dev/null @@ -1,24 +0,0 @@ -from __future__ import annotations - - -from utils.target_artifacts.entry_helpers import text_entry -from utils.target_artifacts.models import SimpleTarget -from utils.target_artifacts.resolvers import OciDigestResolver - -DEFAULT_IMAGE = "otel/opentelemetry-collector-contrib:0.137.0" - - -class Dev(SimpleTarget): - inputs = ( - OciDigestResolver( - name="collector_image", - image=DEFAULT_IMAGE, - variable_name="OTEL_COLLECTOR_IMAGE", - ), - ) - entries = (text_entry("otel_collector-image", "{collector_image.reference}"),) - - -class Prod(SimpleTarget): - inputs = (OciDigestResolver(name="collector_image", image=DEFAULT_IMAGE),) - entries = (text_entry("otel_collector-image", "{collector_image.reference}"),) diff --git a/utils/build/docker/php/artifact.py b/utils/build/docker/php/artifact.py deleted file mode 100644 index 42461c39685..00000000000 --- a/utils/build/docker/php/artifact.py +++ /dev/null @@ -1,55 +0,0 @@ -from __future__ import annotations - -import re - -from utils.target_artifacts.entry_helpers import provider_fetch_entries, text_entry -from utils.target_artifacts.models import ( - ArtifactEntry, - BranchReference, - GitHubReleaseReference, -) -from utils.target_artifacts.resolvers import GitHubBranchResolver, GitHubLatestReleaseResolver - - -def _normalize_branch_for_image_tag(branch_name: str) -> str: - value = re.sub(r"[^a-z0-9]+", "-", branch_name.lower()) - value = re.sub(r"-+", "-", value).strip("-") - return value[:63].rstrip("-") - - -class Dev: - def artifact_inputs(self, env: dict[str, str]) -> tuple[GitHubBranchResolver]: - return ( - GitHubBranchResolver( - name="library_branch", - repository="DataDog/dd-trace-php", - variable_name="LIBRARY_TARGET_BRANCH", - default_value="master", - ), - ) - - def artifact_entries( - self, - resolved_inputs: dict[str, BranchReference], - ) -> tuple[ArtifactEntry, ArtifactEntry]: - resolved_branch = resolved_inputs["library_branch"] - fetch_selector = ( - f"ghcr.io/datadog/dd-trace-php/dd-library-php:{_normalize_branch_for_image_tag(resolved_branch.branch)}" - ) - return provider_fetch_entries( - fetch_filename="php-package-image", - fetch_selector=fetch_selector, - marker_filename="php-package-selection", - bounded_selector=resolved_branch.sha, - ) - - -class Prod: - def artifact_inputs(self, env: dict[str, str]) -> tuple[GitHubLatestReleaseResolver]: - return (GitHubLatestReleaseResolver(name="release", repository="DataDog/dd-trace-php"),) - - def artifact_entries( - self, - resolved_inputs: dict[str, GitHubReleaseReference], - ) -> tuple[ArtifactEntry]: - return (text_entry("php-load-from-release", resolved_inputs["release"].tag_name),) diff --git a/utils/build/docker/php/common/install_ddtrace.sh b/utils/build/docker/php/common/install_ddtrace.sh index 40ff25da7ec..a05b7411263 100755 --- a/utils/build/docker/php/common/install_ddtrace.sh +++ b/utils/build/docker/php/common/install_ddtrace.sh @@ -52,8 +52,8 @@ if [ -d /opt/php/nts ]; then elif [[ $IS_APACHE -eq 0 ]]; then PHP_VERSION=$(php -r "echo PHP_MAJOR_VERSION.'.'.PHP_MINOR_VERSION;") INI_FILE=/etc/php/$PHP_VERSION/fpm/conf.d/98-ddtrace.ini - mkdir -p "$(dirname "$INI_FILE")" - chmod 777 "$(dirname "$INI_FILE")" + mkdir -p $(dirname $INI_FILE) + chmod 777 $(dirname $INI_FILE) fi # Always install from package first (to get recommended.json and other files) @@ -63,13 +63,8 @@ if [ "$PKG" != "" ] && [ ! -f "$SETUP" ]; then fi if [ "$PKG" == "" ]; then - if [ -f /binaries/php-load-from-release ]; then - RELEASE_TAG=$(cat /binaries/php-load-from-release) - curl -LO "https://github.com/DataDog/dd-trace-php/releases/download/${RELEASE_TAG}/datadog-setup.php" - else - # Download latest release for compatibility when artifact staging has not run. - curl -LO https://github.com/DataDog/dd-trace-php/releases/latest/download/datadog-setup.php - fi + #Download latest release + curl -LO https://github.com/DataDog/dd-trace-php/releases/latest/download/datadog-setup.php SETUP=datadog-setup.php unset PKG @@ -106,26 +101,26 @@ else fi # After package installation, override with custom ddtrace.so if present -if [ -f "$DDTRACE_SO" ]; then +if [ -f $DDTRACE_SO ]; then echo "Overriding package ddtrace.so with custom binary from $DDTRACE_SO" # Find and replace the installed ddtrace.so with custom one INSTALLED_DDTRACE=$(find /root /opt /usr/lib/php -name ddtrace.so 2>/dev/null | grep -v /binaries | head -1) if [ -n "$INSTALLED_DDTRACE" ]; then echo "Found installed ddtrace.so at $INSTALLED_DDTRACE, replacing with custom binary" - cp -f "$DDTRACE_SO" "$INSTALLED_DDTRACE" + cp -f $DDTRACE_SO $INSTALLED_DDTRACE else echo "Warning: Could not find installed ddtrace.so to replace" fi fi # After package installation, override with custom ddappsec.so and helper if present -if [ -f "$DDAPPSEC_SO" ] && [ -f "$APPSEC_HELPER_SO" ]; then +if [ -f $DDAPPSEC_SO ] && [ -f $APPSEC_HELPER_SO ]; then echo "Overriding package ddappsec.so and helper with custom binaries" # Find and replace the installed ddappsec.so INSTALLED_DDAPPSEC=$(find /root /opt /usr/lib/php -name ddappsec.so 2>/dev/null | grep -v /binaries | head -1) if [ -n "$INSTALLED_DDAPPSEC" ]; then echo "Found installed ddappsec.so at $INSTALLED_DDAPPSEC, replacing with custom binary" - cp -f "$DDAPPSEC_SO" "$INSTALLED_DDAPPSEC" + cp -f $DDAPPSEC_SO $INSTALLED_DDAPPSEC else echo "Warning: Could not find installed ddappsec.so to replace" fi @@ -134,44 +129,44 @@ if [ -f "$DDAPPSEC_SO" ] && [ -f "$APPSEC_HELPER_SO" ]; then INSTALLED_HELPER=$(find /root /opt -name libddappsec-helper.so 2>/dev/null | grep -v /binaries | head -1) if [ -n "$INSTALLED_HELPER" ]; then echo "Found installed helper at $INSTALLED_HELPER, replacing with custom binary" - cp -f "$APPSEC_HELPER_SO" "$INSTALLED_HELPER" + cp -f $APPSEC_HELPER_SO $INSTALLED_HELPER else echo "Warning: Could not find installed libddappsec-helper.so to replace" fi fi # Install the Rust helper alongside the C++ helper so DD_APPSEC_HELPER_RUST_REDIRECTION works -if [ -f "$APPSEC_RUST_HELPER_SO" ]; then +if [ -f $APPSEC_RUST_HELPER_SO ]; then INSTALLED_HELPER=$(find /root /opt -name libddappsec-helper.so 2>/dev/null | grep -v /binaries | head -1) if [ -n "$INSTALLED_HELPER" ]; then echo "Installing Rust helper at $(dirname "$INSTALLED_HELPER")/libddappsec-helper-rust.so" - cp -f "$APPSEC_RUST_HELPER_SO" "$(dirname "$INSTALLED_HELPER")/libddappsec-helper-rust.so" + cp -f $APPSEC_RUST_HELPER_SO "$(dirname "$INSTALLED_HELPER")/libddappsec-helper-rust.so" else echo "Warning: Could not find installed libddappsec-helper.so to install Rust helper alongside" fi fi -if [ -f "$LIBDDWAF_SO" ]; then +if [ -f $LIBDDWAF_SO ]; then echo "Copying libddwaf.so from /binaries" INSTALLED_HELPER=$(find /root /opt -name libddappsec-helper.so 2>/dev/null | grep -v /binaries | head -1) if [ -n "$INSTALLED_HELPER" ]; then echo "Found installed helper at $INSTALLED_HELPER, installing custom libddwaf.so alongside" - cp -v "$LIBDDWAF_SO" "$(dirname "$INSTALLED_HELPER")" + cp -v $LIBDDWAF_SO "$(dirname "$INSTALLED_HELPER")" else echo "Warning: Could not find installed libddappsec-helper.so" fi fi -if test -f "$INI_FILE"; then +if test -f $INI_FILE; then #There is a bug on 0.98.1 which disable explicitly appsec when it shouldnt. Delete this line when hotfix - sed -i "/datadog.appsec.enabled/s/^/;/g" "$INI_FILE" + sed -i "/datadog.appsec.enabled/s/^/;/g" $INI_FILE #Parametric tests don't need appsec - [ -n "${NO_EXTRACT_VERSION+x}" ] && echo "datadog.appsec.enabled = Off" >> "$INI_FILE" + [ ! -z ${NO_EXTRACT_VERSION+x} ] && echo "datadog.appsec.enabled = Off" >> $INI_FILE fi #Ensure parametric test compatibility -[ -n "${NO_EXTRACT_VERSION+x}" ] && exit 0 +[ ! -z ${NO_EXTRACT_VERSION+x} ] && exit 0 #Extract version info php -d error_reporting='' -d extension=ddtrace.so -d extension=ddappsec.so -r 'echo phpversion("ddtrace");' > \ diff --git a/utils/build/docker/python_lambda/artifact.py b/utils/build/docker/python_lambda/artifact.py deleted file mode 100644 index d65785296c7..00000000000 --- a/utils/build/docker/python_lambda/artifact.py +++ /dev/null @@ -1,42 +0,0 @@ -from __future__ import annotations - - -from utils.target_artifacts.entry_helpers import gha_artifact_entry, text_entry -from utils.target_artifacts.models import ( - ArtifactEntry, - GitHubActionsArtifactReference, - GitHubReleaseReference, -) -from utils.target_artifacts.resolvers import GitHubActionsArtifactResolver, GitHubLatestReleaseResolver - - -class Dev: - def artifact_inputs(self, env: dict[str, str]) -> tuple[GitHubActionsArtifactResolver]: - return ( - GitHubActionsArtifactResolver( - name="workflow_artifact", - repository="DataDog/datadog-lambda-python", - workflow="build_layer.yml", - artifact_name="datadog-lambda-python-3.13-amd64", - variable_name="LIBRARY_TARGET_BRANCH", - default_value="main", - ignore_failed_workflow=False, - ), - ) - - def artifact_entries( - self, - resolved_inputs: dict[str, GitHubActionsArtifactReference], - ) -> tuple[ArtifactEntry]: - return (gha_artifact_entry("python-lambda-github-actions-artifact.json", resolved_inputs["workflow_artifact"]),) - - -class Prod: - def artifact_inputs(self, env: dict[str, str]) -> tuple[GitHubLatestReleaseResolver]: - return (GitHubLatestReleaseResolver(name="release", repository="DataDog/datadog-lambda-python"),) - - def artifact_entries( - self, - resolved_inputs: dict[str, GitHubReleaseReference], - ) -> tuple[ArtifactEntry]: - return (text_entry("python-lambda-load-from-release", resolved_inputs["release"].tag_name),) diff --git a/utils/build/docker/python_lambda/install_datadog_lambda.sh b/utils/build/docker/python_lambda/install_datadog_lambda.sh index 6c1a3bbbe3f..45ba9172fca 100755 --- a/utils/build/docker/python_lambda/install_datadog_lambda.sh +++ b/utils/build/docker/python_lambda/install_datadog_lambda.sh @@ -9,34 +9,14 @@ if [ "$(find . -maxdepth 1 -name "*.zip" | wc -l)" = "1" ]; then echo "Install datadog_lambda from ${path}" unzip "${path}" -d /opt else + echo "Fetching from latest GitHub release..." ARCH=$(uname -m | sed 's/x86_64/amd64/' | sed 's/aarch64/arm64/') - ZIPFILE=datadog_lambda_py-"$ARCH"-3.13.zip - if [ -f python-lambda-github-actions-artifact.json ]; then - echo "Fetching from staged GitHub Actions artifact metadata..." - ARCHIVE_URL=$(jq -r '.archive_download_url' python-lambda-github-actions-artifact.json) - if [ -z "$ARCHIVE_URL" ] || [ "$ARCHIVE_URL" = "null" ]; then - echo "Staged GitHub Actions artifact metadata is missing archive_download_url" - exit 1 - fi - GITHUB_AUTH_HEADER=() - if [ -f /run/secrets/github_token ]; then - GITHUB_AUTH_HEADER=(-H "Authorization: Bearer $(cat /run/secrets/github_token)") - fi - curl -fsSL "${GITHUB_AUTH_HEADER[@]}" -o /tmp/python-lambda-artifact.zip "$ARCHIVE_URL" - mkdir -p /tmp/python-lambda-artifact - unzip -o /tmp/python-lambda-artifact.zip -d /tmp/python-lambda-artifact - cp "$(find /tmp/python-lambda-artifact -name "$ZIPFILE" | head -1)" . - elif [ -f python-lambda-load-from-release ]; then - RELEASE_TAG=$(cat python-lambda-load-from-release) - curl -fsSLO "https://github.com/DataDog/datadog-lambda-python/releases/download/${RELEASE_TAG}/${ZIPFILE}" - else - echo "Fetching from latest GitHub release..." - curl -fsSLO "https://github.com/DataDog/datadog-lambda-python/releases/latest/download/${ZIPFILE}" - fi + echo https://github.com/DataDog/datadog-lambda-python/releases/latest/download/datadog_lambda_py-"$ARCH"-3.13.zip + curl -fsSLO https://github.com/DataDog/datadog-lambda-python/releases/latest/download/datadog_lambda_py-"$ARCH"-3.13.zip unzip -o datadog_lambda_py-"$ARCH"-3.13.zip -d /opt - if [ ! -f "$ZIPFILE" ]; then - echo "Failed to download ${ZIPFILE}" + if [ ! -f datadog_lambda_py-"$ARCH"-3.13.zip ]; then + echo "Failed to download datadog_lambda_py-""$ARCH""-3.13.zip" exit 1 fi fi diff --git a/utils/build/docker/python_otel/artifact.py b/utils/build/docker/python_otel/artifact.py deleted file mode 100644 index 302f746060f..00000000000 --- a/utils/build/docker/python_otel/artifact.py +++ /dev/null @@ -1,17 +0,0 @@ -from __future__ import annotations - - -from utils.target_artifacts.entry_helpers import text_entry -from utils.target_artifacts.models import SimpleTarget -from utils.target_artifacts.resolvers import PypiLatestResolver - -PACKAGE_NAME = "opentelemetry-distro" - - -class Dev(SimpleTarget): - inputs = (PypiLatestResolver(name="otel_package", package=PACKAGE_NAME),) - entries = (text_entry("python-otel-load-from-pip", f"{PACKAGE_NAME}[otlp]=={{otel_package.version}}"),) - - -class Prod(Dev): - pass diff --git a/utils/build/docker/python_otel/flask-poc-otel.Dockerfile b/utils/build/docker/python_otel/flask-poc-otel.Dockerfile index 17e392f973a..ab3b5ee8e81 100644 --- a/utils/build/docker/python_otel/flask-poc-otel.Dockerfile +++ b/utils/build/docker/python_otel/flask-poc-otel.Dockerfile @@ -7,14 +7,10 @@ RUN pip uninstall -y psycopg2-binary RUN pip install psycopg2 ############# -COPY binaries* /binaries/ -RUN if [ -f /binaries/python-otel-load-from-pip ]; then \ - pip install "$(cat /binaries/python-otel-load-from-pip)"; \ - else \ - pip install opentelemetry-distro[otlp]==0.49b0; \ - fi +RUN pip install opentelemetry-distro[otlp]==0.49b0 WORKDIR /app +COPY binaries* /binaries/ COPY utils/build/docker/python/flask /app COPY utils/build/docker/python_otel/flask-poc-otel/app.py /app @@ -26,3 +22,4 @@ RUN pip freeze | grep opentelemetry ENV OTEL_PYTHON_LOGGING_AUTO_INSTRUMENTATION_ENABLED=true ENV FLASK_APP=app.py CMD ./app.sh + diff --git a/utils/build/docker/ruby/artifact.py b/utils/build/docker/ruby/artifact.py deleted file mode 100644 index f49c5534225..00000000000 --- a/utils/build/docker/ruby/artifact.py +++ /dev/null @@ -1,34 +0,0 @@ -from __future__ import annotations - - -from utils.target_artifacts.entry_helpers import text_entry -from utils.target_artifacts.models import SimpleTarget -from utils.target_artifacts.resolvers import GitHubBranchResolver, RubygemsLatestResolver - - -class Dev(SimpleTarget): - inputs = ( - GitHubBranchResolver( - name="library_branch", - repository="DataDog/dd-trace-rb", - variable_name="LIBRARY_TARGET_BRANCH", - default_value="master", - ), - ) - entries = ( - text_entry( - "ruby-load-from-bundle-add", - "gem 'datadog', require: 'datadog/auto_instrument', " - "git: 'https://github.com/DataDog/dd-trace-rb.git', ref: '{library_branch.sha}'", - ), - ) - - -class Prod(SimpleTarget): - inputs = (RubygemsLatestResolver(name="datadog", package="datadog"),) - entries = ( - text_entry( - "ruby-load-from-bundle-add", - "gem 'datadog', '{datadog.version}', require: 'datadog/auto_instrument'", - ), - ) diff --git a/utils/build/docker/ruby_lambda/artifact.py b/utils/build/docker/ruby_lambda/artifact.py deleted file mode 100644 index f1540113f75..00000000000 --- a/utils/build/docker/ruby_lambda/artifact.py +++ /dev/null @@ -1,23 +0,0 @@ -from __future__ import annotations - - -from utils.target_artifacts.entry_helpers import text_entry -from utils.target_artifacts.models import SimpleTarget -from utils.target_artifacts.resolvers import GitHubBranchResolver, GitHubLatestReleaseResolver - - -class Dev(SimpleTarget): - inputs = ( - GitHubBranchResolver( - name="library_branch", - repository="DataDog/datadog-lambda-rb", - variable_name="LIBRARY_TARGET_BRANCH", - default_value="main", - ), - ) - entries = (text_entry("ruby-lambda-load-from-git", "https://github.com/DataDog/datadog-lambda-rb@{library_branch.sha}"),) - - -class Prod(SimpleTarget): - inputs = (GitHubLatestReleaseResolver(name="release", repository="DataDog/datadog-lambda-rb"),) - entries = (text_entry("ruby-lambda-load-from-release", "{release.tag_name}"),) diff --git a/utils/build/docker/ruby_lambda/install_datadog_lambda.sh b/utils/build/docker/ruby_lambda/install_datadog_lambda.sh index 83c4c7c7bc7..19459fda501 100755 --- a/utils/build/docker/ruby_lambda/install_datadog_lambda.sh +++ b/utils/build/docker/ruby_lambda/install_datadog_lambda.sh @@ -28,30 +28,15 @@ elif [ "$(find . -maxdepth 1 -name '*.zip' | wc -l)" = "1" ]; then path=$(readlink -f "$(find . -maxdepth 1 -name '*.zip')") echo "Install datadog-lambda from ${path}" unzip "${path}" -d /opt -elif [ -f ruby-lambda-load-from-git ]; then - TARGET=$(cat ruby-lambda-load-from-git) - URL=$(echo "$TARGET" | cut -d "@" -f 1) - REF=$(echo "$TARGET" | cut -d "@" -f 2) - echo "Install datadog-lambda from ${TARGET}" - git clone "$URL" datadog-lambda-rb - git -C datadog-lambda-rb checkout "$REF" - cd datadog-lambda-rb - gem build datadog-lambda - gem install datadog-lambda-*.gem --install-dir "${GEM_DIR}" --no-document else + echo "Fetching from latest GitHub release..." ARCH=$(uname -m | sed 's/x86_64/amd64/' | sed 's/aarch64/arm64/') RUBY_MINOR=$(ruby -e 'puts RUBY_VERSION.split(".")[0..1].join(".")') # NOTE: Release assets are datadog-lambda_ruby--.zip # just one-dot (3.4, not 3.4.0). # The old datadog_lambda_rb-* name still resolved, but to a stale layer # with no AppSec - if [ -f ruby-lambda-load-from-release ]; then - RELEASE_TAG=$(cat ruby-lambda-load-from-release) - URL="https://github.com/DataDog/datadog-lambda-rb/releases/download/${RELEASE_TAG}/datadog-lambda_ruby-${ARCH}-${RUBY_MINOR}.zip" - else - echo "Fetching from latest GitHub release..." - URL="https://github.com/DataDog/datadog-lambda-rb/releases/latest/download/datadog-lambda_ruby-${ARCH}-${RUBY_MINOR}.zip" - fi + URL="https://github.com/DataDog/datadog-lambda-rb/releases/latest/download/datadog-lambda_ruby-${ARCH}-${RUBY_MINOR}.zip" echo "${URL}" curl -fsSLO "${URL}" ZIPFILE="datadog-lambda_ruby-${ARCH}-${RUBY_MINOR}.zip" diff --git a/utils/build/docker/rust/artifact.py b/utils/build/docker/rust/artifact.py deleted file mode 100644 index 8e3210cf240..00000000000 --- a/utils/build/docker/rust/artifact.py +++ /dev/null @@ -1,28 +0,0 @@ -from __future__ import annotations - - -from utils.target_artifacts.entry_helpers import text_entry -from utils.target_artifacts.models import SimpleTarget -from utils.target_artifacts.resolvers import CratesLatestResolver, GitHubBranchResolver - - -class Dev(SimpleTarget): - inputs = ( - GitHubBranchResolver( - name="library_branch", - repository="DataDog/dd-trace-rs", - variable_name="LIBRARY_TARGET_BRANCH", - default_value="main", - ), - ) - entries = (text_entry("rust-load-from-git", "{library_branch.sha}"),) - - -class Prod(SimpleTarget): - inputs = ( - CratesLatestResolver( - name="datadog_opentelemetry", - package="datadog-opentelemetry", - ), - ) - entries = (text_entry("rust-load-from-crates", "{datadog_opentelemetry.version}"),) diff --git a/utils/build/docker/rust/install_ddtrace.sh b/utils/build/docker/rust/install_ddtrace.sh index 94385bdf354..b9c9f4267ed 100755 --- a/utils/build/docker/rust/install_ddtrace.sh +++ b/utils/build/docker/rust/install_ddtrace.sh @@ -31,13 +31,10 @@ fail() { if [ -e /binaries/rust-load-from-git ]; then rev_or_branch=$(/dev/null 2>&1; then + echo "Clone $REPO_URL -b $rev_or_branch into /binaries/dd-trace-rs" + if ! git clone -b "$rev_or_branch" "$REPO_URL" /binaries/dd-trace-rs >/dev/null 2>&1; then fail "could not clone dd-trace-rs ref '$rev_or_branch'. Check that the ref exists and is accessible." fi - if ! git -C /binaries/dd-trace-rs checkout "$rev_or_branch" >/dev/null 2>&1; then - fail "could not checkout dd-trace-rs ref '$rev_or_branch'. Check that the ref exists and is accessible." - fi fi if [ -e /binaries/dd-trace-rs ]; then @@ -77,13 +74,7 @@ else # remove previous dependency on datadog-opentelemetry and add the new one from crates.io cargo remove datadog-opentelemetry >/dev/null 2>&1 || true - if [ -e /binaries/rust-load-from-crates ]; then - crate_version=$(/dev/null 2>&1; then + if ! cargo add datadog-opentelemetry --features metrics-http,metrics-grpc,logs-http,logs-grpc >/dev/null 2>&1; then fail "could not install datadog-opentelemetry from crates.io. Check network access and the selected package version." fi fi diff --git a/utils/ci/gitlab/build_pipeline.py b/utils/ci/gitlab/build_pipeline.py index d0771d2966c..6f8984416bf 100644 --- a/utils/ci/gitlab/build_pipeline.py +++ b/utils/ci/gitlab/build_pipeline.py @@ -2,16 +2,11 @@ import argparse import json -import os import sys from pathlib import Path -from typing import TYPE_CHECKING from jinja2 import Environment, FileSystemLoader, select_autoescape -if TYPE_CHECKING: - from collections.abc import Callable - _env = Environment(loader=FileSystemLoader(Path(__file__).resolve().parent), autoescape=select_autoescape()) _template = _env.get_template("system-tests.yml.j2") @@ -31,50 +26,6 @@ def noop_stub(stage: str) -> str: """ -def _ensure_quoted(s: str) -> str: - ret = s - if not s.startswith('"'): - ret = '"' + ret - if not s.endswith('"'): - ret = ret + '"' - return ret - - -def _trace(name: str, command: str, library: str, weblog: str = "", scenario: str = "") -> str: - return "".join( - [ - "datadog-ci trace ", - f"--name {_ensure_quoted(name)} ", - f"--tags system-tests.library:{library} ", - f"--tags system-tests.weblog:{weblog} " if weblog else "", - f"--tags system-tests.scenario:{scenario} " if scenario else "", - f"-- {command}", - ] - ) - - -def _generate_build_renderer(*, push_main: bool = False, push_lib_main: bool = False) -> Callable[[str, str], str]: - def _render_build(library: str, weblog: str) -> str: - registry_base = "registry.ddbuild.io/system-tests/cache" - ref = f"{registry_base}/{library}/{weblog}" - command = "".join( - [ - f"./build.sh {library} ", - "-i weblog --save-to-binaries ", - f"--weblog-variant {weblog} ", - '--extra-docker-args "', - f"--cache-from=type=registry,ref={ref}:main ", - f"--cache-from=type=registry,ref={ref}:lib_main ", - f"--cache-to=type=registry,ref={ref}:main,mode=max " if push_main else "", - f"--cache-to=type=registry,ref={ref}:lib_main,mode=max" if push_lib_main else "", - '"', - ] - ) - return _trace(f'"build {library} {weblog}"', command, library, weblog) - - return _render_build - - def render_library( library: str, params: dict, @@ -84,12 +35,10 @@ def render_library( ci_image: str, ref: str, push_to_test_optimization: bool, + docker_auth: bool, binaries_artifact_path: str, binaries_artifacts: str, pipeline_start_time: str, - ci_project_name: str = "", - ci_commit_branch: str = "", - ci_default_branch: str = "", ) -> str: parallel_weblogs = params.get("endtoend_defs", {}).get("parallel_weblogs", []) parallel_jobs = params.get("endtoend_defs", {}).get("parallel_jobs", []) @@ -100,7 +49,6 @@ def render_library( for scenario in job.get("scenarios", []) ] binaries_artifact = params["miscs"]["binaries_artifact"] - ci_environment = params["miscs"].get("ci_environment", "prod") parametric = params["parametric"] # Build the full list of artifact jobs for cross-pipeline downloads. # If binaries_artifacts is provided, use it; otherwise fall back to the single job. @@ -110,13 +58,6 @@ def render_library( binaries_artifacts_list = [binaries_artifact] else: binaries_artifacts_list = [] - - is_default_branch = ci_commit_branch in {"main", "master", ci_default_branch} - render_build = _generate_build_renderer( - push_main=(ci_project_name == "system-tests" and is_default_branch), - push_lib_main=(ci_project_name != "system-tests" and is_default_branch), - ) - return _template.render( scenario_pairs=scenario_pairs, stage=stage, @@ -125,15 +66,13 @@ def render_library( binaries_artifact=binaries_artifact, binaries_artifacts_list=binaries_artifacts_list, binaries_artifact_path=binaries_artifact_path, - ci_environment=ci_environment, parametric=parametric, ci_image=ci_image, ref=ref, push_to_test_optimization=push_to_test_optimization, skip_header=skip_header, + docker_auth_enabled=docker_auth, pipeline_start_time=pipeline_start_time, - render_build=render_build, - trace=_trace, ) @@ -147,12 +86,10 @@ def build( ref: str = "", push_to_test_optimization: bool = False, chunks: int = 3, + docker_auth: bool = False, binaries_artifact_path: str = "", binaries_artifacts: str = "", pipeline_start_time: str = "", - ci_project_name: str = "", - ci_commit_branch: str = "", - ci_default_branch: str = "", ) -> None: """Render pipeline chunk files into *output_dir*, one per chunk.""" output_dir.mkdir(parents=True, exist_ok=True) @@ -189,12 +126,10 @@ def build( ci_image=ci_image, ref=ref, push_to_test_optimization=push_to_test_optimization, + docker_auth=docker_auth, binaries_artifact_path=binaries_artifact_path, binaries_artifacts=binaries_artifacts, pipeline_start_time=pipeline_start_time, - ci_project_name=ci_project_name, - ci_commit_branch=ci_commit_branch, - ci_default_branch=ci_default_branch, ) ) @@ -246,12 +181,10 @@ def main(argv: list[str] | None = None) -> int: ref=args.ref, push_to_test_optimization=args.push_to_test_optimization == "true", chunks=args.chunks, + docker_auth=args.docker_auth == "true", binaries_artifact_path=args.binaries_artifact_path, binaries_artifacts=args.binaries_artifacts, pipeline_start_time=args.pipeline_start_time, - ci_project_name=os.getenv("CI_PROJECT_NAME", ""), - ci_commit_branch=os.getenv("CI_COMMIT_BRANCH", ""), - ci_default_branch=os.getenv("CI_DEFAULT_BRANCH", ""), ) return 0 diff --git a/utils/ci/gitlab/main.yml b/utils/ci/gitlab/main.yml index 381ec75be1c..cec5958ba53 100644 --- a/utils/ci/gitlab/main.yml +++ b/utils/ci/gitlab/main.yml @@ -51,6 +51,9 @@ spec: condition: description: "GitLab CI rule expression controlling whether system-tests jobs run (e.g. '$NIGHTLY_BUILD == \"true\"'). Defaults to always-run." default: "null == null" + docker_auth: + description: "Whether to authenticate calls to docker hub" + default: "false" split_pipeline: description: "Split jobs across multiple child pipelines" type: boolean @@ -125,6 +128,7 @@ system_tests_build_pipeline: force_execute=$(echo "$[[ inputs.force_execute ]],$FORCE_EXECUTE" | tr ',' '\n' | sed '/^$/d' | tr '\n' ',' | sed 's/,$//') parametric_job_count="${SYSTEM_TESTS_PARAMETRIC_JOB_COUNT:-$[[ inputs.parametric_job_count ]]}" push_to_test_optimization="${SYSTEM_TESTS_PUSH_TO_TEST_OPTIMIZATION:-$[[ inputs.push_to_test_optimization ]]}" + docker_auth="${SYSTEM_TESTS_DOCKER_AUTH:-$[[ inputs.docker_auth ]]}" skip_empty_scenario="${SYSTEM_TESTS_SKIP_EMPTY_SCENARIO:-$[[ inputs.skip_empty_scenarios ]]}" echo "libraries: $libraries" echo "scenarios: $scenarios" @@ -143,7 +147,7 @@ system_tests_build_pipeline: done chunks=1 if [ "$SYSTEM_TESTS_SPLIT_PIPELINE" = "true" ]; then chunks=3; fi - python3 utils/ci/gitlab/build_pipeline.py --stage $[[ inputs.stage ]] --ci-image "$CI_IMAGE" --ref "$[[ inputs.ref ]]" --push-to-test-optimization "$push_to_test_optimization" --libraries "$libraries" --params-dir . --output-dir . --chunks $chunks --binaries-artifact-path "$binaries_artifact_path" --binaries-artifacts "$binaries_artifacts" --pipeline-start-time "$SYSTEM_TESTS_PIPELINE_START_TIME" + python3 utils/ci/gitlab/build_pipeline.py --stage $[[ inputs.stage ]] --ci-image "$CI_IMAGE" --ref "$[[ inputs.ref ]]" --push-to-test-optimization "$push_to_test_optimization" --libraries "$libraries" --params-dir . --output-dir . --chunks $chunks --docker-auth "$docker_auth" --binaries-artifact-path "$binaries_artifact_path" --binaries-artifacts "$binaries_artifacts" --pipeline-start-time "$SYSTEM_TESTS_PIPELINE_START_TIME" artifacts: paths: - system-tests/generated-pipeline-chunk-*.yml diff --git a/utils/ci/gitlab/system-tests.yml.j2 b/utils/ci/gitlab/system-tests.yml.j2 index df786faa7d0..695aa868f83 100644 --- a/utils/ci/gitlab/system-tests.yml.j2 +++ b/utils/ci/gitlab/system-tests.yml.j2 @@ -1,3 +1,16 @@ +{% macro docker_auth() %} + - section_start "docker_auth" "Docker hub auth" + - export DOCKER_LOGIN=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-write --with-decryption --query "Parameter.Value" --out text) + - export DOCKER_LOGIN_PASS=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-pass-write --with-decryption --query "Parameter.Value" --out text) + - | + for i in 1 2 3; do + echo "$DOCKER_LOGIN_PASS" | docker login --username "$DOCKER_LOGIN" --password-stdin && break + if [ "$i" -eq 3 ]; then echo "docker login failed after 3 attempts"; exit 1; fi + echo "docker login failed (attempt $i), retrying in $((i*5))s..." + sleep $((i*5)) + done + - section_end "docker_auth" +{% endmacro %} {% macro copy_binaries(path) %} - section_start "copy_binaries" "Copying pre-built binaries into binaries/" - mkdir -p binaries @@ -17,11 +30,9 @@ echo "SYSTEM_TESTS_GENERATED_PIPELINE_START_TIME not set or not numeric ('$SYSTEM_TESTS_GENERATED_PIPELINE_START_TIME'), skipping metric emission" fi {% endmacro %} -{% macro stage_target_artifacts() %} - - section_start "target_artifacts" "Staging target artifacts" - - python3 utils/scripts/stage-target-artifacts.py {{library}} {{ci_environment}} - - section_end "target_artifacts" -{% endmacro %} +{% macro trace(name, command, library, weblog=none, scenario=none) -%} +datadog-ci trace --name "{{name}}" --tags system-tests.library:{{library}}{% if weblog %} --tags system-tests.weblog:{{weblog}}{% endif %}{% if scenario %} --tags system-tests.scenario:{{scenario}}{% endif %} -- {{command}} +{%- endmacro %} {% if not skip_header %} workflow: name: "System-tests end to end" @@ -93,14 +104,15 @@ system_tests_build_{{library}}_{{variant}}: {% endfor %} {% endif %} script: + {% if docker_auth_enabled %} + {{ docker_auth() }} + {% endif %} {% if binaries_artifacts_list and binaries_artifact_path %} {{ copy_binaries(binaries_artifact_path) }} - {% elif not binaries_artifacts_list %} - {{ stage_target_artifacts() }} {% endif %} - section_start "build" "Building weblog" false {{ job_tag("build") }} - - {{ render_build(library, variant) }} + - {{ trace("build " ~ library ~ " " ~ variant, "./build.sh " ~ library ~ " -i weblog --save-to-binaries --weblog-variant " ~ variant, library, weblog=variant) }} - mv binaries .. - section_end "build" artifacts: @@ -134,6 +146,9 @@ system_tests_run_{{library}}_{{scenario}}_{{variant}}: {% endfor %} {% endif %} script: + {% if docker_auth_enabled %} + {{ docker_auth() }} + {% endif %} - section_start "weblog_setup" "Setting up the weblog" {% if build_required %} - mv ../binaries/* binaries/ @@ -185,8 +200,6 @@ system_tests_run_{{library}}_PARAMETRIC_{{job_index}}: {{ job_tag("run") }} {% if binaries_artifacts_list and binaries_artifact_path %} {{ copy_binaries(binaries_artifact_path) }} - {% elif not binaries_artifacts_list %} - {{ stage_target_artifacts() }} {% endif %} - {{ trace("run " ~ library ~ " PARAMETRIC " ~ job_index, "./run.sh PARAMETRIC -L " ~ library ~ " --splits=" ~ parametric.job_count ~ " --group=" ~ job_index, library, scenario="PARAMETRIC") }} - section_end "run" diff --git a/utils/scripts/compute_libraries_and_scenarios.py b/utils/scripts/compute_libraries_and_scenarios.py index 9d10b83de14..7337ebabc2c 100644 --- a/utils/scripts/compute_libraries_and_scenarios.py +++ b/utils/scripts/compute_libraries_and_scenarios.py @@ -31,7 +31,7 @@ ALL_LIBRARIES = LIBRARIES | OTEL_LIBRARIES GITHUB_EXCLUDED_LIBRARIES = {"c"} GITLAB_PR_LIBRARIES = {"c"} -GITLAB_MAIN: set[str] = {"python"} +GITLAB_MAIN = {"python"} def check_scenarios(scenarios: set[str]) -> bool: diff --git a/utils/scripts/docker_base_image.sh b/utils/scripts/docker_base_image.sh index 7c8e5849cc9..3a86ac8faf4 100755 --- a/utils/scripts/docker_base_image.sh +++ b/utils/scripts/docker_base_image.sh @@ -6,18 +6,18 @@ set -eu image="$1" target_dir="$2" -mkdir --parent "$target_dir" +mkdir --parent $target_dir echo "Extracting Docker base image $image to folder $target_dir" -docker pull "$image" -docker save -o "$target_dir/image.tar" "$image" -tar xf "$target_dir/image.tar" -C "$target_dir" -layers=$(jq -r '.[0].Layers[]' "$target_dir/manifest.json") +docker pull $image +docker save -o $target_dir/image.tar $image +tar xf $target_dir/image.tar -C $target_dir +layers=$(jq -r '.[0].Layers[]' $target_dir/manifest.json) for i in $layers; do - tar xf "$target_dir/$i" -C "$target_dir" + tar xf $target_dir/$i -C $target_dir done #Done! clean -rm -rf "$target_dir/image.tar" "$target_dir/manifest.json" "$target_dir/oci-layout" "$target_dir/index.json" -rm -rf "$target_dir/blobs/" +rm -rf $target_dir/image.tar $target_dir/manifest.json $target_dir/oci-layout $target_dir/index.json +rm -rf $target_dir/blobs/ diff --git a/utils/scripts/load-binary.sh b/utils/scripts/load-binary.sh index 4439ac0b3c4..6e0f6b630a7 100755 --- a/utils/scripts/load-binary.sh +++ b/utils/scripts/load-binary.sh @@ -4,65 +4,411 @@ # This product includes software developed at Datadog (https://www.datadoghq.com/). # Copyright 2021 Datadog, Inc. + +########################################################################################## +# The purpose of this script is to download the latest development version of a component. +# +# Binaries sources: +# +# * Agent: Docker hub datadog/agent-dev:master-py3 +# * cpp_httpd: Github action artifact +# * Golang: github.com/DataDog/dd-trace-go/v2@main +# * .NET: ghcr.io/datadog/dd-trace-dotnet +# * Java: S3 +# * Java Lambda: S3 (same binary as Java) +# * PHP: ghcr.io/datadog/dd-trace-php +# * Node.js: Direct from github source +# * Node.js Lambda: Fetch from GitHub Actions artifact +# * C++: Direct from github source +# * Python: S3 https://dd-trace-py-builds.s3.amazonaws.com//index.html +# * Ruby: Direct from github source +# * WAF: Direct from github source, but not working, as this repo is now private +# * Python Lambda: Fetch from GitHub Actions artifact +# * Ruby Lambda: Clone locally the github repo +# * Rust: Clone locally the github repo +########################################################################################## + set -eu +assert_version_is_dev() { + + if [ "$VERSION" = 'dev' ]; then + return 0 + fi + + echo "Don't know how to load version $VERSION for $TARGET" + + exit 1 +} + +assert_target_branch_is_not_set() { + + if [[ -z "${LIBRARY_TARGET_BRANCH:-}" ]]; then + return 0 + fi + + echo "It is not possible to specify the '$LIBRARY_TARGET_BRANCH' target branch for $TARGET library yet" + + exit 1 +} + +ghcr_login_if_token_set() { + if [ -n "$GITHUB_TOKEN" ]; then + echo "Log to GHCR with token" + echo "$GITHUB_TOKEN" | docker login ghcr.io --password-stdin -u "actor" # username is ignored + fi +} + +resolve_github_branch_sha() { + local repository="$1" + local branch="$2" + local encoded_branch + local response + local sha + + encoded_branch=$(jq -rn --arg value "$branch" '$value | @uri') + if ! response=$(curl --fail --location --silent --show-error \ + "${GITHUB_AUTH_HEADER[@]}" \ + "https://api.github.com/repos/${repository}/branches/${encoded_branch}"); then + echo "Unable to resolve branch '${branch}' in ${repository}" >&2 + exit 1 + fi + + sha=$(jq -r '.commit.sha // empty' <<< "$response") + if [[ ! "$sha" =~ ^[0-9a-f]{40}$ ]]; then + echo "Branch '${branch}' in ${repository} did not resolve to a commit SHA" >&2 + exit 1 + fi + + printf '%s' "$sha" +} + +validate_oci_image() { + local image="$1" + + if ! docker manifest inspect "$image" >/dev/null; then + echo "OCI package does not exist or is not accessible: ${image}" >&2 + exit 1 + fi +} + +get_github_action_artifact() { + rm -rf artifacts artifacts.zip + + SLUG=$1 + WORKFLOW=$2 + BRANCH=$3 + ARTIFACT_NAME=$4 + PATTERN=$5 + IGNORE_FAILED_WORKFLOW=${6:-true} # 6th arg, with default "true" + + # query filter seems not to be working ?? + WORKFLOWS=$(curl --silent --fail --show-error -H "Authorization: token $GITHUB_TOKEN" "https://api.github.com/repos/$SLUG/actions/workflows/$WORKFLOW/runs?per_page=100") + + if [ "$IGNORE_FAILED_WORKFLOW" = "true" ]; then + QUERY="[.workflow_runs[] | select(.conclusion != \"failure\" and .head_branch == \"$BRANCH\" and .status == \"completed\")][0]" + else + QUERY="[.workflow_runs[] | select(.head_branch == \"$BRANCH\" and .status == \"completed\")][0]" + fi + + # this wil fail if there are more than 100 artifacts + ARTIFACT_URL=$(echo "$WORKFLOWS" | jq -r "$QUERY | .artifacts_url") + ARTIFACT_URL="$ARTIFACT_URL?per_page=100" + + HTML_URL=$(echo "$WORKFLOWS" | jq -r "$QUERY | .html_url") + echo "Load artifacts for $HTML_URL" + ARTIFACTS=$(curl --silent -H "Authorization: token $GITHUB_TOKEN" "$ARTIFACT_URL") + ARCHIVE_URL=$(echo "$ARTIFACTS" | jq -r --arg ARTIFACT_NAME "$ARTIFACT_NAME" '.artifacts | map(select(.name | contains($ARTIFACT_NAME))) | .[0].archive_download_url') + echo "Load archive $ARCHIVE_URL" + + curl -H "Authorization: token $GITHUB_TOKEN" --output artifacts.zip -L "$ARCHIVE_URL" + + mkdir -p artifacts/ + unzip artifacts.zip -d artifacts/ + + find artifacts/ -type f -name "$PATTERN" -exec cp '{}' . ';' + + rm -rf artifacts artifacts.zip +} + +get_github_release_asset() { + SLUG=$1 + PATTERN=$2 + + release=$(curl --silent --fail --show-error -H "Authorization: token $GITHUB_TOKEN" "https://api.github.com/repos/$SLUG/releases/latest") + + name=$(echo "$release" | jq -r ".assets[].name | select(test(\"$PATTERN\"))") + url=$(echo "$release" | jq -r ".assets[].browser_download_url | select(test(\"$PATTERN\"))") + + echo "Load $url" + + curl -H "Authorization: token $GITHUB_TOKEN" --output "$name" -L "$url" +} + if test -f ".env"; then # shellcheck source=/dev/null source .env fi -TARGET=${1:-} -VERSION=${2:-dev} +TARGET=$1 +VERSION=${2:-'dev'} BINARIES_DIR=${BINARIES_DIR:-binaries} -GITHUB_TOKEN=${GITHUB_TOKEN:-} -if [[ -z "$TARGET" ]]; then - echo "Usage: $0 [dev|prod|custom]" >&2 - exit 1 +GITHUB_TOKEN="${GITHUB_TOKEN:-}" +GITHUB_AUTH_HEADER=() +if [ -n "$GITHUB_TOKEN" ]; then + GITHUB_AUTH_HEADER=(-H "Authorization: Bearer $GITHUB_TOKEN") fi -assert_version_is_dev() { - if [[ "$VERSION" == "dev" ]]; then - return 0 +echo "Load $VERSION binary for $TARGET" + +if [ "$TARGET" = "python" ]; then + python3 utils/scripts/stage-target-artifacts.py \ + "$TARGET" "$VERSION" \ + --binaries-dir "$BINARIES_DIR" \ + --repo-root . + exit 0 +fi + +cd "$BINARIES_DIR/" + +if [ "$TARGET" = "c" ]; then + if [ "$VERSION" = "prod" ]; then + if [[ -n "${LIBRARY_TARGET_BRANCH:-}" || -n "${AUTO_INJECT_TARGET_BRANCH:-}" ]]; then + echo "Target branches can only be used with the development c packages" >&2 + exit 1 + fi + + C_LIBRARY_IMAGE="install.datadoghq.com/apm-library-c-package:latest" + C_INJECTOR_IMAGE="install.datadoghq.com/apm-inject-package:latest" + elif [ "$VERSION" = "dev" ]; then + if [[ -n "${LIBRARY_TARGET_BRANCH:-}" ]]; then + C_LIBRARY_SHA=$(resolve_github_branch_sha "DataDog/dd-trace-c" "$LIBRARY_TARGET_BRANCH") + C_LIBRARY_IMAGE="installtesting.datad0g.com/apm-library-c-package:${C_LIBRARY_SHA}" + else + C_LIBRARY_IMAGE="install.datadoghq.com/apm-library-c-package:latest" + fi + + if [[ -n "${AUTO_INJECT_TARGET_BRANCH:-}" ]]; then + C_INJECTOR_SHA=$(resolve_github_branch_sha "DataDog/auto_inject" "$AUTO_INJECT_TARGET_BRANCH") + C_INJECTOR_IMAGE="installtesting.datad0g.com/apm-inject-package:${C_INJECTOR_SHA}" + else + C_INJECTOR_IMAGE="install.datadoghq.com/apm-inject-package:latest" + fi + else + echo "Don't know how to load version $VERSION for $TARGET" >&2 + exit 1 fi - echo "Don't know how to load version $VERSION for $TARGET" >&2 - exit 1 -} + validate_oci_image "$C_LIBRARY_IMAGE" + validate_oci_image "$C_INJECTOR_IMAGE" -assert_target_branch_is_not_set() { - if [[ -z "${LIBRARY_TARGET_BRANCH:-}" ]]; then - return 0 + printf '%s\n' "$C_LIBRARY_IMAGE" > c-library-image + printf '%s\n' "$C_INJECTOR_IMAGE" > c-injector-image + echo "Using dd-trace-c package ${C_LIBRARY_IMAGE}" + echo "Using auto-inject package ${C_INJECTOR_IMAGE}" + +elif [ "$TARGET" = "java" ] || [ "$TARGET" = "java_lambda" ]; then + assert_version_is_dev + + LIBRARY_TARGET_BRANCH="${LIBRARY_TARGET_BRANCH:-master}" + + curl --fail --location --silent --show-error --output dd-java-agent.jar "https://s3.us-east-1.amazonaws.com/dd-trace-java-builds/${LIBRARY_TARGET_BRANCH}/dd-java-agent.jar" + +elif [ "$TARGET" = "dotnet" ]; then + assert_version_is_dev + + LIBRARY_TARGET_BRANCH="${LIBRARY_TARGET_BRANCH:-latest_snapshot}" + # Normalize branch name for image tag: replace '/' with '_' + NORMALIZED_BRANCH=$(echo "$LIBRARY_TARGET_BRANCH" | sed 's/\//_/g') + + rm -rf ./*.tar.gz + ghcr_login_if_token_set + + ../utils/scripts/docker_base_image.sh "ghcr.io/datadog/dd-trace-dotnet/dd-trace-dotnet:${NORMALIZED_BRANCH}" . + +elif [ "$TARGET" = "ruby" ]; then + assert_version_is_dev + + LIBRARY_TARGET_BRANCH="${LIBRARY_TARGET_BRANCH:-master}" + echo "gem 'datadog', require: 'datadog/auto_instrument', git: 'https://github.com/Datadog/dd-trace-rb.git', branch: '$LIBRARY_TARGET_BRANCH'" > ruby-load-from-bundle-add + echo "Using $(cat ruby-load-from-bundle-add)" + +elif [ "$TARGET" = "php" ]; then + rm -rf ./*.tar.gz + mkdir -p temp + + if [ "${VERSION:-}" = 'prod' ]; then + ../utils/scripts/docker_base_image.sh ghcr.io/datadog/dd-trace-php/dd-library-php:latest ./temp + + elif [ -n "${LIBRARY_TARGET_BRANCH:-}" ]; then + # Match GitLab's CI_COMMIT_REF_SLUG: lowercase, non-alphanumeric → '-', collapse, truncate to 63 bytes and trim + NORMALIZED_BRANCH=$(echo "$LIBRARY_TARGET_BRANCH" | tr '[:upper:]' '[:lower:]' | sed 's/[^a-z0-9]/-/g;s/-\+/-/g;s/^-//;s/-$//' | cut -c1-63 | sed 's/-$//') + ghcr_login_if_token_set + ../utils/scripts/docker_base_image.sh \ + "ghcr.io/datadog/dd-trace-php/dd-library-php:${NORMALIZED_BRANCH}" \ + ./temp + + elif [ "${VERSION:-}" = 'dev' ]; then + URL="https://s3.us-east-1.amazonaws.com/dd-trace-php-builds/latest/datadog-setup.php" + echo "Downloading datadog-setup.php from: $URL" + curl --fail --location --silent --show-error --output ./temp/datadog-setup.php "$URL" + echo "datadog-setup.php downloaded" + + VERSION_HASH=$(grep "define('RELEASE_VERSION'" ./temp/datadog-setup.php | sed -E "s/.*urlencode\('([^']+)'\).*/\1/") + if [ -z "$VERSION_HASH" ]; then + echo "Failed to extract VERSION_HASH from datadog-setup.php" + exit 1 + fi + + VERSION_HASH_ENCODED=${VERSION_HASH//+/%2B} + URL="https://s3.us-east-1.amazonaws.com/dd-trace-php-builds/${VERSION_HASH_ENCODED}/dd-library-php-${VERSION_HASH_ENCODED}-$(arch)-linux-gnu.tar.gz" + echo "Downloading dd-library-php from: $URL" + curl --fail --location --silent --show-error --output "./temp/dd-library-php-${VERSION_HASH}-$(arch)-linux-gnu.tar.gz" "$URL" + echo "dd-library-php $(arch) downloaded" + + else + echo "Don't know how to load version ${VERSION:-} for $TARGET" + exit 1 fi - echo "It is not possible to specify the '$LIBRARY_TARGET_BRANCH' target branch for $TARGET library yet" >&2 + mv ./temp/dd-library-php*.tar.gz . && mv ./temp/datadog-setup.php . && rm -rf ./temp + +elif [ "$TARGET" = "golang" ]; then + assert_version_is_dev + rm -rf golang-load-from-go-get + set -o pipefail + + LIBRARY_TARGET_BRANCH="${LIBRARY_TARGET_BRANCH:-main}" + echo "load last commit on $LIBRARY_TARGET_BRANCH for DataDog/dd-trace-go" + COMMIT_ID=$(curl -sS --fail "${GITHUB_AUTH_HEADER[@]}" "https://api.github.com/repos/DataDog/dd-trace-go/branches/$LIBRARY_TARGET_BRANCH" | jq -r .commit.sha) + + echo "Using github.com/DataDog/dd-trace-go/v2@$COMMIT_ID" + { + echo "github.com/DataDog/dd-trace-go/v2@$COMMIT_ID" + echo "github.com/DataDog/dd-trace-go/contrib/database/sql/v2@$COMMIT_ID" + echo "github.com/DataDog/dd-trace-go/contrib/net/http/v2@$COMMIT_ID" + echo "github.com/DataDog/dd-trace-go/contrib/google.golang.org/grpc/v2@$COMMIT_ID" + echo "github.com/DataDog/dd-trace-go/contrib/99designs/gqlgen/v2@$COMMIT_ID" + echo "github.com/DataDog/dd-trace-go/contrib/gin-gonic/gin/v2@$COMMIT_ID" + echo "github.com/DataDog/dd-trace-go/contrib/graphql-go/graphql/v2@$COMMIT_ID" + echo "github.com/DataDog/dd-trace-go/contrib/graph-gophers/graphql-go/v2@$COMMIT_ID" + echo "github.com/DataDog/dd-trace-go/contrib/go-chi/chi.v5/v2@$COMMIT_ID" + echo "github.com/DataDog/dd-trace-go/contrib/IBM/sarama/v2@$COMMIT_ID" + echo "github.com/DataDog/dd-trace-go/contrib/labstack/echo.v4/v2@$COMMIT_ID" + echo "github.com/DataDog/dd-trace-go/contrib/sirupsen/logrus/v2@$COMMIT_ID" + } > golang-load-from-go-get + + echo "Using github.com/DataDog/orchestrion@latest" + echo "github.com/DataDog/orchestrion@latest" > orchestrion-load-from-go-get + + # envoy integration + echo "Using ghcr.io/datadog/dd-trace-go/service-extensions-callout:dev" + echo "ghcr.io/datadog/dd-trace-go/service-extensions-callout:dev" > golang-service-extensions-callout-image + + # haproxy integration + echo "Using ghcr.io/datadog/dd-trace-go/haproxy-spoa:dev" + echo "ghcr.io/datadog/dd-trace-go/haproxy-spoa:dev" > golang-haproxy-spoa-image + + # azure apim integration + echo "Using ghcr.io/datadog/dd-trace-go/apim-callout:dev" + echo "ghcr.io/datadog/dd-trace-go/apim-callout:dev" > golang-apim-callout-image + +elif [ "$TARGET" = "cpp" ]; then + assert_version_is_dev + # PROFILER: The main version is stored in s3, though we can not access this in CI + # Not handled for now for system-tests. this handles artifact for parametric + LIBRARY_TARGET_BRANCH="${LIBRARY_TARGET_BRANCH:-main}" + echo "https://github.com/DataDog/dd-trace-cpp@$LIBRARY_TARGET_BRANCH" > cpp-load-from-git + echo "Using $(cat cpp-load-from-git)" + +elif [ "$TARGET" = "cpp_httpd" ]; then + assert_version_is_dev + get_github_action_artifact "DataDog/httpd-datadog" "dev.yml" "main" "mod_datadog_artifact" "mod_datadog.so" + +elif [ "$TARGET" = "cpp_kong" ]; then + assert_version_is_dev + LIBRARY_TARGET_BRANCH="${LIBRARY_TARGET_BRANCH:-main}" + echo "Cloning kong-plugin-ddtrace branch ${LIBRARY_TARGET_BRANCH}" + git clone --depth 1 --branch "$LIBRARY_TARGET_BRANCH" \ + https://github.com/DataDog/kong-plugin-ddtrace.git kong-plugin-ddtrace + echo "Using kong-plugin-ddtrace@$(git -C kong-plugin-ddtrace rev-parse --short HEAD)" + +elif [ "$TARGET" = "cpp_nginx" ]; then + assert_version_is_dev + get_github_action_artifact "DataDog/nginx-datadog" "system-tests.yml" "master" "binaries" "binaries.zip" "false" + +elif [ "$TARGET" = "agent" ]; then + assert_version_is_dev + AGENT_TARGET_BRANCH="${AGENT_TARGET_BRANCH:-master-py3}" + echo "datadog/agent-dev:$AGENT_TARGET_BRANCH" > agent-image + echo "Using $(cat agent-image) image" + +elif [ "$TARGET" = "nodejs" ]; then + assert_version_is_dev + + LIBRARY_TARGET_BRANCH="${LIBRARY_TARGET_BRANCH:-master}" + # NPM builds the package, so we put a trigger file that tells install script to get package from github#master + echo "DataDog/dd-trace-js#$LIBRARY_TARGET_BRANCH" > nodejs-load-from-npm + echo "Using $(cat nodejs-load-from-npm)" + +elif [ "$TARGET" = "rust" ]; then + assert_version_is_dev + + LIBRARY_TARGET_BRANCH="${LIBRARY_TARGET_BRANCH:-main}" + echo "$LIBRARY_TARGET_BRANCH" > rust-load-from-git + echo "Using $(cat rust-load-from-git)" + +elif [ "$TARGET" = "waf_rule_set_v1" ]; then exit 1 -} -load_waf_rule_set() { - mkdir -p "$BINARIES_DIR" - curl --fail --location --silent --show-error \ +elif [ "$TARGET" = "waf_rule_set_v2" ]; then + assert_version_is_dev + assert_target_branch_is_not_set + curl --silent \ -H "Authorization: token $GITHUB_TOKEN" \ -H "Accept: application/vnd.github.v3.raw" \ - --output "$BINARIES_DIR/waf_rule_set.json" \ + --output "waf_rule_set.json" \ https://api.github.com/repos/DataDog/appsec-event-rules/contents/build/recommended.json -} -echo "Load $VERSION artifact entries for $TARGET" +elif [ "$TARGET" = "waf_rule_set" ]; then + assert_version_is_dev + assert_target_branch_is_not_set + curl --fail --output "waf_rule_set.json" \ + -H "Authorization: token $GITHUB_TOKEN" \ + -H "Accept: application/vnd.github.v3.raw" \ + https://api.github.com/repos/DataDog/appsec-event-rules/contents/build/recommended.json -case "$TARGET" in - waf_rule_set_v1) - exit 1 - ;; - waf_rule_set|waf_rule_set_v2) - assert_version_is_dev - assert_target_branch_is_not_set - load_waf_rule_set - ;; - *) - python3 utils/scripts/stage-target-artifacts.py \ - "$TARGET" "$VERSION" \ - --binaries-dir "$BINARIES_DIR" \ - --repo-root . - ;; -esac +elif [ "$TARGET" = "python_lambda" ]; then + assert_version_is_dev + + LIBRARY_TARGET_BRANCH="${LIBRARY_TARGET_BRANCH:-main}" + get_github_action_artifact "DataDog/datadog-lambda-python" "build_layer.yml" "$LIBRARY_TARGET_BRANCH" "datadog-lambda-python-3.13-amd64" "datadog_lambda_py-amd64-3.13.zip" "false" + +elif [ "$TARGET" = "nodejs_lambda" ]; then + assert_version_is_dev + + LIBRARY_TARGET_BRANCH="${LIBRARY_TARGET_BRANCH:-main}" + get_github_action_artifact "DataDog/datadog-lambda-js" "build_layer.yml" "$LIBRARY_TARGET_BRANCH" "datadog_lambda_node18.12" "datadog_lambda_node18.12.zip" "false" + +elif [ "$TARGET" = "ruby_lambda" ]; then + assert_version_is_dev + + LIBRARY_TARGET_BRANCH="${LIBRARY_TARGET_BRANCH:-main}" + echo "Cloning datadog-lambda-rb branch ${LIBRARY_TARGET_BRANCH}" + rm -rf datadog-lambda-rb + git clone --depth 1 --branch "$LIBRARY_TARGET_BRANCH" \ + https://github.com/DataDog/datadog-lambda-rb.git datadog-lambda-rb + echo "Using datadog-lambda-rb@$(git -C datadog-lambda-rb rev-parse --short HEAD)" + +elif [ "$TARGET" = "otel_collector" ]; then + assert_version_is_dev + assert_target_branch_is_not_set + + echo "otel/opentelemetry-collector-contrib:nightly" > otel_collector-image + echo "Using $(cat otel_collector-image) image" + +else + echo "Unknown target: $1" + exit 1 +fi; diff --git a/utils/target_artifacts/orchestrator.py b/utils/target_artifacts/orchestrator.py index fb157dca9b6..0108e5d3965 100644 --- a/utils/target_artifacts/orchestrator.py +++ b/utils/target_artifacts/orchestrator.py @@ -76,6 +76,9 @@ def write_artifact_entries( new_entries = _dedupe_entries(entries) owner = {"target": target, "environment": environment} + for filename in manifest_entries: + _validate_filename(filename) + for filename in new_entries: _validate_filename(filename) existing_owner = manifest_entries.get(filename, {}).get("owner") @@ -125,7 +128,10 @@ def _read_manifest(binaries_dir: Path) -> dict[str, Any]: path = binaries_dir / MANIFEST_FILENAME if not path.exists(): return {"version": MANIFEST_VERSION, "entries": {}} - payload = json.loads(path.read_text(encoding="utf-8")) + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise TargetArtifactError(f"Unable to read artifact manifest {path}") from exc if not isinstance(payload, dict): raise TargetArtifactError(f"Artifact manifest {path} is not an object") if payload.get("version") != MANIFEST_VERSION: @@ -157,7 +163,7 @@ def _dedupe_entries(entries: tuple[ArtifactEntry, ...]) -> dict[str, ArtifactEnt def _validate_filename(filename: str) -> None: path = Path(filename) - if path.is_absolute() or ".." in path.parts or filename == MANIFEST_FILENAME: + if not filename or path.name != filename or filename == MANIFEST_FILENAME: raise TargetArtifactError(f"Invalid artifact entry filename '{filename}'") From fec53671d234463ca69834de1a8c8fdcd898d9b8 Mon Sep 17 00:00:00 2001 From: datadog-bits <263423550+datadog-bits@users.noreply.github.com> Date: Tue, 15 Sep 2026 09:08:00 +0000 Subject: [PATCH 10/12] Address artifact staging review feedback Co-authored-by: nccatoni <222672590+nccatoni@users.noreply.github.com> --- docs/execute/binaries.md | 6 +- .../internals/target-artifact-staging-spec.md | 9 +- tests/test_the_test/test_load_binary.py | 3 +- tests/test_the_test/test_target_artifacts.py | 186 +++++++++++++++++- utils/build/docker/python/artifact.py | 16 +- utils/target_artifacts/entry_helpers.py | 13 +- utils/target_artifacts/models.py | 7 +- utils/target_artifacts/orchestrator.py | 74 ++++++- 8 files changed, 294 insertions(+), 20 deletions(-) diff --git a/docs/execute/binaries.md b/docs/execute/binaries.md index b4cdaf330f5..447c9364c9e 100644 --- a/docs/execute/binaries.md +++ b/docs/execute/binaries.md @@ -18,8 +18,10 @@ python3 utils/scripts/stage-target-artifacts.py python ``` Staging writes bounded text selectors and records generated-file ownership in -`binaries/.target-artifacts-manifest.json`. It refuses to overwrite manual files in -`binaries/`. Other targets continue to use their existing loading behavior until +`binaries/.target-artifacts-manifest.json`. It refuses to overwrite manual files, +changed generated entries, symlinks, or conflicting selectors in `binaries/`. +Switching to `custom` removes unchanged generated Python selectors while preserving +manual payloads. Other targets continue to use their existing loading behavior until they are migrated separately. diff --git a/docs/internals/target-artifact-staging-spec.md b/docs/internals/target-artifact-staging-spec.md index aaf85cb13cb..73098f845e3 100644 --- a/docs/internals/target-artifact-staging-spec.md +++ b/docs/internals/target-artifact-staging-spec.md @@ -14,17 +14,20 @@ The shared orchestrator owns external lookups and writes the generated entries. also maintains `binaries/.target-artifacts-manifest.json`, which records the owner and content hash of every generated file. Staging: +- verifies that previously generated entries still match their recorded hashes; - refreshes entries previously owned by the same target; - removes stale entries owned by that target; - preserves entries owned by other targets; and -- refuses to overwrite unowned files or entries owned by another target. +- refuses to overwrite unowned files, changed generated entries, symlinks, conflicting + selectors, or entries owned by another target. Selectors should be bounded, such as a commit SHA, release tag, package version, or OCI digest. If an installer must consume a mutable provider selector, the target must also emit a bounded selection marker with `provider_fetch_entries`. -The `custom` environment is a no-op because an upstream or local artifact bundle is -already the source of truth. +The `custom` environment does not resolve or create selectors because an upstream or +local artifact bundle is already the source of truth. It removes unchanged generated +selectors previously owned by the target so they cannot override that custom payload. ## Commands diff --git a/tests/test_the_test/test_load_binary.py b/tests/test_the_test/test_load_binary.py index f5ff335d1cc..3a1ae673dc1 100644 --- a/tests/test_the_test/test_load_binary.py +++ b/tests/test_the_test/test_load_binary.py @@ -4,7 +4,7 @@ from pathlib import Path import subprocess -from utils import scenarios +from utils import features, scenarios from utils.target_artifacts.orchestrator import MANIFEST_FILENAME @@ -163,6 +163,7 @@ def test_missing_package_fails_with_clear_error(self, tmp_path: Path) -> None: @scenarios.test_the_test +@features.not_reported class Test_LoadBinaryPython: def test_development_branch_uses_target_artifact_staging(self, tmp_path: Path) -> None: binaries_dir = tmp_path / "binaries" diff --git a/tests/test_the_test/test_target_artifacts.py b/tests/test_the_test/test_target_artifacts.py index 3c26ea29328..ca32754e2e3 100644 --- a/tests/test_the_test/test_target_artifacts.py +++ b/tests/test_the_test/test_target_artifacts.py @@ -8,7 +8,7 @@ import pytest import requests -from utils import scenarios +from utils import features, scenarios from utils.target_artifacts.models import ( ArtifactResolver, BranchReference, @@ -160,6 +160,7 @@ def _manifest_entries(binaries_dir: Path) -> dict[str, object]: @scenarios.test_the_test +@features.not_reported class Test_TargetArtifactStaging: def test_custom_environment_is_noop(self, tmp_path: Path) -> None: binaries_dir = tmp_path / "binaries" @@ -174,6 +175,33 @@ def test_custom_environment_is_noop(self, tmp_path: Path) -> None: assert not binaries_dir.exists() + def test_custom_environment_clears_owned_selectors(self, tmp_path: Path) -> None: + _write_target_module( + tmp_path, + """ +from utils.target_artifacts.entry_helpers import text_entry + +class Dev: + def artifact_inputs(self, env): + return () + + def artifact_entries(self, resolved_inputs): + return (text_entry("generated", "selector"),) + +class Prod(Dev): + pass +""", + ) + binaries_dir = tmp_path / "binaries" + stage_target("fake", "dev", repo_root=tmp_path, binaries_dir=binaries_dir) + (binaries_dir / "manual.whl").write_text("payload", encoding="utf-8") + + stage_target("fake", "custom", repo_root=tmp_path, binaries_dir=binaries_dir) + + assert not (binaries_dir / "generated").exists() + assert (binaries_dir / "manual.whl").read_text(encoding="utf-8") == "payload" + assert _manifest_entries(binaries_dir) == {} + def test_manifest_refreshes_owned_files_and_preserves_other_targets(self, tmp_path: Path) -> None: module_path = tmp_path / "utils" / "build" / "docker" / "fake" module_path.mkdir(parents=True) @@ -253,6 +281,158 @@ class Prod(Dev): assert (binaries_dir / "manual").read_text(encoding="utf-8") == "user\n" + @pytest.mark.parametrize("next_environment", ["dev", "custom"]) + def test_changed_owned_file_is_not_replaced_or_removed(self, tmp_path: Path, next_environment: str) -> None: + _write_target_module( + tmp_path, + """ +from utils.target_artifacts.entry_helpers import text_entry + +class Dev: + def artifact_inputs(self, env): + return () + + def artifact_entries(self, resolved_inputs): + return (text_entry("generated", "original"),) + +class Prod(Dev): + pass +""", + ) + binaries_dir = tmp_path / "binaries" + stage_target("fake", "dev", repo_root=tmp_path, binaries_dir=binaries_dir) + generated = binaries_dir / "generated" + generated.write_text("changed\n", encoding="utf-8") + + with pytest.raises(TargetArtifactError, match="Refusing to modify changed artifact entry 'generated'"): + stage_target( + "fake", + next_environment, + repo_root=tmp_path, + binaries_dir=binaries_dir, + ) + + assert generated.read_text(encoding="utf-8") == "changed\n" + + @pytest.mark.parametrize("link_target", ["existing", "broken"]) + def test_artifact_entry_symlink_is_rejected(self, tmp_path: Path, link_target: str) -> None: + _write_target_module( + tmp_path, + """ +from utils.target_artifacts.entry_helpers import text_entry + +class Dev: + def artifact_inputs(self, env): + return () + + def artifact_entries(self, resolved_inputs): + return (text_entry("selector", "generated"),) + +class Prod(Dev): + pass +""", + ) + binaries_dir = tmp_path / "binaries" + binaries_dir.mkdir() + external = tmp_path / "external" + if link_target == "existing": + external.write_text("outside\n", encoding="utf-8") + (binaries_dir / "selector").symlink_to(external) + + with pytest.raises(TargetArtifactError, match="symlink"): + stage_target("fake", "dev", repo_root=tmp_path, binaries_dir=binaries_dir) + + assert link_target != "broken" or not external.exists() + if link_target == "existing": + assert external.read_text(encoding="utf-8") == "outside\n" + + def test_duplicate_resolver_names_fail_before_resolution( + self, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + _write_target_module( + tmp_path, + """ +from utils.target_artifacts.resolvers import EnvResolver + +class Dev: + def artifact_inputs(self, env): + return (EnvResolver(name="duplicate"), EnvResolver(name="duplicate")) + + def artifact_entries(self, resolved_inputs): + return () + +class Prod(Dev): + pass +""", + ) + resolve_calls = 0 + + def fake_resolve(_resolver: EnvResolver, _env: dict[str, str]) -> LiteralValue: + nonlocal resolve_calls + resolve_calls += 1 + return LiteralValue(name="duplicate", value="value") + + monkeypatch.setattr(EnvResolver, "resolve", fake_resolve) + + with pytest.raises(TargetArtifactError, match=r"Duplicate artifact input name.*duplicate"): + stage_target("fake", "dev", repo_root=tmp_path, binaries_dir=tmp_path / "binaries") + + assert resolve_calls == 0 + + def test_dynamic_target_module_supports_dataclasses(self, tmp_path: Path) -> None: + _write_target_module( + tmp_path, + """ +from __future__ import annotations +from dataclasses import dataclass + +@dataclass +class Dev: + value: str = "selector" + + def artifact_inputs(self, env): + return () + + def artifact_entries(self, resolved_inputs): + return () + +class Prod(Dev): + pass +""", + ) + + target_environment = load_target_environment(tmp_path, "fake", "dev") + + assert target_environment.value == "selector" # type: ignore[attr-defined] + + def test_conflicting_unowned_selector_is_rejected(self, tmp_path: Path) -> None: + _write_target_module( + tmp_path, + """ +from utils.target_artifacts.entry_helpers import text_entry + +class Dev: + def artifact_inputs(self, env): + return () + + def artifact_entries(self, resolved_inputs): + return (text_entry("prod-selector", "prod", conflicting_filenames=("dev-selector",)),) + +class Prod(Dev): + pass +""", + ) + binaries_dir = tmp_path / "binaries" + binaries_dir.mkdir() + (binaries_dir / "dev-selector").write_text("manual\n", encoding="utf-8") + + with pytest.raises(TargetArtifactError, match="conflicting selector 'dev-selector' is not owned"): + stage_target("fake", "prod", repo_root=tmp_path, binaries_dir=binaries_dir) + + assert not (binaries_dir / "prod-selector").exists() + @pytest.mark.parametrize("filename", ["", "../outside", "nested/entry", MANIFEST_FILENAME]) def test_invalid_entry_filename_is_rejected(self, tmp_path: Path, filename: str) -> None: _write_target_module( @@ -347,6 +527,7 @@ def test_artifact_resolver_docstring_names_resolved_input_type( @scenarios.test_the_test +@features.not_reported class Test_TargetArtifactResolvers: def test_github_requests_include_auth_header_when_token_is_provided( self, @@ -925,6 +1106,7 @@ def fake_run( @scenarios.test_the_test +@features.not_reported class Test_TargetArtifactModules: @pytest.mark.parametrize("environment", ["dev", "prod"]) def test_python_staging_emits_a_bounded_selector(self, environment: str) -> None: @@ -942,6 +1124,8 @@ def test_python_staging_emits_a_bounded_selector(self, environment: str) -> None if environment == "dev": assert entries[0].filename == "python-load-from-s3" assert entries[0].content == f"{SHA}\n" + assert entries[0].conflicting_filenames == ("python-load-from-pip",) else: assert entries[0].filename == "python-load-from-pip" assert entries[0].content == "ddtrace==1.2.3\n" + assert entries[0].conflicting_filenames == ("python-load-from-s3",) diff --git a/utils/build/docker/python/artifact.py b/utils/build/docker/python/artifact.py index 9af29c5006f..f50768ad560 100644 --- a/utils/build/docker/python/artifact.py +++ b/utils/build/docker/python/artifact.py @@ -15,9 +15,21 @@ class Dev(SimpleTarget): default_value="main", ), ) - entries = (text_entry("python-load-from-s3", "{library_branch.sha}"),) + entries = ( + text_entry( + "python-load-from-s3", + "{library_branch.sha}", + conflicting_filenames=("python-load-from-pip",), + ), + ) class Prod(SimpleTarget): inputs = (PypiLatestResolver(name="ddtrace", package="ddtrace"),) - entries = (text_entry("python-load-from-pip", "ddtrace=={ddtrace.version}"),) + entries = ( + text_entry( + "python-load-from-pip", + "ddtrace=={ddtrace.version}", + conflicting_filenames=("python-load-from-s3",), + ), + ) diff --git a/utils/target_artifacts/entry_helpers.py b/utils/target_artifacts/entry_helpers.py index 55fc6048668..f9257e1adb3 100644 --- a/utils/target_artifacts/entry_helpers.py +++ b/utils/target_artifacts/entry_helpers.py @@ -9,8 +9,17 @@ ) -def text_entry(filename: str, content: str) -> ArtifactEntry: - return ArtifactEntry(filename=filename, content=f"{content.rstrip()}\n") +def text_entry( + filename: str, + content: str, + *, + conflicting_filenames: tuple[str, ...] = (), +) -> ArtifactEntry: + return ArtifactEntry( + filename=filename, + content=f"{content.rstrip()}\n", + conflicting_filenames=conflicting_filenames, + ) def json_entry(filename: str, payload: dict[str, object]) -> ArtifactEntry: diff --git a/utils/target_artifacts/models.py b/utils/target_artifacts/models.py index 5c227265936..7ed4f98e881 100644 --- a/utils/target_artifacts/models.py +++ b/utils/target_artifacts/models.py @@ -10,6 +10,7 @@ class TargetArtifactError(Exception): class ArtifactEntry: filename: str content: str + conflicting_filenames: tuple[str, ...] = () @dataclass(frozen=True) @@ -126,6 +127,10 @@ def artifact_inputs(self, _env: dict[str, str]) -> tuple[ArtifactResolver, ...]: def artifact_entries(self, resolved_inputs: dict[str, ResolvedArtifactInput]) -> tuple[ArtifactEntry, ...]: return tuple( - ArtifactEntry(filename=entry.filename, content=entry.content.format(**resolved_inputs)) + ArtifactEntry( + filename=entry.filename, + content=entry.content.format(**resolved_inputs), + conflicting_filenames=entry.conflicting_filenames, + ) for entry in self.entries ) diff --git a/utils/target_artifacts/orchestrator.py b/utils/target_artifacts/orchestrator.py index 0108e5d3965..4b47a3265ac 100644 --- a/utils/target_artifacts/orchestrator.py +++ b/utils/target_artifacts/orchestrator.py @@ -4,6 +4,7 @@ import importlib.util import json import os +import sys from pathlib import Path from typing import TYPE_CHECKING, Any @@ -34,16 +35,21 @@ def stage_target( env = dict(os.environ if process_env is None else process_env) - if environment == "custom": - return if environment not in {"dev", "prod"}: + if environment == "custom": + manifest_path = output_dir / MANIFEST_FILENAME + if manifest_path.exists() or manifest_path.is_symlink(): + write_artifact_entries(output_dir, target, environment, ()) + return raise TargetArtifactError(f"Unknown target artifact environment: {environment}") target_environment = load_target_environment(root, target, environment) - resolved_inputs = { - artifact_resolver.name: artifact_resolver.resolve(env) - for artifact_resolver in target_environment.artifact_inputs(env) - } + artifact_inputs = target_environment.artifact_inputs(env) + input_names = [artifact_resolver.name for artifact_resolver in artifact_inputs] + duplicate_names = sorted({name for name in input_names if input_names.count(name) > 1}) + if duplicate_names: + raise TargetArtifactError(f"Duplicate artifact input name(s): {', '.join(duplicate_names)}") + resolved_inputs = {artifact_resolver.name: artifact_resolver.resolve(env) for artifact_resolver in artifact_inputs} entries = target_environment.artifact_entries(resolved_inputs) write_artifact_entries(output_dir, target, environment, entries) @@ -79,10 +85,16 @@ def write_artifact_entries( for filename in manifest_entries: _validate_filename(filename) - for filename in new_entries: + for filename, metadata in manifest_entries.items(): + if _same_target(metadata.get("owner"), target): + _validate_owned_file(binaries_dir / filename, filename, metadata) + + for filename, entry in new_entries.items(): _validate_filename(filename) existing_owner = manifest_entries.get(filename, {}).get("owner") path = binaries_dir / filename + if path.is_symlink(): + raise TargetArtifactError(f"Refusing to write artifact entry through symlink '{filename}'") if existing_owner is not None and not _same_target(existing_owner, target): owner_target = ( existing_owner.get("target", "") if isinstance(existing_owner, dict) else "" @@ -90,6 +102,25 @@ def write_artifact_entries( raise TargetArtifactError(f"Artifact entry '{filename}' is already owned by target '{owner_target}'") if path.exists() and existing_owner is None: raise TargetArtifactError(f"Refusing to overwrite unowned artifact entry '{filename}'") + for conflicting_filename in entry.conflicting_filenames: + _validate_filename(conflicting_filename) + if conflicting_filename in new_entries: + raise TargetArtifactError( + f"Artifact entries '{filename}' and '{conflicting_filename}' conflict with each other" + ) + conflicting_path = binaries_dir / conflicting_filename + if conflicting_path.is_symlink(): + raise TargetArtifactError( + f"Refusing artifact entry '{filename}' because conflicting selector " + f"'{conflicting_filename}' is a symlink" + ) + if conflicting_path.exists(): + conflicting_owner = manifest_entries.get(conflicting_filename, {}).get("owner") + if not _same_target(conflicting_owner, target): + raise TargetArtifactError( + f"Refusing artifact entry '{filename}' because conflicting selector " + f"'{conflicting_filename}' is not owned by target '{target}'" + ) for filename, metadata in list(manifest_entries.items()): owner_data = metadata.get("owner") @@ -120,12 +151,23 @@ def _load_module(module_path: Path, module_name: str) -> ModuleType: if spec is None or spec.loader is None: raise TargetArtifactError(f"Unable to import target artifact module at {module_path}") module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) + previous_module = sys.modules.get(module_name) + sys.modules[module_name] = module + try: + spec.loader.exec_module(module) + except BaseException: + if previous_module is None: + del sys.modules[module_name] + else: + sys.modules[module_name] = previous_module + raise return module def _read_manifest(binaries_dir: Path) -> dict[str, Any]: path = binaries_dir / MANIFEST_FILENAME + if path.is_symlink(): + raise TargetArtifactError(f"Artifact manifest {path} must not be a symlink") if not path.exists(): return {"version": MANIFEST_VERSION, "entries": {}} try: @@ -167,5 +209,21 @@ def _validate_filename(filename: str) -> None: raise TargetArtifactError(f"Invalid artifact entry filename '{filename}'") +def _validate_owned_file(path: Path, filename: str, metadata: dict[str, Any]) -> None: + if path.is_symlink(): + raise TargetArtifactError(f"Refusing to modify owned artifact entry symlink '{filename}'") + if not path.exists(): + return + expected_hash = metadata.get("sha256") + if not isinstance(expected_hash, str): + raise TargetArtifactError(f"Owned artifact entry '{filename}' has no valid recorded hash") + try: + actual_hash = hashlib.sha256(path.read_bytes()).hexdigest() + except OSError as exc: + raise TargetArtifactError(f"Unable to verify owned artifact entry '{filename}'") from exc + if actual_hash != expected_hash: + raise TargetArtifactError(f"Refusing to modify changed artifact entry '{filename}'") + + def _same_target(owner: object, target: str) -> bool: return isinstance(owner, dict) and owner.get("target") == target From 37d40538ca0f8b17202a8b4fde0afb6a3538b3cb Mon Sep 17 00:00:00 2001 From: datadog-bits <263423550+datadog-bits@users.noreply.github.com> Date: Tue, 15 Sep 2026 09:15:37 +0000 Subject: [PATCH 11/12] Remove test-the-test feature decorators Co-authored-by: nccatoni <222672590+nccatoni@users.noreply.github.com> --- tests/test_the_test/test_load_binary.py | 3 +-- tests/test_the_test/test_target_artifacts.py | 5 +---- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/tests/test_the_test/test_load_binary.py b/tests/test_the_test/test_load_binary.py index 3a1ae673dc1..f5ff335d1cc 100644 --- a/tests/test_the_test/test_load_binary.py +++ b/tests/test_the_test/test_load_binary.py @@ -4,7 +4,7 @@ from pathlib import Path import subprocess -from utils import features, scenarios +from utils import scenarios from utils.target_artifacts.orchestrator import MANIFEST_FILENAME @@ -163,7 +163,6 @@ def test_missing_package_fails_with_clear_error(self, tmp_path: Path) -> None: @scenarios.test_the_test -@features.not_reported class Test_LoadBinaryPython: def test_development_branch_uses_target_artifact_staging(self, tmp_path: Path) -> None: binaries_dir = tmp_path / "binaries" diff --git a/tests/test_the_test/test_target_artifacts.py b/tests/test_the_test/test_target_artifacts.py index ca32754e2e3..05f321890d3 100644 --- a/tests/test_the_test/test_target_artifacts.py +++ b/tests/test_the_test/test_target_artifacts.py @@ -8,7 +8,7 @@ import pytest import requests -from utils import features, scenarios +from utils import scenarios from utils.target_artifacts.models import ( ArtifactResolver, BranchReference, @@ -160,7 +160,6 @@ def _manifest_entries(binaries_dir: Path) -> dict[str, object]: @scenarios.test_the_test -@features.not_reported class Test_TargetArtifactStaging: def test_custom_environment_is_noop(self, tmp_path: Path) -> None: binaries_dir = tmp_path / "binaries" @@ -527,7 +526,6 @@ def test_artifact_resolver_docstring_names_resolved_input_type( @scenarios.test_the_test -@features.not_reported class Test_TargetArtifactResolvers: def test_github_requests_include_auth_header_when_token_is_provided( self, @@ -1106,7 +1104,6 @@ def fake_run( @scenarios.test_the_test -@features.not_reported class Test_TargetArtifactModules: @pytest.mark.parametrize("environment", ["dev", "prod"]) def test_python_staging_emits_a_bounded_selector(self, environment: str) -> None: From ec6437270325ec3c13b9b36a3795e82ac8dae841 Mon Sep 17 00:00:00 2001 From: datadog-bits <263423550+datadog-bits@users.noreply.github.com> Date: Fri, 18 Sep 2026 15:13:51 +0000 Subject: [PATCH 12/12] Automate dev and prod selector conflicts Co-authored-by: nccatoni <222672590+nccatoni@users.noreply.github.com> --- docs/execute/binaries.md | 2 + .../internals/target-artifact-staging-spec.md | 11 +- tests/test_the_test/test_target_artifacts.py | 175 +++++++++++++++++- utils/build/docker/python/artifact.py | 16 +- utils/target_artifacts/entry_helpers.py | 3 - utils/target_artifacts/models.py | 9 +- utils/target_artifacts/orchestrator.py | 83 ++++++--- 7 files changed, 241 insertions(+), 58 deletions(-) diff --git a/docs/execute/binaries.md b/docs/execute/binaries.md index 447c9364c9e..8c46a4ff3b7 100644 --- a/docs/execute/binaries.md +++ b/docs/execute/binaries.md @@ -20,6 +20,8 @@ python3 utils/scripts/stage-target-artifacts.py python Staging writes bounded text selectors and records generated-file ownership in `binaries/.target-artifacts-manifest.json`. It refuses to overwrite manual files, changed generated entries, symlinks, or conflicting selectors in `binaries/`. +Development and production selectors for the same target are treated as mutually +exclusive automatically. Switching to `custom` removes unchanged generated Python selectors while preserving manual payloads. Other targets continue to use their existing loading behavior until they are migrated separately. diff --git a/docs/internals/target-artifact-staging-spec.md b/docs/internals/target-artifact-staging-spec.md index 73098f845e3..fcc73db54ea 100644 --- a/docs/internals/target-artifact-staging-spec.md +++ b/docs/internals/target-artifact-staging-spec.md @@ -7,8 +7,9 @@ targets continue to use `utils/scripts/load-binary.sh` until migrated separately ## Contract Each migrated target provides `utils/build/docker//artifact.py` with `Dev` -and `Prod` implementations. They declare resolver inputs and map the resolved values -to text entries without performing network or filesystem side effects themselves. +and `Prod` implementations. They declare every filename they may emit, declare +resolver inputs, and map the resolved values to text entries without performing +network or filesystem side effects themselves. The shared orchestrator owns external lookups and writes the generated entries. It also maintains `binaries/.target-artifacts-manifest.json`, which records the owner @@ -21,6 +22,12 @@ and content hash of every generated file. Staging: - refuses to overwrite unowned files, changed generated entries, symlinks, conflicting selectors, or entries owned by another target. +The filenames declared by a target's `Dev` and `Prod` implementations define its +selector family. Staging rejects any manual selector in that family that the selected +environment did not emit, while allowing multiple entries emitted together to coexist. +Individual entries do not need to name their conflicts, and declaring filenames does +not resolve the inactive environment's external inputs. + Selectors should be bounded, such as a commit SHA, release tag, package version, or OCI digest. If an installer must consume a mutable provider selector, the target must also emit a bounded selection marker with `provider_fetch_entries`. diff --git a/tests/test_the_test/test_target_artifacts.py b/tests/test_the_test/test_target_artifacts.py index 05f321890d3..e94aff6717b 100644 --- a/tests/test_the_test/test_target_artifacts.py +++ b/tests/test_the_test/test_target_artifacts.py @@ -181,6 +181,9 @@ def test_custom_environment_clears_owned_selectors(self, tmp_path: Path) -> None from utils.target_artifacts.entry_helpers import text_entry class Dev: + def artifact_entry_filenames(self): + return ("generated",) + def artifact_inputs(self, env): return () @@ -210,6 +213,9 @@ def test_manifest_refreshes_owned_files_and_preserves_other_targets(self, tmp_pa from utils.target_artifacts.entry_helpers import text_entry class Dev: + def artifact_entry_filenames(self): + return ("kept", "stale") + def artifact_inputs(self, env): return () @@ -217,6 +223,9 @@ def artifact_entries(self, resolved_inputs): return (text_entry("kept", "one"), text_entry("stale", "old")) class Prod: + def artifact_entry_filenames(self): + return ("kept",) + def artifact_inputs(self, env): return () @@ -232,6 +241,9 @@ def artifact_entries(self, resolved_inputs): from utils.target_artifacts.entry_helpers import text_entry class Dev: + def artifact_entry_filenames(self): + return ("other",) + def artifact_inputs(self, env): return () @@ -261,6 +273,9 @@ def test_unowned_file_is_not_overwritten(self, tmp_path: Path) -> None: from utils.target_artifacts.entry_helpers import text_entry class Dev: + def artifact_entry_filenames(self): + return ("manual",) + def artifact_inputs(self, env): return () @@ -288,6 +303,9 @@ def test_changed_owned_file_is_not_replaced_or_removed(self, tmp_path: Path, nex from utils.target_artifacts.entry_helpers import text_entry class Dev: + def artifact_entry_filenames(self): + return ("generated",) + def artifact_inputs(self, env): return () @@ -321,6 +339,9 @@ def test_artifact_entry_symlink_is_rejected(self, tmp_path: Path, link_target: s from utils.target_artifacts.entry_helpers import text_entry class Dev: + def artifact_entry_filenames(self): + return ("selector",) + def artifact_inputs(self, env): return () @@ -356,6 +377,9 @@ def test_duplicate_resolver_names_fail_before_resolution( from utils.target_artifacts.resolvers import EnvResolver class Dev: + def artifact_entry_filenames(self): + return () + def artifact_inputs(self, env): return (EnvResolver(name="duplicate"), EnvResolver(name="duplicate")) @@ -391,6 +415,9 @@ def test_dynamic_target_module_supports_dataclasses(self, tmp_path: Path) -> Non class Dev: value: str = "selector" + def artifact_entry_filenames(self): + return () + def artifact_inputs(self, env): return () @@ -406,32 +433,159 @@ class Prod(Dev): assert target_environment.value == "selector" # type: ignore[attr-defined] - def test_conflicting_unowned_selector_is_rejected(self, tmp_path: Path) -> None: + def test_manual_dev_selector_blocks_prod_without_resolving_dev(self, tmp_path: Path) -> None: _write_target_module( tmp_path, """ from utils.target_artifacts.entry_helpers import text_entry class Dev: + def artifact_entry_filenames(self): + return ("dev-selector",) + def artifact_inputs(self, env): - return () + raise AssertionError("dev inputs must not be inspected while staging prod") def artifact_entries(self, resolved_inputs): - return (text_entry("prod-selector", "prod", conflicting_filenames=("dev-selector",)),) + raise AssertionError("dev entries must not be produced while staging prod") -class Prod(Dev): - pass +class Prod: + def artifact_entry_filenames(self): + return ("prod-selector",) + + def artifact_inputs(self, env): + return () + + def artifact_entries(self, resolved_inputs): + return (text_entry("prod-selector", "prod"),) """, ) binaries_dir = tmp_path / "binaries" binaries_dir.mkdir() (binaries_dir / "dev-selector").write_text("manual\n", encoding="utf-8") - with pytest.raises(TargetArtifactError, match="conflicting selector 'dev-selector' is not owned"): + with pytest.raises(TargetArtifactError, match=r"conflicting selector 'dev-selector'.*not owned"): stage_target("fake", "prod", repo_root=tmp_path, binaries_dir=binaries_dir) assert not (binaries_dir / "prod-selector").exists() + def test_manual_prod_selector_blocks_dev_without_resolving_prod(self, tmp_path: Path) -> None: + _write_target_module( + tmp_path, + """ +from utils.target_artifacts.entry_helpers import text_entry + +class Dev: + def artifact_entry_filenames(self): + return ("dev-selector",) + + def artifact_inputs(self, env): + return () + + def artifact_entries(self, resolved_inputs): + return (text_entry("dev-selector", "dev"),) + +class Prod: + def artifact_entry_filenames(self): + return ("prod-selector",) + + def artifact_inputs(self, env): + raise AssertionError("prod inputs must not be inspected while staging dev") + + def artifact_entries(self, resolved_inputs): + raise AssertionError("prod entries must not be produced while staging dev") +""", + ) + binaries_dir = tmp_path / "binaries" + binaries_dir.mkdir() + (binaries_dir / "prod-selector").write_text("manual\n", encoding="utf-8") + + with pytest.raises(TargetArtifactError, match=r"conflicting selector 'prod-selector'.*not owned"): + stage_target("fake", "dev", repo_root=tmp_path, binaries_dir=binaries_dir) + + assert not (binaries_dir / "dev-selector").exists() + + def test_selected_environment_can_emit_multiple_entries(self, tmp_path: Path) -> None: + _write_target_module( + tmp_path, + """ +from utils.target_artifacts.entry_helpers import text_entry + +class Dev: + def artifact_entry_filenames(self): + return ("dev-selector", "dev-marker") + + def artifact_inputs(self, env): + return () + + def artifact_entries(self, resolved_inputs): + return (text_entry("dev-selector", "dev"), text_entry("dev-marker", "bounded")) + +class Prod: + def artifact_entry_filenames(self): + return ("prod-selector",) + + def artifact_inputs(self, env): + return () + + def artifact_entries(self, resolved_inputs): + return (text_entry("prod-selector", "prod"),) +""", + ) + binaries_dir = tmp_path / "binaries" + + stage_target("fake", "dev", repo_root=tmp_path, binaries_dir=binaries_dir) + + assert (binaries_dir / "dev-selector").read_text(encoding="utf-8") == "dev\n" + assert (binaries_dir / "dev-marker").read_text(encoding="utf-8") == "bounded\n" + + @pytest.mark.parametrize("filename", ["", "../outside", "nested/entry", MANIFEST_FILENAME]) + def test_invalid_declared_filename_is_rejected_before_resolution(self, tmp_path: Path, filename: str) -> None: + _write_target_module( + tmp_path, + f""" +class Dev: + def artifact_entry_filenames(self): + return ({filename!r},) + + def artifact_inputs(self, env): + raise AssertionError("invalid declarations must fail before resolution") + + def artifact_entries(self, resolved_inputs): + return () + +class Prod(Dev): + pass +""", + ) + + with pytest.raises(TargetArtifactError, match="Invalid artifact entry filename"): + stage_target("fake", "dev", repo_root=tmp_path, binaries_dir=tmp_path / "binaries") + + def test_undeclared_emitted_filename_is_rejected(self, tmp_path: Path) -> None: + _write_target_module( + tmp_path, + """ +from utils.target_artifacts.entry_helpers import text_entry + +class Dev: + def artifact_entry_filenames(self): + return ("declared",) + + def artifact_inputs(self, env): + return () + + def artifact_entries(self, resolved_inputs): + return (text_entry("undeclared", "value"),) + +class Prod(Dev): + pass +""", + ) + + with pytest.raises(TargetArtifactError, match=r"emitted undeclared artifact entry filename.*undeclared"): + stage_target("fake", "dev", repo_root=tmp_path, binaries_dir=tmp_path / "binaries") + @pytest.mark.parametrize("filename", ["", "../outside", "nested/entry", MANIFEST_FILENAME]) def test_invalid_entry_filename_is_rejected(self, tmp_path: Path, filename: str) -> None: _write_target_module( @@ -440,6 +594,9 @@ def test_invalid_entry_filename_is_rejected(self, tmp_path: Path, filename: str) from utils.target_artifacts.entry_helpers import text_entry class Dev: + def artifact_entry_filenames(self): + return ({filename!r},) + def artifact_inputs(self, env): return () @@ -459,6 +616,9 @@ def test_manifest_cannot_delete_files_outside_binaries(self, tmp_path: Path) -> tmp_path, """ class Dev: + def artifact_entry_filenames(self): + return () + def artifact_inputs(self, env): return () @@ -1118,11 +1278,10 @@ def test_python_staging_emits_a_bounded_selector(self, environment: str) -> None entries = target_environment.artifact_entries(resolved) assert len(entries) == 1 + assert target_environment.artifact_entry_filenames() == (entries[0].filename,) if environment == "dev": assert entries[0].filename == "python-load-from-s3" assert entries[0].content == f"{SHA}\n" - assert entries[0].conflicting_filenames == ("python-load-from-pip",) else: assert entries[0].filename == "python-load-from-pip" assert entries[0].content == "ddtrace==1.2.3\n" - assert entries[0].conflicting_filenames == ("python-load-from-s3",) diff --git a/utils/build/docker/python/artifact.py b/utils/build/docker/python/artifact.py index f50768ad560..9af29c5006f 100644 --- a/utils/build/docker/python/artifact.py +++ b/utils/build/docker/python/artifact.py @@ -15,21 +15,9 @@ class Dev(SimpleTarget): default_value="main", ), ) - entries = ( - text_entry( - "python-load-from-s3", - "{library_branch.sha}", - conflicting_filenames=("python-load-from-pip",), - ), - ) + entries = (text_entry("python-load-from-s3", "{library_branch.sha}"),) class Prod(SimpleTarget): inputs = (PypiLatestResolver(name="ddtrace", package="ddtrace"),) - entries = ( - text_entry( - "python-load-from-pip", - "ddtrace=={ddtrace.version}", - conflicting_filenames=("python-load-from-s3",), - ), - ) + entries = (text_entry("python-load-from-pip", "ddtrace=={ddtrace.version}"),) diff --git a/utils/target_artifacts/entry_helpers.py b/utils/target_artifacts/entry_helpers.py index f9257e1adb3..52fe781ecfd 100644 --- a/utils/target_artifacts/entry_helpers.py +++ b/utils/target_artifacts/entry_helpers.py @@ -12,13 +12,10 @@ def text_entry( filename: str, content: str, - *, - conflicting_filenames: tuple[str, ...] = (), ) -> ArtifactEntry: return ArtifactEntry( filename=filename, content=f"{content.rstrip()}\n", - conflicting_filenames=conflicting_filenames, ) diff --git a/utils/target_artifacts/models.py b/utils/target_artifacts/models.py index 7ed4f98e881..eb658d0155b 100644 --- a/utils/target_artifacts/models.py +++ b/utils/target_artifacts/models.py @@ -10,7 +10,6 @@ class TargetArtifactError(Exception): class ArtifactEntry: filename: str content: str - conflicting_filenames: tuple[str, ...] = () @dataclass(frozen=True) @@ -95,6 +94,10 @@ def resolve(self, env: dict[str, str], /) -> ResolvedArtifactInput: @runtime_checkable class TargetArtifactEnvironment(Protocol): + def artifact_entry_filenames(self) -> tuple[str, ...]: + """Declare every artifact entry filename this environment can emit.""" + ... + def artifact_inputs( self, env: dict[str, str], @@ -125,12 +128,14 @@ class SimpleTarget: def artifact_inputs(self, _env: dict[str, str]) -> tuple[ArtifactResolver, ...]: return self.inputs + def artifact_entry_filenames(self) -> tuple[str, ...]: + return tuple(entry.filename for entry in self.entries) + def artifact_entries(self, resolved_inputs: dict[str, ResolvedArtifactInput]) -> tuple[ArtifactEntry, ...]: return tuple( ArtifactEntry( filename=entry.filename, content=entry.content.format(**resolved_inputs), - conflicting_filenames=entry.conflicting_filenames, ) for entry in self.entries ) diff --git a/utils/target_artifacts/orchestrator.py b/utils/target_artifacts/orchestrator.py index 4b47a3265ac..1be950a4c0f 100644 --- a/utils/target_artifacts/orchestrator.py +++ b/utils/target_artifacts/orchestrator.py @@ -43,7 +43,12 @@ def stage_target( return raise TargetArtifactError(f"Unknown target artifact environment: {environment}") - target_environment = load_target_environment(root, target, environment) + target_environments = load_target_environments(root, target) + target_environment = target_environments[environment] + environment_filenames = { + name: _validate_declared_filenames(target, name, target_env.artifact_entry_filenames()) + for name, target_env in target_environments.items() + } artifact_inputs = target_environment.artifact_inputs(env) input_names = [artifact_resolver.name for artifact_resolver in artifact_inputs] duplicate_names = sorted({name for name in input_names if input_names.count(name) > 1}) @@ -51,24 +56,36 @@ def stage_target( raise TargetArtifactError(f"Duplicate artifact input name(s): {', '.join(duplicate_names)}") resolved_inputs = {artifact_resolver.name: artifact_resolver.resolve(env) for artifact_resolver in artifact_inputs} entries = target_environment.artifact_entries(resolved_inputs) - write_artifact_entries(output_dir, target, environment, entries) + undeclared_filenames = sorted({entry.filename for entry in entries} - set(environment_filenames[environment])) + if undeclared_filenames: + raise TargetArtifactError( + f"{target}.{environment} emitted undeclared artifact entry filename(s): {', '.join(undeclared_filenames)}" + ) + selector_filenames = tuple(filename for filenames in environment_filenames.values() for filename in filenames) + write_artifact_entries(output_dir, target, environment, entries, selector_filenames=selector_filenames) def load_target_environment(repo_root: Path, target: str, environment: str) -> TargetArtifactEnvironment: + return load_target_environments(repo_root, target)[environment] + + +def load_target_environments(repo_root: Path, target: str) -> dict[str, TargetArtifactEnvironment]: module_path = repo_root / "utils" / "build" / "docker" / target / "artifact.py" if not module_path.exists(): raise TargetArtifactError(f"No target artifact module found for '{target}' at {module_path}") module = _load_module(module_path, f"system_tests_target_artifacts_{target}") - class_name = "Dev" if environment == "dev" else "Prod" - environment_class = getattr(module, class_name, None) - if environment_class is None: - raise TargetArtifactError(f"Target artifact module for '{target}' does not define {class_name}") - - instance = environment_class() - if not isinstance(instance, TargetArtifactEnvironment): - raise TargetArtifactError(f"{target}.{class_name} does not implement TargetArtifactEnvironment") - return instance + result: dict[str, TargetArtifactEnvironment] = {} + for environment, class_name in (("dev", "Dev"), ("prod", "Prod")): + environment_class = getattr(module, class_name, None) + if environment_class is None: + raise TargetArtifactError(f"Target artifact module for '{target}' does not define {class_name}") + + instance = environment_class() + if not isinstance(instance, TargetArtifactEnvironment): + raise TargetArtifactError(f"{target}.{class_name} does not implement TargetArtifactEnvironment") + result[environment] = instance + return result def write_artifact_entries( @@ -76,6 +93,8 @@ def write_artifact_entries( target: str, environment: str, entries: tuple[ArtifactEntry, ...], + *, + selector_filenames: tuple[str, ...] = (), ) -> None: manifest = _read_manifest(binaries_dir) manifest_entries = _manifest_entries(manifest) @@ -85,11 +104,14 @@ def write_artifact_entries( for filename in manifest_entries: _validate_filename(filename) + for filename in selector_filenames: + _validate_filename(filename) + for filename, metadata in manifest_entries.items(): if _same_target(metadata.get("owner"), target): _validate_owned_file(binaries_dir / filename, filename, metadata) - for filename, entry in new_entries.items(): + for filename in new_entries: _validate_filename(filename) existing_owner = manifest_entries.get(filename, {}).get("owner") path = binaries_dir / filename @@ -102,25 +124,17 @@ def write_artifact_entries( raise TargetArtifactError(f"Artifact entry '{filename}' is already owned by target '{owner_target}'") if path.exists() and existing_owner is None: raise TargetArtifactError(f"Refusing to overwrite unowned artifact entry '{filename}'") - for conflicting_filename in entry.conflicting_filenames: - _validate_filename(conflicting_filename) - if conflicting_filename in new_entries: - raise TargetArtifactError( - f"Artifact entries '{filename}' and '{conflicting_filename}' conflict with each other" - ) - conflicting_path = binaries_dir / conflicting_filename - if conflicting_path.is_symlink(): + + for selector_filename in set(selector_filenames) - set(new_entries): + selector_path = binaries_dir / selector_filename + if selector_path.is_symlink(): + raise TargetArtifactError(f"Refusing conflicting selector symlink '{selector_filename}'") + if selector_path.exists(): + selector_owner = manifest_entries.get(selector_filename, {}).get("owner") + if not _same_target(selector_owner, target): raise TargetArtifactError( - f"Refusing artifact entry '{filename}' because conflicting selector " - f"'{conflicting_filename}' is a symlink" + f"Refusing conflicting selector '{selector_filename}' because it is not owned by target '{target}'" ) - if conflicting_path.exists(): - conflicting_owner = manifest_entries.get(conflicting_filename, {}).get("owner") - if not _same_target(conflicting_owner, target): - raise TargetArtifactError( - f"Refusing artifact entry '{filename}' because conflicting selector " - f"'{conflicting_filename}' is not owned by target '{target}'" - ) for filename, metadata in list(manifest_entries.items()): owner_data = metadata.get("owner") @@ -203,6 +217,17 @@ def _dedupe_entries(entries: tuple[ArtifactEntry, ...]) -> dict[str, ArtifactEnt return result +def _validate_declared_filenames(target: str, environment: str, filenames: tuple[str, ...]) -> tuple[str, ...]: + duplicates = sorted({filename for filename in filenames if filenames.count(filename) > 1}) + if duplicates: + raise TargetArtifactError( + f"{target}.{environment} declares duplicate artifact entry filename(s): {', '.join(duplicates)}" + ) + for filename in filenames: + _validate_filename(filename) + return filenames + + def _validate_filename(filename: str) -> None: path = Path(filename) if not filename or path.name != filename or filename == MANIFEST_FILENAME: