diff --git a/.github/chainguard/self.gitlab-update-agent-version.sts.yaml b/.github/chainguard/self.gitlab-update-agent-version.sts.yaml new file mode 100644 index 00000000000..0b43039563b --- /dev/null +++ b/.github/chainguard/self.gitlab-update-agent-version.sts.yaml @@ -0,0 +1,18 @@ +--- +# Policy for: update_agent_version in DataDog/system-tests/.gitlab-ci.yml +issuer: https://gitlab.ddbuild.io + +subject_pattern: "project_path:DataDog/system-tests:ref_type:branch:ref:main" + +claim_pattern: + project_path: "DataDog/system-tests" + ref: "main" + ref_type: "branch" + ref_path: "refs/heads/main" + ref_protected: "true" + pipeline_source: "schedule" + ci_config_ref_uri: "gitlab\\.ddbuild\\.io/DataDog/system-tests//\\.gitlab-ci\\.yml@refs/heads/main" + +permissions: + contents: write + pull_requests: write diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7364a938230..1bede807187 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -280,6 +280,24 @@ check_merge_labels: rules: - if: $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "main" +update_agent_version: + image: $CI_IMAGE + tags: + - arch:amd64 + stage: system-tests-utils + resource_group: update-agent-version + id_tokens: + DDOCTOSTS_ID_TOKEN: + aud: dd-octo-sts + needs: + - job: build_ci_image + artifacts: false + optional: true + script: + - python3 utils/scripts/update_agent_version.py + rules: + - if: '$SCHEDULED_JOB == "nightly" && $CI_PIPELINE_SOURCE == "schedule" && $CI_COMMIT_BRANCH == "main"' + generate_system_tests_lambda_proxy_image: image: registry.ddbuild.io/ci/libdatadog-build/ci_docker_base:100425777 tags: ["docker-in-docker:amd64"] diff --git a/tests/test_the_test/test_update_agent_version.py b/tests/test_the_test/test_update_agent_version.py new file mode 100644 index 00000000000..213672d9728 --- /dev/null +++ b/tests/test_the_test/test_update_agent_version.py @@ -0,0 +1,330 @@ +import os +import subprocess +from pathlib import Path + +import pytest + +from utils import scenarios +from utils.scripts.update_agent_version import ( + AGENT_VERSION_LOCK, + AUTOMATION_BRANCH, + GitHubApi, + automate_update, + enable_auto_merge, + normalize_version, + publish_update, + revoke_token, + run_automation, + update_agent_version, +) + + +def write_lock(root: Path) -> None: + lock = root / AGENT_VERSION_LOCK + lock.parent.mkdir(parents=True) + lock.write_text("# Pinned Agent version, updated automatically\nDD_AGENT_VERSION=7.78.4\n") + + +@scenarios.test_the_test +def test_update_agent_version_updates_lock(tmp_path: Path) -> None: + write_lock(tmp_path) + (tmp_path / AGENT_VERSION_LOCK).write_text("This content is replaced completely.\n") + + assert update_agent_version(tmp_path, "v8.0.1") + assert (tmp_path / AGENT_VERSION_LOCK).read_text() == ( + "# Pinned Agent version, updated automatically\nDD_AGENT_VERSION=8.0.1\n" + ) + + assert not update_agent_version(tmp_path, "8.0.1") + + +@scenarios.test_the_test +@pytest.mark.parametrize("version", ["7.82", "7.82.3-rc.1", "latest"]) +def test_normalize_version_rejects_unsupported_versions(version: str) -> None: + with pytest.raises(ValueError, match="Expected a stable Agent version"): + normalize_version(version) + + +@scenarios.test_the_test +def test_agent_version_consumers_load_lock() -> None: + root = Path(__file__).resolve().parents[2] + virtual_machine = root / "utils/build/virtual_machine" + auto_inject = root / "utils/build/virtual_machine/provisions/auto-inject" + + lock_lines = (virtual_machine / "agent.lock").read_text().splitlines() + assert lock_lines[0] == "# Pinned Agent version, updated automatically" + assert len(lock_lines) == 2 + locked_version = lock_lines[1].removeprefix("DD_AGENT_VERSION=") + assert lock_lines[1] == f"DD_AGENT_VERSION={normalize_version(locked_version)}" + assert "agent:${DD_AGENT_VERSION}" in (auto_inject / "docker/docker-compose-agent-prod.yml").read_text() + + compose_path = "utils/build/virtual_machine/provisions/auto-inject/docker/docker-compose-agent-prod.yml" + lock_path = "utils/build/virtual_machine/agent.lock" + provision_root = virtual_machine + compose_copy_points = [path for path in provision_root.rglob("*.yml") if compose_path in path.read_text()] + assert compose_copy_points + for copy_point in compose_copy_points: + assert lock_path in copy_point.read_text() + + lock_consumers = ( + auto_inject / "auto-inject_installer_manual.yml", + auto_inject / "repositories/autoinstall/execute_install_script.sh", + root / "utils/build/virtual_machine/provisions/local-auto-inject-install-script/provision.yml", + root / "utils/build/virtual_machine/weblogs/common/pull_agent_image.sh", + ) + for consumer in lock_consumers: + content = consumer.read_text() + assert "agent.lock" in content + assert "DD_AGENT_VERSION" in content + assert "install_script_agent7.sh" not in content + + compose_launchers = ( + root / "utils/build/virtual_machine/weblogs/common/create_and_run_app_container.sh", + root / "utils/build/virtual_machine/weblogs/common/create_and_run_app_multicontainer.sh", + root / "utils/build/virtual_machine/weblogs/java/test-app-java-buildpack/" + "test-app-java_docker_compose_run_buildpack.sh", + ) + for launcher in compose_launchers: + content = launcher.read_text() + assert 'AGENT_LOCK="agent.lock"' in content + assert '. "./${AGENT_LOCK}"' in content + + +@scenarios.test_the_test +def test_pull_agent_image_resolves_version_from_lock(tmp_path: Path) -> None: + root = Path(__file__).resolve().parents[2] + (tmp_path / "agent.lock").write_text("DD_AGENT_VERSION=8.0.1\n") + (tmp_path / "docker-compose-agent-prod.yml").write_text( + "services:\n datadog:\n image: gcr.io/datadoghq/agent:${DD_AGENT_VERSION}\n" + ) + + fake_bin = tmp_path / "bin" + fake_bin.mkdir() + call_log = tmp_path / "sudo.log" + fake_sudo = fake_bin / "sudo" + fake_sudo.write_text('#!/bin/sh\nprintf "%s\\n" "$*" >> "$CALL_LOG"\n') + fake_sudo.chmod(0o755) + env = os.environ.copy() + env.update( + { + "CALL_LOG": str(call_log), + "DOCKER_PULL_MAX_RETRIES": "1", + "PATH": f"{fake_bin}:{env['PATH']}", + } + ) + + subprocess.run( + ["bash", str(root / "utils/build/virtual_machine/weblogs/common/pull_agent_image.sh")], + cwd=tmp_path, + check=True, + env=env, + ) + + assert call_log.read_text() == "docker pull gcr.io/datadoghq/agent:8.0.1\n" + + +@scenarios.test_the_test +def test_automate_update_publishes_latest_version(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + write_lock(tmp_path) + published: list[tuple[Path, str]] = [] + github = GitHubApi("token") + monkeypatch.setattr("utils.scripts.update_agent_version.latest_agent_version", lambda _github: "8.0.1") + monkeypatch.setattr( + "utils.scripts.update_agent_version.publish_update", + lambda root, version, _github, _env: published.append((root, version)), + ) + + assert automate_update(tmp_path, github, {}) + assert published == [(tmp_path, "8.0.1")] + + +@scenarios.test_the_test +@pytest.mark.parametrize("version", ["7.82.3", "7.82.2", "6.53.4"]) +def test_automate_update_skips_publish_when_not_newer( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, version: str +) -> None: + write_lock(tmp_path) + update_agent_version(tmp_path, "7.82.3") + github = GitHubApi("token") + monkeypatch.setattr( + "utils.scripts.update_agent_version.publish_update", + lambda _root, _version, _github, _env: pytest.fail("publish should not run"), + ) + + assert not automate_update(tmp_path, github, {}, version) + assert (tmp_path / AGENT_VERSION_LOCK).read_text().endswith("DD_AGENT_VERSION=7.82.3\n") + + +@scenarios.test_the_test +@pytest.mark.parametrize("existing_pr", ["", "1234"]) +def test_publish_update_creates_only_missing_pr( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, existing_pr: str +) -> None: + commands: list[list[str]] = [] + + class FakeGitHubApi(GitHubApi): + def __init__(self) -> None: + super().__init__("token") + self.calls: list[tuple[str, str]] = [] + self.descriptions: list[dict[str, object]] = [] + + def request(self, method: str, path: str, data: dict[str, object] | None = None) -> object: + self.calls.append((method, path)) + if method == "GET": + return [{"node_id": "PR_node_id", "number": int(existing_pr)}] if existing_pr else [] + if path == "/graphql": + assert data is not None + # GitHub rejects enabling auto-merge on a PR that already has it, i.e. on every refresh. + if existing_pr: + return { + "data": {"enablePullRequestAutoMerge": None}, + "errors": [{"message": "Pull request Auto merge is already enabled."}], + } + return {"data": {"enablePullRequestAutoMerge": {"pullRequest": {"number": 42}}}} + assert data is not None + self.descriptions.append(data) + return {"node_id": "PR_node_id"} + + def fake_run( + _root: Path, + args: list[str], + *, + capture_output: bool = False, + env: dict[str, str] | None = None, + ) -> subprocess.CompletedProcess[str]: + assert env == {"GH_TOKEN": "token"} + commands.append(args) + return subprocess.CompletedProcess(args, 0, stdout="" if capture_output else None) + + monkeypatch.setattr("utils.scripts.update_agent_version.run_command", fake_run) + + github = FakeGitHubApi() + publish_update(tmp_path, "7.82.3", github, {"GH_TOKEN": "token"}) + + assert ["git", "add", str(AGENT_VERSION_LOCK)] in commands + assert ["git", "push", "--force", "--set-upstream", "origin", AUTOMATION_BRANCH] in commands + assert not any(command[0] == "gh" for command in commands) + assert any(method == "POST" and path.endswith("/pulls") for method, path in github.calls) is (not existing_pr) + if existing_pr: + assert ("PATCH", f"/repos/DataDog/system-tests/pulls/{existing_pr}") in github.calls + # Whether it is created or refreshed, the PR describes the version that was just pushed. + assert [description["title"] for description in github.descriptions] == ["APMSP-3752 Update Agent to 7.82.3"] + assert github.calls[-1] == ("POST", "/graphql") + + +def fake_github(result: object) -> GitHubApi: + class FakeGitHubApi(GitHubApi): + def request(self, method: str, path: str, data: dict[str, object] | None = None) -> object: # noqa: ARG002 + return result + + return FakeGitHubApi("token") + + +@scenarios.test_the_test +@pytest.mark.parametrize( + "result", + [ + {"errors": [{"message": "Pull request Auto merge is not allowed for this repository"}]}, + { + "errors": [ + {"message": "Pull request Auto merge is already enabled."}, + {"message": "Something else went wrong"}, + ] + }, + ], +) +def test_enable_auto_merge_reports_real_failures(result: object) -> None: + with pytest.raises(RuntimeError, match="failed to enable pull request auto-merge"): + enable_auto_merge(fake_github(result), "PR_node_id") + + +@scenarios.test_the_test +@pytest.mark.parametrize("result", [[], {"errors": "boom"}]) +def test_enable_auto_merge_rejects_invalid_responses(result: object) -> None: + with pytest.raises(TypeError, match="invalid auto-merge response"): + enable_auto_merge(fake_github(result), "PR_node_id") + + +@scenarios.test_the_test +def test_run_automation_revokes_token_after_failure(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + commands: list[list[str]] = [] + + def fake_run( + _root: Path, + args: list[str], + *, + capture_output: bool = False, + env: dict[str, str] | None = None, + ) -> subprocess.CompletedProcess[str]: + assert env is None + commands.append(args) + stdout = "secret-token" if capture_output else "" + return subprocess.CompletedProcess(args, 0, stdout=stdout) + + def fail_update(_root: Path, _github: GitHubApi, env: dict[str, str], _version: str | None) -> bool: + assert env["GH_TOKEN"] == "secret-token" + raise RuntimeError("publish failed") + + monkeypatch.setattr("utils.scripts.update_agent_version.run_command", fake_run) + monkeypatch.setattr("utils.scripts.update_agent_version.automate_update", fail_update) + + with pytest.raises(RuntimeError, match="publish failed"): + run_automation(tmp_path) + + assert commands[-1] == ["dd-octo-sts", "revoke", "-t", "secret-token"] + + +@scenarios.test_the_test +def test_revoke_token_hides_token_and_command_output( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + token = "secret-token" + + def fail_run( + _root: Path, + args: list[str], + *, + capture_output: bool = False, + env: dict[str, str] | None = None, + ) -> subprocess.CompletedProcess[str]: + assert capture_output + assert env is None + raise subprocess.CalledProcessError( + 1, + args, + output=f"stdout containing {token}", + stderr=f"stderr containing {token}", + ) + + monkeypatch.setattr("utils.scripts.update_agent_version.run_command", fail_run) + + with pytest.raises(RuntimeError, match="token revocation failed with exit code 1") as error: + revoke_token(tmp_path, token) + + captured = capsys.readouterr() + assert token not in str(error.value) + assert captured.out == "" + assert captured.err == "" + + +@scenarios.test_the_test +def test_run_automation_rejects_empty_token(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + commands: list[list[str]] = [] + + def fake_run( + _root: Path, + args: list[str], + *, + capture_output: bool = False, + env: dict[str, str] | None = None, + ) -> subprocess.CompletedProcess[str]: + assert env is None + commands.append(args) + return subprocess.CompletedProcess(args, 0, stdout="" if capture_output else None) + + monkeypatch.setattr("utils.scripts.update_agent_version.run_command", fake_run) + + with pytest.raises(RuntimeError, match="empty GitHub token"): + run_automation(tmp_path) + + assert not any(command[:2] == ["dd-octo-sts", "revoke"] for command in commands) diff --git a/utils/build/virtual_machine/agent.lock b/utils/build/virtual_machine/agent.lock new file mode 100644 index 00000000000..309976b046d --- /dev/null +++ b/utils/build/virtual_machine/agent.lock @@ -0,0 +1,2 @@ +# Pinned Agent version, updated automatically +DD_AGENT_VERSION=7.78.4 diff --git a/utils/build/virtual_machine/provisions/auto-inject/auto-inject_installer_manual.yml b/utils/build/virtual_machine/provisions/auto-inject/auto-inject_installer_manual.yml index 0c83daa82d3..4850db186c4 100644 --- a/utils/build/virtual_machine/provisions/auto-inject/auto-inject_installer_manual.yml +++ b/utils/build/virtual_machine/provisions/auto-inject/auto-inject_installer_manual.yml @@ -5,13 +5,16 @@ local_path: utils/build/virtual_machine/provisions/auto-inject/tracer_debug/debug_config.yaml - name: copy-docker-config local_path: utils/build/virtual_machine/provisions/auto-inject/docker/docker_config.yaml + - name: copy-agent-version-lock + local_path: utils/build/virtual_machine/agent.lock - name: copy-binaries local_path: binaries/ remote-command: | - # Pin to 7.78.4 agent release. APMSP-3059 - export DD_AGENT_MAJOR_VERSION=7 - export DD_AGENT_MINOR_VERSION=78.4 + . ./agent.lock + export DD_AGENT_MAJOR_VERSION="${DD_AGENT_VERSION%%.*}" + export DD_AGENT_MINOR_VERSION="${DD_AGENT_VERSION#*.}" + AGENT_INSTALL_SCRIPT="install_script_agent${DD_AGENT_MAJOR_VERSION}.sh" # Check if Docker is installed and ensure it's running if command -v docker >/dev/null 2>&1; then echo "Docker is installed, ensuring service is enabled and running..." @@ -118,21 +121,21 @@ sudo cat /etc/datadog-agent/application_monitoring.yaml || true # Check if install script exists locally in binaries folder - if [ -f "install_script_agent7.sh" ]; then + if [ -f "${AGENT_INSTALL_SCRIPT}" ]; then echo "*** Execute installation script from provided binaries ***" - cp install_script_agent7.sh install_script.sh + cp "${AGENT_INSTALL_SCRIPT}" install_script.sh chmod +x install_script.sh else echo "Download installation script from S3" # dns install.datadoghq.com or the network interfaces are no very steady, so we need to retry # On the old machines the curl command doesn't support the --retry-connrefused flag, we implement the retry policy in a loop - # standard exec: DD_REPO_URL=${DD_injection_repo_url} bash -c "$(curl -L https://dd-agent.s3.amazonaws.com/scripts/install_script_agent7.sh)" + # standard exec: DD_REPO_URL=${DD_injection_repo_url} bash -c "$(curl -L https://dd-agent.s3.amazonaws.com/scripts/${AGENT_INSTALL_SCRIPT})" MAX_RETRIES=5 RETRY_DELAY=5 COUNTER=0 while [ $COUNTER -lt $MAX_RETRIES ]; do - curl -L https://dd-agent.s3.amazonaws.com/scripts/install_script_agent7.sh -o install_script.sh && break + curl -L "https://dd-agent.s3.amazonaws.com/scripts/${AGENT_INSTALL_SCRIPT}" -o install_script.sh && break echo "Retrying in $RETRY_DELAY seconds..." sleep $RETRY_DELAY ((COUNTER++)) diff --git a/utils/build/virtual_machine/provisions/auto-inject/docker/docker-compose-agent-prod.yml b/utils/build/virtual_machine/provisions/auto-inject/docker/docker-compose-agent-prod.yml index ee6c3fe3d54..e7637cb8097 100644 --- a/utils/build/virtual_machine/provisions/auto-inject/docker/docker-compose-agent-prod.yml +++ b/utils/build/virtual_machine/provisions/auto-inject/docker/docker-compose-agent-prod.yml @@ -1,8 +1,7 @@ services: datadog: container_name: dd-agent - # Pin to 7.78.4 agent release. APMSP-3059 - image: gcr.io/datadoghq/agent:7.78.4 + image: gcr.io/datadoghq/agent:${DD_AGENT_VERSION} environment: - DD_API_KEY=${DD_API_KEY} - DD_SITE=datadoghq.com diff --git a/utils/build/virtual_machine/provisions/auto-inject/repositories/autoinstall/execute_install_script.sh b/utils/build/virtual_machine/provisions/auto-inject/repositories/autoinstall/execute_install_script.sh index 193949ffa3c..fcfd3a40d5e 100755 --- a/utils/build/virtual_machine/provisions/auto-inject/repositories/autoinstall/execute_install_script.sh +++ b/utils/build/virtual_machine/provisions/auto-inject/repositories/autoinstall/execute_install_script.sh @@ -2,8 +2,14 @@ # This script is needed only for this reason: https://datadoghq.atlassian.net/browse/AP-2165 -if [ -z "$INSTALLER_URL" ]; then - INSTALLER_URL="https://dd-agent.s3.amazonaws.com/scripts/install_script_agent7.sh" +# shellcheck source=/dev/null +. ./agent.lock +export DD_AGENT_MAJOR_VERSION="${DD_AGENT_VERSION%%.*}" +export DD_AGENT_MINOR_VERSION="${DD_AGENT_VERSION#*.}" +AGENT_INSTALL_SCRIPT="install_script_agent${DD_AGENT_MAJOR_VERSION}.sh" + +if [ -z "${INSTALLER_URL:-}" ]; then + INSTALLER_URL="https://dd-agent.s3.amazonaws.com/scripts/${AGENT_INSTALL_SCRIPT}" fi if [ "$DD_APM_INSTRUMENTATION_ENABLED" == "docker" ]; then @@ -62,9 +68,9 @@ fi sudo sh -c "sudo mkdir -p /etc/datadog-agent && printf \"api_key: ${DD_API_KEY}\nsite: datadoghq.com\n\" > /etc/datadog-agent/datadog.yaml" -if [ -f "install_script_agent7.sh" ]; then +if [ -f "${AGENT_INSTALL_SCRIPT}" ]; then echo "*** Execute installation script from provided binaries ***" - cp install_script_agent7.sh install_script.sh + cp "${AGENT_INSTALL_SCRIPT}" install_script.sh chmod +x install_script.sh else echo "Download installation script from S3" diff --git a/utils/build/virtual_machine/provisions/container-auto-inject-install-script/auto-inject_container_script.yml b/utils/build/virtual_machine/provisions/container-auto-inject-install-script/auto-inject_container_script.yml index 542a81e216e..f354185057e 100644 --- a/utils/build/virtual_machine/provisions/container-auto-inject-install-script/auto-inject_container_script.yml +++ b/utils/build/virtual_machine/provisions/container-auto-inject-install-script/auto-inject_container_script.yml @@ -9,6 +9,8 @@ local_path: utils/build/virtual_machine/provisions/auto-inject/tracer_debug/debug_config.yaml - name: copy-agent-docker-compose local_path: utils/build/virtual_machine/provisions/auto-inject/docker/docker-compose-agent-prod.yml + - name: copy-agent-version-lock + local_path: utils/build/virtual_machine/agent.lock - name: copy-binaries local_path: binaries/ remote-command: | @@ -25,4 +27,4 @@ printf "DD_APM_RECEIVER_SOCKET=/opt/datadog/apm/inject/run/apm.socket\nDD_DOGSTATSD_SOCKET=/opt/datadog/apm/inject/run/dsd.socket\nDD_USE_DOGSTATSD=true\n" | sudo tee /var/run/datadog-installer/environment DD_APM_INSTRUMENTATION_ENABLED=docker bash execute_install_script.sh sudo cp docker_config.yaml /etc/datadog-agent/inject/docker_config.yaml - sudo cp debug_config.yaml /etc/datadog-agent/inject/debug_config.yaml \ No newline at end of file + sudo cp debug_config.yaml /etc/datadog-agent/inject/debug_config.yaml diff --git a/utils/build/virtual_machine/provisions/container-auto-inject-install-script/provision.yml b/utils/build/virtual_machine/provisions/container-auto-inject-install-script/provision.yml index b605ffb6134..30ac97e29c3 100644 --- a/utils/build/virtual_machine/provisions/container-auto-inject-install-script/provision.yml +++ b/utils/build/virtual_machine/provisions/container-auto-inject-install-script/provision.yml @@ -15,7 +15,7 @@ provision_steps: - prepare-docker #Prepare the docker environment - amazon-ecr-credential-helper # Install AWS ECR helper to download images from ECR - patch-docker-daemon #Patch the docker daemon to avoid networking issues/ip conflicts incident-31160 - - install-agent #Install the agent (allways latest release) + - install-agent #Install the agent (version pinned in utils/build/virtual_machine/agent.lock) - autoinjection_install_script #Install the auto-injection softaware 'datadog-apm-inject' and 'datadog-apm-library-$DD_LANG' init-config: @@ -41,9 +41,9 @@ install-agent: copy_files: - name: copy-agent-docker-compose local_path: utils/build/virtual_machine/provisions/auto-inject/docker/docker-compose-agent-prod.yml + - name: copy-agent-version-lock + local_path: utils/build/virtual_machine/agent.lock remote-command: cat docker-compose-agent-prod.yml autoinjection_install_script: install: !include utils/build/virtual_machine/provisions/container-auto-inject-install-script/auto-inject_container_script.yml - - diff --git a/utils/build/virtual_machine/provisions/host-auto-inject-install-script/auto-inject_host_script.yml b/utils/build/virtual_machine/provisions/host-auto-inject-install-script/auto-inject_host_script.yml index 59d1c5376bf..eb98f275934 100644 --- a/utils/build/virtual_machine/provisions/host-auto-inject-install-script/auto-inject_host_script.yml +++ b/utils/build/virtual_machine/provisions/host-auto-inject-install-script/auto-inject_host_script.yml @@ -5,6 +5,8 @@ local_path: utils/build/virtual_machine/provisions/auto-inject/repositories/autoinstall/execute_install_script.sh - name: copy-tracer-debug-config local_path: utils/build/virtual_machine/provisions/auto-inject/tracer_debug/debug_config.yaml + - name: copy-agent-version-lock + local_path: utils/build/virtual_machine/agent.lock - name: copy-binaries local_path: binaries/ remote-command: | diff --git a/utils/build/virtual_machine/provisions/local-auto-inject-install-script/provision.yml b/utils/build/virtual_machine/provisions/local-auto-inject-install-script/provision.yml index 0f4b2e29904..053c38fa5a7 100644 --- a/utils/build/virtual_machine/provisions/local-auto-inject-install-script/provision.yml +++ b/utils/build/virtual_machine/provisions/local-auto-inject-install-script/provision.yml @@ -12,7 +12,7 @@ vm_logs: #Mandatory: Steps to install provision provision_steps: - init-config #Very first machine actions, like disable auto updates - - install-agent #Install the agent (allways latest release) + - install-agent #Install the agent (version pinned in utils/build/virtual_machine/agent.lock) - autoinjection_install_script #Install the auto-injection softaware 'datadog-apm-inject' and 'datadog-apm-library-$DD_LANG' using the agent install script - install-local-apm-library @@ -24,8 +24,15 @@ init-config: install-agent: install: - os_type: linux + copy_files: + - name: copy-agent-version-lock + local_path: utils/build/virtual_machine/agent.lock remote-command: | - REPO_URL=$DD_agent_repo_url DD_AGENT_DIST_CHANNEL=$DD_agent_dist_channel DD_AGENT_MAJOR_VERSION=$DD_agent_major_version bash -c "$(curl -L https://dd-agent.s3.amazonaws.com/scripts/install_script_agent7.sh)" + . ./agent.lock + export DD_AGENT_MAJOR_VERSION="${DD_AGENT_VERSION%%.*}" + export DD_AGENT_MINOR_VERSION="${DD_AGENT_VERSION#*.}" + AGENT_INSTALL_SCRIPT="install_script_agent${DD_AGENT_MAJOR_VERSION}.sh" + REPO_URL=$DD_agent_repo_url DD_AGENT_DIST_CHANNEL=$DD_agent_dist_channel bash -c "$(curl -L https://dd-agent.s3.amazonaws.com/scripts/${AGENT_INSTALL_SCRIPT})" autoinjection_install_script: install: !include utils/build/virtual_machine/provisions/host-auto-inject-install-script/auto-inject_host_script.yml diff --git a/utils/build/virtual_machine/weblogs/common/create_and_run_app_container.sh b/utils/build/virtual_machine/weblogs/common/create_and_run_app_container.sh index 11e4a53c9be..ac6f861727f 100755 --- a/utils/build/virtual_machine/weblogs/common/create_and_run_app_container.sh +++ b/utils/build/virtual_machine/weblogs/common/create_and_run_app_container.sh @@ -14,6 +14,7 @@ set -e readonly DIAGNOSTICS_LOG="${HOME}/dd-agent-diagnostics.log" readonly SCRIPT_MARKER="create_and_run_app_container.sh diagnostics-v3" readonly AGENT_COMPOSE="docker-compose-agent-prod.yml" +readonly AGENT_LOCK="agent.lock" readonly WEBLOG_LOG_DIR="/var/log/datadog_weblog" readonly DOCKER_BUILD_MAX_RETRIES=3 PULL_AGENT_IMAGE_SCRIPT="$(dirname "$0")/pull_agent_image.sh" @@ -24,6 +25,15 @@ agent_is_enabled() { [ -f "${AGENT_COMPOSE}" ] } +load_agent_version() { + agent_is_enabled || return 0 + + set -a + # shellcheck source=/dev/null + . "./${AGENT_LOCK}" + set +a +} + # --------------------------------------------------------------------------- # Diagnostics (collected on every run, and on failure, to help debugging in CI) # --------------------------------------------------------------------------- @@ -39,7 +49,7 @@ dump_dd_agent_diagnostics() { { echo "..:: DD-AGENT DIAGNOSTICS (${SCRIPT_MARKER}) ::.." date -u '+%Y-%m-%dT%H:%M:%SZ' - sudo docker-compose -f "${AGENT_COMPOSE}" ps 2>&1 || true + sudo -E docker-compose -f "${AGENT_COMPOSE}" ps 2>&1 || true if sudo docker inspect dd-agent >/dev/null 2>&1; then echo "..:: DD-AGENT HEALTH ::.." sudo docker inspect dd-agent --format '{{json .State.Health}}' 2>&1 || true @@ -50,7 +60,7 @@ dump_dd_agent_diagnostics() { sudo docker ps -a 2>&1 || true fi echo "..:: DD-AGENT LOGS (docker-compose logs) ::.." - sudo docker-compose -f "${AGENT_COMPOSE}" logs --no-color datadog 2>&1 || true + sudo -E docker-compose -f "${AGENT_COMPOSE}" logs --no-color datadog 2>&1 || true } 2>&1 | tee -a "${DIAGNOSTICS_LOG}" # Mirror diagnostics into the log folder downloaded by the test harness. @@ -152,7 +162,7 @@ print_services_output() { sudo docker-compose ps if agent_is_enabled; then echo "..:: DATADOG AGENT OUTPUT ::.." - sudo docker-compose -f "${AGENT_COMPOSE}" logs datadog + sudo -E docker-compose -f "${AGENT_COMPOSE}" logs datadog fi echo "..:: WEBLOG APP OUTPUT ::.." sudo docker-compose logs @@ -163,6 +173,8 @@ print_services_output() { # Main # --------------------------------------------------------------------------- main() { + load_agent_version + # Always dump agent diagnostics on failure too (trap is deduplicated against the success dump). trap _on_exit EXIT diff --git a/utils/build/virtual_machine/weblogs/common/create_and_run_app_multicontainer.sh b/utils/build/virtual_machine/weblogs/common/create_and_run_app_multicontainer.sh index 2b6170e2d9a..3c756c998a3 100755 --- a/utils/build/virtual_machine/weblogs/common/create_and_run_app_multicontainer.sh +++ b/utils/build/virtual_machine/weblogs/common/create_and_run_app_multicontainer.sh @@ -4,6 +4,7 @@ set -e readonly AGENT_COMPOSE="docker-compose-agent-prod.yml" +readonly AGENT_LOCK="agent.lock" PULL_AGENT_IMAGE_SCRIPT="$(dirname "$0")/pull_agent_image.sh" readonly PULL_AGENT_IMAGE_SCRIPT @@ -20,6 +21,11 @@ sudo rm -rf system-tests || true sudo docker-compose -f docker-compose.yml build --parallel if [ -f "${AGENT_COMPOSE}" ]; then + set -a + # shellcheck source=/dev/null + . "./${AGENT_LOCK}" + set +a + # Agent may be installed in a different way. Pull with retries before compose up # so GCR rate limits / timeouts do not fail the provision on the first attempt. bash "${PULL_AGENT_IMAGE_SCRIPT}" diff --git a/utils/build/virtual_machine/weblogs/common/pull_agent_image.sh b/utils/build/virtual_machine/weblogs/common/pull_agent_image.sh index f472d89d7fa..5e842b123e0 100644 --- a/utils/build/virtual_machine/weblogs/common/pull_agent_image.sh +++ b/utils/build/virtual_machine/weblogs/common/pull_agent_image.sh @@ -14,6 +14,7 @@ fi set -e readonly AGENT_COMPOSE="${1:-${AGENT_COMPOSE:-docker-compose-agent-prod.yml}}" +readonly AGENT_LOCK="${AGENT_LOCK:-agent.lock}" readonly DOCKER_PULL_MAX_RETRIES="${DOCKER_PULL_MAX_RETRIES:-3}" if [ ! -f "${AGENT_COMPOSE}" ]; then @@ -21,8 +22,26 @@ if [ ! -f "${AGENT_COMPOSE}" ]; then exit 0 fi +if [ ! -f "${AGENT_LOCK}" ]; then + echo "Agent version lock ${AGENT_LOCK} not found" + exit 1 +fi + +# shellcheck source=/dev/null +. "${AGENT_LOCK}" +if [ -z "${DD_AGENT_VERSION:-}" ]; then + echo "DD_AGENT_VERSION is missing from ${AGENT_LOCK}" + exit 1 +fi + agent_compose_image() { - awk '/^[[:space:]]*image:[[:space:]]*/ { print $2; exit }' "${AGENT_COMPOSE}" + awk -v version="${DD_AGENT_VERSION}" ' + /^[[:space:]]*image:[[:space:]]*/ { + gsub(/\$\{DD_AGENT_VERSION\}/, version, $2) + print $2 + exit + } + ' "${AGENT_COMPOSE}" } pull_docker_image() { diff --git a/utils/build/virtual_machine/weblogs/java/test-app-java-buildpack/test-app-java_docker_compose_run_buildpack.sh b/utils/build/virtual_machine/weblogs/java/test-app-java-buildpack/test-app-java_docker_compose_run_buildpack.sh index 8af81e92841..9d7134bd871 100755 --- a/utils/build/virtual_machine/weblogs/java/test-app-java-buildpack/test-app-java_docker_compose_run_buildpack.sh +++ b/utils/build/virtual_machine/weblogs/java/test-app-java-buildpack/test-app-java_docker_compose_run_buildpack.sh @@ -3,6 +3,8 @@ set -e +readonly AGENT_LOCK="agent.lock" + # Function to retry commands up to 3 times retry_command() { local max_attempts=3 @@ -52,6 +54,11 @@ retry_command "sudo ./gradlew -PdockerImageRepo=system-tests/local -PdockerImage echo "**************** RUN SERVICES*****************" if [ -f docker-compose-agent-prod.yml ]; then + set -a + # shellcheck source=/dev/null + . "./${AGENT_LOCK}" + set +a + # Agent may be installed in a different way. Pull with retries before compose # up so GCR rate limits / timeouts do not fail the provision on the first attempt. bash "$(dirname "$0")/pull_agent_image.sh" @@ -73,7 +80,7 @@ echo "**************** RUNNING DOCKER SERVICES *****************" sudo docker-compose ps if [ -f docker-compose-agent-prod.yml ]; then echo "**************** DATADOG AGENT OUTPUT ********************" - sudo docker-compose -f docker-compose-agent-prod.yml logs datadog + sudo -E docker-compose -f docker-compose-agent-prod.yml logs datadog fi echo "**************** WEBLOG APP OUTPUT********************" sudo docker-compose logs diff --git a/utils/scripts/libraries_and_scenarios_rules.yml b/utils/scripts/libraries_and_scenarios_rules.yml index 7ec983d3bed..0584e4db5e0 100644 --- a/utils/scripts/libraries_and_scenarios_rules.yml +++ b/utils/scripts/libraries_and_scenarios_rules.yml @@ -286,6 +286,9 @@ patterns: libraries: null utils/scripts/ssi_wizards/*: scenario_groups: null + utils/scripts/update_agent_version.py: + scenario_groups: null + libraries: null utils/scripts/update_change_log.sh: scenario_groups: null libraries: null diff --git a/utils/scripts/update_agent_version.py b/utils/scripts/update_agent_version.py new file mode 100755 index 00000000000..2454b23ed7e --- /dev/null +++ b/utils/scripts/update_agent_version.py @@ -0,0 +1,216 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +import argparse +import json +import os +import re +import subprocess +import urllib.parse +import urllib.request +from pathlib import Path +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from collections.abc import Mapping, Sequence + + +VERSION_PATTERN = re.compile(r"^[0-9]+\.[0-9]+\.[0-9]+$") +AUTOMATION_BRANCH = "apmsp-3752/update-agent-version" +REPOSITORY = "DataDog/system-tests" +OCTO_STS_POLICY = "self.gitlab-update-agent-version" +GITHUB_API_URL = "https://api.github.com" +AUTO_MERGE_ALREADY_ENABLED = "auto merge is already enabled" + +AGENT_VERSION_LOCK = Path("utils/build/virtual_machine/agent.lock") + + +def normalize_version(version: str) -> str: + normalized = version.removeprefix("v") + if VERSION_PATTERN.fullmatch(normalized) is None: + raise ValueError(f"Expected a stable Agent version, got: {version}") + return normalized + + +def version_key(version: str) -> tuple[int, int, int]: + major, minor, patch = normalize_version(version).split(".") + return int(major), int(minor), int(patch) + + +def locked_agent_version(root: Path) -> str: + for line in (root / AGENT_VERSION_LOCK).read_text().splitlines(): + if line.startswith("DD_AGENT_VERSION="): + return normalize_version(line.removeprefix("DD_AGENT_VERSION=")) + raise ValueError(f"No DD_AGENT_VERSION found in {AGENT_VERSION_LOCK}") + + +def update_agent_version(root: Path, version: str) -> bool: + normalized = normalize_version(version) + lock_path = root / AGENT_VERSION_LOCK + updated = f"# Pinned Agent version, updated automatically\nDD_AGENT_VERSION={normalized}\n" + if lock_path.read_text() == updated: + return False + lock_path.write_text(updated) + return True + + +def run_command( + root: Path, + args: Sequence[str], + *, + capture_output: bool = False, + env: Mapping[str, str] | None = None, +) -> subprocess.CompletedProcess[str]: + return subprocess.run(args, cwd=root, check=True, capture_output=capture_output, text=True, env=env) + + +class GitHubApi: + def __init__(self, token: str) -> None: + self.token = token + + def request(self, method: str, path: str, data: dict[str, object] | None = None) -> object: + request = urllib.request.Request( # noqa: S310 + f"{GITHUB_API_URL}{path}", + data=json.dumps(data).encode() if data is not None else None, + method=method, + headers={ + "Accept": "application/vnd.github+json", + "Authorization": f"Bearer {self.token}", + "Content-Type": "application/json", + "X-GitHub-Api-Version": "2022-11-28", + }, + ) + with urllib.request.urlopen(request) as response: # noqa: S310 + return json.load(response) + + +def latest_agent_version(github: GitHubApi) -> str: + release = github.request("GET", "/repos/DataDog/datadog-agent/releases/latest") + if not isinstance(release, dict) or not isinstance(release.get("tag_name"), str): + raise TypeError("GitHub returned an invalid latest Agent release") + return normalize_version(release["tag_name"]) + + +def enable_auto_merge(github: GitHubApi, pull_request_node_id: str) -> None: + result = github.request( + "POST", + "/graphql", + { + "query": "mutation($pullRequestId: ID!) { enablePullRequestAutoMerge(input: {" + "pullRequestId: $pullRequestId, mergeMethod: SQUASH}) { pullRequest { number } } }", + "variables": {"pullRequestId": pull_request_node_id}, + }, + ) + if not isinstance(result, dict): + raise TypeError("GitHub returned an invalid auto-merge response") + errors = result.get("errors") or [] + if not isinstance(errors, list): + raise TypeError("GitHub returned an invalid auto-merge response") + # A refreshed PR keeps the auto-merge enabled by a previous run, and GitHub rejects enabling it twice. + if any( + not isinstance(error, dict) or AUTO_MERGE_ALREADY_ENABLED not in str(error.get("message", "")).lower() + for error in errors + ): + raise RuntimeError("GitHub failed to enable pull request auto-merge") + + +def publish_update(root: Path, version: str, github: GitHubApi, env: Mapping[str, str]) -> None: + run_command(root, ["git", "remote", "set-url", "origin", f"https://github.com/{REPOSITORY}.git"], env=env) + run_command(root, ["git", "switch", "--force-create", AUTOMATION_BRANCH], env=env) + run_command(root, ["git", "add", str(AGENT_VERSION_LOCK)], env=env) + run_command(root, ["git", "config", "user.name", "github-actions[bot]"], env=env) + run_command(root, ["git", "config", "user.email", "github-actions[bot]@users.noreply.github.com"], env=env) + run_command( + root, + ["git", "config", "credential.helper", "!f() { echo username=x-access-token; echo password=$GH_TOKEN; }; f"], + env=env, + ) + run_command(root, ["git", "commit", "-m", f"APMSP-3752 update Agent to {version}"], env=env) + run_command(root, ["git", "push", "--force", "--set-upstream", "origin", AUTOMATION_BRANCH], env=env) + + head = urllib.parse.quote(f"DataDog:{AUTOMATION_BRANCH}", safe="") + pull_requests = github.request("GET", f"/repos/{REPOSITORY}/pulls?head={head}&state=open") + if not isinstance(pull_requests, list): + raise TypeError("GitHub returned an invalid pull request list") + description: dict[str, object] = { + "title": f"APMSP-3752 Update Agent to {version}", + "body": "Automated daily update of the Agent version pinned by SSI tests. " + "The PR will merge automatically after all required checks pass.", + } + if pull_requests: + # The branch is force-pushed, so the open PR now describes the previous version: overwrite it. + existing = pull_requests[0] + if not isinstance(existing, dict) or not isinstance(existing.get("number"), int): + raise TypeError("GitHub returned an invalid pull request list") + pull_request = github.request("PATCH", f"/repos/{REPOSITORY}/pulls/{existing['number']}", description) + else: + pull_request = github.request( + "POST", + f"/repos/{REPOSITORY}/pulls", + {"base": "main", "head": AUTOMATION_BRANCH, **description}, + ) + if not isinstance(pull_request, dict) or not isinstance(pull_request.get("node_id"), str): + raise TypeError("GitHub returned an invalid pull request") + enable_auto_merge(github, pull_request["node_id"]) + + +def automate_update(root: Path, github: GitHubApi, env: Mapping[str, str], version: str | None = None) -> bool: + normalized = normalize_version(version) if version is not None else latest_agent_version(github) + locked = locked_agent_version(root) + if version_key(normalized) <= version_key(locked): + # GitHub reports the most recently published release as the latest one, not the highest + # version: a patch released on an older branch must not roll the pin backward. + print(f"Agent pin {locked} is not older than {normalized}") + return False + + if not update_agent_version(root, normalized): + print(f"Agent pins already current: {normalized}") + return False + + publish_update(root, normalized, github, env) + print(f"Published Agent pin update: {normalized}") + return True + + +def revoke_token(root: Path, token: str) -> None: + try: + run_command(root, ["dd-octo-sts", "revoke", "-t", token], capture_output=True) + except subprocess.CalledProcessError as error: + raise RuntimeError(f"dd-octo-sts token revocation failed with exit code {error.returncode}") from None + + +def run_automation(root: Path, version: str | None = None) -> bool: + scope_args = ["--scope", REPOSITORY, "--policy", OCTO_STS_POLICY] + run_command(root, ["dd-octo-sts", "version"]) + run_command(root, ["dd-octo-sts", "debug", *scope_args]) + token = run_command(root, ["dd-octo-sts", "token", *scope_args], capture_output=True).stdout.strip() + if not token: + raise RuntimeError("dd-octo-sts returned an empty GitHub token") + + github_env = os.environ.copy() + github_env["GH_TOKEN"] = token + github = GitHubApi(token) + try: + return automate_update(root, github, github_env, version) + finally: + revoke_token(root, token) + + +def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Publish an automated update of the Agent version pinned by SSI scenarios" + ) + parser.add_argument("--version", help="Override the latest stable Agent version") + parser.add_argument("--root", type=Path, default=Path.cwd(), help=argparse.SUPPRESS) + return parser.parse_args(argv) + + +def main(argv: Sequence[str] | None = None) -> int: + args = parse_args(argv) + run_automation(args.root, args.version) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())