From b3a129f41360e6e2f2358358f714b0135230f424 Mon Sep 17 00:00:00 2001 From: Michael Richey Date: Tue, 22 Sep 2026 15:45:52 -0400 Subject: [PATCH 1/2] feat: add --alter-logs-indexes-retention-days flag Add a sync/migrate flag that overrides num_retention_days on logs indexes where the field is present, mirroring the existing --alter-flex-logs-retention-days flag (which overrides num_flex_logs_retention_days). Both use insert-if-present semantics: the override is applied only when the flag is set and the field already exists on the resource, so a resource lacking the field is never modified. Refactor pre_resource_action_hook to drive both overrides from a single _RETENTION_OVERRIDES table of (config attribute, resource field) pairs, keeping the flex behavior byte-for-byte identical while making future retention overrides trivial to add. --- datadog_sync/commands/shared/options.py | 8 ++ datadog_sync/model/logs_indexes.py | 16 +++- datadog_sync/utils/configuration.py | 3 + tests/unit/test_logs_indexes_retention.py | 100 ++++++++++++++++++++++ 4 files changed, 124 insertions(+), 3 deletions(-) create mode 100644 tests/unit/test_logs_indexes_retention.py diff --git a/datadog_sync/commands/shared/options.py b/datadog_sync/commands/shared/options.py index b084fa21..b040fb95 100644 --- a/datadog_sync/commands/shared/options.py +++ b/datadog_sync/commands/shared/options.py @@ -577,6 +577,14 @@ def click_config_file_provider(ctx: Context, opts: CustomOptionClass, value: Non help="Override num_flex_logs_retention_days on logs indexes where the field is present.", cls=CustomOptionClass, ), + option( + "--alter-logs-indexes-retention-days", + required=False, + type=IntRange(min=30), + default=None, + help="Override num_retention_days on logs indexes where the field is present.", + cls=CustomOptionClass, + ), option( "--skip-monitors-with-restricted-roles", required=False, diff --git a/datadog_sync/model/logs_indexes.py b/datadog_sync/model/logs_indexes.py index f50da5c5..32449391 100644 --- a/datadog_sync/model/logs_indexes.py +++ b/datadog_sync/model/logs_indexes.py @@ -26,6 +26,15 @@ class LogsIndexes(BaseResource): # Additional LogsIndexes specific attributes logs_indexes_order_url: str = "/api/v1/logs/config/index-order" + # Pairs of (Configuration attribute, logs-index resource field) that + # pre_resource_action_hook overrides when the flag is set and the field is + # already present on the resource (insert-if-present semantics; never adds + # the field to a resource that lacks it). + _RETENTION_OVERRIDES = ( + ("alter_flex_logs_retention_days", "num_flex_logs_retention_days"), + ("alter_logs_indexes_retention_days", "num_retention_days"), + ) + async def get_resources(self, client: CustomClient) -> List[Dict]: resp = await client.get(self.resource_config.base_path) @@ -47,9 +56,10 @@ async def import_resource(self, _id: Optional[str] = None, resource: Optional[Di return resource["name"], resource async def pre_resource_action_hook(self, _id, resource: Dict) -> None: - retention_days = self.config.alter_flex_logs_retention_days - if retention_days is not None and "num_flex_logs_retention_days" in resource: - resource["num_flex_logs_retention_days"] = retention_days + for config_attr, field in self._RETENTION_OVERRIDES: + retention_days = getattr(self.config, config_attr) + if retention_days is not None and field in resource: + resource[field] = retention_days async def pre_apply_hook(self) -> None: pass diff --git a/datadog_sync/utils/configuration.py b/datadog_sync/utils/configuration.py index ef7c8ffc..9a009f42 100644 --- a/datadog_sync/utils/configuration.py +++ b/datadog_sync/utils/configuration.py @@ -80,6 +80,7 @@ class Configuration(object): allow_self_lockout: bool datadog_host_override: Optional[str] = None alter_flex_logs_retention_days: Optional[int] = None + alter_logs_indexes_retention_days: Optional[int] = None emit_json: bool = False # Opt-in: drop principal/role references that are absent from BOTH destination and # source state (permanently gone -- e.g. deleted before this org's first-ever import) @@ -549,6 +550,7 @@ def build_config(cmd: Command, **kwargs: Optional[Any]) -> Configuration: allow_self_lockout = kwargs.get("allow_self_lockout", False) datadog_host_override = kwargs.get("datadog_host_override") alter_flex_logs_retention_days = kwargs.get("alter_flex_logs_retention_days") + alter_logs_indexes_retention_days = kwargs.get("alter_logs_indexes_retention_days") # Parse allow_partial_permissions_roles allow_partial_permissions_roles = [] @@ -860,6 +862,7 @@ def build_config(cmd: Command, **kwargs: Optional[Any]) -> Configuration: allow_self_lockout=allow_self_lockout, datadog_host_override=datadog_host_override, alter_flex_logs_retention_days=alter_flex_logs_retention_days, + alter_logs_indexes_retention_days=alter_logs_indexes_retention_days, emit_json=emit_json, command=cmd.value, allow_partial_permissions_roles=allow_partial_permissions_roles, diff --git a/tests/unit/test_logs_indexes_retention.py b/tests/unit/test_logs_indexes_retention.py new file mode 100644 index 00000000..280ac4bc --- /dev/null +++ b/tests/unit/test_logs_indexes_retention.py @@ -0,0 +1,100 @@ +# Unless explicitly stated otherwise all files in this repository are licensed +# under the 3-clause BSD style license (see LICENSE). +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019 Datadog, Inc. + +import asyncio +import importlib +from unittest.mock import patch + +import pytest +from click.testing import CliRunner + +from datadog_sync.cli import cli +from datadog_sync.constants import Command +from datadog_sync.model.logs_indexes import LogsIndexes + + +@pytest.mark.parametrize( + "command,module_name,expected_command", + [ + ("sync", "datadog_sync.commands.sync", Command.SYNC), + ("migrate", "datadog_sync.commands.migrate", Command.MIGRATE), + ], +) +def test_cli_accepts_alter_logs_indexes_retention_days(command, module_name, expected_command): + runner = CliRunner(mix_stderr=False) + command_module = importlib.import_module(module_name) + + with patch.object(command_module, "run_cmd") as mock_run_cmd: + result = runner.invoke(cli, [command, "--alter-logs-indexes-retention-days=30"]) + + assert result.exit_code == 0, result.output + mock_run_cmd.assert_called_once() + called_command, kwargs = mock_run_cmd.call_args.args[0], mock_run_cmd.call_args.kwargs + assert called_command == expected_command + assert kwargs["alter_logs_indexes_retention_days"] == 30 + + +@pytest.mark.parametrize("value", ["invalid", "-1", "0", "29"]) +def test_cli_rejects_invalid_logs_indexes_retention_days(value): + runner = CliRunner(mix_stderr=False) + sync_module = importlib.import_module("datadog_sync.commands.sync") + + with patch.object(sync_module, "run_cmd") as mock_run_cmd: + result = runner.invoke( + cli, + ["sync", f"--alter-logs-indexes-retention-days={value}"], + ) + + assert result.exit_code != 0 + mock_run_cmd.assert_not_called() + + +def test_alter_logs_indexes_retention_days_overrides_existing_field(mock_config): + mock_config.alter_logs_indexes_retention_days = 90 + resource = {"name": "index-test", "num_retention_days": 30} + + asyncio.run(LogsIndexes(mock_config).pre_resource_action_hook("index-test", resource)) + + assert resource["num_retention_days"] == 90 + + +def test_alter_logs_indexes_retention_days_does_not_add_missing_field(mock_config): + mock_config.alter_logs_indexes_retention_days = 90 + resource = {"name": "index-test"} + + asyncio.run(LogsIndexes(mock_config).pre_resource_action_hook("index-test", resource)) + + assert "num_retention_days" not in resource + + +def test_unset_alter_logs_indexes_retention_days_preserves_existing_field(mock_config): + mock_config.alter_logs_indexes_retention_days = None + resource = {"name": "index-test", "num_retention_days": 30} + + asyncio.run(LogsIndexes(mock_config).pre_resource_action_hook("index-test", resource)) + + assert resource["num_retention_days"] == 30 + + +def test_both_retention_flags_apply_independently(mock_config): + mock_config.alter_logs_indexes_retention_days = 60 + mock_config.alter_flex_logs_retention_days = 90 + resource = {"name": "index-test", "num_retention_days": 7, "num_flex_logs_retention_days": 30} + + asyncio.run(LogsIndexes(mock_config).pre_resource_action_hook("index-test", resource)) + + assert resource["num_retention_days"] == 60 + assert resource["num_flex_logs_retention_days"] == 90 + + +def test_logs_indexes_flag_does_not_touch_flex_field(mock_config): + mock_config.alter_logs_indexes_retention_days = 60 + mock_config.alter_flex_logs_retention_days = None + resource = {"name": "index-test", "num_retention_days": 7, "num_flex_logs_retention_days": 30} + + asyncio.run(LogsIndexes(mock_config).pre_resource_action_hook("index-test", resource)) + + assert resource["num_retention_days"] == 60 + assert resource["num_flex_logs_retention_days"] == 30 From 12d4fdb00629c7c38779436c4fd0969d66262b64 Mon Sep 17 00:00:00 2001 From: Michael Richey Date: Tue, 22 Sep 2026 16:14:52 -0400 Subject: [PATCH 2/2] test(logs_indexes): cover migrate-path invalid retention values Parametrize test_cli_rejects_invalid_logs_indexes_retention_days across both sync and migrate, mirroring the acceptance test, so the shared --alter-logs-indexes-retention-days option is rejected for invalid values on both commands. --- tests/unit/test_logs_indexes_retention.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/tests/unit/test_logs_indexes_retention.py b/tests/unit/test_logs_indexes_retention.py index 280ac4bc..dbc67121 100644 --- a/tests/unit/test_logs_indexes_retention.py +++ b/tests/unit/test_logs_indexes_retention.py @@ -36,15 +36,22 @@ def test_cli_accepts_alter_logs_indexes_retention_days(command, module_name, exp assert kwargs["alter_logs_indexes_retention_days"] == 30 +@pytest.mark.parametrize( + "command,module_name", + [ + ("sync", "datadog_sync.commands.sync"), + ("migrate", "datadog_sync.commands.migrate"), + ], +) @pytest.mark.parametrize("value", ["invalid", "-1", "0", "29"]) -def test_cli_rejects_invalid_logs_indexes_retention_days(value): +def test_cli_rejects_invalid_logs_indexes_retention_days(command, module_name, value): runner = CliRunner(mix_stderr=False) - sync_module = importlib.import_module("datadog_sync.commands.sync") + command_module = importlib.import_module(module_name) - with patch.object(sync_module, "run_cmd") as mock_run_cmd: + with patch.object(command_module, "run_cmd") as mock_run_cmd: result = runner.invoke( cli, - ["sync", f"--alter-logs-indexes-retention-days={value}"], + [command, f"--alter-logs-indexes-retention-days={value}"], ) assert result.exit_code != 0