diff --git a/datadog_sync/commands/shared/options.py b/datadog_sync/commands/shared/options.py index b084fa21..b040fb95 100644 --- a/datadog_sync/commands/shared/options.py +++ b/datadog_sync/commands/shared/options.py @@ -577,6 +577,14 @@ def click_config_file_provider(ctx: Context, opts: CustomOptionClass, value: Non help="Override num_flex_logs_retention_days on logs indexes where the field is present.", cls=CustomOptionClass, ), + option( + "--alter-logs-indexes-retention-days", + required=False, + type=IntRange(min=30), + default=None, + help="Override num_retention_days on logs indexes where the field is present.", + cls=CustomOptionClass, + ), option( "--skip-monitors-with-restricted-roles", required=False, diff --git a/datadog_sync/model/logs_indexes.py b/datadog_sync/model/logs_indexes.py index f50da5c5..32449391 100644 --- a/datadog_sync/model/logs_indexes.py +++ b/datadog_sync/model/logs_indexes.py @@ -26,6 +26,15 @@ class LogsIndexes(BaseResource): # Additional LogsIndexes specific attributes logs_indexes_order_url: str = "/api/v1/logs/config/index-order" + # Pairs of (Configuration attribute, logs-index resource field) that + # pre_resource_action_hook overrides when the flag is set and the field is + # already present on the resource (insert-if-present semantics; never adds + # the field to a resource that lacks it). + _RETENTION_OVERRIDES = ( + ("alter_flex_logs_retention_days", "num_flex_logs_retention_days"), + ("alter_logs_indexes_retention_days", "num_retention_days"), + ) + async def get_resources(self, client: CustomClient) -> List[Dict]: resp = await client.get(self.resource_config.base_path) @@ -47,9 +56,10 @@ async def import_resource(self, _id: Optional[str] = None, resource: Optional[Di return resource["name"], resource async def pre_resource_action_hook(self, _id, resource: Dict) -> None: - retention_days = self.config.alter_flex_logs_retention_days - if retention_days is not None and "num_flex_logs_retention_days" in resource: - resource["num_flex_logs_retention_days"] = retention_days + for config_attr, field in self._RETENTION_OVERRIDES: + retention_days = getattr(self.config, config_attr) + if retention_days is not None and field in resource: + resource[field] = retention_days async def pre_apply_hook(self) -> None: pass diff --git a/datadog_sync/utils/configuration.py b/datadog_sync/utils/configuration.py index ef7c8ffc..9a009f42 100644 --- a/datadog_sync/utils/configuration.py +++ b/datadog_sync/utils/configuration.py @@ -80,6 +80,7 @@ class Configuration(object): allow_self_lockout: bool datadog_host_override: Optional[str] = None alter_flex_logs_retention_days: Optional[int] = None + alter_logs_indexes_retention_days: Optional[int] = None emit_json: bool = False # Opt-in: drop principal/role references that are absent from BOTH destination and # source state (permanently gone -- e.g. deleted before this org's first-ever import) @@ -549,6 +550,7 @@ def build_config(cmd: Command, **kwargs: Optional[Any]) -> Configuration: allow_self_lockout = kwargs.get("allow_self_lockout", False) datadog_host_override = kwargs.get("datadog_host_override") alter_flex_logs_retention_days = kwargs.get("alter_flex_logs_retention_days") + alter_logs_indexes_retention_days = kwargs.get("alter_logs_indexes_retention_days") # Parse allow_partial_permissions_roles allow_partial_permissions_roles = [] @@ -860,6 +862,7 @@ def build_config(cmd: Command, **kwargs: Optional[Any]) -> Configuration: allow_self_lockout=allow_self_lockout, datadog_host_override=datadog_host_override, alter_flex_logs_retention_days=alter_flex_logs_retention_days, + alter_logs_indexes_retention_days=alter_logs_indexes_retention_days, emit_json=emit_json, command=cmd.value, allow_partial_permissions_roles=allow_partial_permissions_roles, diff --git a/tests/unit/test_logs_indexes_retention.py b/tests/unit/test_logs_indexes_retention.py new file mode 100644 index 00000000..dbc67121 --- /dev/null +++ b/tests/unit/test_logs_indexes_retention.py @@ -0,0 +1,107 @@ +# Unless explicitly stated otherwise all files in this repository are licensed +# under the 3-clause BSD style license (see LICENSE). +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019 Datadog, Inc. + +import asyncio +import importlib +from unittest.mock import patch + +import pytest +from click.testing import CliRunner + +from datadog_sync.cli import cli +from datadog_sync.constants import Command +from datadog_sync.model.logs_indexes import LogsIndexes + + +@pytest.mark.parametrize( + "command,module_name,expected_command", + [ + ("sync", "datadog_sync.commands.sync", Command.SYNC), + ("migrate", "datadog_sync.commands.migrate", Command.MIGRATE), + ], +) +def test_cli_accepts_alter_logs_indexes_retention_days(command, module_name, expected_command): + runner = CliRunner(mix_stderr=False) + command_module = importlib.import_module(module_name) + + with patch.object(command_module, "run_cmd") as mock_run_cmd: + result = runner.invoke(cli, [command, "--alter-logs-indexes-retention-days=30"]) + + assert result.exit_code == 0, result.output + mock_run_cmd.assert_called_once() + called_command, kwargs = mock_run_cmd.call_args.args[0], mock_run_cmd.call_args.kwargs + assert called_command == expected_command + assert kwargs["alter_logs_indexes_retention_days"] == 30 + + +@pytest.mark.parametrize( + "command,module_name", + [ + ("sync", "datadog_sync.commands.sync"), + ("migrate", "datadog_sync.commands.migrate"), + ], +) +@pytest.mark.parametrize("value", ["invalid", "-1", "0", "29"]) +def test_cli_rejects_invalid_logs_indexes_retention_days(command, module_name, value): + runner = CliRunner(mix_stderr=False) + command_module = importlib.import_module(module_name) + + with patch.object(command_module, "run_cmd") as mock_run_cmd: + result = runner.invoke( + cli, + [command, f"--alter-logs-indexes-retention-days={value}"], + ) + + assert result.exit_code != 0 + mock_run_cmd.assert_not_called() + + +def test_alter_logs_indexes_retention_days_overrides_existing_field(mock_config): + mock_config.alter_logs_indexes_retention_days = 90 + resource = {"name": "index-test", "num_retention_days": 30} + + asyncio.run(LogsIndexes(mock_config).pre_resource_action_hook("index-test", resource)) + + assert resource["num_retention_days"] == 90 + + +def test_alter_logs_indexes_retention_days_does_not_add_missing_field(mock_config): + mock_config.alter_logs_indexes_retention_days = 90 + resource = {"name": "index-test"} + + asyncio.run(LogsIndexes(mock_config).pre_resource_action_hook("index-test", resource)) + + assert "num_retention_days" not in resource + + +def test_unset_alter_logs_indexes_retention_days_preserves_existing_field(mock_config): + mock_config.alter_logs_indexes_retention_days = None + resource = {"name": "index-test", "num_retention_days": 30} + + asyncio.run(LogsIndexes(mock_config).pre_resource_action_hook("index-test", resource)) + + assert resource["num_retention_days"] == 30 + + +def test_both_retention_flags_apply_independently(mock_config): + mock_config.alter_logs_indexes_retention_days = 60 + mock_config.alter_flex_logs_retention_days = 90 + resource = {"name": "index-test", "num_retention_days": 7, "num_flex_logs_retention_days": 30} + + asyncio.run(LogsIndexes(mock_config).pre_resource_action_hook("index-test", resource)) + + assert resource["num_retention_days"] == 60 + assert resource["num_flex_logs_retention_days"] == 90 + + +def test_logs_indexes_flag_does_not_touch_flex_field(mock_config): + mock_config.alter_logs_indexes_retention_days = 60 + mock_config.alter_flex_logs_retention_days = None + resource = {"name": "index-test", "num_retention_days": 7, "num_flex_logs_retention_days": 30} + + asyncio.run(LogsIndexes(mock_config).pre_resource_action_hook("index-test", resource)) + + assert resource["num_retention_days"] == 60 + assert resource["num_flex_logs_retention_days"] == 30