diff --git a/README.md b/README.md index a4ef942..2d81ac0 100644 --- a/README.md +++ b/README.md @@ -50,6 +50,14 @@ Use `--pretty` for indented JSON and `--raw` to print the Datadog response witho ```sh ddcli logs search --query 'service:web error' --from now-15m --to now --limit 50 +ddcli logs indexes order +ddcli logs indexes list --query production +ddcli logs indexes get bandzoogle-production +ddcli logs indexes patch-exclusions patch.json --dry-run +ddcli logs indexes patch-exclusions patch.json +ddcli logs pipelines order +ddcli logs pipelines list --query openresty +ddcli logs pipelines get PIPELINE_ID ddcli synthetics list --query checkout --limit 25 ddcli synthetics get abc-def-ghi ddcli metrics list --query system.cpu @@ -81,6 +89,57 @@ ddcli scopes --command cost Time flags accept `now`, relative values like `now-15m`, RFC3339 timestamps, Unix seconds, or Unix milliseconds. Logs and spans pass time strings through to Datadog; metrics and Error Tracking convert them to the epoch formats required by their APIs. +## Log configuration audit + +The `logs indexes` and `logs pipelines` commands are read-only. Index responses +include routing filters, daily quotas, Standard/Flex retention, and ordered +exclusion filters with sample rates. Pipeline responses preserve the API's +processor order and include filters, parser rules, remappers, and nested +processors. + +Use the explicit order commands when routing order matters: + +```sh +ddcli logs indexes order --pretty +ddcli logs indexes get bandzoogle-production --pretty +ddcli logs pipelines order --pretty +ddcli logs pipelines list --query openresty --pretty +``` + +These commands require `logs_read_config`. Datadog also requires an +administrator-owned application key for pipeline configuration reads. + +### Preconditioned exclusion patch + +`logs indexes patch-exclusions` is the only log-configuration write command. It +fetches the named index, requires every named exclusion to occur exactly once +with the exact expected query, replaces only those query strings, and sends one +index update request containing all current updateable properties. Missing, +duplicate, stale, unknown, empty, and no-op patch entries fail without writing. + +Patch files are non-secret JSON: + +```json +{ + "index_name": "example-index", + "replacements": [ + { + "name": "Example exclusion", + "expected_query": "service:example status:info", + "replacement_query": "service:example status:(info OR ok)" + } + ] +} +``` + +Always run `--dry-run` first. It performs the read and all precondition checks, +then prints the exact before/after queries and preserved invariants without an +update. The guarded read requires the `logs_read_config` RBAC permission. +Applying through the V1 `UpdateLogsIndex` endpoint requires +`logs_modify_indexes`, shown as **Logs Modify Indexes** in the Datadog UI. +These are role or scoped application-key permissions; Datadog does not offer +them as OAuth client scopes. + ## Dashboard apply `dashboards apply` validates a canonical Datadog dashboard JSON file and creates @@ -100,7 +159,10 @@ matching also needs `dashboards_read`. ## Required Permissions -Use `ddcli scopes` to print the Datadog permissions and OAuth scopes needed by each command group. This command does not require Datadog credentials. +Use `ddcli scopes` to print the Datadog RBAC permissions and, where available, +separate OAuth scopes needed by each command group. Commands marked with no +OAuth scope require application-key authentication. This command does not +require Datadog credentials. Datadog API keys identify the organization. Access is controlled by the application key owner's role permissions, scoped application key permissions, or OAuth access token scopes. diff --git a/cmd/logs.go b/cmd/logs.go index 8c986c7..00cfd06 100644 --- a/cmd/logs.go +++ b/cmd/logs.go @@ -1,7 +1,10 @@ package cmd import ( + "strings" + "github.com/DataDog/datadog-api-client-go/v2/api/datadog" + "github.com/DataDog/datadog-api-client-go/v2/api/datadogV1" "github.com/DataDog/datadog-api-client-go/v2/api/datadogV2" "github.com/bandzoogle/datadog-cli/internal/output" "github.com/spf13/cobra" @@ -9,7 +12,7 @@ import ( var logsCmd = &cobra.Command{ Use: "logs", - Short: "Search Datadog logs", + Short: "Search logs and manage log configuration", } var logsSearchCmd = &cobra.Command{ @@ -23,9 +26,59 @@ var logsSearchCmd = &cobra.Command{ RunE: runLogsSearch, } +var logsIndexesCmd = &cobra.Command{ + Use: "indexes", + Short: "Inspect log indexes and exclusion filters", +} + +var logsIndexesListCmd = &cobra.Command{ + Use: "list", + Short: "List log indexes in evaluation order", + RunE: runLogsIndexesList, +} + +var logsIndexesGetCmd = &cobra.Command{ + Use: "get ", + Short: "Get a log index including exclusion filters", + Args: cobra.ExactArgs(1), + RunE: runLogsIndexesGet, +} + +var logsIndexesOrderCmd = &cobra.Command{ + Use: "order", + Short: "Get authoritative log index routing order", + RunE: runLogsIndexesOrder, +} + +var logsPipelinesCmd = &cobra.Command{ + Use: "pipelines", + Short: "Inspect log processing pipelines", +} + +var logsPipelinesListCmd = &cobra.Command{ + Use: "list", + Short: "List log pipelines in evaluation order", + RunE: runLogsPipelinesList, +} + +var logsPipelinesGetCmd = &cobra.Command{ + Use: "get ", + Short: "Get a log pipeline including processors", + Args: cobra.ExactArgs(1), + RunE: runLogsPipelinesGet, +} + +var logsPipelinesOrderCmd = &cobra.Command{ + Use: "order", + Short: "Get authoritative log pipeline order", + RunE: runLogsPipelinesOrder, +} + func init() { rootCmd.AddCommand(logsCmd) - logsCmd.AddCommand(logsSearchCmd) + logsCmd.AddCommand(logsSearchCmd, logsIndexesCmd, logsPipelinesCmd) + logsIndexesCmd.AddCommand(logsIndexesListCmd, logsIndexesGetCmd, logsIndexesOrderCmd) + logsPipelinesCmd.AddCommand(logsPipelinesListCmd, logsPipelinesGetCmd, logsPipelinesOrderCmd) logsSearchCmd.Flags().String("query", "", "Log search query") logsSearchCmd.Flags().String("from", "now-15m", "Start time, e.g. now-15m or RFC3339") @@ -33,6 +86,134 @@ func init() { logsSearchCmd.Flags().Int32("limit", 50, "Maximum logs to return") logsSearchCmd.Flags().String("cursor", "", "Pagination cursor from a previous response") logsSearchCmd.Flags().StringSlice("index", nil, "Log indexes to search; repeat or comma-separate") + logsIndexesListCmd.Flags().String("query", "", "Client-side filter for index name") + logsPipelinesListCmd.Flags().String("query", "", "Client-side filter for pipeline name or ID") +} + +func runLogsIndexesList(cmd *cobra.Command, args []string) error { + client, err := datadogClient(cmd) + if err != nil { + return err + } + query, _ := cmd.Flags().GetString("query") + api := datadogV1.NewLogsIndexesApi(client.API) + indexes, response, err := api.ListLogIndexes(client.Context) + if err != nil { + return apiError("logs indexes list", response, err) + } + if query != "" { + indexes.Indexes = filterLogIndexes(indexes.Indexes, query) + } + order, orderResponse, err := api.GetLogsIndexOrder(client.Context) + if err != nil { + return apiError("logs indexes order", orderResponse, err) + } + return output.WriteEnvelope(cmd.OutOrStdout(), + map[string]any{"command": "logs indexes list", "filter": query}, + meta(client.Site, map[string]any{"count": len(indexes.Indexes)}, response), + map[string]any{"indexes": indexes, "order": order}, + outputOptions(), + ) +} + +func runLogsIndexesGet(cmd *cobra.Command, args []string) error { + client, err := datadogClient(cmd) + if err != nil { + return err + } + name := args[0] + api := datadogV1.NewLogsIndexesApi(client.API) + index, response, err := api.GetLogsIndex(client.Context, name) + if err != nil { + return apiError("logs indexes get", response, err) + } + return output.WriteEnvelope(cmd.OutOrStdout(), + map[string]any{"command": "logs indexes get", "index_name": name}, + meta(client.Site, nil, response), + index, + outputOptions(), + ) +} + +func runLogsIndexesOrder(cmd *cobra.Command, args []string) error { + client, err := datadogClient(cmd) + if err != nil { + return err + } + api := datadogV1.NewLogsIndexesApi(client.API) + order, response, err := api.GetLogsIndexOrder(client.Context) + if err != nil { + return apiError("logs indexes order", response, err) + } + return output.WriteEnvelope(cmd.OutOrStdout(), + map[string]any{"command": "logs indexes order"}, + meta(client.Site, nil, response), + order, + outputOptions(), + ) +} + +func runLogsPipelinesList(cmd *cobra.Command, args []string) error { + client, err := datadogClient(cmd) + if err != nil { + return err + } + query, _ := cmd.Flags().GetString("query") + api := datadogV1.NewLogsPipelinesApi(client.API) + pipelines, response, err := api.ListLogsPipelines(client.Context) + if err != nil { + return apiError("logs pipelines list", response, err) + } + if query != "" { + pipelines = filterLogPipelines(pipelines, query) + } + order, orderResponse, err := api.GetLogsPipelineOrder(client.Context) + if err != nil { + return apiError("logs pipelines order", orderResponse, err) + } + return output.WriteEnvelope(cmd.OutOrStdout(), + map[string]any{"command": "logs pipelines list", "filter": query}, + meta(client.Site, map[string]any{"count": len(pipelines)}, response), + map[string]any{"pipelines": pipelines, "order": order}, + outputOptions(), + ) +} + +func runLogsPipelinesGet(cmd *cobra.Command, args []string) error { + client, err := datadogClient(cmd) + if err != nil { + return err + } + id := args[0] + api := datadogV1.NewLogsPipelinesApi(client.API) + pipeline, response, err := api.GetLogsPipeline(client.Context, id) + if err != nil { + return apiError("logs pipelines get", response, err) + } + return output.WriteEnvelope(cmd.OutOrStdout(), + map[string]any{"command": "logs pipelines get", "pipeline_id": id}, + meta(client.Site, nil, response), + pipeline, + outputOptions(), + ) +} + +func runLogsPipelinesOrder(cmd *cobra.Command, args []string) error { + client, err := datadogClient(cmd) + if err != nil { + return err + } + api := datadogV1.NewLogsPipelinesApi(client.API) + order, response, err := api.GetLogsPipelineOrder(client.Context) + if err != nil { + return apiError("logs pipelines order", response, err) + } + return output.WriteEnvelope(cmd.OutOrStdout(), + map[string]any{"command": "logs pipelines order"}, + meta(client.Site, nil, response), + order, + outputOptions(), + ) } func runLogsSearch(cmd *cobra.Command, args []string) error { @@ -83,3 +264,26 @@ func runLogsSearch(cmd *cobra.Command, args []string) error { outputOptions(), ) } + +func filterLogIndexes(indexes []datadogV1.LogsIndex, query string) []datadogV1.LogsIndex { + query = strings.ToLower(strings.TrimSpace(query)) + filtered := make([]datadogV1.LogsIndex, 0, len(indexes)) + for _, index := range indexes { + if strings.Contains(strings.ToLower(index.GetName()), query) { + filtered = append(filtered, index) + } + } + return filtered +} + +func filterLogPipelines(pipelines []datadogV1.LogsPipeline, query string) []datadogV1.LogsPipeline { + query = strings.ToLower(strings.TrimSpace(query)) + filtered := make([]datadogV1.LogsPipeline, 0, len(pipelines)) + for _, pipeline := range pipelines { + if strings.Contains(strings.ToLower(pipeline.GetName()), query) || + strings.Contains(strings.ToLower(pipeline.GetId()), query) { + filtered = append(filtered, pipeline) + } + } + return filtered +} diff --git a/cmd/logs_config_test.go b/cmd/logs_config_test.go new file mode 100644 index 0000000..68944ec --- /dev/null +++ b/cmd/logs_config_test.go @@ -0,0 +1,53 @@ +package cmd + +import ( + "testing" + + "github.com/DataDog/datadog-api-client-go/v2/api/datadogV1" +) + +func TestFilterLogIndexesPreservesOrder(t *testing.T) { + indexes := []datadogV1.LogsIndex{ + logIndex("bandzoogle-production"), + logIndex("bandzoogle-development"), + logIndex("other"), + } + + got := filterLogIndexes(indexes, "bandzoogle") + + if len(got) != 2 { + t.Fatalf("expected two indexes, got %d", len(got)) + } + if got[0].GetName() != "bandzoogle-production" || + got[1].GetName() != "bandzoogle-development" { + t.Fatalf("unexpected filtered order: %#v", got) + } +} + +func TestFilterLogPipelinesMatchesNameOrID(t *testing.T) { + pipelines := []datadogV1.LogsPipeline{ + logPipeline("abc-123", "OpenResty"), + logPipeline("def-456", "Rails"), + } + + byName := filterLogPipelines(pipelines, "openresty") + if len(byName) != 1 || byName[0].GetId() != "abc-123" { + t.Fatalf("unexpected name match: %#v", byName) + } + + byID := filterLogPipelines(pipelines, "DEF") + if len(byID) != 1 || byID[0].GetName() != "Rails" { + t.Fatalf("unexpected ID match: %#v", byID) + } +} + +func logIndex(name string) datadogV1.LogsIndex { + index := datadogV1.NewLogsIndex(*datadogV1.NewLogsFilter(), name) + return *index +} + +func logPipeline(id, name string) datadogV1.LogsPipeline { + pipeline := datadogV1.NewLogsPipeline(name) + pipeline.SetId(id) + return *pipeline +} diff --git a/cmd/logs_patch.go b/cmd/logs_patch.go new file mode 100644 index 0000000..d6954f1 --- /dev/null +++ b/cmd/logs_patch.go @@ -0,0 +1,244 @@ +package cmd + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "os" + "strings" + + "github.com/DataDog/datadog-api-client-go/v2/api/datadogV1" + "github.com/bandzoogle/datadog-cli/internal/output" + "github.com/spf13/cobra" +) + +type logExclusionPatch struct { + IndexName string `json:"index_name"` + Replacements []logExclusionQueryReplacement `json:"replacements"` +} + +type logExclusionQueryReplacement struct { + Name string `json:"name"` + ExpectedQuery string `json:"expected_query"` + ReplacementQuery string `json:"replacement_query"` +} + +type logExclusionQueryChange struct { + Name string `json:"name"` + Before string `json:"before"` + After string `json:"after"` +} + +var logsIndexesPatchExclusionsCmd = &cobra.Command{ + Use: "patch-exclusions ", + Short: "Apply preconditioned exclusion-query replacements", + Long: `Fetch an index, require every named exclusion to have its exact expected +query, then replace only those query strings in one index update request. + +The command rejects missing, duplicate, stale, empty, and no-op replacements. +All unrelated index properties and exclusion fields are preserved. + +The guarded read requires the Logs Configuration Read RBAC permission +(logs_read_config). The V1 index update requires Logs Modify Indexes +(logs_modify_indexes). Datadog does not offer these Logs RBAC permissions as +OAuth client scopes.`, + Args: cobra.ExactArgs(1), + RunE: runLogsIndexesPatchExclusions, +} + +func init() { + logsIndexesCmd.AddCommand(logsIndexesPatchExclusionsCmd) + logsIndexesPatchExclusionsCmd.Flags().Bool("dry-run", false, "Validate and print the exact diff without updating Datadog") +} + +func runLogsIndexesPatchExclusions(cmd *cobra.Command, args []string) error { + patch, err := loadLogExclusionPatch(args[0]) + if err != nil { + return err + } + client, err := datadogClient(cmd) + if err != nil { + return err + } + api := datadogV1.NewLogsIndexesApi(client.API) + current, getResponse, err := api.GetLogsIndex(client.Context, patch.IndexName) + if err != nil { + return apiError("logs indexes patch-exclusions get", getResponse, err) + } + updatedFilters, changes, err := applyLogExclusionPatch(current.ExclusionFilters, patch) + if err != nil { + return err + } + invariants := logIndexInvariants(current) + dryRun, _ := cmd.Flags().GetBool("dry-run") + if dryRun { + return output.WriteEnvelope(cmd.OutOrStdout(), + map[string]any{"command": "logs indexes patch-exclusions", "file": args[0]}, + meta(client.Site, map[string]any{"action": "dry-run"}, getResponse), + map[string]any{ + "index_name": patch.IndexName, + "changes": changes, + "invariants": invariants, + }, + outputOptions(), + ) + } + + update := logIndexUpdateRequest(current, updatedFilters) + result, updateResponse, err := api.UpdateLogsIndex(client.Context, patch.IndexName, update) + if err != nil { + return apiError("logs indexes patch-exclusions update", updateResponse, err) + } + return output.WriteEnvelope(cmd.OutOrStdout(), + map[string]any{"command": "logs indexes patch-exclusions", "file": args[0]}, + meta(client.Site, map[string]any{"action": "update"}, updateResponse), + map[string]any{ + "index_name": patch.IndexName, + "changes": changes, + "invariants": invariants, + "index": result, + }, + outputOptions(), + ) +} + +func loadLogExclusionPatch(path string) (logExclusionPatch, error) { + var patch logExclusionPatch + body, err := os.ReadFile(path) + if err != nil { + return patch, fmt.Errorf("read log exclusion patch: %w", err) + } + decoder := json.NewDecoder(bytes.NewReader(body)) + decoder.DisallowUnknownFields() + if err := decoder.Decode(&patch); err != nil { + return patch, fmt.Errorf("parse log exclusion patch: %w", err) + } + if err := decoder.Decode(&struct{}{}); err != io.EOF { + return patch, fmt.Errorf("parse log exclusion patch: trailing JSON content") + } + if strings.TrimSpace(patch.IndexName) == "" { + return patch, fmt.Errorf("index_name is required") + } + if len(patch.Replacements) == 0 { + return patch, fmt.Errorf("at least one replacement is required") + } + return patch, nil +} + +func applyLogExclusionPatch( + exclusions []datadogV1.LogsExclusion, + patch logExclusionPatch, +) ([]datadogV1.LogsExclusion, []logExclusionQueryChange, error) { + replacements := make(map[string]logExclusionQueryReplacement, len(patch.Replacements)) + for _, replacement := range patch.Replacements { + if strings.TrimSpace(replacement.Name) == "" { + return nil, nil, fmt.Errorf("replacement name is required") + } + if replacement.ExpectedQuery == "" || replacement.ReplacementQuery == "" { + return nil, nil, fmt.Errorf("replacement %q requires non-empty expected_query and replacement_query", replacement.Name) + } + if replacement.ExpectedQuery == replacement.ReplacementQuery { + return nil, nil, fmt.Errorf("replacement %q is a no-op", replacement.Name) + } + if _, exists := replacements[replacement.Name]; exists { + return nil, nil, fmt.Errorf("duplicate replacement for exclusion %q", replacement.Name) + } + replacements[replacement.Name] = replacement + } + + counts := make(map[string]int, len(replacements)) + for _, exclusion := range exclusions { + if _, exists := replacements[exclusion.Name]; exists { + counts[exclusion.Name]++ + } + } + for name := range replacements { + switch counts[name] { + case 0: + return nil, nil, fmt.Errorf("exclusion %q is missing", name) + case 1: + default: + return nil, nil, fmt.Errorf("exclusion %q appears %d times", name, counts[name]) + } + } + + updated := append([]datadogV1.LogsExclusion(nil), exclusions...) + changes := make([]logExclusionQueryChange, 0, len(replacements)) + for i, exclusion := range updated { + replacement, exists := replacements[exclusion.Name] + if !exists { + continue + } + filter, ok := exclusion.GetFilterOk() + if !ok { + return nil, nil, fmt.Errorf("exclusion %q has no filter", exclusion.Name) + } + query, ok := filter.GetQueryOk() + if !ok { + return nil, nil, fmt.Errorf("exclusion %q has no query", exclusion.Name) + } + if *query != replacement.ExpectedQuery { + return nil, nil, fmt.Errorf( + "stale exclusion %q: expected query %q, got %q", + exclusion.Name, + replacement.ExpectedQuery, + *query, + ) + } + filterCopy := *filter + filterCopy.SetQuery(replacement.ReplacementQuery) + exclusion.SetFilter(filterCopy) + updated[i] = exclusion + changes = append(changes, logExclusionQueryChange{ + Name: exclusion.Name, + Before: replacement.ExpectedQuery, + After: replacement.ReplacementQuery, + }) + } + return updated, changes, nil +} + +func logIndexUpdateRequest( + index datadogV1.LogsIndex, + exclusions []datadogV1.LogsExclusion, +) datadogV1.LogsIndexUpdateRequest { + request := datadogV1.NewLogsIndexUpdateRequest(index.Filter) + request.SetExclusionFilters(exclusions) + if value, ok := index.GetDailyLimitOk(); ok { + request.SetDailyLimit(*value) + } + if value, ok := index.GetDailyLimitResetOk(); ok { + request.SetDailyLimitReset(*value) + } + if value, ok := index.GetDailyLimitWarningThresholdPercentageOk(); ok { + request.SetDailyLimitWarningThresholdPercentage(*value) + } + if value, ok := index.GetNumRetentionDaysOk(); ok { + request.SetNumRetentionDays(*value) + } + if value, ok := index.GetNumFlexLogsRetentionDaysOk(); ok { + request.SetNumFlexLogsRetentionDays(*value) + } + if value, ok := index.GetTagsOk(); ok { + request.SetTags(*value) + } + return *request +} + +func logIndexInvariants(index datadogV1.LogsIndex) map[string]any { + return map[string]any{ + "daily_limit": optionalInt64(index.GetDailyLimitOk()), + "filter_query": index.Filter.GetQuery(), + "num_retention_days": optionalInt64(index.GetNumRetentionDaysOk()), + "num_flex_logs_retention_days": optionalInt64(index.GetNumFlexLogsRetentionDaysOk()), + "exclusion_count": len(index.ExclusionFilters), + } +} + +func optionalInt64(value *int64, ok bool) any { + if !ok { + return nil + } + return *value +} diff --git a/cmd/logs_patch_test.go b/cmd/logs_patch_test.go new file mode 100644 index 0000000..64917b7 --- /dev/null +++ b/cmd/logs_patch_test.go @@ -0,0 +1,178 @@ +package cmd + +import ( + "os" + "path/filepath" + "reflect" + "strings" + "testing" + + "github.com/DataDog/datadog-api-client-go/v2/api/datadogV1" +) + +func TestApplyLogExclusionPatchChangesOnlyExpectedQueries(t *testing.T) { + exclusions := []datadogV1.LogsExclusion{ + logExclusion("first", "query one", 1, true), + logExclusion("unrelated", "leave me", 0.95, false), + logExclusion("second", "query two", 1, true), + } + patch := logExclusionPatch{ + IndexName: "production", + Replacements: []logExclusionQueryReplacement{ + {Name: "second", ExpectedQuery: "query two", ReplacementQuery: "new two"}, + {Name: "first", ExpectedQuery: "query one", ReplacementQuery: "new one"}, + }, + } + + updated, changes, err := applyLogExclusionPatch(exclusions, patch) + if err != nil { + t.Fatal(err) + } + if len(changes) != 2 || changes[0].Name != "first" || changes[1].Name != "second" { + t.Fatalf("expected changes in live exclusion order, got %#v", changes) + } + firstFilter := updated[0].GetFilter() + secondFilter := updated[2].GetFilter() + if firstFilter.GetQuery() != "new one" || + secondFilter.GetQuery() != "new two" { + t.Fatalf("queries were not replaced: %#v", updated) + } + if !reflect.DeepEqual(updated[1], exclusions[1]) { + t.Fatal("unrelated exclusion changed") + } + expectedFirst := exclusions[0] + filter := expectedFirst.GetFilter() + filter.SetQuery("new one") + expectedFirst.SetFilter(filter) + if !reflect.DeepEqual(updated[0], expectedFirst) { + t.Fatal("changed exclusion fields other than query") + } + originalFilter := exclusions[0].GetFilter() + if originalFilter.GetQuery() != "query one" { + t.Fatal("input exclusions were mutated") + } +} + +func TestApplyLogExclusionPatchRejectsStaleQuery(t *testing.T) { + exclusions := []datadogV1.LogsExclusion{ + logExclusion("target", "current", 1, true), + } + patch := patchFor("target", "expected", "replacement") + + _, _, err := applyLogExclusionPatch(exclusions, patch) + + if err == nil || !strings.Contains(err.Error(), "stale exclusion") { + t.Fatalf("expected stale query error, got %v", err) + } +} + +func TestApplyLogExclusionPatchRejectsMissingAndDuplicateLiveFilters(t *testing.T) { + patch := patchFor("target", "expected", "replacement") + + _, _, missingErr := applyLogExclusionPatch(nil, patch) + if missingErr == nil || !strings.Contains(missingErr.Error(), "is missing") { + t.Fatalf("expected missing error, got %v", missingErr) + } + + exclusions := []datadogV1.LogsExclusion{ + logExclusion("target", "expected", 1, true), + logExclusion("target", "expected", 1, true), + } + _, _, duplicateErr := applyLogExclusionPatch(exclusions, patch) + if duplicateErr == nil || !strings.Contains(duplicateErr.Error(), "appears 2 times") { + t.Fatalf("expected duplicate live filter error, got %v", duplicateErr) + } +} + +func TestApplyLogExclusionPatchRejectsDuplicatePatchFilters(t *testing.T) { + patch := logExclusionPatch{ + IndexName: "production", + Replacements: []logExclusionQueryReplacement{ + {Name: "target", ExpectedQuery: "one", ReplacementQuery: "two"}, + {Name: "target", ExpectedQuery: "two", ReplacementQuery: "three"}, + }, + } + + _, _, err := applyLogExclusionPatch(nil, patch) + + if err == nil || !strings.Contains(err.Error(), "duplicate replacement") { + t.Fatalf("expected duplicate replacement error, got %v", err) + } +} + +func TestLogIndexUpdateRequestPreservesUpdateableProperties(t *testing.T) { + index := datadogV1.NewLogsIndex(*datadogV1.NewLogsFilterWithDefaults(), "production") + index.Filter.SetQuery("account:production") + index.SetDailyLimit(500_000_000) + index.SetDailyLimitWarningThresholdPercentage(80) + index.SetNumRetentionDays(7) + index.SetNumFlexLogsRetentionDays(30) + index.SetTags([]string{"team:server-admin"}) + exclusions := []datadogV1.LogsExclusion{ + logExclusion("target", "replacement", 1, true), + } + + request := logIndexUpdateRequest(*index, exclusions) + + requestFilter := request.GetFilter() + if requestFilter.GetQuery() != "account:production" || + request.GetDailyLimit() != 500_000_000 || + request.GetDailyLimitWarningThresholdPercentage() != 80 || + request.GetNumRetentionDays() != 7 || + request.GetNumFlexLogsRetentionDays() != 30 || + !reflect.DeepEqual(request.GetTags(), []string{"team:server-admin"}) || + !reflect.DeepEqual(request.GetExclusionFilters(), exclusions) { + t.Fatalf("update request did not preserve index properties: %#v", request) + } +} + +func TestLoadLogExclusionPatchIsStrict(t *testing.T) { + path := filepath.Join(t.TempDir(), "patch.json") + err := os.WriteFile(path, []byte(`{ + "index_name": "production", + "replacements": [{ + "name": "target", + "expected_query": "one", + "replacement_query": "two" + }], + "unexpected": true + }`), 0o600) + if err != nil { + t.Fatal(err) + } + + _, err = loadLogExclusionPatch(path) + + if err == nil || !strings.Contains(err.Error(), "unknown field") { + t.Fatalf("expected strict JSON error, got %v", err) + } +} + +func TestLogExclusionPatchHelpNamesRBACPermissions(t *testing.T) { + help := logsIndexesPatchExclusionsCmd.Long + if !strings.Contains(help, "logs_read_config") || + !strings.Contains(help, "logs_modify_indexes") { + t.Fatalf("expected exact Logs RBAC permissions in help, got:\n%s", help) + } + if strings.Contains(help, "logs_write_config") { + t.Fatalf("help includes nonexistent permission:\n%s", help) + } +} + +func patchFor(name, expected, replacement string) logExclusionPatch { + return logExclusionPatch{ + IndexName: "production", + Replacements: []logExclusionQueryReplacement{ + {Name: name, ExpectedQuery: expected, ReplacementQuery: replacement}, + }, + } +} + +func logExclusion(name, query string, sampleRate float64, enabled bool) datadogV1.LogsExclusion { + filter := datadogV1.NewLogsExclusionFilter(sampleRate) + filter.SetQuery(query) + exclusion := datadogV1.NewLogsExclusion(name) + exclusion.SetFilter(*filter) + exclusion.SetIsEnabled(enabled) + return *exclusion +} diff --git a/cmd/scopes.go b/cmd/scopes.go index de102ca..00fdbf7 100644 --- a/cmd/scopes.go +++ b/cmd/scopes.go @@ -13,7 +13,7 @@ type scopeInfo struct { Command string `json:"command"` API string `json:"api"` Permission string `json:"permission"` - OAuthScope string `json:"oauth_scope"` + OAuthScope string `json:"oauth_scope,omitempty"` Notes []string `json:"notes,omitempty"` } @@ -21,6 +21,7 @@ type scopesResponse struct { Summary []string `json:"summary"` Required []scopeInfo `json:"required"` Unique []string `json:"unique"` + OAuthScopes []string `json:"oauth_scopes"` RecommendedRole []string `json:"recommended_role"` } @@ -47,12 +48,14 @@ func runScopes(cmd *cobra.Command, args []string) error { resp := scopesResponse{ Summary: []string{ "DD_API_KEY identifies the Datadog organization.", - "DD_APP_KEY or DD_APPLICATION_KEY must belong to a user/service account with the listed permissions.", - "DD_ACCESS_TOKEN must include the listed OAuth scopes when using bearer-token auth.", + "DD_APP_KEY or DD_APPLICATION_KEY must belong to a user/service account with the listed RBAC permissions.", + "Scoped application keys can be assigned any listed RBAC permission.", + "DD_ACCESS_TOKEN must include each listed OAuth scope; commands without an OAuth scope require application-key authentication.", "Use the smallest subset that matches the commands you plan to run.", }, - Required: required, - Unique: uniquePermissions(required), + Required: required, + Unique: uniquePermissions(required), + OAuthScopes: uniqueOAuthScopes(required), RecommendedRole: []string{ "Create a service account for ddcli.", "Grant only the read permissions for the command groups you need.", @@ -82,14 +85,26 @@ func renderScopesText(resp scopesResponse, filter string) string { b.WriteString("Datadog permissions for ddcli:\n\n") } + b.WriteString("RBAC / scoped application-key permissions:\n") b.WriteString(strings.Join(resp.Unique, "\n")) + b.WriteString("\n\nOAuth scopes:\n") + if len(resp.OAuthScopes) == 0 { + b.WriteString("(none)") + } else { + b.WriteString(strings.Join(resp.OAuthScopes, "\n")) + } b.WriteString("\n\n") - b.WriteString("Grant these to the DD_APP_KEY owner, scoped application key, or OAuth token.\n") + b.WriteString("Grant RBAC permissions to the DD_APP_KEY owner or scoped application key.\n") + b.WriteString("OAuth tokens can run only commands with a listed OAuth scope.\n") b.WriteString("DD_API_KEY only identifies the Datadog organization.\n\n") b.WriteString("Command coverage:\n") for _, item := range resp.Required { - fmt.Fprintf(&b, "- %s: %s\n", item.Command, item.Permission) + oauthScope := item.OAuthScope + if oauthScope == "" { + oauthScope = "unavailable" + } + fmt.Fprintf(&b, "- %s: RBAC %s; OAuth %s\n", item.Command, item.Permission, oauthScope) } b.WriteString("\nSetup notes:\n") @@ -105,9 +120,36 @@ func requiredScopes() []scopeInfo { Command: "logs search", API: "Logs Search API", Permission: "logs_read_data", - OAuthScope: "logs_read_data", Notes: []string{ "Actual log visibility may also be narrowed by log indexes and restriction queries.", + "Datadog does not offer this Logs RBAC permission as an OAuth client scope.", + }, + }, + { + Command: "logs indexes|pipelines list|get|order", + API: "Logs Configuration API", + Permission: "logs_read_config", + Notes: []string{ + "Pipeline configuration endpoints require an administrator-owned application key.", + "Datadog does not offer this Logs RBAC permission as an OAuth client scope.", + }, + }, + { + Command: "logs indexes patch-exclusions (guarded read)", + API: "Logs Indexes API", + Permission: "logs_read_config", + Notes: []string{ + "Datadog UI name: Logs Configuration Read.", + "Datadog does not offer this Logs RBAC permission as an OAuth client scope.", + }, + }, + { + Command: "logs indexes patch-exclusions (index update)", + API: "Logs Indexes API", + Permission: "logs_modify_indexes", + Notes: []string{ + "Datadog UI name: Logs Modify Indexes.", + "Datadog does not offer this Logs RBAC permission as an OAuth client scope.", }, }, { @@ -231,3 +273,18 @@ func uniquePermissions(scopes []scopeInfo) []string { sort.Strings(out) return out } + +func uniqueOAuthScopes(scopes []scopeInfo) []string { + seen := map[string]bool{} + for _, scope := range scopes { + if scope.OAuthScope != "" { + seen[scope.OAuthScope] = true + } + } + out := make([]string, 0, len(seen)) + for scope := range seen { + out = append(out, scope) + } + sort.Strings(out) + return out +} diff --git a/cmd/scopes_test.go b/cmd/scopes_test.go index 8d72d8b..cdc6338 100644 --- a/cmd/scopes_test.go +++ b/cmd/scopes_test.go @@ -1,6 +1,7 @@ package cmd import ( + "reflect" "strings" "testing" ) @@ -34,23 +35,57 @@ func TestFilterScopesByCommandPrefix(t *testing.T) { } } +func TestLogIndexPatchUsesRBACPermissionWithoutOAuthScope(t *testing.T) { + got := filterScopes(requiredScopes(), "logs indexes patch-exclusions") + if len(got) != 2 { + t.Fatalf("expected two patch permission rows, got %d", len(got)) + } + permissions := uniquePermissions(got) + want := []string{"logs_modify_indexes", "logs_read_config"} + if !reflect.DeepEqual(permissions, want) { + t.Fatalf("expected permissions %#v, got %#v", want, permissions) + } + for _, item := range got { + if item.OAuthScope != "" { + t.Fatalf("expected no OAuth scope, got %s", item.OAuthScope) + } + } +} + +func TestUniqueOAuthScopesOmitsUnavailableScopes(t *testing.T) { + got := uniqueOAuthScopes([]scopeInfo{ + {Permission: "logs_modify_indexes"}, + {Permission: "dashboards_read", OAuthScope: "dashboards_read"}, + {Permission: "dashboards_write", OAuthScope: "dashboards_write"}, + {Permission: "other", OAuthScope: "dashboards_read"}, + }) + want := []string{"dashboards_read", "dashboards_write"} + if !reflect.DeepEqual(got, want) { + t.Fatalf("expected %#v, got %#v", want, got) + } +} + func TestRenderScopesTextShowsUniquePermissionsTogether(t *testing.T) { resp := scopesResponse{ Required: []scopeInfo{ - {Command: "metrics query", Permission: "metrics_read"}, - {Command: "dashboards list", Permission: "dashboards_read"}, + {Command: "metrics query", Permission: "metrics_read", OAuthScope: "metrics_read"}, + {Command: "logs indexes patch-exclusions", Permission: "logs_modify_indexes"}, }, - Unique: []string{"dashboards_read", "metrics_read"}, + Unique: []string{"logs_modify_indexes", "metrics_read"}, + OAuthScopes: []string{"metrics_read"}, RecommendedRole: []string{ "Create a service account for ddcli.", }, } got := renderScopesText(resp, "") - if !strings.Contains(got, "dashboards_read\nmetrics_read") { + if !strings.Contains(got, "logs_modify_indexes\nmetrics_read") { t.Fatalf("expected permissions grouped together, got:\n%s", got) } - if !strings.Contains(got, "- metrics query: metrics_read") { + if !strings.Contains(got, "- metrics query: RBAC metrics_read; OAuth metrics_read") { + t.Fatalf("expected OAuth command coverage, got:\n%s", got) + } + if !strings.Contains(got, "- logs indexes patch-exclusions: RBAC logs_modify_indexes; OAuth unavailable") { t.Fatalf("expected command coverage, got:\n%s", got) } }