diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 9883559..15a4a72 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -72,6 +72,7 @@ stackstate-backup-cli/ - `cmd/victoriametrics/`: VictoriaMetrics backup/restore commands (list, restore, check-and-finalize) - `cmd/settings/`: Settings backup/restore commands (list, restore, check-and-finalize) - `cmd/version/`: Version information +- `cmd/replication/`: Read-only HA replication observations and bounded waiting **Dependency Rules**: - ✅ Can import: `internal/app/*` (preferred), all other `internal/` packages @@ -126,6 +127,7 @@ appCtx.NewCHClient(backupAPIPort, dbPort) // ClickHouse client factory **Key Packages**: - `portforward/`: Manages Kubernetes port-forwarding lifecycle +- `replication/`: Discovers namespace workloads and evaluates fixed database queries, without backup configuration or restore operations - `scale/`: Deployment and StatefulSet scaling workflows with detailed logging - `restore/`: Restore job orchestration (confirmation, job lifecycle, finalization, resource management) - `restorelock/`: Prevents parallel restore operations using Kubernetes annotations diff --git a/README.md b/README.md index bac5f87..5446853 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,7 @@ This CLI tool replaces the legacy Bash-based backup/restore scripts with a singl - Stackgraph backups and restores - VictoriaMetrics backups and restores - Settings backups and restores +- Read-only database replication checks selected from the deployed topology ## Installation @@ -44,6 +45,19 @@ sts-backup [command] [subcommand] [flags] ## Commands +### replication check + +Inspect HDFS, Elasticsearch, Kafka, ClickHouse and ZooKeeper replication in one namespace: + +```bash +sts-backup replication check --namespace observability +sts-backup replication check --namespace observability --wait --output json +``` + +This command has its own flags and does not require backup configuration. +See [Replication checks](docs/replication.md) for status and exit-code semantics, +authentication and the maintenance checks outside its scope. + ### version Display version information. diff --git a/cmd/replication/replication.go b/cmd/replication/replication.go new file mode 100644 index 0000000..b375009 --- /dev/null +++ b/cmd/replication/replication.go @@ -0,0 +1,195 @@ +// Package replication exposes read-only database replication checks. +package replication + +import ( + "context" + "encoding/json" + "fmt" + "io" + "os" + "os/signal" + "strings" + "syscall" + "text/tabwriter" + "time" + + "github.com/spf13/cobra" + + "github.com/stackvista/stackstate-backup-cli/internal/app" + checker "github.com/stackvista/stackstate-backup-cli/internal/orchestration/replication" +) + +const ( + defaultTimeout = 10 * time.Minute + defaultInterval = 10 * time.Second + defaultStableFor = 30 * time.Second + tablePadding = 2 +) + +type flags struct { + options checker.Options + kubeconfig string + output string + wait bool + timeout time.Duration + interval time.Duration + stableFor time.Duration +} + +// Cmd creates the replication command independently of backup configuration. +func Cmd() *cobra.Command { + command := &cobra.Command{Use: "replication", Short: "Inspect database replication without changing cluster state"} + f := &flags{interval: defaultInterval} + check := &cobra.Command{ + Use: "check", Short: "Check observed replication; return nonzero unless all selected checks pass", + Long: "Check chart-managed HDFS, Elasticsearch, Kafka, ClickHouse and ZooKeeper replication. " + + "This is a point-in-time observation, not permission to remove a node. " + + "Uses StatefulSet configuration to select applicable checks. " + + "Requires pods/exec access; only fixed read-only database queries are executed.", + Args: cobra.NoArgs, SilenceUsage: true, + RunE: func(command *cobra.Command, _ []string) error { return run(command, f) }, + } + check.Flags().StringVarP(&f.options.Namespace, "namespace", "n", "", "Kubernetes namespace (required)") + check.Flags().StringVar(&f.kubeconfig, "kubeconfig", "", "Kubeconfig path; uses normal kubeconfig or in-cluster credentials") + check.Flags().StringSliceVar(&f.options.Components, "components", []string{"hdfs", "elasticsearch", "kafka", "clickhouse", "zookeeper"}, "Components to check") + check.Flags().StringVarP(&f.output, "output", "o", "table", "Output format: table or json") + check.Flags().BoolVar(&f.wait, "wait", false, "Wait for sustained healthy replication") + check.Flags().DurationVar(&f.timeout, "timeout", defaultTimeout, "Overall deadline, including queries") + check.Flags().DurationVar(&f.stableFor, "stable-for", defaultStableFor, "Required healthy observation period in wait mode") + _ = check.MarkFlagRequired("namespace") + command.AddCommand(check) + return command +} + +func (f *flags) validate() error { + if f.output != "table" && f.output != "json" { + return fmt.Errorf("output must be table or json") + } + if f.timeout <= 0 || f.stableFor < 0 { + return fmt.Errorf("timeout must be positive; stable-for cannot be negative") + } + if f.wait && f.stableFor >= f.timeout { + return fmt.Errorf("stable-for must be shorter than timeout") + } + return f.options.Validate() +} + +func run(command *cobra.Command, f *flags) error { + if err := f.validate(); err != nil { + return err + } + probe, err := app.NewReplicationChecker(f.kubeconfig, f.options) + if err != nil { + return err + } + ctx, stop := signal.NotifyContext(command.Context(), os.Interrupt, syscall.SIGTERM) + defer stop() + ctx, cancel := context.WithTimeout(ctx, f.timeout) + defer cancel() + report, checkErr := observe(ctx, probe.Check, f, command.ErrOrStderr(), probe.Verify) + return finishReport(command.OutOrStdout(), f.output, report, checkErr) +} + +func finishReport(writer io.Writer, format string, report checker.Report, checkErr error) error { + if checkErr != nil { + report.Status = checker.Unknown + report.Error = checkErr.Error() + } + if err := writeReport(writer, format, report); err != nil { + return err + } + if checkErr != nil { + return checkErr + } + if report.Status != checker.Healthy && report.Status != checker.NotApplicable { + return fmt.Errorf("replication is %s; see the report", report.Status) + } + return nil +} + +func observe(ctx context.Context, check func(context.Context) checker.Report, f *flags, progress io.Writer, verify func(context.Context, checker.Report) checker.Report) (checker.Report, error) { + if !f.wait { + report := check(ctx) + if ctx.Err() != nil { + return checker.Report{Namespace: report.Namespace}, fmt.Errorf("replication check ended: %w", ctx.Err()) + } + if verify != nil && report.Status == checker.Healthy { + _, _ = fmt.Fprintf(progress, "[%s] Running final replication verification.\n", time.Now().UTC().Format(time.RFC3339)) + verified := verify(ctx, report) + if ctx.Err() != nil { + return report, fmt.Errorf("replication verification ended: %w", ctx.Err()) + } + report = verified + } + return report, nil + } + _, _ = fmt.Fprintf(progress, "[%s] Waiting for all applicable checks to remain healthy for %s (timeout %s).\n", + time.Now().UTC().Format(time.RFC3339), f.stableFor, f.timeout) + return checker.Wait(ctx, check, f.interval, f.stableFor, func(report checker.Report, state checker.WaitProgress) { + timestamp := state.ObservedAt.Format(time.RFC3339) + for _, check := range report.Checks { + _, _ = fmt.Fprintf(progress, "[%s] %s %s: %s\n", timestamp, check.Component, check.Status, strings.Join(check.Messages, "; ")) + } + _, _ = fmt.Fprintf(progress, "[%s] %s\n", timestamp, stabilityMessage(report, state)) + }, verify) +} + +func stabilityMessage(report checker.Report, state checker.WaitProgress) string { + switch { + case state.Verifying: + return "All applicable checks stable; running final replication verification." + case state.RetryAfter > 0: + return fmt.Sprintf("Final verification did not pass; stability period reset; next observation in %s.", state.RetryAfter) + case report.Status == checker.NotApplicable: + return "No applicable replication checks; configured component availability checks passed." + case state.Reset && report.Status == checker.Healthy: + return fmt.Sprintf("Database topology, readiness or runtime changed; stability period restarted (0s/%s).", state.Required) + case state.Complete: + return fmt.Sprintf("All applicable checks healthy; stability period satisfied (%s/%s).", + state.HealthyFor.Round(time.Millisecond), state.Required) + case report.Status == checker.Healthy: + return fmt.Sprintf("All applicable checks healthy; verifying stability: %s/%s (%s remaining).", + state.HealthyFor.Round(time.Millisecond), state.Required, (state.Required - state.HealthyFor).Round(time.Millisecond)) + case state.Reset: + return "Stability period reset; waiting for all selected checks to become healthy." + default: + return "Waiting for all selected checks to become healthy; stability period has not started." + } +} + +func writeReport(writer io.Writer, format string, report checker.Report) error { + if format == "json" { + if err := json.NewEncoder(writer).Encode(report); err != nil { + return fmt.Errorf("write JSON report: %w", err) + } + return nil + } + if _, err := fmt.Fprintf(writer, "[%s] Replication result: %s\n", time.Now().UTC().Format(time.RFC3339), report.Status); err != nil { + return fmt.Errorf("write report status: %w", err) + } + if report.Error != "" { + if _, err := fmt.Fprintln(writer, report.Error); err != nil { + return fmt.Errorf("write report error: %w", err) + } + } + if len(report.Checks) == 0 { + _, err := fmt.Fprintln(writer, "No completed observation.") + return err + } + if _, err := fmt.Fprintf(writer, "Last completed observation started: %s\n", report.CheckedAt.Format(time.RFC3339)); err != nil { + return fmt.Errorf("write observation timestamp: %w", err) + } + table := tabwriter.NewWriter(writer, 0, 0, tablePadding, ' ', 0) + if _, err := fmt.Fprintln(table, "COMPONENT\tSTATUS\tDETAILS"); err != nil { + return fmt.Errorf("write report header: %w", err) + } + for _, check := range report.Checks { + if _, err := fmt.Fprintf(table, "%s\t%s\t%s\n", check.Component, check.Status, strings.Join(check.Messages, "; ")); err != nil { + return fmt.Errorf("write report row: %w", err) + } + } + if err := table.Flush(); err != nil { + return fmt.Errorf("flush report: %w", err) + } + return nil +} diff --git a/cmd/replication/replication_test.go b/cmd/replication/replication_test.go new file mode 100644 index 0000000..f6bbe1b --- /dev/null +++ b/cmd/replication/replication_test.go @@ -0,0 +1,183 @@ +package replication + +import ( + "bytes" + "context" + "encoding/json" + "strings" + "testing" + "testing/synctest" + "time" + + "github.com/spf13/pflag" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + checker "github.com/stackvista/stackstate-backup-cli/internal/orchestration/replication" +) + +func TestJSONOutputIsSeparateFromProgress(t *testing.T) { + var stdout, stderr bytes.Buffer + report, err := observe(context.Background(), func(context.Context) checker.Report { + return checker.Report{Status: checker.Healthy, Checks: []checker.Result{ + {Component: "kafka", Status: checker.Healthy, Messages: []string{"all assigned replicas in sync"}}, + }} + }, &flags{wait: true, interval: time.Millisecond}, &stderr, nil) + require.NoError(t, err) + require.NoError(t, writeReport(&stdout, "json", report)) + var decoded checker.Report + require.NoError(t, json.Unmarshal(stdout.Bytes(), &decoded)) + assert.Equal(t, checker.Healthy, decoded.Status) + assert.Contains(t, stderr.String(), "kafka healthy") +} + +func TestCheckValidatesBeforeConnecting(t *testing.T) { + command := Cmd() + command.SetArgs([]string{"check", "--namespace=test", "--components=kafka", "--output=invalid"}) + command.SetOut(&bytes.Buffer{}) + command.SetErr(&bytes.Buffer{}) + err := command.Execute() + require.ErrorContains(t, err, "output must be table or json") +} + +func TestHelpDoesNotRequireBackupConfiguration(t *testing.T) { + command := Cmd() + command.SetArgs([]string{"check", "--help"}) + var output bytes.Buffer + command.SetOut(&output) + require.NoError(t, command.Execute()) + assert.NotContains(t, output.String(), "--release") + assert.NotContains(t, output.String(), "--secret") + assert.NotContains(t, output.String(), "--configmap") + assert.Contains(t, output.String(), "hdfs,elasticsearch,kafka,clickhouse,zookeeper") +} + +func TestCheckExposesOnlyOperationalFlags(t *testing.T) { + check, _, err := Cmd().Find([]string{"check"}) + require.NoError(t, err) + var names []string + check.Flags().VisitAll(func(flag *pflag.Flag) { names = append(names, flag.Name) }) + assert.ElementsMatch(t, []string{"namespace", "kubeconfig", "components", "output", "wait", "timeout", "stable-for"}, names) +} + +func TestRemovedFlagsAreRejectedBeforeConnecting(t *testing.T) { + for _, name := range []string{ + "interval", "request-timeout", "hdfs-audit-timeout", "kafka-bootstrap-server", "kafka-client-properties", + "elasticsearch-scheme", "elasticsearch-ca", "elasticsearch-server-name", + } { + t.Run(name, func(t *testing.T) { + command := Cmd() + command.SetOut(&bytes.Buffer{}) + command.SetErr(&bytes.Buffer{}) + command.SetArgs([]string{"check", "--namespace=test", "--" + name + "=unused"}) + require.ErrorContains(t, command.Execute(), "unknown flag: --"+name) + }) + } +} + +func TestReportAndExitAgree(t *testing.T) { + tests := []struct { + name, status, expected string + checkErr error + }{ + {"healthy", checker.Healthy, checker.Healthy, nil}, + {"not applicable", checker.NotApplicable, checker.NotApplicable, nil}, + {"degraded", checker.Degraded, checker.Degraded, nil}, + {"unknown", checker.Unknown, checker.Unknown, nil}, + {"timeout after healthy sample", checker.Healthy, checker.Unknown, context.DeadlineExceeded}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + var output bytes.Buffer + err := finishReport(&output, "json", checker.Report{Status: test.status}, test.checkErr) + var report checker.Report + require.NoError(t, json.Unmarshal(output.Bytes(), &report)) + assert.Equal(t, test.expected, report.Status) + if test.expected == checker.Healthy || test.expected == checker.NotApplicable { + require.NoError(t, err) + } else { + require.Error(t, err) + } + if test.checkErr != nil { + require.ErrorIs(t, err, test.checkErr) + assert.NotEmpty(t, report.Error) + } + }) + } +} + +func TestWaitOutputShowsTimestampsAndStabilityProgress(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + var stderr bytes.Buffer + report, err := observe(context.Background(), func(context.Context) checker.Report { + return checker.Report{CheckedAt: time.Now().UTC(), Status: checker.Healthy, + Checks: []checker.Result{{Component: "kafka", Status: checker.Healthy, Messages: []string{"all replicas in sync"}}}} + }, &flags{wait: true, interval: 10 * time.Second, stableFor: 30 * time.Second, timeout: time.Minute}, &stderr, nil) + require.NoError(t, err) + assert.Equal(t, checker.Healthy, report.Status) + for _, line := range strings.Split(strings.TrimSpace(stderr.String()), "\n") { + assert.Regexp(t, `^\[\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z\] `, line) + } + assert.Contains(t, stderr.String(), "verifying stability: 0s/30s (30s remaining)") + assert.Contains(t, stderr.String(), "verifying stability: 10s/30s (20s remaining)") + assert.Contains(t, stderr.String(), "stability period satisfied (30s/30s)") + }) +} + +func TestCancelledTableLabelsLastCompletedObservation(t *testing.T) { + var stdout bytes.Buffer + report := checker.Report{CheckedAt: time.Date(2026, time.September, 11, 12, 0, 0, 0, time.UTC), Status: checker.Healthy, + Checks: []checker.Result{{Component: "kafka", Status: checker.Healthy, Messages: []string{"all replicas in sync"}}}} + err := finishReport(&stdout, "table", report, context.Canceled) + require.ErrorIs(t, err, context.Canceled) + assert.Contains(t, stdout.String(), "Replication result: unknown") + assert.Contains(t, stdout.String(), "context canceled") + assert.Contains(t, stdout.String(), "Last completed observation started: 2026-09-11T12:00:00Z") + assert.Contains(t, stdout.String(), "all replicas in sync") +} + +func TestCancellationBeforeFirstObservation(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + var stdout, stderr bytes.Buffer + report, err := observe(ctx, func(context.Context) checker.Report { + cancel() + return checker.Report{Namespace: "test", Checks: []checker.Result{{Messages: []string{"aborted request URL"}}}} + }, &flags{wait: true, interval: time.Second}, &stderr, nil) + require.ErrorIs(t, err, context.Canceled) + assert.Equal(t, "test", report.Namespace) + assert.Empty(t, report.Checks) + assert.NotContains(t, stderr.String(), "aborted") + require.ErrorIs(t, finishReport(&stdout, "table", report, err), context.Canceled) + assert.Contains(t, stdout.String(), "No completed observation.") +} + +func TestHealthyTopologyChangeExplainsStabilityReset(t *testing.T) { + message := stabilityMessage(checker.Report{Status: checker.Healthy}, checker.WaitProgress{Reset: true, Required: 30 * time.Second}) + assert.Contains(t, message, "topology, readiness or runtime changed") + assert.Contains(t, message, "stability period restarted (0s/30s)") +} + +func TestSingleObservationRequiresFinalVerification(t *testing.T) { + var progress bytes.Buffer + calls := 0 + report, err := observe(context.Background(), func(context.Context) checker.Report { + return checker.Report{Status: checker.Healthy} + }, &flags{}, &progress, func(_ context.Context, report checker.Report) checker.Report { + calls++ + report.Status = checker.Unknown + return report + }) + require.NoError(t, err) + assert.Equal(t, 1, calls) + assert.Equal(t, checker.Unknown, report.Status) + var output bytes.Buffer + require.Error(t, finishReport(&output, "json", report, nil)) + assert.Contains(t, progress.String(), "Running final replication verification") +} + +func TestAuditProgressDoesNotPrematurelyClaimSuccess(t *testing.T) { + assert.Contains(t, stabilityMessage(checker.Report{Status: checker.Healthy}, + checker.WaitProgress{Verifying: true}), "running final replication verification") + assert.Contains(t, stabilityMessage(checker.Report{Status: checker.Unknown}, + checker.WaitProgress{RetryAfter: 30 * time.Second}), "next observation in 30s") +} diff --git a/cmd/root.go b/cmd/root.go index f2d7686..4ae0b28 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -6,6 +6,7 @@ import ( "github.com/spf13/cobra" "github.com/stackvista/stackstate-backup-cli/cmd/clickhouse" "github.com/stackvista/stackstate-backup-cli/cmd/elasticsearch" + "github.com/stackvista/stackstate-backup-cli/cmd/replication" "github.com/stackvista/stackstate-backup-cli/cmd/settings" "github.com/stackvista/stackstate-backup-cli/cmd/stackgraph" "github.com/stackvista/stackstate-backup-cli/cmd/version" @@ -56,12 +57,13 @@ func init() { // Add commands that don't need backup config flags rootCmd.AddCommand(version.Cmd()) + rootCmd.AddCommand(replication.Cmd()) } var rootCmd = &cobra.Command{ Use: "sts-backup", - Short: "Backup and restore tool for SUSE Observability platform", - Long: `A CLI tool for managing backups and restores for SUSE Observability platform running on Kubernetes.`, + Short: "Backup, restore and replication checks for SUSE Observability", + Long: `A CLI tool for managing backups, restores and database replication checks for SUSE Observability running on Kubernetes.`, } func Execute() { diff --git a/docs/replication.md b/docs/replication.md new file mode 100644 index 0000000..3152836 --- /dev/null +++ b/docs/replication.md @@ -0,0 +1,286 @@ +# Check database replication + +`sts-backup replication check` inspects the chart-managed databases in a +namespace containing one SUSE Observability installation. It does not require +the backup ConfigMap, backup Secret, or enabled backups. + +```bash +sts-backup replication check --namespace observability +``` + +For testing from a source checkout: + +```bash +go run . replication check --namespace observability --wait +``` + +Replace `observability` with the installation namespace. The command uses the +current kubeconfig context; use `--kubeconfig ` to select another config. + +The command exposes these options: + +| Flag | Purpose | +|---|---| +| `--namespace`, `-n` | Installation namespace (required). | +| `--kubeconfig` | Alternative kubeconfig path. | +| `--components` | Comma-separated databases to check; defaults to all five. | +| `--output`, `-o` | `table` (default) or `json`. | +| `--wait` | Wait for sustained healthy observations. | +| `--timeout` | Overall deadline, including the final audit; defaults to `10m`. | +| `--stable-for` | Required healthy observation period with `--wait`; defaults to `30s`. | + +The command returns exit code **0** when every selected component is `healthy` +or `not_applicable`. Any degraded, missing, inaccessible, unsupported or +incompletely described component returns exit code **1**. A missing component +is never silently skipped. + +Checks follow the deployed topology: StatefulSet desired replica counts already +reflect the sizing profile and its overrides. A component configured with one +replica per database group is `not_applicable` after its Kubernetes availability +checks pass. HBase mono is also `not_applicable` for distributed HDFS checks. +This works with Helm or GitOps deployments without reading Helm release records, +Secrets or a profile name. It does not infer configuration from the number of +surviving pods. + +Use JSON for automation: + +```bash +sts-backup replication check \ + --namespace observability \ + --output json +``` + +The report identifies the namespace, observation time, selected +components, status (`healthy`, `degraded`, `unknown` or `not_applicable`) and +diagnostic messages. The status describes only the selected checks. If all +selected components are `not_applicable`, the overall status is also +`not_applicable`; exit code zero does not establish application redundancy. + +## Wait for recovery + +```bash +sts-backup replication check \ + --namespace observability \ + --wait --timeout 15m --stable-for 30s \ + --output json > replication.json +``` + +Wait mode requires consecutive successful observations spanning `--stable-for`, +with all applicable replication checks healthy and all selected workloads +available. Components marked `not_applicable` do not prevent success. +If every selected component is `not_applicable`, wait mode completes after the +first successful availability observation. +The default is 30 seconds, starting when the first fully healthy observation +completes. Earlier rounds with any `unknown` or `degraded` component do not +count. Observations normally pause for ten seconds between rounds, shortened +when less time remains in the healthy period. An unsuccessful observation resets +the period. +Changes to relevant database topology, readiness or container identity also +reset the period between observations, even if each observation is healthy. +Progress explains when such a change restarts the timer. + +Progress includes UTC timestamps and shows the elapsed and remaining healthy +period, resets, and successful completion. Seeing every component healthy +does not mean the stability period has already elapsed. Database queries take +time; the checker needs another completed healthy observation to confirm the +period, rather than exiting on a timer alone. +Once stable, an applicable HDFS check also needs its final block audit to pass. +Progress announces final verification before reporting completion. A failed +audit resets stability and delays the next observation by 30 seconds. + +To finish on the first fully healthy observation, explicitly use +`--wait --stable-for 0s`. + +Progress goes to stderr; stdout contains one final report. Table output includes +the report time and the start time of the last completed observation. JSON +retains its `checkedAt` timestamp. The overall deadline includes database +queries, the final audit and the checks after it. Ordinary API requests and +database probes are capped internally at 30 seconds, within the overall +deadline. The HDFS audit uses the remaining overall time without a separate cap. + +Ctrl+C stops further queries. Cancellation or timeout returns nonzero and sets +the overall result to `unknown`, retaining the last completed observation +instead of replacing it with errors from interrupted queries. If no observation +completed, the report says so. Healthy component results in a cancelled report +describe that previous observation; the wait did not finish successfully. + +Select an explicit subset if a database is intentionally disabled: + +```bash +sts-backup replication check \ + --namespace observability \ + --components hdfs,elasticsearch,kafka +``` + +This selection does not validate the omitted database. + +## What is checked + +The checker discovers StatefulSets using `app.kubernetes.io/name` +(`hbase`, `elasticsearch`, `kafka`, `clickhouse`, `zookeeper`) and +`app.kubernetes.io/component` within the namespace. The HDFS components +`hdfs-nn` and `hdfs-dn` distinguish the NameNode and DataNodes from the +SecondaryNameNode. It assumes one SUSE Observability installation per +namespace; no Helm release name is required. It verifies the desired pods +exist, belong to those StatefulSets, are Ready, and are not terminating or +undergoing a rollout. +It queries database state and compares relevant Kubernetes state before and +afterward. Changes to selected StatefulSet membership, desired replicas, +generation, rollout revisions or convergence invalidate the observation. +So do changes to their Pods' identity, ownership, node assignment, readiness +transitions or termination, and database container identity, start time or +restart count. + +Resource versions, unrelated annotations and readiness heartbeat timestamps do +not invalidate observations. Workloads outside the selected checks are excluded. +These comparisons detect changes visible in the sampled state; they are not a +continuous Kubernetes event watch. + +| Component | Replication evidence | +|---|---| +| HDFS | Configured default and minimum write replication are at least two; all expected DataNodes are live; the NameNode is out of safe mode; no missing, corrupt, under-replicated or pending-replication blocks are reported by JMX. Before overall success, a file/block audit verifies completed blocks against their replication targets. | +| Elasticsearch | Expected members are present; health is green; every returned index has at least one replica shard; no shards are unassigned, initializing or relocating. | +| Kafka | Every described partition has at least two distinct assigned replicas, complete ISR membership and an in-sync leader. Summaries and partition descriptions must agree. `__consumer_offsets` must exist. `__transaction_state` is validated when present; verified absence is reported without failing the check. | +| ClickHouse | Every discovered member is queried. Replicated table groups have their expected active replicas, live coordination sessions and no read-only members. Replication logs are caught up, and no replication queue tasks other than background `MERGE_PARTS` remain. Missing or duplicate table replicas and current query exceptions fail the check. | +| ZooKeeper | At least three voting members are available. Every member reports the expected voting membership; exactly one is leader and the rest are followers. The leader reports all expected followers synchronized and is checked again after sampling the ensemble. | + +Kafka creates `__transaction_state` lazily when transactions are used. If it +is not listed, a separate read-only topic configuration query must confirm +absence; permission failures or ambiguous results produce `unknown`, retaining +any partition or ISR problems already found. The checker never creates topics. +It does not validate the broker defaults that would govern +a future transaction topic. A present transaction topic still needs at least two +replicas and complete ISR. + +For replicated ClickHouse deployments, the check requires replicated-table +evidence and does not classify an empty result as healthy. Unreplicated tables +are outside its scope. When both log pointers are zero, the checker queries +`system.zookeeper` to distinguish a genuinely empty replication log from an +unprocessed first log entry. Query failures remain `unknown`; pending queue +tasks, inactive replicas and coordination errors still fail. + +ClickHouse can retain `last_queue_update_exception` after recovery. The checker +reports that history without failing otherwise healthy replication. Current +coordination-query errors, expired sessions and replication backlog still fail. + +ZooKeeper is checked by default. To inspect it alone: + +```bash +sts-backup replication check -n observability --components zookeeper --wait +``` + +ZooKeeper's `ruok` response does not prove that the ensemble has recovered. +The checker reads `mntr` from every member instead, and requires full voting +membership to recover rather than accepting a surviving majority. Wait mode +applies the same healthy observation period as for the other databases. + +## Final HDFS audit + +When all selected lightweight checks pass, the checker runs a read-only HDFS +audit before returning success. With `--wait`, this happens after the stability +period, not on every poll. HDFS marked `not_applicable` does not require an audit. + +The audit uses `hdfs fsck / -files -blocks -openforwrite -includeSnapshots`. +It examines NameNode file and block metadata, not the contents of HFiles or WALs. +Every audited file must have a replication target of at least two, and completed +blocks must have enough live replicas to meet that file's target. Completed +blocks in open files and snapshot references are included. Counts can include +the same underlying block referenced by multiple snapshots. + +For under-construction blocks, Hadoop reports expected pipeline membership. +The checker requires that count to meet the file's target and explicitly reports +that persistence of the latest writes is not verified. It does not stop writers, +roll WALs or require every WAL to close. A healthy audit is not proof that every +active pipeline has durably replicated its latest bytes. + +Use a larger overall `--timeout` for namespaces whose file/block audit needs +more time. The same deadline includes the earlier observations and the checks +after the audit: + +```bash +sts-backup replication check -n observability --wait --timeout 15m +``` + +Output is parsed as a stream, with bounded line size and diagnostic storage. +The parser requires matching file/block counts and a complete final summary; +`fsck`'s `HEALTHY` line alone is insufficient. Incomplete, timed-out or unsupported +reports return `unknown`. Erasure-coded files and symlink records are unsupported. +Parser errors include a line number and a bounded, escaped excerpt of the +rejected record. +After a successful audit, all selected lightweight checks run again and relevant +Kubernetes state must still match. The audit is a sampled scan, not an atomic +filesystem snapshot; its cost depends on file/block count and NameNode load. + +## Access and supported layouts + +The command uses the normal kubeconfig selection or a Kubernetes service +account when running in a Pod. It requires listing Pods and StatefulSets and +creating `pods/exec` requests in the target namespace. + +**`pods/exec` permission allows arbitrary commands in pods.** The checker +itself executes only fixed read-only queries. Use a dedicated identity and +grant this permission only in the installation's namespace. It does not scale +workloads, change PDBs, annotate resources, repair databases or invoke restore +operations. No backup credentials are loaded. + +Queries use the database tools already present in the product containers: +`hdfs` and `curl` in the NameNode, `curl` in Elasticsearch, +`kafka-topics.sh` and `kafka-configs.sh` in Kafka, `clickhouse-client` in ClickHouse, and Bash TCP +access to ZooKeeper. +Custom images, container names, external databases, alternative NameNode +topologies and overridden database name/component labels are not supported by +these initial adapters. Failed discovery or unsupported response formats +produce `unknown`, not success. + +The initial HTTP adapters use the chart's NameNode and Elasticsearch ports. +The ZooKeeper adapter requires the chart's plaintext loopback client port +and `mntr` in its four-letter-command whitelist. It supports the chart's +single ensemble of voting participants; external ensembles, observer layouts, +weighted quorums and TLS-only client listeners are outside its scope. Missing +membership or synchronization metrics produce `unknown`. No configuration +changes, HTTP AdminServer or database writes are needed. + +Kafka uses the chart's plaintext listener at `localhost:9092` inside the broker +pod. Elasticsearch uses `http://127.0.0.1:9200` inside its pod and the pod's +`ELASTIC_PASSWORD` when present. These connections require no CLI configuration. +Kafka SASL/TLS and Elasticsearch HTTPS customizations are unsupported. +Failed connections or permission failures return `unknown`. + +ClickHouse uses the pod's `CLICKHOUSE_ADMIN_USER`, +`CLICKHOUSE_ADMIN_PASSWORD` and `CLICKHOUSE_TCP_PORT`. Credentials stay inside +the pod. Query stderr is suppressed because database tools can echo credentials; +when a query fails, inspect the component's configuration through your normal +administrative procedure. Ordinary query output is size-limited and excess output +is treated as unverified; the final HDFS report uses the streaming parser described +above. + +## Maintenance boundary + +This is a sampled replication report, **not a “safe to remove this node” +certificate or a maintenance lock**. It does not prevent another process from +starting maintenance, provide an atomic database snapshot, or prove continued +health between observations. + +The current StatefulSet specification is the configuration baseline. The checker +cannot distinguish an intentional replica-count override from an accidental or +temporary scale-down in that specification, recover an original sizing profile, +or detect an entirely deleted shard from surviving workloads alone. Keep the +desired topology intact during node maintenance. A missing pod or a workload +still configured for several replicas cannot become `not_applicable` just +because fewer replicas are Ready. + +`not_applicable` verifies Kubernetes availability only. It does not establish +database health or storage-level redundancy for a single-replica component. + +This first version does not validate Longhorn volume health or placement, +spare capacity, HBase region assignment and WAL recovery, quorum survival +after a specific node is removed, VictoriaMetrics redundancy, backup freshness, +or the consequences of removing a particular node. The HDFS audit verifies +completed blocks but does not prove persistence of the latest active WAL writes. +It is not a complete implementation +of the product's node-maintenance checklist. + +Continue to serialize maintenance, preserve storage redundancy, follow the +documented recovery procedure and check these additional requirements. +Run this checker after the affected node or replacement can schedule workloads. +Only proceed when the report and the remaining maintenance checks pass. diff --git a/internal/app/replication.go b/internal/app/replication.go new file mode 100644 index 0000000..0cf8937 --- /dev/null +++ b/internal/app/replication.go @@ -0,0 +1,17 @@ +package app + +import ( + "fmt" + + "github.com/stackvista/stackstate-backup-cli/internal/clients/k8s" + "github.com/stackvista/stackstate-backup-cli/internal/orchestration/replication" +) + +// NewReplicationChecker requires Kubernetes access, but no backup ConfigMap or Secret. +func NewReplicationChecker(kubeconfig string, options replication.Options) (*replication.Checker, error) { + client, err := k8s.NewClient(kubeconfig, false) + if err != nil { + return nil, fmt.Errorf("create Kubernetes client: %w", err) + } + return replication.New(client, options) +} diff --git a/internal/clients/k8s/exec.go b/internal/clients/k8s/exec.go new file mode 100644 index 0000000..24fea7b --- /dev/null +++ b/internal/clients/k8s/exec.go @@ -0,0 +1,61 @@ +package k8s + +import ( + "bytes" + "context" + "fmt" + "io" + "net/http" + + corev1 "k8s.io/api/core/v1" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/tools/remotecommand" +) + +const maxExecOutput = 8 << 20 + +type boundedBuffer struct { + buffer bytes.Buffer + err error +} + +func (b *boundedBuffer) Write(p []byte) (int, error) { + if len(p) > maxExecOutput-b.buffer.Len() { + b.err = fmt.Errorf("query output exceeds %d bytes", maxExecOutput) + return 0, b.err + } + return b.buffer.Write(p) +} + +// Exec runs a command without a TTY, bounded by the caller's context and output limit. +func (c *Client) Exec(ctx context.Context, namespace, pod, container string, command []string) ([]byte, error) { + var output boundedBuffer + if err := c.ExecTo(ctx, namespace, pod, container, command, &output); err != nil { + return nil, err + } + if output.err != nil { + return nil, output.err + } + return output.buffer.Bytes(), nil +} + +// ExecTo streams stdout to a caller-supplied writer without retaining the report. +func (c *Client) ExecTo(ctx context.Context, namespace, pod, container string, command []string, output io.Writer) error { + request := c.clientset.CoreV1().RESTClient().Post(). + Namespace(namespace).Resource("pods").Name(pod).SubResource("exec"). + VersionedParams(&corev1.PodExecOptions{ + Container: container, + Command: command, + Stdout: true, + Stderr: true, + }, scheme.ParameterCodec) + executor, err := remotecommand.NewSPDYExecutor(c.restConfig, http.MethodPost, request.URL()) + if err != nil { + return fmt.Errorf("create pod executor: %w", err) + } + // Database tools can echo authentication details in stderr. + if err := executor.StreamWithContext(ctx, remotecommand.StreamOptions{Stdout: output, Stderr: io.Discard}); err != nil { + return fmt.Errorf("query pod %s/%s: %w", namespace, pod, err) + } + return nil +} diff --git a/internal/clients/k8s/exec_test.go b/internal/clients/k8s/exec_test.go new file mode 100644 index 0000000..8f1cad2 --- /dev/null +++ b/internal/clients/k8s/exec_test.go @@ -0,0 +1,20 @@ +package k8s + +import ( + "io" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestExecOutputLimit(t *testing.T) { + var output boundedBuffer + n, err := output.Write([]byte("report")) + require.NoError(t, err) + assert.Equal(t, 6, n) + _, err = io.Copy(&output, io.LimitReader(strings.NewReader(strings.Repeat("x", maxExecOutput)), int64(maxExecOutput))) + require.ErrorContains(t, err, "query output exceeds") + assert.LessOrEqual(t, output.buffer.Len(), maxExecOutput) +} diff --git a/internal/orchestration/replication/applicability.go b/internal/orchestration/replication/applicability.go new file mode 100644 index 0000000..8edc4bd --- /dev/null +++ b/internal/orchestration/replication/applicability.go @@ -0,0 +1,119 @@ +package replication + +import ( + "fmt" + + appsv1 "k8s.io/api/apps/v1" +) + +const ( + hdfsComponent = "hdfs" + clickhouseComponent = "clickhouse" + nameNodeComponent = "hdfs-nn" + dataNodeComponent = "hdfs-dn" + monoComponent = "stackgraph" +) + +func workloadComponent(workload appsv1.StatefulSet) string { + name := workload.Labels["app.kubernetes.io/name"] + component := workload.Labels["app.kubernetes.io/component"] + if name == "hbase" { + switch component { + case nameNodeComponent, dataNodeComponent, monoComponent: + return hdfsComponent + default: + return "" + } + } + switch name { + case "elasticsearch", clickhouseComponent, "kafka", "zookeeper": + return name + default: + return "" + } +} + +func databaseContainer(workload appsv1.StatefulSet) string { + if workloadComponent(workload) == hdfsComponent { + switch workload.Labels["app.kubernetes.io/component"] { + case nameNodeComponent: + return "namenode" + case dataNodeComponent: + return "datanode" + case monoComponent: + return monoComponent + } + } + return workloadComponent(workload) +} + +// Use desired replicas, never the number of surviving or Ready pods. +func (i inventory) applicability(component string) *Result { + expected := make(map[string]appsv1.StatefulSet) + for _, workload := range i.workloads { + if workloadComponent(workload) == component { + if _, duplicate := expected[workload.Name]; duplicate { + return resultPointer(component, Unknown, "duplicate database workload") + } + expected[workload.Name] = workload + } + } + if len(expected) == 0 { + return resultPointer(component, Unknown, "no supported database StatefulSet found; select components explicitly if this database is intentionally disabled") + } + for _, workload := range expected { + container := databaseContainer(workload) + if !hasContainer(workload.Spec.Template.Spec.Containers, container) { + return resultPointer(component, Unknown, fmt.Sprintf("%s has an unsupported container layout", workload.Name)) + } + if _, err := i.workloadPods(workload); err != nil { + return resultPointer(component, Unknown, err.Error()) + } + } + return replicationApplicability(component, expected) +} + +func replicationApplicability(component string, expected map[string]appsv1.StatefulSet) *Result { + total, single, shards := 0, 0, 0 + namenodes := 0 + for _, workload := range expected { + if component == hdfsComponent { + switch workload.Labels["app.kubernetes.io/component"] { + case monoComponent: + if len(expected) != 1 || *workload.Spec.Replicas != 1 { + return resultPointer(component, Unknown, "unsupported HBase mono layout") + } + return resultPointer(component, NotApplicable, "HBase mono layout does not use distributed HDFS; availability checked; storage redundancy not checked") + case dataNodeComponent: + case nameNodeComponent: + if *workload.Spec.Replicas != 1 { + return resultPointer(component, Unknown, "multiple NameNode replicas are not supported") + } + namenodes++ + continue + default: + continue + } + } + total += int(*workload.Spec.Replicas) + shards++ + if *workload.Spec.Replicas == 1 { + single++ + } + } + if component == hdfsComponent && (namenodes != 1 || shards == 0) { + return resultPointer(component, Unknown, "distributed HDFS layout requires a NameNode and DataNodes") + } + if total == 1 || component == clickhouseComponent && shards == single { + return resultPointer(component, NotApplicable, "StatefulSet configuration has one replica per database group; availability checked; application replication not applicable; storage redundancy not checked") + } + if component == clickhouseComponent && single > 0 { + return resultPointer(component, Unknown, "mixed single-replica and replicated ClickHouse shards are not supported") + } + return nil +} + +func resultPointer(component, status, message string) *Result { + value := result(component, status, message) + return &value +} diff --git a/internal/orchestration/replication/applicability_test.go b/internal/orchestration/replication/applicability_test.go new file mode 100644 index 0000000..ab56bc9 --- /dev/null +++ b/internal/orchestration/replication/applicability_test.go @@ -0,0 +1,108 @@ +package replication + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes/fake" + "k8s.io/utils/ptr" +) + +func TestSingleReplicaConfiguration(t *testing.T) { + tests := []struct { + component, application, role, container string + }{ + {"clickhouse", "clickhouse", "clickhouse", "clickhouse"}, + {"kafka", "kafka", "kafka", "kafka"}, + {"elasticsearch", "elasticsearch", "master", "elasticsearch"}, + {"zookeeper", "zookeeper", "zookeeper", "zookeeper"}, + {"hdfs", "hbase", "stackgraph", "stackgraph"}, + } + for _, test := range tests { + t.Run(test.component, func(t *testing.T) { + objects := databaseObjects(test.application, test.role, test.container, 1) + client := fake.NewSimpleClientset(objects...) + kube := &fakeKubernetes{client: client} + options := testOptions() + options.Components = []string{test.component} + probe, err := New(kube, options) + require.NoError(t, err) + report := probe.Check(context.Background()) + assert.Equal(t, NotApplicable, report.Status, report) + assert.Zero(t, kube.calls) + for _, action := range client.Actions() { + assert.Equal(t, "list", action.GetVerb()) + assert.Contains(t, []string{"pods", "statefulsets"}, action.GetResource().Resource) + } + }) + } +} + +func TestReplicationUsesConfiguredTopology(t *testing.T) { + kube := &fakeKubernetes{ + client: fake.NewSimpleClientset(databaseObjects("clickhouse", "clickhouse", "clickhouse", 3)...), + exec: func(_ context.Context, _, pod, _ string, _ []string) ([]byte, error) { + row := clickhouseFixture() + row["replica_name"], row["total_replicas"], row["active_replicas"] = pod, "3", "3" + return encode(t, map[string]any{"data": []any{row}}), nil + }, + } + options := testOptions() + options.Components = []string{"clickhouse"} + probe, err := New(kube, options) + require.NoError(t, err) + report := probe.Check(context.Background()) + assert.Equal(t, Healthy, report.Status, report) + assert.Equal(t, 3, kube.calls) +} + +func TestMissingEvidenceCannotBecomeNotApplicable(t *testing.T) { + tests := []struct { + name string + change func([]runtime.Object) []runtime.Object + }{ + {"missing workload", func(_ []runtime.Object) []runtime.Object { return nil }}, + {"missing pod", func(objects []runtime.Object) []runtime.Object { return objects[:1] }}, + {"not ready", func(objects []runtime.Object) []runtime.Object { + objects[1].(*corev1.Pod).Status.Conditions[0].Status = corev1.ConditionFalse + return objects + }}, + {"one survivor of three", func(objects []runtime.Object) []runtime.Object { + objects[0].(*appsv1.StatefulSet).Spec.Replicas = ptr.To(int32(3)) + return objects + }}, + {"scaled to zero", func(objects []runtime.Object) []runtime.Object { + objects[0].(*appsv1.StatefulSet).Spec.Replicas = ptr.To(int32(0)) + return objects + }}, + {"unsupported container", func(objects []runtime.Object) []runtime.Object { + objects[0].(*appsv1.StatefulSet).Spec.Template.Spec.Containers[0].Name = "custom" + return objects + }}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + objects := test.change(databaseObjects("clickhouse", "clickhouse", "clickhouse", 1)) + kube := &fakeKubernetes{client: fake.NewSimpleClientset(objects...)} + options := testOptions() + options.Components = []string{"clickhouse"} + probe, err := New(kube, options) + require.NoError(t, err) + assert.Equal(t, Unknown, probe.Check(context.Background()).Status) + assert.Zero(t, kube.calls) + }) + } +} + +func TestNotApplicableAggregation(t *testing.T) { + assert.Equal(t, NotApplicable, reportStatus([]Result{{Status: NotApplicable}})) + assert.Equal(t, Healthy, reportStatus([]Result{{Status: Healthy}, {Status: NotApplicable}})) + assert.Equal(t, Degraded, reportStatus([]Result{{Status: Degraded}, {Status: NotApplicable}})) + assert.Equal(t, Unknown, reportStatus([]Result{{Status: Unknown}, {Status: NotApplicable}})) + assert.Equal(t, Unknown, reportStatus(nil)) +} diff --git a/internal/orchestration/replication/checker.go b/internal/orchestration/replication/checker.go new file mode 100644 index 0000000..538039d --- /dev/null +++ b/internal/orchestration/replication/checker.go @@ -0,0 +1,106 @@ +package replication + +import ( + "context" + "fmt" + "time" +) + +const requestTimeout = 30 * time.Second + +// Checker observes databases without invoking backup or restore operations. +type Checker struct { + kube Kubernetes + options Options +} + +// New creates a checker for the installation in one namespace. +func New(kube Kubernetes, options Options) (*Checker, error) { + if err := options.Validate(); err != nil { + return nil, err + } + return &Checker{kube: kube, options: options}, nil +} + +// Check queries every selected component and rechecks Kubernetes membership afterward. +func (c *Checker) Check(ctx context.Context) Report { + report := Report{ + CheckedAt: time.Now().UTC(), Namespace: c.options.Namespace, + Checks: make([]Result, 0, len(c.options.Components)), + } + before, err := c.discover(ctx) + if err == nil { + report.topology = before.fingerprint(c.options.Components) + } + for _, component := range c.options.Components { + if ctx.Err() != nil { + report.Status = Unknown + return report + } + if err != nil { + report.Checks = append(report.Checks, result(component, Unknown, err.Error())) + continue + } + if applicability := before.applicability(component); applicability != nil { + report.Checks = append(report.Checks, *applicability) + } else { + report.Checks = append(report.Checks, c.checkComponent(ctx, before, component)) + } + } + if ctx.Err() != nil { + report.Status = Unknown + return report + } + if err == nil { + after, afterErr := c.discover(ctx) + if afterErr != nil || report.topology != after.fingerprint(c.options.Components) { + message := "database topology, readiness or runtime changed during the checks; repeat the observation" + if afterErr != nil { + message = afterErr.Error() + } + for n := range report.Checks { + report.Checks[n].Status = Unknown + report.Checks[n].Messages = append(report.Checks[n].Messages, message) + } + } + } + report.Status = reportStatus(report.Checks) + return report +} + +func (c *Checker) query(ctx context.Context, pod, container string, command []string) ([]byte, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + ctx, cancel := context.WithTimeout(ctx, requestTimeout) + defer cancel() + data, err := c.kube.Exec(ctx, c.options.Namespace, pod, container, command) + if ctx.Err() != nil { + return nil, ctx.Err() + } + return data, err +} + +func (c *Checker) checkComponent(ctx context.Context, inventory inventory, component string) Result { + switch component { + case "hdfs": + return c.checkHDFS(ctx, inventory) + case "elasticsearch": + return c.checkElasticsearch(ctx, inventory) + case "kafka": + return c.checkKafka(ctx, inventory) + case "clickhouse": + return c.checkClickHouse(ctx, inventory) + case "zookeeper": + return c.checkZooKeeper(ctx, inventory) + default: + return result(component, Unknown, "unsupported component") + } +} + +func expectedMembers(members []member) error { + if len(members) < minReplicas { + return fmt.Errorf("at least %d application replicas are required; discovered %d", minReplicas, len(members)) + } + return nil +} diff --git a/internal/orchestration/replication/checker_test.go b/internal/orchestration/replication/checker_test.go new file mode 100644 index 0000000..4d58b97 --- /dev/null +++ b/internal/orchestration/replication/checker_test.go @@ -0,0 +1,265 @@ +package replication + +import ( + "context" + "fmt" + "io" + "testing" + "testing/synctest" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/kubernetes/fake" + "k8s.io/utils/ptr" +) + +type fakeKubernetes struct { + client kubernetes.Interface + exec func(context.Context, string, string, string, []string) ([]byte, error) + stream func(context.Context, string, string, string, []string, io.Writer) error + calls int +} + +func (f *fakeKubernetes) ExecTo(ctx context.Context, namespace, pod, container string, command []string, output io.Writer) error { + if f.stream != nil { + return f.stream(ctx, namespace, pod, container, command, output) + } + data, err := f.Exec(ctx, namespace, pod, container, command) + if err != nil { + return err + } + _, err = output.Write(data) + return err +} + +func (f *fakeKubernetes) Clientset() kubernetes.Interface { return f.client } + +func (f *fakeKubernetes) Exec(ctx context.Context, namespace, pod, container string, command []string) ([]byte, error) { + f.calls++ + return f.exec(ctx, namespace, pod, container, command) +} + +func testOptions() Options { + return Options{Namespace: "test", Components: []string{"kafka"}} +} + +func kafkaObjects() []runtime.Object { + return databaseObjects("kafka", "kafka", "kafka", 2) +} + +func databaseObjects(application, component, container string, replicas int32) []runtime.Object { + labels := map[string]string{ + "app.kubernetes.io/name": application, "app.kubernetes.io/component": component, "app.kubernetes.io/instance": "arbitrary-release", + } + workload := &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{Name: component, Namespace: "test", UID: types.UID("sts-" + component), Generation: 1, ResourceVersion: "1", Labels: labels}, + Spec: appsv1.StatefulSetSpec{ + Replicas: ptr.To(replicas), + Template: corev1.PodTemplateSpec{Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: container}}}}, + }, + Status: appsv1.StatefulSetStatus{ReadyReplicas: replicas, ObservedGeneration: 1, CurrentRevision: "one", UpdateRevision: "one"}, + } + objects := []runtime.Object{workload} + for n := int32(0); n < replicas; n++ { + name := fmt.Sprintf("%s-%d", component, n) + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, Namespace: "test", UID: types.UID(name), ResourceVersion: "1", Labels: labels, + OwnerReferences: []metav1.OwnerReference{{Kind: "StatefulSet", Name: component, UID: workload.UID, Controller: ptr.To(true)}}, + }, + Spec: corev1.PodSpec{NodeName: fmt.Sprintf("node-%d", n), Containers: []corev1.Container{{Name: container}}}, + Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{ + {Type: corev1.PodReady, Status: corev1.ConditionTrue}, + }}, + } + objects = append(objects, pod) + } + return objects +} + +func TestCheckerQueriesReadyPodsAndDoesNotMutateKubernetes(t *testing.T) { + client := fake.NewSimpleClientset(kafkaObjects()...) + kube := &fakeKubernetes{client: client, exec: func(ctx context.Context, namespace, pod, container string, command []string) ([]byte, error) { + assert.Equal(t, "test", namespace) + assert.Equal(t, "kafka-0", pod) + assert.Equal(t, "kafka", container) + assert.Equal(t, []string{"bash", "-ec", kafkaQuery, "replication-check", "--bootstrap-server", "localhost:9092", "--describe"}, command) + _, deadline := ctx.Deadline() + assert.True(t, deadline) + return []byte(kafkaFixture()), nil + }} + probe, err := New(kube, testOptions()) + require.NoError(t, err) + report := probe.Check(context.Background()) + require.Equal(t, Healthy, report.Status, report) + assert.Equal(t, 1, kube.calls) + for _, action := range client.Actions() { + assert.Equal(t, "list", action.GetVerb()) + } +} + +func TestCheckerRejectsIncompleteTopology(t *testing.T) { + tests := []struct { + name string + mutate func([]runtime.Object) []runtime.Object + }{ + {"missing pod", func(objects []runtime.Object) []runtime.Object { return objects[:2] }}, + {"not ready", func(objects []runtime.Object) []runtime.Object { + objects[1].(*corev1.Pod).Status.Conditions[0].Status = corev1.ConditionFalse + return objects + }}, + {"terminating", func(objects []runtime.Object) []runtime.Object { + objects[1].(*corev1.Pod).DeletionTimestamp = ptr.To(metav1.Now()) + return objects + }}, + {"wrong owner", func(objects []runtime.Object) []runtime.Object { + objects[1].(*corev1.Pod).OwnerReferences[0].UID = "other" + return objects + }}, + {"wrong database label", func(objects []runtime.Object) []runtime.Object { + objects[0].(*appsv1.StatefulSet).Labels = map[string]string{"app.kubernetes.io/name": "other", "app.kubernetes.io/component": "kafka"} + return objects + }}, + {"wrong component label", func(objects []runtime.Object) []runtime.Object { + objects[0].(*appsv1.StatefulSet).Labels["app.kubernetes.io/component"] = "unrelated" + return objects + }}, + {"different namespace", func(objects []runtime.Object) []runtime.Object { + for _, object := range objects { + object.(metav1.Object).SetNamespace("other") + } + return objects + }}, + {"rolling update", func(objects []runtime.Object) []runtime.Object { + objects[0].(*appsv1.StatefulSet).Status.UpdateRevision = "two" + return objects + }}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + kube := &fakeKubernetes{client: fake.NewSimpleClientset(test.mutate(kafkaObjects())...)} + probe, err := New(kube, testOptions()) + require.NoError(t, err) + assert.Equal(t, Unknown, probe.Check(context.Background()).Status) + assert.Zero(t, kube.calls) + }) + } +} + +func TestHDFSDiscoveryDistinguishesSecondaryNameNode(t *testing.T) { + objects := databaseObjects("hbase", "hdfs-nn", "namenode", 1) + objects = append(objects, databaseObjects("hbase", "hdfs-snn", "namenode", 1)...) + objects = append(objects, databaseObjects("hbase", "hdfs-dn", "datanode", 3)...) + kube := &fakeKubernetes{client: fake.NewSimpleClientset(objects...), exec: func(_ context.Context, namespace, pod, container string, _ []string) ([]byte, error) { + assert.Equal(t, "test", namespace) + assert.Equal(t, "hdfs-nn-0", pod) + assert.Equal(t, "namenode", container) + return hdfsFixture(t, func(map[string]any) {}), nil + }} + options := testOptions() + options.Components = []string{"hdfs"} + probe, err := New(kube, options) + require.NoError(t, err) + report := probe.Check(context.Background()) + assert.Equal(t, Healthy, report.Status, report) + assert.Equal(t, 1, kube.calls) +} + +func TestDiscoveryDoesNotRequireHelmReleaseLabel(t *testing.T) { + objects := kafkaObjects() + for _, object := range objects { + delete(object.(metav1.Object).GetLabels(), "app.kubernetes.io/instance") + } + kube := &fakeKubernetes{client: fake.NewSimpleClientset(objects...), exec: func(context.Context, string, string, string, []string) ([]byte, error) { + return []byte(kafkaFixture()), nil + }} + probe, err := New(kube, testOptions()) + require.NoError(t, err) + assert.Equal(t, Healthy, probe.Check(context.Background()).Status) +} + +func TestCheckerRejectsMembershipChangeDuringQueries(t *testing.T) { + client := fake.NewSimpleClientset(kafkaObjects()...) + kube := &fakeKubernetes{client: client, exec: func(ctx context.Context, namespace, pod, _ string, _ []string) ([]byte, error) { + current, err := client.CoreV1().Pods(namespace).Get(ctx, pod, metav1.GetOptions{}) + require.NoError(t, err) + current.UID = "replacement-pod" + _, err = client.CoreV1().Pods(namespace).Update(ctx, current, metav1.UpdateOptions{}) + require.NoError(t, err) + return []byte(kafkaFixture()), nil + }} + probe, err := New(kube, testOptions()) + require.NoError(t, err) + report := probe.Check(context.Background()) + assert.Equal(t, Unknown, report.Status) + assert.Contains(t, report.Checks[0].Messages, "database topology, readiness or runtime changed during the checks; repeat the observation") +} + +func TestQueryFailureCannotPass(t *testing.T) { + kube := &fakeKubernetes{client: fake.NewSimpleClientset(kafkaObjects()...), exec: func(context.Context, string, string, string, []string) ([]byte, error) { + return nil, fmt.Errorf("query not authorized") + }} + probe, err := New(kube, testOptions()) + require.NoError(t, err) + assert.Equal(t, Unknown, probe.Check(context.Background()).Status) +} + +func TestOrdinaryProbeRespectsInternalAndOverallDeadlines(t *testing.T) { + for _, overall := range []time.Duration{5 * time.Second, 2 * time.Minute} { + t.Run(overall.String(), func(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), overall) + defer cancel() + kube := &fakeKubernetes{exec: func(ctx context.Context, _, _, _ string, _ []string) ([]byte, error) { + <-ctx.Done() + return nil, ctx.Err() + }} + probe, err := New(kube, testOptions()) + require.NoError(t, err) + start := time.Now() + _, err = probe.query(ctx, "kafka-0", "kafka", nil) + require.ErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, min(overall, 30*time.Second), time.Since(start)) + if overall > 30*time.Second { + require.NoError(t, ctx.Err(), "a stalled probe must leave time for further observations") + } + }) + }) + } +} + +func TestInvalidScopeRejected(t *testing.T) { + for _, components := range [][]string{nil, {"kafka", "kafka"}, {"not-a-store"}} { + options := testOptions() + options.Components = components + _, err := New(nil, options) + require.Error(t, err) + } +} + +func TestCancellationStopsRemainingQueriesAndDiscovery(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + client := fake.NewSimpleClientset(kafkaObjects()...) + kube := &fakeKubernetes{client: client, exec: func(context.Context, string, string, string, []string) ([]byte, error) { + cancel() + return nil, fmt.Errorf("aborted request with query URL") + }} + options := testOptions() + options.Components = []string{"kafka", "zookeeper", "clickhouse"} + probe, err := New(kube, options) + require.NoError(t, err) + report := probe.Check(ctx) + assert.Equal(t, Unknown, report.Status) + require.Len(t, report.Checks, 1) + assert.Equal(t, []string{"context canceled"}, report.Checks[0].Messages) + assert.Equal(t, 1, kube.calls) + assert.Len(t, client.Actions(), 2, "no second discovery after cancellation") +} diff --git a/internal/orchestration/replication/clickhouse.go b/internal/orchestration/replication/clickhouse.go new file mode 100644 index 0000000..f520882 --- /dev/null +++ b/internal/orchestration/replication/clickhouse.go @@ -0,0 +1,188 @@ +package replication + +import ( + "context" + "encoding/json" + "fmt" + "sort" +) + +const clickhouseSQL = `SELECT database, table, zookeeper_path, replica_name, +is_readonly, is_session_expired, total_replicas, active_replicas, +log_max_index, log_pointer, absolute_delay, last_queue_update_exception, zookeeper_exception, +ifNull(pending_data_tasks, 0) AS pending_data_tasks +FROM system.replicas +LEFT JOIN ( + SELECT database, table, countIf(type != 'MERGE_PARTS') AS pending_data_tasks + FROM system.replication_queue GROUP BY database, table +) AS queues USING (database, table) +WHERE database NOT IN ('system', 'INFORMATION_SCHEMA', 'information_schema') +FORMAT JSON` + +const clickhouseQuery = `export CLICKHOUSE_PASSWORD="${CLICKHOUSE_ADMIN_PASSWORD:?missing ClickHouse credentials}" +exec clickhouse-client --host 127.0.0.1 --port "${CLICKHOUSE_TCP_PORT:-9000}" \ + --user "${CLICKHOUSE_ADMIN_USER:?missing ClickHouse user}" --readonly 1 --query "$1" "${@:2}"` + +const clickhouseEmptyLogSQL = `SELECT count() AS entries FROM system.zookeeper WHERE path = {log_path:String} FORMAT JSON` + +type replicaObservation struct { + path string + name string + pod string + total int64 + historicalQueueErr bool + needsLogVerification bool + problems []string +} + +func (c *Checker) checkClickHouse(ctx context.Context, inventory inventory) Result { + members, err := inventory.members("clickhouse", "clickhouse", "clickhouse") + if err != nil { + return result("clickhouse", Unknown, err.Error()) + } + var observations []replicaObservation + for _, member := range members { + if member.replicas < minReplicas { + return result("clickhouse", Degraded, fmt.Sprintf("%s belongs to a shard with fewer than two replicas", member.pod.Name)) + } + command := []string{"bash", "-ec", clickhouseQuery, "replication-check", clickhouseSQL} + data, err := c.query(ctx, member.pod.Name, "clickhouse", command) + if err != nil { + return result("clickhouse", Unknown, err.Error()) + } + rows, err := parseClickHouse(data, member.pod.Name, member.replicas) + if err != nil { + return result("clickhouse", Unknown, fmt.Sprintf("%s: %v", member.pod.Name, err)) + } + if err := c.verifyEmptyLogs(ctx, member.pod.Name, rows); err != nil { + return result("clickhouse", Unknown, err.Error()) + } + observations = append(observations, rows...) + } + return evaluateClickHouse(observations) +} + +func (c *Checker) verifyEmptyLogs(ctx context.Context, pod string, rows []replicaObservation) error { + for n := range rows { + if !rows[n].needsLogVerification { + continue + } + command := []string{"bash", "-ec", clickhouseQuery, "replication-check", clickhouseEmptyLogSQL, "--param_log_path=" + rows[n].path + "/log"} + data, err := c.query(ctx, pod, "clickhouse", command) + if err != nil { + return fmt.Errorf("%s: could not verify empty replication log: %w", pod, err) + } + var response struct { + Data []map[string]json.RawMessage `json:"data"` + } + if err := json.Unmarshal(data, &response); err != nil || len(response.Data) != 1 { + return fmt.Errorf("%s: invalid empty replication log evidence", pod) + } + entries, err := number(response.Data[0], "entries") + if err != nil { + return fmt.Errorf("%s: invalid empty replication log evidence", pod) + } + rows[n].needsLogVerification = false + if entries > 0 { + rows[n].problems = append(rows[n].problems, "replication log contains entries not yet pulled into the local queue") + } + } + return nil +} + +func parseClickHouse(data []byte, pod string, expected int) ([]replicaObservation, error) { + var response struct { + Data []map[string]json.RawMessage `json:"data"` + } + if err := json.Unmarshal(data, &response); err != nil || len(response.Data) == 0 { + return nil, fmt.Errorf("no replicated-table evidence returned") + } + var observations []replicaObservation + for _, row := range response.Data { + observation, err := parseReplica(row, pod, expected) + if err != nil { + return nil, err + } + observations = append(observations, observation) + } + return observations, nil +} + +func parseReplica(row map[string]json.RawMessage, pod string, expected int) (replicaObservation, error) { + fields := make(map[string]string) + for _, key := range []string{"database", "table", "zookeeper_path", "replica_name", "last_queue_update_exception", "zookeeper_exception"} { + value, err := text(row, key) + if err != nil { + return replicaObservation{}, err + } + fields[key] = value + } + counts, err := numbers(row, "is_readonly", "is_session_expired", "total_replicas", "active_replicas", + "log_max_index", "log_pointer", "pending_data_tasks", "absolute_delay") + if err != nil { + return replicaObservation{}, err + } + if fields["zookeeper_path"] == "" || fields["replica_name"] == "" || fields["database"] == "" || fields["table"] == "" { + return replicaObservation{}, fmt.Errorf("missing replicated-table identity") + } + observation := replicaObservation{path: fields["zookeeper_path"], name: fields["replica_name"], pod: pod, total: counts["total_replicas"]} + // ClickHouse retains this exception even after subsequent queue updates succeed. + observation.historicalQueueErr = fields["last_queue_update_exception"] != "" + if observation.total < minReplicas || observation.total != int64(expected) || counts["active_replicas"] != observation.total { + observation.problems = append(observation.problems, fmt.Sprintf("%d/%d replicas active; %d expected", counts["active_replicas"], observation.total, expected)) + } + if counts["is_readonly"] != 0 || counts["is_session_expired"] != 0 { + observation.problems = append(observation.problems, "replica is read-only or coordination session expired") + } + observation.needsLogVerification = counts["log_pointer"] == 0 && counts["log_max_index"] == 0 + if counts["log_max_index"] > 0 && counts["log_pointer"] <= counts["log_max_index"] || counts["pending_data_tasks"] != 0 { + observation.problems = append(observation.problems, fmt.Sprintf("replication backlog: %d data tasks; reported delay %ds", counts["pending_data_tasks"], counts["absolute_delay"])) + } + if fields["zookeeper_exception"] != "" { + observation.problems = append(observation.problems, "coordination query reported an exception") + } + return observation, nil +} + +func evaluateClickHouse(observations []replicaObservation) Result { + groups := make(map[string][]replicaObservation) + var problems []string + historicalErrors := 0 + for _, observation := range observations { + if observation.needsLogVerification { + return result("clickhouse", Unknown, "zero log pointers require verification that the replication log is empty") + } + if observation.historicalQueueErr { + historicalErrors++ + } + groups[observation.path] = append(groups[observation.path], observation) + for _, problem := range observation.problems { + problems = append(problems, fmt.Sprintf("%s %s: %s", observation.pod, observation.path, problem)) + } + } + if len(groups) == 0 { + return result("clickhouse", Unknown, "no replicated tables found") + } + for path, replicas := range groups { + names := make(map[string]bool) + pods := make(map[string]bool) + for _, replica := range replicas { + if names[replica.name] || pods[replica.pod] || replica.total != replicas[0].total { + return result("clickhouse", Unknown, "duplicate or inconsistent replica evidence for "+path) + } + names[replica.name], pods[replica.pod] = true, true + } + if int64(len(replicas)) != replicas[0].total { + problems = append(problems, fmt.Sprintf("%s: queried %d/%d table replicas", path, len(replicas), replicas[0].total)) + } + } + if len(problems) > 0 { + sort.Strings(problems) + return Result{Component: "clickhouse", Status: Degraded, Messages: problems} + } + report := result("clickhouse", Healthy, fmt.Sprintf("%d replicated table groups checked on every member; no pending data replication tasks", len(groups))) + if historicalErrors > 0 { + report.Messages = append(report.Messages, fmt.Sprintf("%d table replicas retain a previous queue-update exception; current replication checks pass", historicalErrors)) + } + return report +} diff --git a/internal/orchestration/replication/discovery.go b/internal/orchestration/replication/discovery.go new file mode 100644 index 0000000..d68922d --- /dev/null +++ b/internal/orchestration/replication/discovery.go @@ -0,0 +1,116 @@ +package replication + +import ( + "context" + "fmt" + "io" + "slices" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" +) + +// Kubernetes is the read/query surface needed by the checker. +type Kubernetes interface { + Clientset() kubernetes.Interface + Exec(context.Context, string, string, string, []string) ([]byte, error) + ExecTo(context.Context, string, string, string, []string, io.Writer) error +} + +type inventory struct { + workloads []appsv1.StatefulSet + pods []corev1.Pod +} + +type member struct { + pod corev1.Pod + replicas int +} + +func (c *Checker) discover(ctx context.Context) (inventory, error) { + ctx, cancel := context.WithTimeout(ctx, requestTimeout) + defer cancel() + options := metav1.ListOptions{LabelSelector: "app.kubernetes.io/name in (hbase,elasticsearch,kafka,clickhouse,zookeeper)"} + sets, err := c.kube.Clientset().AppsV1().StatefulSets(c.options.Namespace).List(ctx, options) + if err != nil { + return inventory{}, fmt.Errorf("list database StatefulSets: %w", err) + } + pods, err := c.kube.Clientset().CoreV1().Pods(c.options.Namespace).List(ctx, options) + if err != nil { + return inventory{}, fmt.Errorf("list database pods: %w", err) + } + return inventory{workloads: sets.Items, pods: pods.Items}, nil +} + +func hasContainer(containers []corev1.Container, name string) bool { + return slices.ContainsFunc(containers, func(container corev1.Container) bool { return container.Name == name }) +} + +func (i inventory) members(name, component, container string) ([]member, error) { + var members []member + found := false + for _, workload := range i.workloads { + if workload.Labels["app.kubernetes.io/name"] != name || + (component != "" && workload.Labels["app.kubernetes.io/component"] != component) { + continue + } + if !hasContainer(workload.Spec.Template.Spec.Containers, container) { + continue + } + found = true + pods, err := i.workloadPods(workload) + if err != nil { + return nil, err + } + for _, pod := range pods { + members = append(members, member{pod: pod, replicas: int(*workload.Spec.Replicas)}) + } + } + if !found { + return nil, fmt.Errorf("no chart-managed StatefulSet for name=%q component=%q with container %q; check namespace, selected components and chart layout", name, component, container) + } + slices.SortFunc(members, func(a, b member) int { + if a.pod.Name < b.pod.Name { + return -1 + } + if a.pod.Name > b.pod.Name { + return 1 + } + return 0 + }) + return members, nil +} + +func (i inventory) workloadPods(workload appsv1.StatefulSet) ([]corev1.Pod, error) { + if workload.Spec.Replicas == nil || *workload.Spec.Replicas < 1 { + return nil, fmt.Errorf("%s has no desired replicas", workload.Name) + } + expected := *workload.Spec.Replicas + if workload.DeletionTimestamp != nil || workload.Status.ObservedGeneration < workload.Generation || + workload.Status.ReadyReplicas != expected || workload.Status.CurrentRevision != workload.Status.UpdateRevision { + return nil, fmt.Errorf("%s is not converged: %d/%d replicas ready", workload.Name, workload.Status.ReadyReplicas, expected) + } + var pods []corev1.Pod + for _, pod := range i.pods { + owner := metav1.GetControllerOf(&pod) + if owner == nil || owner.UID != workload.UID || owner.Kind != "StatefulSet" { + continue + } + if pod.DeletionTimestamp != nil || pod.Spec.NodeName == "" || !podReady(pod) { + return nil, fmt.Errorf("%s is terminating, unscheduled or not Ready", pod.Name) + } + pods = append(pods, pod) + } + if len(pods) != int(expected) { + return nil, fmt.Errorf("%s has %d/%d current pods", workload.Name, len(pods), expected) + } + return pods, nil +} + +func podReady(pod corev1.Pod) bool { + return pod.Status.Phase == corev1.PodRunning && slices.ContainsFunc(pod.Status.Conditions, func(condition corev1.PodCondition) bool { + return condition.Type == corev1.PodReady && condition.Status == corev1.ConditionTrue + }) +} diff --git a/internal/orchestration/replication/elasticsearch.go b/internal/orchestration/replication/elasticsearch.go new file mode 100644 index 0000000..0bf0c86 --- /dev/null +++ b/internal/orchestration/replication/elasticsearch.go @@ -0,0 +1,72 @@ +package replication + +import ( + "context" + "encoding/json" + "fmt" + "sort" +) + +const elasticsearchQuery = `set -- --fail --silent --show-error --max-time 20 +if [ -n "${ELASTIC_PASSWORD:-}" ]; then + set -- "$@" --user "elastic:${ELASTIC_PASSWORD}" +fi +exec curl "$@" 'http://127.0.0.1:9200/_cluster/health?level=indices'` + +func (c *Checker) checkElasticsearch(ctx context.Context, inventory inventory) Result { + members, err := inventory.members("elasticsearch", "", "elasticsearch") + if err != nil { + return result("elasticsearch", Unknown, err.Error()) + } + if err := expectedMembers(members); err != nil { + return result("elasticsearch", Degraded, err.Error()) + } + command := []string{"bash", "-ec", elasticsearchQuery} + data, err := c.query(ctx, members[0].pod.Name, "elasticsearch", command) + if err != nil { + return result("elasticsearch", Unknown, err.Error()) + } + return evaluateElasticsearch(data, len(members)) +} + +func evaluateElasticsearch(data []byte, expected int) Result { + var fields map[string]json.RawMessage + if err := json.Unmarshal(data, &fields); err != nil { + return result("elasticsearch", Unknown, "invalid cluster health response") + } + counts, err := numbers(fields, "number_of_nodes", "unassigned_shards", "initializing_shards", "relocating_shards") + if err != nil { + return result("elasticsearch", Unknown, err.Error()) + } + status, err := text(fields, "status") + if err != nil { + return result("elasticsearch", Unknown, err.Error()) + } + var indices map[string]map[string]json.RawMessage + if err := json.Unmarshal(fields["indices"], &indices); err != nil || len(indices) == 0 { + return result("elasticsearch", Unknown, "no index replication evidence returned") + } + var problems []string + if status != "green" || counts["number_of_nodes"] != int64(expected) { + problems = append(problems, fmt.Sprintf("cluster status %s; %d/%d expected nodes", status, counts["number_of_nodes"], expected)) + } + for _, key := range []string{"unassigned_shards", "initializing_shards", "relocating_shards"} { + if counts[key] > 0 { + problems = append(problems, fmt.Sprintf("%s=%d", key, counts[key])) + } + } + for name, index := range indices { + replicas, err := number(index, "number_of_replicas") + if err != nil { + return result("elasticsearch", Unknown, fmt.Sprintf("index %s: %v", name, err)) + } + if replicas < minReplicas-1 { + problems = append(problems, fmt.Sprintf("index %s has no replica shard", name)) + } + } + if len(problems) > 0 { + sort.Strings(problems) + return Result{Component: "elasticsearch", Status: Degraded, Messages: problems} + } + return result("elasticsearch", Healthy, fmt.Sprintf("%d indices have replica shards; all shards allocated with no recovery or relocation", len(indices))) +} diff --git a/internal/orchestration/replication/evaluators_test.go b/internal/orchestration/replication/evaluators_test.go new file mode 100644 index 0000000..2b35710 --- /dev/null +++ b/internal/orchestration/replication/evaluators_test.go @@ -0,0 +1,187 @@ +package replication + +import ( + "encoding/json" + "fmt" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func encode(t *testing.T, value any) []byte { + t.Helper() + data, err := json.Marshal(value) + require.NoError(t, err) + return data +} + +func hdfsFixture(t *testing.T, mutate func(map[string]any)) []byte { + t.Helper() + fields := map[string]any{ + "name": "Hadoop:service=NameNode,name=FSNamesystem", + "UnderReplicatedBlocks": 0, "MissingBlocks": 0, "CorruptBlocks": 0, "PendingReplicationBlocks": 0, "NumLiveDataNodes": 3, "Safemode": "", + } + mutate(fields) + return encode(t, map[string]any{"configuredReplication": 3, "minimumReplication": 2, "jmx": map[string]any{"beans": []any{fields}}}) +} + +func TestHDFSEvidence(t *testing.T) { + tests := []struct { + name string + mutate func(map[string]any) + status string + }{ + {"recovered", func(map[string]any) {}, Healthy}, + {"replicating", func(m map[string]any) { m["UnderReplicatedBlocks"] = 12 }, Degraded}, + {"missing block", func(m map[string]any) { m["MissingBlocks"] = 1 }, Degraded}, + {"corrupt block", func(m map[string]any) { m["CorruptBlocks"] = 1 }, Degraded}, + {"pending replication", func(m map[string]any) { m["PendingReplicationBlocks"] = 2 }, Degraded}, + {"missing datanode", func(m map[string]any) { m["NumLiveDataNodes"] = 2 }, Degraded}, + {"safe mode", func(m map[string]any) { m["Safemode"] = "ON" }, Degraded}, + {"missing metric", func(m map[string]any) { delete(m, "UnderReplicatedBlocks") }, Unknown}, + {"null metric", func(m map[string]any) { m["CorruptBlocks"] = nil }, Unknown}, + {"negative metric", func(m map[string]any) { m["MissingBlocks"] = -1 }, Unknown}, + {"unrelated bean", func(m map[string]any) { m["name"] = "Hadoop:service=Other" }, Unknown}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + assert.Equal(t, test.status, evaluateHDFS(hdfsFixture(t, test.mutate), 3).Status) + }) + } + data := strings.Replace(string(hdfsFixture(t, func(map[string]any) {})), `"configuredReplication":3`, `"configuredReplication":1`, 1) + assert.Equal(t, Degraded, evaluateHDFS([]byte(data), 3).Status) + assert.Equal(t, Unknown, evaluateHDFS([]byte(`{"beans":[]}`), 3).Status) +} + +func esFixture() map[string]any { + return map[string]any{ + "status": "green", "number_of_nodes": 3, "unassigned_shards": 0, "initializing_shards": 0, "relocating_shards": 0, + "indices": map[string]any{"events": map[string]any{"number_of_replicas": 1}}, + } +} + +func TestElasticsearchEvidence(t *testing.T) { + tests := []struct { + name string + mutate func(map[string]any) + status string + }{ + {"recovered", func(map[string]any) {}, Healthy}, + {"yellow", func(m map[string]any) { m["status"] = "yellow" }, Degraded}, + {"node missing", func(m map[string]any) { m["number_of_nodes"] = 2 }, Degraded}, + {"initializing", func(m map[string]any) { m["initializing_shards"] = 1 }, Degraded}, + {"relocating", func(m map[string]any) { m["relocating_shards"] = 1 }, Degraded}, + {"unassigned", func(m map[string]any) { m["unassigned_shards"] = 1 }, Degraded}, + {"green without replicas", func(m map[string]any) { + m["indices"] = map[string]any{"events": map[string]any{"number_of_replicas": 0}} + }, Degraded}, + {"empty cluster", func(m map[string]any) { m["indices"] = map[string]any{} }, Unknown}, + {"partial response", func(m map[string]any) { delete(m, "initializing_shards") }, Unknown}, + {"missing index replication", func(m map[string]any) { m["indices"] = map[string]any{"events": map[string]any{}} }, Unknown}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + fields := esFixture() + test.mutate(fields) + assert.Equal(t, test.status, evaluateElasticsearch(encode(t, fields), 3).Status) + }) + } +} + +func kafkaFixture() string { + var lines []string + for _, topic := range []string{"events", "__consumer_offsets", "__transaction_state"} { + lines = append(lines, fmt.Sprintf("Topic: %s TopicId: x PartitionCount: 1 ReplicationFactor: 2 Configs: min.insync.replicas=1", topic)) + lines = append(lines, fmt.Sprintf(" Topic: %s Partition: 0 Leader: 0 Replicas: 0,1 Isr: 1,0", topic)) + } + return strings.Join(lines, "\n") +} + +func TestKafkaEvidence(t *testing.T) { + tests := []struct { + name, from, to, status string + }{ + {"recovered", "Isr: 1,0", "Isr: 1,0", Healthy}, + {"single replica transaction state", "Leader: 0 Replicas: 0,1 Isr: 1,0", "Leader: 0 Replicas: 0 Isr: 0", Degraded}, + {"missing ISR", "Isr: 1,0", "Isr: 0", Degraded}, + {"wrong ISR", "Isr: 1,0", "Isr: 2,0", Degraded}, + {"missing leader", "Leader: 0", "Leader: -1", Degraded}, + {"duplicate replica", "Replicas: 0,1", "Replicas: 0,0", Degraded}, + {"empty ISR", "Isr: 1,0", "Isr:", Degraded}, + {"unused transactions", "__transaction_state", "another-topic", Healthy}, + {"missing consumer offsets", "__consumer_offsets", "another-topic", Unknown}, + {"incomplete partitions", "PartitionCount: 1", "PartitionCount: 2", Unknown}, + {"wrong partition ID", "Partition: 0", "Partition: 5", Unknown}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + output := strings.ReplaceAll(kafkaFixture(), test.from, test.to) + assert.Equal(t, test.status, evaluateKafka(output).Status) + }) + } + assert.Equal(t, Unknown, evaluateKafka("").Status) + assert.Equal(t, Unknown, evaluateKafka(kafkaFixture()+"\n"+kafkaFixture()).Status) +} + +func clickhouseFixture() map[string]any { + return map[string]any{ + "database": "otel", "table": "traces", "zookeeper_path": "/clickhouse/tables/shard0/traces", + "replica_name": "replica0", "is_readonly": 0, "is_session_expired": 0, + "total_replicas": "2", "active_replicas": "2", "log_max_index": "100", "log_pointer": "101", + "pending_data_tasks": "0", "absolute_delay": "0", "last_queue_update_exception": "", "zookeeper_exception": "", + } +} + +func TestClickHouseReplicaEvidence(t *testing.T) { + tests := []struct { + name string + mutate func(map[string]any) + status string + }{ + {"recovered", func(map[string]any) {}, Healthy}, + {"merge backlog alone", func(m map[string]any) { m["queue_size"] = 8; m["merges_in_queue"] = 8 }, Healthy}, + {"lagging log", func(m map[string]any) { m["log_pointer"] = "100" }, Degraded}, + {"data backlog", func(m map[string]any) { m["pending_data_tasks"] = "1" }, Degraded}, + {"read only", func(m map[string]any) { m["is_readonly"] = 1 }, Degraded}, + {"expired session", func(m map[string]any) { m["is_session_expired"] = 1 }, Degraded}, + {"inactive replica", func(m map[string]any) { m["active_replicas"] = "1" }, Degraded}, + {"wrong replication", func(m map[string]any) { m["total_replicas"] = "1" }, Degraded}, + {"coordination exception", func(m map[string]any) { m["zookeeper_exception"] = "connection failed" }, Degraded}, + {"historical queue exception after recovery", func(m map[string]any) { m["last_queue_update_exception"] = "previous Keeper error" }, Healthy}, + {"queue exception with backlog", func(m map[string]any) { + m["last_queue_update_exception"] = "Keeper error" + m["log_pointer"] = "100" + }, Degraded}, + {"missing backlog evidence", func(m map[string]any) { delete(m, "pending_data_tasks") }, Unknown}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + row := clickhouseFixture() + test.mutate(row) + observations, err := parseClickHouse(encode(t, map[string]any{"data": []any{row}}), "pod0", 2) + if test.status == Unknown { + require.Error(t, err) + return + } + require.NoError(t, err) + peer := observations[0] + peer.pod, peer.name, peer.problems = "pod1", "replica1", nil + report := evaluateClickHouse(append(observations, peer)) + assert.Equal(t, test.status, report.Status) + if test.name == "historical queue exception after recovery" { + assert.Contains(t, strings.Join(report.Messages, " "), "previous queue-update exception") + } + }) + } +} + +func TestClickHouseChecksEveryTableReplica(t *testing.T) { + observations, err := parseClickHouse(encode(t, map[string]any{"data": []any{clickhouseFixture()}}), "pod0", 2) + require.NoError(t, err) + assert.Equal(t, Degraded, evaluateClickHouse(observations).Status) + assert.Equal(t, Unknown, evaluateClickHouse(append(observations, observations...)).Status) + _, err = parseClickHouse([]byte(`{"data":[]}`), "pod0", 2) + require.Error(t, err) +} diff --git a/internal/orchestration/replication/hdfs.go b/internal/orchestration/replication/hdfs.go new file mode 100644 index 0000000..ab18cce --- /dev/null +++ b/internal/orchestration/replication/hdfs.go @@ -0,0 +1,91 @@ +package replication + +import ( + "context" + "encoding/json" + "fmt" +) + +const hdfsQuery = `unset HADOOP_OPTS +printf '{"configuredReplication":' +hdfs getconf -confKey dfs.replication +printf ',"minimumReplication":' +hdfs getconf -confKey dfs.namenode.replication.min +printf ',"jmx":' +curl --fail --silent --show-error --max-time 20 'http://127.0.0.1:50070/jmx' +printf '}'` + +func (c *Checker) checkHDFS(ctx context.Context, inventory inventory) Result { + namenodes, err := inventory.members("hbase", "hdfs-nn", "namenode") + if err != nil { + return result("hdfs", Unknown, err.Error()) + } + if len(namenodes) != 1 { + return result("hdfs", Unknown, "expected the chart's single NameNode; external or HA NameNode layouts are not supported") + } + datanodes, err := inventory.members("hbase", "hdfs-dn", "datanode") + if err != nil { + return result("hdfs", Unknown, err.Error()) + } + if err := expectedMembers(datanodes); err != nil { + return result("hdfs", Degraded, err.Error()) + } + data, err := c.query(ctx, namenodes[0].pod.Name, "namenode", []string{"bash", "-ec", hdfsQuery}) + if err != nil { + return result("hdfs", Unknown, err.Error()) + } + return evaluateHDFS(data, len(datanodes)) +} + +func evaluateHDFS(data []byte, expected int) Result { + var response struct { + Replication *int `json:"configuredReplication"` + Minimum *int `json:"minimumReplication"` + JMX struct { + Beans []map[string]json.RawMessage `json:"beans"` + } `json:"jmx"` + } + if err := json.Unmarshal(data, &response); err != nil || response.Replication == nil || response.Minimum == nil { + return result("hdfs", Unknown, "invalid HDFS replication/JMX response") + } + fields := make(map[string]json.RawMessage) + for _, bean := range response.JMX.Beans { + name, _ := text(bean, "name") + switch name { + case "Hadoop:service=NameNode,name=FSNamesystem", "Hadoop:service=NameNode,name=FSNamesystemState", "Hadoop:service=NameNode,name=NameNodeInfo": + for key, value := range bean { + fields[key] = value + } + } + } + counts, err := numbers(fields, "UnderReplicatedBlocks", "MissingBlocks", "CorruptBlocks", "PendingReplicationBlocks", "NumLiveDataNodes") + if err != nil { + return result("hdfs", Unknown, err.Error()) + } + safemode, err := text(fields, "Safemode") + if err != nil { + return result("hdfs", Unknown, err.Error()) + } + var problems []string + if *response.Replication < minReplicas { + problems = append(problems, fmt.Sprintf("configured block replication is %d; at least %d required", *response.Replication, minReplicas)) + } + if *response.Minimum < minReplicas { + problems = append(problems, fmt.Sprintf("minimum write replication is %d; at least %d required", *response.Minimum, minReplicas)) + } + if safemode != "" { + problems = append(problems, "NameNode is in safe mode") + } + if counts["NumLiveDataNodes"] != int64(expected) { + problems = append(problems, fmt.Sprintf("%d/%d expected DataNodes are live", counts["NumLiveDataNodes"], expected)) + } + for _, key := range []string{"UnderReplicatedBlocks", "MissingBlocks", "CorruptBlocks", "PendingReplicationBlocks"} { + if counts[key] != 0 { + problems = append(problems, fmt.Sprintf("%s=%d", key, counts[key])) + } + } + if len(problems) > 0 { + return Result{Component: "hdfs", Status: Degraded, Messages: problems} + } + return result("hdfs", Healthy, fmt.Sprintf("%d DataNodes live; no missing, corrupt, under-replicated or pending-replication blocks", expected)) +} diff --git a/internal/orchestration/replication/hdfs_audit.go b/internal/orchestration/replication/hdfs_audit.go new file mode 100644 index 0000000..c56038f --- /dev/null +++ b/internal/orchestration/replication/hdfs_audit.go @@ -0,0 +1,84 @@ +package replication + +import ( + "context" + "fmt" + "slices" +) + +// Summary counters include departing replicas and incomplete WAL blocks; audit per-block live replicas instead. +const hdfsAuditQuery = `unset HADOOP_OPTS +exec hdfs fsck / -files -blocks -openforwrite -includeSnapshots` + +// Verify audits HDFS before success, then rechecks the selected databases. +func (c *Checker) Verify(ctx context.Context, previous Report) Report { + if previous.Status != Healthy || !slices.ContainsFunc(previous.Checks, func(check Result) bool { + return check.Component == hdfsComponent && check.Status == Healthy + }) { + return previous + } + report := previous + report.Checks = slices.Clone(previous.Checks) + before, err := c.discover(ctx) + if err != nil { + return replaceHDFS(report, result(hdfsComponent, Unknown, err.Error())) + } + if before.fingerprint(c.options.Components) != previous.topology { + return invalidateAudit(report) + } + namenodes, err := before.members("hbase", nameNodeComponent, "namenode") + if err != nil || len(namenodes) != 1 { + return replaceHDFS(report, result(hdfsComponent, Unknown, "HDFS audit requires one available NameNode")) + } + audit := c.auditHDFS(ctx, namenodes[0].pod.Name) + if audit.Status != Healthy { + return replaceHDFS(report, audit) + } + current := c.Check(ctx) + current.CheckedAt = previous.CheckedAt + if current.topology != previous.topology { + return invalidateAudit(current) + } + for n := range current.Checks { + if current.Checks[n].Component == hdfsComponent && current.Checks[n].Status == Healthy { + current.Checks[n].Messages = append(current.Checks[n].Messages, audit.Messages...) + } + } + return current +} + +func (c *Checker) auditHDFS(ctx context.Context, pod string) Result { + parser := &fsckParser{} + err := c.kube.ExecTo(ctx, c.options.Namespace, pod, "namenode", []string{"bash", "-ec", hdfsAuditQuery}, parser) + if ctx.Err() != nil { + return result(hdfsComponent, Unknown, fmt.Sprintf("HDFS block audit did not complete: %v", ctx.Err())) + } + audit := parser.result() + if err != nil && audit.Status != Degraded { + failure := result(hdfsComponent, Unknown, fmt.Sprintf("HDFS block audit execution failed: %v", err)) + if audit.Status == Unknown { + failure.Messages = append(failure.Messages, audit.Messages...) + } + return failure + } + return audit +} + +func replaceHDFS(report Report, audit Result) Report { + for n := range report.Checks { + if report.Checks[n].Component == hdfsComponent { + report.Checks[n] = audit + } + } + report.Status = reportStatus(report.Checks) + return report +} + +func invalidateAudit(report Report) Report { + report.Status = Unknown + for n := range report.Checks { + report.Checks[n].Status = Unknown + report.Checks[n].Messages = append(report.Checks[n].Messages, "database state changed around the HDFS audit; repeat the observation") + } + return report +} diff --git a/internal/orchestration/replication/hdfs_audit_test.go b/internal/orchestration/replication/hdfs_audit_test.go new file mode 100644 index 0000000..09c11ef --- /dev/null +++ b/internal/orchestration/replication/hdfs_audit_test.go @@ -0,0 +1,155 @@ +package replication + +import ( + "context" + "fmt" + "io" + "strings" + "testing" + "testing/synctest" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/fake" +) + +func TestHDFSAuditVerification(t *testing.T) { + tests := []struct { + name, status string + }{ + {"healthy", Healthy}, {"truncated", Unknown}, {"query failure", Unknown}, {"rejected record and query failure", Unknown}, + {"replication one", Degraded}, {"deadline", Unknown}, {"topology changed", Unknown}, {"health changed", Degraded}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + objects := databaseObjects("hbase", "hdfs-nn", "namenode", 1) + objects = append(objects, databaseObjects("hbase", "hdfs-dn", "datanode", 3)...) + client := fake.NewSimpleClientset(objects...) + audits := 0 + kube := &fakeKubernetes{client: client, + exec: func(context.Context, string, string, string, []string) ([]byte, error) { + return hdfsFixture(t, func(fields map[string]any) { + if test.name == "health changed" && audits > 0 { + fields["UnderReplicatedBlocks"] = 1 + } + }), nil + }, + stream: func(ctx context.Context, namespace, pod, container string, command []string, out io.Writer) error { + audits++ + assert.Equal(t, "hdfs-nn-0", pod) + assert.Equal(t, "namenode", container) + assert.Equal(t, []string{"bash", "-ec", hdfsAuditQuery}, command) + if test.name == "deadline" { + <-ctx.Done() + return ctx.Err() + } + data := fsckFixture() + switch test.name { + case "truncated": + data = strings.Split(data, "Status:")[0] + case "replication one": + data = strings.Replace(data, "replication=2", "replication=1", 1) + case "query failure": + return fmt.Errorf("query failed") + case "rejected record and query failure": + _, err := io.WriteString(out, fsckStart+"unexpected format\n") + require.NoError(t, err) + return fmt.Errorf("query failed") + case "topology changed": + p, err := client.CoreV1().Pods(namespace).Get(ctx, pod, metav1.GetOptions{}) + require.NoError(t, err) + p.Status.ContainerStatuses = []corev1.ContainerStatus{{Name: "namenode", RestartCount: 1}} + _, err = client.CoreV1().Pods(namespace).Update(ctx, p, metav1.UpdateOptions{}) + require.NoError(t, err) + } + _, err := io.WriteString(out, data) + return err + }, + } + options := testOptions() + options.Components = []string{"hdfs"} + probe, err := New(kube, options) + require.NoError(t, err) + before := probe.Check(context.Background()) + require.Equal(t, Healthy, before.Status) + timeout := time.Second + if test.name == "deadline" { + timeout = 10 * time.Millisecond + } + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + after := probe.Verify(ctx, before) + assert.Equal(t, test.status, after.Status, after) + assert.Equal(t, 1, audits) + assert.Equal(t, Healthy, before.Checks[0].Status, "verification must not mutate the prior observation") + if test.name == "rejected record and query failure" { + messages := strings.Join(after.Checks[0].Messages, "; ") + assert.Contains(t, messages, "query failed") + assert.Contains(t, messages, `fsck line 3: unsupported fsck file or block record; record "unexpected format"`) + } + if test.status == Healthy { + assert.Contains(t, strings.Join(after.Checks[0].Messages, "; "), "completed block entries") + assert.Equal(t, 2, kube.calls, "lightweight health is rechecked after the audit") + } + }) + } +} + +func TestHDFSAuditUsesRemainingOverallDeadline(t *testing.T) { + for _, overall := range []time.Duration{time.Minute, 5 * time.Minute} { + t.Run(overall.String(), func(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), overall) + defer cancel() + deadline, _ := ctx.Deadline() + kube := &fakeKubernetes{stream: func(ctx context.Context, _, _, _ string, _ []string, out io.Writer) error { + actual, ok := ctx.Deadline() + require.True(t, ok) + assert.Equal(t, deadline, actual, "the audit must not have a separate timeout") + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(3 * time.Minute): + _, err := io.WriteString(out, fsckFixture()) + return err + } + }} + probe, err := New(kube, testOptions()) + require.NoError(t, err) + time.Sleep(30 * time.Second) + start := time.Now() + report := probe.auditHDFS(ctx, "namenode-0") + if overall == time.Minute { + assert.Equal(t, Unknown, report.Status) + assert.Contains(t, report.Messages[0], "context deadline exceeded") + assert.Equal(t, 30*time.Second, time.Since(start)) + } else { + assert.Equal(t, Healthy, report.Status, report) + assert.Equal(t, 3*time.Minute, time.Since(start), "audits longer than two minutes can complete") + } + }) + }) + } +} + +func TestNoHDFSAuditForInapplicableOrUnhealthyChecks(t *testing.T) { + probe, err := New(&fakeKubernetes{client: fake.NewSimpleClientset()}, testOptions()) + require.NoError(t, err) + for _, report := range []Report{ + {Status: NotApplicable}, + {Status: Degraded, Checks: []Result{{Component: "hdfs", Status: Healthy}}}, + {Status: Healthy, Checks: []Result{{Component: "hdfs", Status: NotApplicable}}}, + {Status: Healthy, Checks: []Result{{Component: "kafka", Status: Healthy}}}, + } { + assert.Equal(t, report, probe.Verify(context.Background(), report)) + } +} + +func TestHDFSRequiresMinimumWriteReplication(t *testing.T) { + data := string(hdfsFixture(t, func(map[string]any) {})) + assert.Equal(t, Degraded, evaluateHDFS([]byte(strings.Replace(data, `"minimumReplication":2`, `"minimumReplication":1`, 1)), 3).Status) + assert.Equal(t, Unknown, evaluateHDFS([]byte(strings.Replace(data, `"minimumReplication":2`, `"otherMinimum":2`, 1)), 3).Status) +} diff --git a/internal/orchestration/replication/hdfs_fsck.go b/internal/orchestration/replication/hdfs_fsck.go new file mode 100644 index 0000000..e6ead1d --- /dev/null +++ b/internal/orchestration/replication/hdfs_fsck.go @@ -0,0 +1,249 @@ +package replication + +import ( + "bytes" + "fmt" + "regexp" + "strconv" + "strings" +) + +const ( + maxFsckLine = 64 << 10 + maxFsckDiagnostics = 8 + maxDiagnosticPath = 160 +) + +var ( + fsckFilePattern = regexp.MustCompile(`^(/.+) [0-9]+ bytes, replicated: replication=([0-9]+), ([0-9]+) block\(s\)(:|, OPENFORWRITE:)(.*)$`) + fsckBlockPattern = regexp.MustCompile(`^([0-9]+)\. (\S+:blk_-?[0-9]+_[0-9]+) len=[0-9]+(?: (Live_repl|Expected_repl)=([0-9]+)| (MISSING!))(.*)$`) + fsckNumber = regexp.MustCompile(`^([0-9]+)(?:\s|$)`) +) + +type fsckFile struct { + path string + target, declared int64 + seen int64 + open, expecting bool + underConstruction bool +} + +type fsckParser struct { + pending []byte + err error + lines int64 + started, statusSeen, ended, done bool + status string + section string + file *fsckFile + files, completed, open int64 + summaryFiles, summaryBlocks bool + ecFiles, ecBlocks bool + problems int64 + messages []string +} + +func (p *fsckParser) Write(data []byte) (int, error) { + size := len(data) + for len(data) > 0 && p.err == nil { + end := bytes.IndexByte(data, '\n') + if end < 0 { + end = len(data) + } + if len(p.pending)+end > maxFsckLine { + if len(p.pending) < maxDiagnosticPath { + p.pending = append(p.pending, data[:min(end, maxDiagnosticPath-len(p.pending)+1)]...) + } + p.lines++ + p.err = p.lineError(fmt.Errorf("fsck line exceeds the supported size")) + break + } + p.pending = append(p.pending, data[:end]...) + if end == len(data) { + break + } + p.consumeLine() + data = data[end+1:] + } + // Keep draining after a parse error; only the bounded diagnostic is retained. + return size, nil +} + +func (p *fsckParser) consumeLine() { + p.lines++ + if err := p.line(strings.TrimSpace(string(p.pending))); err != nil { + p.err = p.lineError(err) + } + p.pending = p.pending[:0] +} + +func (p *fsckParser) lineError(err error) error { + return fmt.Errorf("fsck line %d: %w; record %q", p.lines, err, diagnosticPath(string(p.pending))) +} + +func (p *fsckParser) result() Result { + if p.err == nil && len(p.pending) != 0 { + p.consumeLine() + p.pending = nil + } + if p.err != nil { + return result("hdfs", Unknown, p.err.Error()) + } + if !p.done || !p.summaryFiles || !p.summaryBlocks || !p.ecFiles || !p.ecBlocks { + return result("hdfs", Unknown, "incomplete fsck report; block replication could not be verified") + } + if p.problems > 0 { + return Result{Component: "hdfs", Status: Degraded, + Messages: append([]string{fmt.Sprintf("HDFS block audit found %d replication problems", p.problems)}, p.messages...)} + } + report := result("hdfs", Healthy, fmt.Sprintf("HDFS audit: %d files and %d completed block entries meet replication targets of at least two (including snapshot references)", p.files, p.completed)) + if p.open > 0 { + report.Messages = append(report.Messages, fmt.Sprintf("%d under-construction blocks have expected pipeline membership meeting their targets; persistence of their latest writes is not verified", p.open)) + } + return report +} + +func (p *fsckParser) line(line string) error { + if line == "" { + return nil + } + if p.done { + return fmt.Errorf("unexpected output after fsck completion") + } + if !p.started { + if !strings.HasPrefix(line, "FSCK started by ") || !strings.Contains(line, " for path / at ") { + return fmt.Errorf("missing fsck start marker for the filesystem root") + } + p.started = true + return nil + } + if p.statusSeen { + return p.summary(line) + } + if line == "Status: HEALTHY" || line == "Status: CORRUPT" { + if err := p.finishFile(); err != nil { + return err + } + p.statusSeen, p.status = true, strings.TrimPrefix(line, "Status: ") + if p.status == "CORRUPT" { + p.problem("fsck reports missing or corrupt data") + } + return nil + } + if match := fsckFilePattern.FindStringSubmatch(line); match != nil { + return p.startFile(match) + } + if strings.HasPrefix(line, "/") && strings.HasSuffix(line, " ") { + return p.finishFile() + } + if line == "Under Construction Block:" { + if p.file == nil || !p.file.open || p.file.expecting || p.file.underConstruction { + return fmt.Errorf("unexpected under-construction block marker") + } + p.file.expecting = true + return nil + } + if match := fsckBlockPattern.FindStringSubmatch(line); match != nil { + return p.block(match) + } + if p.file != nil && fsckWarning(line) { + p.problem("fsck reports a file replication or placement problem") + return nil + } + return fmt.Errorf("unsupported fsck file or block record") +} + +func (p *fsckParser) finishFile() error { + if p.file != nil && (p.file.seen != p.file.declared || p.file.expecting) { + return fmt.Errorf("fsck file block count does not match its block records") + } + p.file = nil + return nil +} + +func (p *fsckParser) startFile(match []string) error { + if err := p.finishFile(); err != nil { + return err + } + target, err := strconv.ParseInt(match[2], 10, 64) + if err != nil || target < 1 { + return fmt.Errorf("invalid fsck replication target") + } + blocks, err := strconv.ParseInt(match[3], 10, 64) + if err != nil { + return fmt.Errorf("invalid fsck file block count") + } + p.file = &fsckFile{path: match[1], target: target, declared: blocks, open: match[4] == ", OPENFORWRITE:"} + p.files++ + if target < minReplicas { + p.problem(fmt.Sprintf("file %q has replication target %d", diagnosticPath(match[1]), target)) + } + tail := strings.TrimSpace(match[5]) + if tail != "" && tail != "OK" { + if !fsckWarning(tail) { + return fmt.Errorf("unsupported fsck file status") + } + p.problem("fsck reports a file replication or placement problem") + } + return nil +} + +func (p *fsckParser) block(match []string) error { + index, err := strconv.ParseInt(match[1], 10, 64) + if err != nil || p.file == nil || p.file.seen != index || p.file.seen >= p.file.declared || p.file.underConstruction { + return fmt.Errorf("unexpected or duplicate fsck block record") + } + p.file.seen++ + if match[5] == "MISSING!" { + if p.file.expecting { + return fmt.Errorf("missing pipeline evidence for under-construction block") + } + p.completed++ + p.problem(fmt.Sprintf("block %s is missing", match[2])) + return nil + } + replicas, err := strconv.ParseInt(match[4], 10, 64) + if err != nil { + return fmt.Errorf("invalid fsck replica count") + } + expected := match[3] == "Expected_repl" + if expected != p.file.expecting { + return fmt.Errorf("fsck live replicas and expected pipeline evidence are inconsistent") + } + if expected { + p.open++ + p.file.expecting, p.file.underConstruction = false, true + } else { + p.completed++ + } + if replicas < p.file.target { + p.problem(fmt.Sprintf("block %s in %q has %d/%d %s", match[2], diagnosticPath(p.file.path), replicas, p.file.target, match[3])) + } + if strings.TrimSpace(match[6]) != "" { + return fmt.Errorf("unsupported fsck block details") + } + return nil +} + +func fsckWarning(line string) bool { + for _, marker := range []string{"Under replicated ", "Replica placement policy is violated", "CORRUPT", "MISSING"} { + if strings.HasPrefix(line, marker) || strings.Contains(line, ": "+marker) { + return true + } + } + return false +} + +func diagnosticPath(path string) string { + if len(path) > maxDiagnosticPath { + return path[:maxDiagnosticPath] + "..." + } + return path +} + +func (p *fsckParser) problem(message string) { + p.problems++ + if len(p.messages) < maxFsckDiagnostics { + p.messages = append(p.messages, message) + } +} diff --git a/internal/orchestration/replication/hdfs_fsck_summary.go b/internal/orchestration/replication/hdfs_fsck_summary.go new file mode 100644 index 0000000..5857c3a --- /dev/null +++ b/internal/orchestration/replication/hdfs_fsck_summary.go @@ -0,0 +1,135 @@ +package replication + +import ( + "fmt" + "strconv" + "strings" +) + +const ( + fsckReplicated = "replicated" + fsckEC = "ec" +) + +func (p *fsckParser) summary(line string) error { + if strings.Contains(line, "not validated") || strings.Contains(line, "Files currently being written:") || + strings.Contains(line, "Total open files size:") { + return fmt.Errorf("fsck excluded open-file evidence from the audit") + } + switch line { + case "Replicated Blocks:": + if p.section != "" { + return fmt.Errorf("duplicate fsck replication summary") + } + p.section = fsckReplicated + return nil + case "Erasure Coded Block Groups:": + if p.section != fsckReplicated || !p.summaryFiles || !p.summaryBlocks { + return fmt.Errorf("incomplete fsck replication summary") + } + p.section = fsckEC + return nil + } + if strings.HasPrefix(line, "FSCK ended at ") { + if p.section != fsckEC || !p.ecFiles || !p.ecBlocks || p.ended { + return fmt.Errorf("incomplete fsck completion summary") + } + p.ended = true + return nil + } + if strings.HasPrefix(line, "The filesystem under path ") { + if !p.ended || line != "The filesystem under path '/' is "+p.status { + return fmt.Errorf("missing or inconsistent fsck completion status") + } + p.done = true + return nil + } + if p.ended { + return fmt.Errorf("unexpected output after fsck summary") + } + if strings.HasPrefix(line, "Total files:") { + return p.fileSummary(line) + } + if strings.HasPrefix(line, "Total blocks (validated):") { + count, err := fsckSummaryCount(line) + if err != nil || p.section != fsckReplicated || p.summaryBlocks || count != p.completed+p.open { + return fmt.Errorf("fsck summary block count does not match audited blocks") + } + p.summaryBlocks = true + return nil + } + if strings.HasPrefix(line, "Total block groups (validated):") { + count, err := fsckSummaryCount(line) + if err != nil || p.section != fsckEC || p.ecBlocks || count != 0 { + return fmt.Errorf("erasure-coded block groups are not supported by the replication audit") + } + p.ecBlocks = true + return nil + } + return p.summaryMetric(line) +} + +func fsckSummaryCount(line string) (int64, error) { + _, value, _ := strings.Cut(line, ":") + match := fsckNumber.FindStringSubmatch(strings.TrimSpace(value)) + if match == nil { + return 0, fmt.Errorf("invalid fsck summary counter") + } + count, err := strconv.ParseInt(match[1], 10, 64) + if err != nil { + return 0, fmt.Errorf("invalid fsck summary counter") + } + return count, nil +} + +func (p *fsckParser) summaryMetric(line string) error { + if strings.Trim(line, "* ") == "" { + return nil + } + for _, prefix := range []string{"Missing blocks:", "Corrupt blocks:", "Missing replicas:", "Blocks queued for replication:", + "Under-replicated blocks:", "Mis-replicated blocks:", "UNDER MIN REPL'D BLOCKS:", "CORRUPT FILES:", "MISSING BLOCKS:", "CORRUPT BLOCKS:"} { + if strings.HasPrefix(line, prefix) { + count, err := fsckSummaryCount(line) + if err != nil { + return err + } + if count > 0 { + p.problem(fmt.Sprintf("%s %d", prefix, count)) + } + return nil + } + } + for _, prefix := range []string{"Number of data-nodes:", "Number of racks:", "Total dirs:", "Total symlinks:", "Total size:", + "Minimally replicated blocks:", "Over-replicated blocks:", "Default replication factor:", "Average block replication:", + "Minimally erasure-coded block groups:", "Over-erasure-coded block groups:", "Under-erasure-coded block groups:", + "Unsatisfactory placement block groups:", "Average block group size:", "Missing block groups:", "Corrupt block groups:", + "Missing internal blocks:", "DecommissionedReplicas:", "DecommissioningReplicas:", "EnteringMaintenanceReplicas:", + "InMaintenanceReplicas:", "MINIMAL BLOCK REPLICATION:", "MISSING SIZE:", "CORRUPT SIZE:"} { + if strings.HasPrefix(line, prefix) { + return nil + } + } + return fmt.Errorf("unsupported fsck summary record") +} + +func (p *fsckParser) fileSummary(line string) error { + count, err := fsckSummaryCount(line) + if err != nil { + return err + } + switch p.section { + case fsckReplicated: + if p.summaryFiles || count != p.files { + return fmt.Errorf("fsck summary file count does not match audited files") + } + p.summaryFiles = true + case fsckEC: + if p.ecFiles || count != 0 { + return fmt.Errorf("erasure-coded files are not supported by the replication audit") + } + p.ecFiles = true + default: + return fmt.Errorf("unexpected fsck file summary") + } + return nil +} diff --git a/internal/orchestration/replication/hdfs_fsck_test.go b/internal/orchestration/replication/hdfs_fsck_test.go new file mode 100644 index 0000000..89a7c6e --- /dev/null +++ b/internal/orchestration/replication/hdfs_fsck_test.go @@ -0,0 +1,189 @@ +package replication + +import ( + "fmt" + "io" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const fsckStart = "FSCK started by checker (auth:SIMPLE) from /127.0.0.1 for path / at Tue Sep 15 08:00:00 UTC 2026\n/ \n" + +func fsckSummary(files, blocks int) string { + return fmt.Sprintf(` +Status: HEALTHY + Number of data-nodes: 3 + Total dirs: 1 + Total symlinks: 0 +Replicated Blocks: + Total files: %d + Total blocks (validated): %d + Missing blocks: 0 + Corrupt blocks: 0 +Erasure Coded Block Groups: + Total files: 0 + Total block groups (validated): 0 +FSCK ended at Tue Sep 15 08:00:01 UTC 2026 in 100 milliseconds +The filesystem under path '/' is HEALTHY`, files, blocks) +} + +func fsckFixture() string { + return fsckStart + `/hbase/table/region/file 100 bytes, replicated: replication=2, 1 block(s): OK +0. BP-test:blk_1_1 len=100 Live_repl=2 +/hbase/WALs/active 100 bytes, replicated: replication=2, 2 block(s), OPENFORWRITE: OK +0. BP-test:blk_2_1 len=80 Live_repl=2 +Under Construction Block: +1. BP-test:blk_3_1 len=20 Expected_repl=2 +` + fsckSummary(2, 3) +} + +func TestFsckEvidence(t *testing.T) { + tests := []struct { + name, status string + change func(string) string + }{ + {"completed blocks and active WAL", Healthy, func(s string) string { return s }}, + {"replication one despite healthy footer", Degraded, func(s string) string { + return strings.Replace(s, "replication=2", "replication=1", 1) + }}, + {"live count below target", Degraded, func(s string) string { return strings.Replace(s, "Live_repl=2", "Live_repl=1", 1) }}, + {"target three still needs three copies", Degraded, func(s string) string { return strings.Replace(s, "replication=2", "replication=3", 1) }}, + {"short active pipeline", Degraded, func(s string) string { return strings.ReplaceAll(s, "Expected_repl=2", "Expected_repl=1") }}, + {"missing block", Degraded, func(s string) string { + s = strings.Replace(s, "Live_repl=2", "MISSING!", 1) + return strings.ReplaceAll(s, "HEALTHY", "CORRUPT") + }}, + {"snapshot references", Healthy, func(s string) string { + return strings.Replace(s, "/hbase/table/region/file", "/hbase/.snapshot/backup/table/region/file", 1) + }}, + {"missing start", Unknown, func(s string) string { return strings.TrimPrefix(s, fsckStart) }}, + {"missing footer", Unknown, func(s string) string { + return strings.Split(s, "The filesystem under path")[0] + }}, + {"missing summary", Unknown, func(s string) string { + return strings.Split(s, "Status:")[0] + "The filesystem under path '/' is HEALTHY" + }}, + {"missing block detail", Unknown, func(s string) string { + return strings.Replace(s, "0. BP-test:blk_1_1 len=100 Live_repl=2\n", "", 1) + }}, + {"duplicate block detail", Unknown, func(s string) string { + return strings.Replace(s, "0. BP-test:blk_1_1 len=100 Live_repl=2\n", + "0. BP-test:blk_1_1 len=100 Live_repl=2\n0. BP-test:blk_1_1 len=100 Live_repl=2\n", 1) + }}, + {"file total mismatch", Unknown, func(s string) string { return strings.Replace(s, "Total files: 2", "Total files: 3", 1) }}, + {"block total mismatch", Unknown, func(s string) string { + return strings.Replace(s, "Total blocks (validated): 3", "Total blocks (validated): 4", 1) + }}, + {"open-file evidence excluded", Unknown, func(s string) string { + return strings.Replace(s, "Total blocks (validated): 3", "Total blocks (validated): 3 (Total open file blocks (not validated): 1)", 1) + }}, + {"missing pipeline marker", Unknown, func(s string) string { return strings.ReplaceAll(s, "Under Construction Block:\n", "") }}, + {"expected count is not a live count", Unknown, func(s string) string { return strings.Replace(s, "Live_repl=2", "Expected_repl=2", 1) }}, + {"unexpected output", Unknown, func(s string) string { return s + "\npermission denied" }}, + {"integer overflow", Unknown, func(s string) string { + return strings.Replace(s, "replication=2", "replication=99999999999999999999999", 1) + }}, + {"erasure coded files", Unknown, func(s string) string { + return strings.Replace(s, "Erasure Coded Block Groups:\n Total files: 0", "Erasure Coded Block Groups:\n Total files: 1", 1) + }}, + {"unsupported layout", Unknown, func(s string) string { + return strings.Replace(s, "replicated: replication=2", "erasure-coded: policy=RS-6-3", 1) + }}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + parser := &fsckParser{} + data := test.change(fsckFixture()) + for len(data) > 0 { + size := min(len(data), 17) + _, err := io.WriteString(parser, data[:size]) + require.NoError(t, err) + data = data[size:] + } + report := parser.result() + assert.Equal(t, test.status, report.Status, report) + if test.name == "completed blocks and active WAL" { + assert.Contains(t, report.Messages[0], "2 completed block entries") + assert.Contains(t, report.Messages[1], "1 under-construction blocks") + assert.Contains(t, report.Messages[1], "latest writes is not verified") + } + }) + } +} + +func TestFsckEmptyFilesystem(t *testing.T) { + parser := &fsckParser{} + _, err := io.WriteString(parser, fsckStart+fsckSummary(0, 0)) + require.NoError(t, err) + assert.Equal(t, Healthy, parser.result().Status) +} + +func TestFsckStreamingAndBoundedDiagnostics(t *testing.T) { + parser := &fsckParser{} + blocks := 200000 + written, err := fmt.Fprintf(parser, "%s/hbase/data 200000 bytes, replicated: replication=2, %d block(s): OK\n", fsckStart, blocks) + require.NoError(t, err) + for n := range blocks { + size, err := fmt.Fprintf(parser, "%d. BP-test:blk_%d_1 len=1 Live_repl=1\n", n, n) + require.NoError(t, err) + written += size + } + _, err = io.WriteString(parser, fsckSummary(1, blocks)) + require.NoError(t, err) + report := parser.result() + assert.Greater(t, written, 8<<20) + assert.Equal(t, Degraded, report.Status) + assert.Equal(t, maxFsckDiagnostics+1, len(report.Messages)) + assert.LessOrEqual(t, cap(parser.pending), maxFsckLine) +} + +func TestFsckRejectsOversizedLine(t *testing.T) { + parser := &fsckParser{} + _, err := io.WriteString(parser, strings.Repeat("x", maxFsckLine+1)) + require.NoError(t, err) + assert.Equal(t, Unknown, parser.result().Status) + assert.LessOrEqual(t, len(parser.pending), maxFsckLine) +} + +func TestFsckRejectedRecordDiagnostics(t *testing.T) { + tests := []struct { + name, record, expected string + }{ + {"unsupported", "unexpected format", `"unexpected format"`}, + {"terminal controls", "invalid\t\x1b[31m\"record\"\r", `"invalid\t\x1b[31m\"record\"\r"`}, + {"bounded excerpt", strings.Repeat("x", maxDiagnosticPath+100), `"` + strings.Repeat("x", maxDiagnosticPath) + `..."`}, + {"oversized", strings.Repeat("x", maxFsckLine+1), `"` + strings.Repeat("x", maxDiagnosticPath) + `..."`}, + } + for _, test := range tests { + for _, newline := range []string{"", "\n"} { + t.Run(fmt.Sprintf("%s/newline=%t", test.name, newline != ""), func(t *testing.T) { + for _, chunkSize := range []int{1, 17, maxFsckLine + 1000} { + parser := &fsckParser{} + data := fsckStart + "\n" + test.record + newline + for len(data) > 0 { + n := min(chunkSize, len(data)) + written, err := io.WriteString(parser, data[:n]) + require.NoError(t, err) + require.Equal(t, n, written) + data = data[n:] + } + report := parser.result() + require.Equal(t, Unknown, report.Status) + require.Len(t, report.Messages, 1) + message := report.Messages[0] + assert.Contains(t, message, "fsck line 4:") + assert.Contains(t, message, "record "+test.expected) + assert.NotContains(t, message, "\x1b") + assert.NotContains(t, message, "\r") + assert.Less(t, len(message), 1000) + _, err := io.WriteString(parser, "\nlater error\n") + require.NoError(t, err) + assert.Equal(t, report, parser.result(), "retain the first rejected record while draining") + } + }) + } + } +} diff --git a/internal/orchestration/replication/json.go b/internal/orchestration/replication/json.go new file mode 100644 index 0000000..ee6b998 --- /dev/null +++ b/internal/orchestration/replication/json.go @@ -0,0 +1,54 @@ +package replication + +import ( + "encoding/json" + "fmt" + "strconv" +) + +func number(fields map[string]json.RawMessage, key string) (int64, error) { + raw, found := fields[key] + if !found || string(raw) == "null" { + return 0, fmt.Errorf("missing numeric field %s", key) + } + var value int64 + if err := json.Unmarshal(raw, &value); err != nil { + var quoted string + if err := json.Unmarshal(raw, "ed); err != nil { + return 0, fmt.Errorf("invalid integer field %s", key) + } + var parseErr error + value, parseErr = strconv.ParseInt(quoted, 10, 64) + if parseErr != nil { + return 0, fmt.Errorf("invalid integer field %s", key) + } + } + if value < 0 { + return 0, fmt.Errorf("invalid nonnegative integer field %s", key) + } + return value, nil +} + +func text(fields map[string]json.RawMessage, key string) (string, error) { + raw, found := fields[key] + if !found || string(raw) == "null" { + return "", fmt.Errorf("missing string field %s", key) + } + var value string + if err := json.Unmarshal(raw, &value); err != nil { + return "", fmt.Errorf("invalid string field %s", key) + } + return value, nil +} + +func numbers(fields map[string]json.RawMessage, keys ...string) (map[string]int64, error) { + values := make(map[string]int64, len(keys)) + for _, key := range keys { + value, err := number(fields, key) + if err != nil { + return nil, err + } + values[key] = value + } + return values, nil +} diff --git a/internal/orchestration/replication/kafka.go b/internal/orchestration/replication/kafka.go new file mode 100644 index 0000000..4887404 --- /dev/null +++ b/internal/orchestration/replication/kafka.go @@ -0,0 +1,172 @@ +package replication + +import ( + "context" + "fmt" + "regexp" + "strconv" + "strings" +) + +// The CLI must not bind the broker's inherited JMX port. +const kafkaQuery = `unset JMX_PORT KAFKA_JMX_OPTS +exec kafka-topics.sh "$@"` + +// --all bypasses Kafka's ACL-filtered topic-list precheck and queries DescribeConfigs directly. +const transactionTopicQuery = `unset JMX_PORT KAFKA_JMX_OPTS +if output="$(kafka-configs.sh "$@" --describe --all --entity-type topics --entity-name __transaction_state 2>&1)"; then + printf 'present\n' +elif [[ "$output" == *AuthorizationException* ]]; then + printf 'unverified\n' +elif [[ "$output" == *UnknownTopicOrPartitionException* ]]; then + printf 'absent\n' +else + printf 'unverified\n' +fi` + +var ( + kafkaTopicPattern = regexp.MustCompile(`\bTopic:\s*(\S+)`) + kafkaPartitionPattern = regexp.MustCompile(`\bPartition:\s*(\d+)\b`) + kafkaCountPattern = regexp.MustCompile(`\bPartitionCount:\s*(\d+)\b`) + kafkaFieldsPattern = regexp.MustCompile(`\b(Leader|Replicas|Isr):\s*([-\d,]+)`) +) + +func (c *Checker) checkKafka(ctx context.Context, inventory inventory) Result { + members, err := inventory.members("kafka", "kafka", "kafka") + if err != nil { + return result("kafka", Unknown, err.Error()) + } + if err := expectedMembers(members); err != nil { + return result("kafka", Degraded, err.Error()) + } + command := []string{"bash", "-ec", kafkaQuery, "replication-check", "--bootstrap-server", "localhost:9092", "--describe"} + data, err := c.query(ctx, members[0].pod.Name, "kafka", command) + if err != nil { + return result("kafka", Unknown, err.Error()) + } + report := evaluateKafka(string(data)) + if report.Status == Unknown || hasTransactionTopic(string(data)) { + return report + } + probe := []string{"bash", "-ec", transactionTopicQuery, "replication-check", "--bootstrap-server", "localhost:9092"} + presence, err := c.query(ctx, members[0].pod.Name, "kafka", probe) + if err != nil || strings.TrimSpace(string(presence)) != "absent" { + report.Status = Unknown + report.Messages = append(report.Messages, "__transaction_state was not listed and its absence could not be verified; check topic permissions or retry") + return report + } + report.Messages = append(report.Messages, "__transaction_state is absent; transaction-topic replication is not applicable to this observation") + return report +} + +func hasTransactionTopic(output string) bool { + for _, match := range kafkaTopicPattern.FindAllStringSubmatch(output, -1) { + if match[1] == "__transaction_state" { + return true + } + } + return false +} + +func brokerIDs(value string) (map[int]bool, error) { + if value == "" { + return nil, fmt.Errorf("missing broker IDs") + } + ids := make(map[int]bool) + for _, part := range strings.Split(value, ",") { + id, err := strconv.Atoi(part) + if err != nil || id < 0 || ids[id] { + return nil, fmt.Errorf("invalid or duplicate broker ID") + } + ids[id] = true + } + return ids, nil +} + +func partitionProblem(line string) string { + fields := make(map[string]string) + for _, match := range kafkaFieldsPattern.FindAllStringSubmatch(line, -1) { + fields[match[1]] = match[2] + } + replicas, err := brokerIDs(fields["Replicas"]) + if err != nil || len(replicas) < minReplicas { + return "fewer than two distinct assigned replicas or invalid assignment" + } + isr, err := brokerIDs(fields["Isr"]) + if err != nil || len(isr) != len(replicas) { + return "assigned replicas are not all in sync" + } + for id := range replicas { + if !isr[id] { + return "assigned replicas are not all in sync" + } + } + leader, err := strconv.Atoi(fields["Leader"]) + if err != nil || !isr[leader] { + return "no available in-sync leader" + } + return "" +} + +func evaluateKafka(output string) Result { + counts := make(map[string]int) + partitions := make(map[string]map[int]bool) + var problems []string + for _, line := range strings.Split(output, "\n") { + topicMatch := kafkaTopicPattern.FindStringSubmatch(line) + if topicMatch == nil { + continue + } + topic := topicMatch[1] + if count := kafkaCountPattern.FindStringSubmatch(line); count != nil { + value, err := strconv.Atoi(count[1]) + if err != nil || value < 1 || counts[topic] != 0 { + return result("kafka", Unknown, "invalid or duplicate topic summary") + } + counts[topic] = value + } + if partition := kafkaPartitionPattern.FindStringSubmatch(line); partition != nil { + id, err := strconv.Atoi(partition[1]) + if err != nil || partitions[topic][id] { + return result("kafka", Unknown, "invalid or duplicate partition description") + } + if partitions[topic] == nil { + partitions[topic] = make(map[int]bool) + } + partitions[topic][id] = true + if problem := partitionProblem(line); problem != "" { + problems = append(problems, fmt.Sprintf("%s partition %d: %s", topic, id, problem)) + } + } + } + if err := completeKafkaEvidence(counts, partitions); err != nil { + return result("kafka", Unknown, err.Error()) + } + report := result("kafka", Healthy, fmt.Sprintf("all partitions of %d topics have at least two replicas, complete ISR and an in-sync leader", len(counts))) + if len(problems) > 0 { + report = Result{Component: "kafka", Status: Degraded, Messages: problems} + } + return report +} + +func completeKafkaEvidence(counts map[string]int, partitions map[string]map[int]bool) error { + for _, topic := range []string{"__consumer_offsets"} { + if counts[topic] == 0 { + return fmt.Errorf("required internal topic %s is absent; initialize its workload and repeat the check", topic) + } + } + if len(counts) != len(partitions) { + return fmt.Errorf("topic summaries and partition descriptions do not match") + } + for topic, count := range counts { + if len(partitions[topic]) != count { + return fmt.Errorf("incomplete partition descriptions for %s", topic) + } + for id := 0; id < count; id++ { + if !partitions[topic][id] { + return fmt.Errorf("missing partition %d for %s", id, topic) + } + } + } + return nil +} diff --git a/internal/orchestration/replication/kafka_test.go b/internal/orchestration/replication/kafka_test.go new file mode 100644 index 0000000..9862ceb --- /dev/null +++ b/internal/orchestration/replication/kafka_test.go @@ -0,0 +1,93 @@ +package replication + +import ( + "context" + "os" + "os/exec" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestKafkaQueryDoesNotInheritBrokerJMXPort(t *testing.T) { + bash, err := exec.LookPath("bash") + if err != nil { + t.Skip("bash is required to exercise the in-pod query") + } + dir := t.TempDir() + script := `#!/bin/sh +test -z "${JMX_PORT:-}" || exit 1 +test -z "${KAFKA_JMX_OPTS:-}" || exit 1 +printf '%s\n' "$KAFKA_OPTS" "$@" +` + path := filepath.Join(dir, "kafka-topics.sh") + require.NoError(t, os.WriteFile(path, []byte(script), 0o600)) + require.NoError(t, os.Chmod(path, 0o700)) + t.Setenv("PATH", dir+string(os.PathListSeparator)+os.Getenv("PATH")) + t.Setenv("JMX_PORT", "5555") + t.Setenv("KAFKA_JMX_OPTS", "-Dcom.sun.management.jmxremote.port=5555") + t.Setenv("KAFKA_OPTS", "-Djava.security.auth.login.config=/mounted/jaas.conf") + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + command := exec.CommandContext(ctx, bash, "-ec", kafkaQuery, "replication-check", + "--bootstrap-server", "localhost:9092", "--describe") + output, err := command.CombinedOutput() + require.NoError(t, err, string(output)) + assert.Equal(t, "-Djava.security.auth.login.config=/mounted/jaas.conf\n--bootstrap-server\nlocalhost:9092\n--describe\n", string(output)) +} + +func TestTransactionTopicQueryUsesDirectConfigEvidence(t *testing.T) { + bash, err := exec.LookPath("bash") + if err != nil { + t.Skip("bash is required to exercise the in-pod query") + } + dir := t.TempDir() + // Model the 4.1 listing precheck separately from the broker's DescribeConfigs response. + script := `#!/bin/sh +test -z "${JMX_PORT:-}" && test -z "${KAFKA_JMX_OPTS:-}" || exit 1 +test "$1" = "--bootstrap-server" && test "$2" = "localhost:9092" || exit 1 +shift 2 +if [ "$KAFKA_TEST_VERSION" = "4.1" ] && [ "$KAFKA_TEST_EXIT" != "0" ]; then + case " $* " in + *" --all "*) ;; + *) printf "The topic '__transaction_state' doesn't exist and doesn't have dynamic config.\n"; exit 0 ;; + esac +fi +printf '%s\n' "$KAFKA_TEST_OUTPUT" +exit "$KAFKA_TEST_EXIT" +` + path := filepath.Join(dir, "kafka-configs.sh") + require.NoError(t, os.WriteFile(path, []byte(script), 0o600)) + require.NoError(t, os.Chmod(path, 0o700)) + t.Setenv("PATH", dir+string(os.PathListSeparator)+os.Getenv("PATH")) + t.Setenv("JMX_PORT", "5555") + t.Setenv("KAFKA_JMX_OPTS", "-Dcom.sun.management.jmxremote.port=5555") + tests := []struct { + name, output, exit, expected string + }{ + {"present", "All configs for topic __transaction_state are:", "0", "present"}, + {"absent", "org.apache.kafka.common.errors.UnknownTopicOrPartitionException: unknown topic", "1", "absent"}, + {"ACL hidden", "org.apache.kafka.common.errors.TopicAuthorizationException: denied", "1", "unverified"}, + {"authorization overrides absence", "TopicAuthorizationException UnknownTopicOrPartitionException", "1", "unverified"}, + {"timeout", "TimeoutException: broker unavailable", "1", "unverified"}, + } + for _, version := range []string{"3.9", "4.1"} { + for _, test := range tests { + t.Run(version+"/"+test.name, func(t *testing.T) { + t.Setenv("KAFKA_TEST_VERSION", version) + t.Setenv("KAFKA_TEST_OUTPUT", test.output) + t.Setenv("KAFKA_TEST_EXIT", test.exit) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + command := exec.CommandContext(ctx, bash, "-ec", transactionTopicQuery, "replication-check", + "--bootstrap-server", "localhost:9092") + output, err := command.CombinedOutput() + require.NoError(t, err, string(output)) + assert.Equal(t, test.expected+"\n", string(output)) + }) + } + } +} diff --git a/internal/orchestration/replication/optional_state_test.go b/internal/orchestration/replication/optional_state_test.go new file mode 100644 index 0000000..22cef45 --- /dev/null +++ b/internal/orchestration/replication/optional_state_test.go @@ -0,0 +1,123 @@ +package replication + +import ( + "context" + "fmt" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "k8s.io/client-go/kubernetes/fake" +) + +func TestTransactionTopicAbsenceRequiresSuccessfulVerification(t *testing.T) { + tests := []struct { + presence string + status string + failure bool + }{ + {"absent\n", Healthy, false}, + {"present\n", Unknown, false}, + {"unverified\n", Unknown, false}, + {"", Unknown, true}, + } + for _, test := range tests { + t.Run(test.presence, func(t *testing.T) { + kube := &fakeKubernetes{client: fake.NewSimpleClientset(kafkaObjects()...), exec: func(_ context.Context, _, _, _ string, command []string) ([]byte, error) { + if command[2] == kafkaQuery { + return []byte(strings.ReplaceAll(kafkaFixture(), "__transaction_state", "nontransactional-topic")), nil + } + assert.Equal(t, transactionTopicQuery, command[2]) + if test.failure { + return nil, fmt.Errorf("cannot query metadata") + } + return []byte(test.presence), nil + }} + probe, err := New(kube, testOptions()) + require.NoError(t, err) + report := probe.Check(context.Background()) + assert.Equal(t, test.status, report.Status, report) + assert.Equal(t, 2, kube.calls) + if test.status == Healthy { + assert.Contains(t, strings.Join(report.Checks[0].Messages, " "), "not applicable") + } else { + assert.NotContains(t, strings.Join(report.Checks[0].Messages, " "), "is absent") + } + }) + } +} + +func TestUnverifiedTransactionTopicPreservesPartitionProblems(t *testing.T) { + for _, presence := range []string{"absent\n", "present\n", "unverified\n", ""} { + t.Run(presence, func(t *testing.T) { + kube := &fakeKubernetes{client: fake.NewSimpleClientset(kafkaObjects()...), exec: func(_ context.Context, _, _, _ string, command []string) ([]byte, error) { + if command[2] == kafkaQuery { + data := strings.ReplaceAll(kafkaFixture(), "__transaction_state", "nontransactional-topic") + return []byte(strings.Replace(data, "Isr: 1,0", "Isr: 0", 1)), nil + } + if presence == "" { + return nil, fmt.Errorf("query failed") + } + return []byte(presence), nil + }} + probe, err := New(kube, testOptions()) + require.NoError(t, err) + report := probe.Check(context.Background()) + messages := strings.Join(report.Checks[0].Messages, "; ") + assert.Contains(t, messages, "events partition 0: assigned replicas are not all in sync") + if presence == "absent\n" { + assert.Equal(t, Degraded, report.Status) + assert.Contains(t, messages, "is absent") + } else { + assert.Equal(t, Unknown, report.Status) + assert.Contains(t, messages, "absence could not be verified") + assert.NotContains(t, messages, "is absent") + } + }) + } +} + +func TestZeroLogPointersRequireAnEmptyKeeperLog(t *testing.T) { + tests := []struct { + name, evidence, status string + pending, readonly int + }{ + {"empty log", `{"data":[{"entries":"0"}]}`, Healthy, 0, 0}, + {"first log entry still unprocessed", `{"data":[{"entries":"1"}]}`, Degraded, 0, 0}, + {"no evidence", `{"data":[]}`, Unknown, 0, 0}, + {"missing count", `{"data":[{}]}`, Unknown, 0, 0}, + {"invalid count", `{"data":[{"entries":-1}]}`, Unknown, 0, 0}, + {"query failure", "", Unknown, 0, 0}, + {"pending queue still fails", `{"data":[{"entries":0}]}`, Degraded, 1, 0}, + {"read only still fails", `{"data":[{"entries":0}]}`, Degraded, 0, 1}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + objects := databaseObjects("clickhouse", "clickhouse", "clickhouse", 2) + kube := &fakeKubernetes{client: fake.NewSimpleClientset(objects...), exec: func(_ context.Context, _, pod, _ string, command []string) ([]byte, error) { + if command[4] == clickhouseSQL { + row := clickhouseFixture() + row["log_pointer"], row["log_max_index"] = "0", "0" + row["replica_name"], row["pending_data_tasks"], row["is_readonly"] = pod, test.pending, test.readonly + return encode(t, map[string]any{"data": []any{row}}), nil + } + assert.Equal(t, clickhouseEmptyLogSQL, command[4]) + assert.Equal(t, "--param_log_path=/clickhouse/tables/shard0/traces/log", command[5]) + if test.evidence == "" { + return nil, fmt.Errorf("Keeper query denied") + } + return []byte(test.evidence), nil + }} + options := testOptions() + options.Components = []string{"clickhouse"} + probe, err := New(kube, options) + require.NoError(t, err) + report := probe.Check(context.Background()) + assert.Equal(t, test.status, report.Status, report) + if test.status != Unknown { + assert.Equal(t, 4, kube.calls, "each replica's empty log is verified") + } + }) + } +} diff --git a/internal/orchestration/replication/report.go b/internal/orchestration/replication/report.go new file mode 100644 index 0000000..c6ee113 --- /dev/null +++ b/internal/orchestration/replication/report.go @@ -0,0 +1,86 @@ +// Package replication checks the observed replication state of chart-managed databases. +// It does not grant permission to disrupt a node or modify database state. +package replication + +import ( + "fmt" + "slices" + "time" +) + +const ( + Healthy = "healthy" + Degraded = "degraded" + Unknown = "unknown" + NotApplicable = "not_applicable" + + minReplicas = 2 +) + +var supportedComponents = []string{"hdfs", "elasticsearch", "kafka", "clickhouse", "zookeeper"} + +// Options identifies the installation and selected databases. +type Options struct { + Namespace string + Components []string +} + +// Validate rejects ambiguous scope. +func (o Options) Validate() error { + if o.Namespace == "" { + return fmt.Errorf("namespace is required") + } + if len(o.Components) == 0 { + return fmt.Errorf("select at least one component") + } + seen := make(map[string]bool) + for _, component := range o.Components { + if !slices.Contains(supportedComponents, component) || seen[component] { + return fmt.Errorf("invalid or duplicate component %q; choose from %v", component, supportedComponents) + } + seen[component] = true + } + return nil +} + +// Result reports one component; missing evidence is unknown, never healthy. +type Result struct { + Component string `json:"component"` + Status string `json:"status"` + Messages []string `json:"messages"` +} + +// Report is a point-in-time observation, not a maintenance lock. +type Report struct { + CheckedAt time.Time `json:"checkedAt"` + Namespace string `json:"namespace"` + Status string `json:"status"` + Error string `json:"error,omitempty"` + Checks []Result `json:"checks"` + topology string +} + +func result(component, status, message string) Result { + return Result{Component: component, Status: status, Messages: []string{message}} +} + +func reportStatus(checks []Result) string { + status := NotApplicable + if len(checks) == 0 { + return Unknown + } + for _, check := range checks { + if check.Status == Unknown { + return Unknown + } + if check.Status == NotApplicable { + continue + } + if check.Status != Healthy { + status = Degraded + } else if status == NotApplicable { + status = Healthy + } + } + return status +} diff --git a/internal/orchestration/replication/snapshot.go b/internal/orchestration/replication/snapshot.go new file mode 100644 index 0000000..6a0f79e --- /dev/null +++ b/internal/orchestration/replication/snapshot.go @@ -0,0 +1,109 @@ +package replication + +import ( + "crypto/sha256" + "encoding/json" + "fmt" + "slices" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" +) + +type workloadState struct { + Name, Application, Role string + UID types.UID + Generation int64 + Replicas *int32 + Observed, Deleting bool + ReadyReplicas int32 + CurrentRevision, UpdateRevision string + Containers []string +} + +type podState struct { + Name, Node string + UID, Owner types.UID + Phase corev1.PodPhase + Deleting bool + Ready corev1.ConditionStatus + ReadyTransition metav1.Time + Container *containerState +} + +type containerState struct { + Name, ID, ImageID string + Restarts int32 + Ready bool + Started *bool + RunningSince metav1.Time +} + +func (i inventory) fingerprint(components []string) string { + var entries []string + containers := make(map[types.UID]string) + for _, workload := range i.workloads { + if !slices.Contains(components, workloadComponent(workload)) { + continue + } + containers[workload.UID] = databaseContainer(workload) + entries = append(entries, snapshotJSON(snapshotWorkload(workload))) + } + for _, pod := range i.pods { + owner := metav1.GetControllerOf(&pod) + if owner == nil || owner.Kind != "StatefulSet" { + continue + } + container, selected := containers[owner.UID] + if selected { + entries = append(entries, snapshotJSON(snapshotPod(pod, owner.UID, container))) + } + } + slices.Sort(entries) + return fmt.Sprintf("%x", sha256.Sum256([]byte(snapshotJSON(entries)))) +} + +func snapshotJSON(value any) string { + data, _ := json.Marshal(value) + return string(data) +} + +func snapshotWorkload(workload appsv1.StatefulSet) workloadState { + state := workloadState{ + Name: workload.Name, UID: workload.UID, Generation: workload.Generation, + Application: workload.Labels["app.kubernetes.io/name"], Role: workload.Labels["app.kubernetes.io/component"], + Replicas: workload.Spec.Replicas, ReadyReplicas: workload.Status.ReadyReplicas, + Observed: workload.Status.ObservedGeneration >= workload.Generation, Deleting: workload.DeletionTimestamp != nil, + CurrentRevision: workload.Status.CurrentRevision, UpdateRevision: workload.Status.UpdateRevision, + } + for _, container := range workload.Spec.Template.Spec.Containers { + state.Containers = append(state.Containers, container.Name) + } + slices.Sort(state.Containers) + return state +} + +func snapshotPod(pod corev1.Pod, owner types.UID, container string) podState { + state := podState{Name: pod.Name, UID: pod.UID, Owner: owner, Node: pod.Spec.NodeName, + Phase: pod.Status.Phase, Deleting: pod.DeletionTimestamp != nil} + for _, condition := range pod.Status.Conditions { + if condition.Type == corev1.PodReady { + state.Ready, state.ReadyTransition = condition.Status, condition.LastTransitionTime + } + } + for _, status := range pod.Status.ContainerStatuses { + if status.Name != container { + continue + } + state.Container = &containerState{ + Name: status.Name, ID: status.ContainerID, ImageID: status.ImageID, + Restarts: status.RestartCount, Ready: status.Ready, Started: status.Started, + } + if status.State.Running != nil { + state.Container.RunningSince = status.State.Running.StartedAt + } + } + return state +} diff --git a/internal/orchestration/replication/snapshot_test.go b/internal/orchestration/replication/snapshot_test.go new file mode 100644 index 0000000..264f754 --- /dev/null +++ b/internal/orchestration/replication/snapshot_test.go @@ -0,0 +1,185 @@ +package replication + +import ( + "context" + "slices" + "testing" + "testing/synctest" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes/fake" + "k8s.io/utils/ptr" +) + +func runningKafkaObjects() []runtime.Object { + objects := kafkaObjects() + for _, object := range objects { + if pod, ok := object.(*corev1.Pod); ok { + pod.Status.Conditions[0].LastTransitionTime = metav1.NewTime(time.Unix(100, 0)) + pod.Status.ContainerStatuses = []corev1.ContainerStatus{ + {Name: "kafka", ContainerID: "containerd://original", Ready: true, Started: ptr.To(true), + State: corev1.ContainerState{Running: &corev1.ContainerStateRunning{StartedAt: metav1.NewTime(time.Unix(90, 0))}}}, + {Name: "metrics", ContainerID: "containerd://metrics", Ready: true}, + } + } + } + return objects +} + +func TestChangesDuringQueries(t *testing.T) { + tests := []struct { + name string + status string + change func(*appsv1.StatefulSet, *corev1.Pod) + }{ + {"metadata and heartbeat", Healthy, func(sts *appsv1.StatefulSet, pod *corev1.Pod) { + sts.Annotations = map[string]string{"housekeeping": "updated"} + pod.Annotations = map[string]string{"housekeeping": "updated"} + pod.Status.Conditions[0].LastProbeTime = metav1.NewTime(time.Unix(110, 0)) + pod.Status.Conditions[0].Message = "updated diagnostic" + }}, + {"sidecar restart while pod stays ready", Healthy, func(_ *appsv1.StatefulSet, pod *corev1.Pod) { + pod.Status.ContainerStatuses[1].RestartCount++ + }}, + {"ready condition", Unknown, func(_ *appsv1.StatefulSet, pod *corev1.Pod) { + pod.Status.Conditions[0].Status = corev1.ConditionFalse + }}, + {"readiness flap recovered", Unknown, func(_ *appsv1.StatefulSet, pod *corev1.Pod) { + pod.Status.Conditions[0].LastTransitionTime = metav1.NewTime(time.Unix(110, 0)) + }}, + {"database restart", Unknown, func(_ *appsv1.StatefulSet, pod *corev1.Pod) { + pod.Status.ContainerStatuses[0].RestartCount++ + }}, + {"container replacement", Unknown, func(_ *appsv1.StatefulSet, pod *corev1.Pod) { + pod.Status.ContainerStatuses[0].ContainerID = "containerd://replacement" + }}, + {"container start time", Unknown, func(_ *appsv1.StatefulSet, pod *corev1.Pod) { + pod.Status.ContainerStatuses[0].State.Running.StartedAt = metav1.NewTime(time.Unix(110, 0)) + }}, + {"node assignment", Unknown, func(_ *appsv1.StatefulSet, pod *corev1.Pod) { + pod.Spec.NodeName = "replacement-node" + }}, + {"termination", Unknown, func(_ *appsv1.StatefulSet, pod *corev1.Pod) { + pod.DeletionTimestamp = ptr.To(metav1.NewTime(time.Unix(110, 0))) + }}, + {"ownership", Unknown, func(_ *appsv1.StatefulSet, pod *corev1.Pod) { + pod.OwnerReferences[0].UID = "another-statefulset" + }}, + {"scaling", Unknown, func(sts *appsv1.StatefulSet, _ *corev1.Pod) { + sts.Spec.Replicas = ptr.To(int32(1)) + }}, + {"observed spec change", Unknown, func(sts *appsv1.StatefulSet, _ *corev1.Pod) { + sts.Generation++ + sts.Status.ObservedGeneration++ + }}, + {"controller not converged", Unknown, func(sts *appsv1.StatefulSet, _ *corev1.Pod) { + sts.Status.ObservedGeneration = 0 + }}, + {"workload readiness", Unknown, func(sts *appsv1.StatefulSet, _ *corev1.Pod) { + sts.Status.ReadyReplicas-- + }}, + {"rollout", Unknown, func(sts *appsv1.StatefulSet, _ *corev1.Pod) { + sts.Status.UpdateRevision = "next" + }}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + client := fake.NewSimpleClientset(runningKafkaObjects()...) + kube := &fakeKubernetes{client: client, exec: func(ctx context.Context, namespace, podName, _ string, _ []string) ([]byte, error) { + pod, err := client.CoreV1().Pods(namespace).Get(ctx, podName, metav1.GetOptions{}) + require.NoError(t, err) + sts, err := client.AppsV1().StatefulSets(namespace).Get(ctx, "kafka", metav1.GetOptions{}) + require.NoError(t, err) + sts.ResourceVersion, pod.ResourceVersion = "2", "2" + test.change(sts, pod) + _, err = client.CoreV1().Pods(namespace).Update(ctx, pod, metav1.UpdateOptions{}) + require.NoError(t, err) + _, err = client.AppsV1().StatefulSets(namespace).Update(ctx, sts, metav1.UpdateOptions{}) + require.NoError(t, err) + return []byte(kafkaFixture()), nil + }} + probe, err := New(kube, testOptions()) + require.NoError(t, err) + assert.Equal(t, test.status, probe.Check(context.Background()).Status) + }) + } +} + +func TestSnapshotsIgnoreUnselectedWorkloadsAndListOrder(t *testing.T) { + const otherDatabase = "zookeeper" + objects := append(runningKafkaObjects(), databaseObjects(otherDatabase, otherDatabase, otherDatabase, 3)...) + kube := &fakeKubernetes{client: fake.NewSimpleClientset(objects...)} + probe, err := New(kube, testOptions()) + require.NoError(t, err) + inventory, err := probe.discover(context.Background()) + require.NoError(t, err) + kafkaBefore := inventory.fingerprint([]string{"kafka"}) + allBefore := inventory.fingerprint([]string{"kafka", otherDatabase}) + slices.Reverse(inventory.workloads) + slices.Reverse(inventory.pods) + assert.Equal(t, allBefore, inventory.fingerprint([]string{"kafka", otherDatabase})) + for n := range inventory.pods { + if inventory.pods[n].Labels["app.kubernetes.io/name"] == otherDatabase { + inventory.pods[n].Status.Conditions[0].Status = corev1.ConditionFalse + } + } + orphan := inventory.pods[len(inventory.pods)-1].DeepCopy() + orphan.UID, orphan.OwnerReferences = "diagnostic-pod", nil + inventory.pods = append(inventory.pods, *orphan) + assert.Equal(t, kafkaBefore, inventory.fingerprint([]string{"kafka"})) + assert.NotEqual(t, allBefore, inventory.fingerprint([]string{"kafka", otherDatabase})) +} + +func TestWaitTracksMeaningfulChangesBetweenHealthyChecks(t *testing.T) { + for _, restart := range []bool{false, true} { + name := "metadata only" + expected := 30 * time.Second + if restart { + name, expected = "database restart", 50*time.Second + } + t.Run(name, func(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + client := fake.NewSimpleClientset(runningKafkaObjects()...) + kube := &fakeKubernetes{client: client, exec: func(context.Context, string, string, string, []string) ([]byte, error) { + return []byte(kafkaFixture()), nil + }} + probe, err := New(kube, testOptions()) + require.NoError(t, err) + start := time.Now() + calls := 0 + var progress []WaitProgress + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + report, err := Wait(ctx, func(ctx context.Context) Report { + calls++ + if calls == 3 { + pod, err := client.CoreV1().Pods("test").Get(ctx, "kafka-0", metav1.GetOptions{}) + require.NoError(t, err) + pod.ResourceVersion = "2" + pod.Annotations = map[string]string{"updated": "yes"} + if restart { + pod.Status.ContainerStatuses[0].RestartCount++ + } + _, err = client.CoreV1().Pods("test").Update(ctx, pod, metav1.UpdateOptions{}) + require.NoError(t, err) + } + return probe.Check(ctx) + }, 10*time.Second, 30*time.Second, func(report Report, state WaitProgress) { + assert.Equal(t, Healthy, report.Status) + progress = append(progress, state) + }, nil) + require.NoError(t, err) + assert.Equal(t, Healthy, report.Status) + assert.Equal(t, expected, time.Since(start)) + assert.Equal(t, restart, progress[2].Reset) + assert.Equal(t, 30*time.Second, progress[len(progress)-1].HealthyFor) + }) + }) + } +} diff --git a/internal/orchestration/replication/wait.go b/internal/orchestration/replication/wait.go new file mode 100644 index 0000000..4b9dce8 --- /dev/null +++ b/internal/orchestration/replication/wait.go @@ -0,0 +1,113 @@ +package replication + +import ( + "context" + "fmt" + "time" +) + +const minimumVerificationRetry = 30 * time.Second + +// WaitProgress describes the stability period after a completed observation. +type WaitProgress struct { + ObservedAt time.Time + HealthyFor time.Duration + Required time.Duration + Reset bool + Complete bool + Verifying bool + RetryAfter time.Duration +} + +type stabilityPeriod struct { + since time.Time + topology string +} + +func (s *stabilityPeriod) observe(report Report, required time.Duration) WaitProgress { + now := time.Now() + progress := WaitProgress{ObservedAt: now.UTC(), Required: required} + switch report.Status { + case NotApplicable: + progress.Complete = true + case Healthy: + if !s.since.IsZero() && s.topology != report.topology { + progress.Reset = true + s.since = now + } + if s.since.IsZero() { + s.since = now + } + progress.HealthyFor = now.Sub(s.since) + progress.Complete = progress.HealthyFor >= required + default: + progress.Reset = !s.since.IsZero() + s.since = time.Time{} + } + s.topology = report.topology + return progress +} + +// Wait polls until replication remains healthy for stableFor or the caller's deadline expires. +// Interrupted observations never replace the last completed report. +func Wait(ctx context.Context, check func(context.Context) Report, interval, stableFor time.Duration, observe func(Report, WaitProgress), verify func(context.Context, Report) Report) (Report, error) { + if interval <= 0 || stableFor < 0 { + return Report{}, fmt.Errorf("interval must be positive and stable-for cannot be negative") + } + var period stabilityPeriod + var report Report + for { + if err := ctx.Err(); err != nil { + return report, fmt.Errorf("replication wait ended: %w", err) + } + next := check(ctx) + if err := ctx.Err(); err != nil { + if report.Namespace == "" { + report.Namespace = next.Namespace + } + return report, fmt.Errorf("replication wait ended: %w", err) + } + report = next + progress := period.observe(report, stableFor) + if progress.Complete && report.Status == Healthy && verify != nil { + progress.Complete, progress.Verifying = false, true + if observe != nil { + observe(report, progress) + } + verified := verify(ctx, report) + if err := ctx.Err(); err != nil { + return report, fmt.Errorf("replication verification ended: %w", err) + } + report = verified + progress.ObservedAt, progress.Verifying = time.Now().UTC(), false + progress.Complete = report.Status == Healthy || report.Status == NotApplicable + if !progress.Complete { + period.since = time.Time{} + progress.Reset, progress.HealthyFor = true, 0 + progress.RetryAfter = max(interval, minimumVerificationRetry) + } + } + if observe != nil { + observe(report, progress) + } + if err := ctx.Err(); err != nil { + return report, fmt.Errorf("replication wait ended: %w", err) + } + if progress.Complete { + return report, nil + } + delay := interval + if progress.RetryAfter > 0 { + delay = progress.RetryAfter + } else if report.Status == Healthy { + delay = min(delay, stableFor-progress.HealthyFor) + } + timer := time.NewTimer(delay) + select { + case <-ctx.Done(): + timer.Stop() + return report, fmt.Errorf("replication wait ended: %w", ctx.Err()) + case <-timer.C: + } + } +} diff --git a/internal/orchestration/replication/wait_test.go b/internal/orchestration/replication/wait_test.go new file mode 100644 index 0000000..6a15710 --- /dev/null +++ b/internal/orchestration/replication/wait_test.go @@ -0,0 +1,226 @@ +package replication + +import ( + "context" + "testing" + "testing/synctest" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestWaitContinuesThroughUnknownAndDegraded(t *testing.T) { + statuses := []string{Unknown, Degraded, Healthy} + calls := 0 + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + report, err := Wait(ctx, func(context.Context) Report { + status := statuses[calls] + calls++ + return Report{Status: status} + }, time.Millisecond, 0, nil, nil) + require.NoError(t, err) + assert.Equal(t, Healthy, report.Status) + assert.Equal(t, 3, calls) +} + +func TestWaitHonorsDeadlineDuringQuery(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Millisecond) + defer cancel() + _, err := Wait(ctx, func(ctx context.Context) Report { + <-ctx.Done() + return Report{Status: Healthy} + }, time.Millisecond, 0, nil, nil) + require.ErrorIs(t, err, context.DeadlineExceeded) +} + +func TestWaitRequiresSustainedHealthyObservations(t *testing.T) { + calls := 0 + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + _, err := Wait(ctx, func(context.Context) Report { + calls++ + if calls == 2 { + return Report{Status: Degraded} + } + return Report{Status: Healthy} + }, time.Millisecond, 5*time.Millisecond, nil, nil) + require.NoError(t, err) + assert.GreaterOrEqual(t, calls, 4) +} + +func TestWaitCancellation(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + _, err := Wait(ctx, func(context.Context) Report { + cancel() + return Report{Status: Unknown} + }, time.Hour, 0, nil, nil) + require.ErrorIs(t, err, context.Canceled) +} + +func TestNoApplicableReplicationChecksDoesNotWaitForever(t *testing.T) { + calls := 0 + report, err := Wait(context.Background(), func(context.Context) Report { + calls++ + return Report{Status: NotApplicable} + }, time.Second, time.Minute, nil, nil) + require.NoError(t, err) + assert.Equal(t, NotApplicable, report.Status) + assert.Equal(t, 1, calls) +} + +func TestWaitExitsAfterDefaultStabilityPeriodWithSlowChecks(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + start := time.Now() + var progress []WaitProgress + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + report, err := Wait(ctx, func(context.Context) Report { + time.Sleep(2 * time.Second) + return Report{Status: Healthy} + }, 10*time.Second, 30*time.Second, func(_ Report, state WaitProgress) { + progress = append(progress, state) + }, nil) + require.NoError(t, err) + assert.Equal(t, Healthy, report.Status) + require.Len(t, progress, 4) + assert.Equal(t, time.Duration(0), progress[0].HealthyFor) + assert.Equal(t, 12*time.Second, progress[1].HealthyFor) + assert.Equal(t, 32*time.Second, progress[3].HealthyFor) + assert.True(t, progress[3].Complete) + assert.Equal(t, 34*time.Second, time.Since(start)) + }) +} + +func TestWaitRechecksAtStabilityDeadlineBeforeLongInterval(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + start := time.Now() + ctx, cancel := context.WithTimeout(context.Background(), 45*time.Second) + defer cancel() + calls := 0 + _, err := Wait(ctx, func(context.Context) Report { + calls++ + return Report{Status: Healthy} + }, time.Minute, 30*time.Second, nil, nil) + require.NoError(t, err) + assert.Equal(t, 2, calls) + assert.Equal(t, 30*time.Second, time.Since(start)) + }) +} + +func TestWaitResetsStabilityProgress(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + calls := 0 + var progress []WaitProgress + _, err := Wait(context.Background(), func(context.Context) Report { + calls++ + if calls == 3 { + return Report{Status: Degraded} + } + return Report{Status: Healthy} + }, 10*time.Second, 30*time.Second, func(_ Report, state WaitProgress) { + progress = append(progress, state) + }, nil) + require.NoError(t, err) + require.Len(t, progress, 7) + assert.True(t, progress[2].Reset) + assert.Zero(t, progress[3].HealthyFor) + assert.Equal(t, 30*time.Second, progress[6].HealthyFor) + assert.True(t, progress[6].Complete) + }) +} + +func TestWaitCancellationPreservesLastCompletedObservation(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + calls, observed := 0, 0 + lastCompleted := Report{Namespace: "test", CheckedAt: time.Now(), Status: Healthy, + Checks: []Result{result("kafka", Healthy, "all partitions in sync")}} + report, err := Wait(ctx, func(context.Context) Report { + calls++ + if calls == 3 { + cancel() + return Report{Status: Unknown, Checks: []Result{result("kafka", Unknown, "aborted query URL")}} + } + return lastCompleted + }, 10*time.Second, 30*time.Second, func(Report, WaitProgress) { observed++ }, nil) + require.ErrorIs(t, err, context.Canceled) + assert.Equal(t, lastCompleted, report) + assert.Equal(t, 2, observed) + }) +} + +func TestWaitAuditsOnlyAfterStabilityAndPacesFailures(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + start := time.Now() + audits := 0 + var auditTimes []time.Duration + var progress []WaitProgress + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + report, err := Wait(ctx, func(context.Context) Report { + return Report{Status: Healthy, topology: "unchanged"} + }, 10*time.Second, 30*time.Second, func(_ Report, state WaitProgress) { + progress = append(progress, state) + }, func(_ context.Context, report Report) Report { + audits++ + auditTimes = append(auditTimes, time.Since(start)) + if audits == 1 { + report.Status = Degraded + } + return report + }) + require.NoError(t, err) + assert.Equal(t, Healthy, report.Status) + assert.Equal(t, []time.Duration{30 * time.Second, 90 * time.Second}, auditTimes) + verifying, retries := 0, 0 + for _, state := range progress { + if state.Verifying { + verifying++ + assert.False(t, state.Complete, "stability alone must not announce completion") + } + if state.RetryAfter > 0 { + retries++ + assert.Equal(t, 30*time.Second, state.RetryAfter) + } + } + assert.Equal(t, 2, verifying) + assert.Equal(t, 1, retries) + assert.True(t, progress[len(progress)-1].Complete) + }) +} + +func TestWaitAuditCancellationPreservesLastObservation(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + last := Report{Status: Healthy, Namespace: "test"} + report, err := Wait(ctx, func(context.Context) Report { return last }, time.Second, 0, nil, + func(ctx context.Context, _ Report) Report { + <-ctx.Done() + return Report{Status: Unknown, Error: "interrupted audit"} + }) + require.ErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, last, report) + }) +} + +func TestAuditFailureWithNoStabilityStillHasRetryDelay(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + start := time.Now() + attempts := 0 + _, err := Wait(context.Background(), func(context.Context) Report { return Report{Status: Healthy} }, + time.Millisecond, 0, nil, func(_ context.Context, report Report) Report { + attempts++ + if attempts == 1 { + report.Status = Unknown + } + return report + }) + require.NoError(t, err) + assert.Equal(t, 2, attempts) + assert.Equal(t, 30*time.Second, time.Since(start)) + }) +} diff --git a/internal/orchestration/replication/zookeeper.go b/internal/orchestration/replication/zookeeper.go new file mode 100644 index 0000000..f04478d --- /dev/null +++ b/internal/orchestration/replication/zookeeper.go @@ -0,0 +1,169 @@ +package replication + +import ( + "context" + "fmt" + "strconv" + "strings" +) + +const ( + minZooKeeperVoters = 3 + zooKeeperLeader = "leader" + zooKeeperFollower = "follower" +) + +const zookeeperQuery = `exec 3<>/dev/tcp/127.0.0.1/2181 +printf mntr >&3 +while true; do + if IFS= read -r -t 5 -u 3 line; then + printf '%s\n' "$line" + else + status=$? + if [ "$status" -eq 1 ] && [ -z "$line" ]; then + break + fi + exit 1 + fi +done` + +type zooKeeperObservation struct { + pod string + role string + peerState string + voters int + synced int +} + +func (c *Checker) checkZooKeeper(ctx context.Context, inventory inventory) Result { + members, err := inventory.members("zookeeper", "zookeeper", "zookeeper") + if err != nil { + return result("zookeeper", Unknown, err.Error()) + } + if len(members) < minZooKeeperVoters { + return result("zookeeper", Degraded, "at least three voting members are required for fault-tolerant ZooKeeper") + } + var observations []zooKeeperObservation + for _, member := range members { + if member.replicas != len(members) { + return result("zookeeper", Unknown, "expected one chart-managed ZooKeeper ensemble") + } + observation, err := c.queryZooKeeper(ctx, member.pod.Name) + if err != nil { + return result("zookeeper", Unknown, err.Error()) + } + observations = append(observations, observation) + } + report := evaluateZooKeeper(observations) + if report.Status != Healthy { + return report + } + // Recheck the leader after sampling followers; elections do not change Pod status. + for n, observation := range observations { + if observation.role != zooKeeperLeader { + continue + } + current, err := c.queryZooKeeper(ctx, observation.pod) + if err != nil { + return result("zookeeper", Unknown, err.Error()) + } + if current.role != zooKeeperLeader { + return result("zookeeper", Unknown, "ZooKeeper leadership changed during the checks; repeat the observation") + } + observations[n] = current + } + return evaluateZooKeeper(observations) +} + +func (c *Checker) queryZooKeeper(ctx context.Context, pod string) (zooKeeperObservation, error) { + data, err := c.query(ctx, pod, "zookeeper", []string{"bash", "-ec", zookeeperQuery}) + if err != nil { + return zooKeeperObservation{}, err + } + observation, err := parseZooKeeper(string(data), pod) + if err != nil { + return zooKeeperObservation{}, fmt.Errorf("%s: %w", pod, err) + } + return observation, nil +} + +func parseZooKeeper(data, pod string) (zooKeeperObservation, error) { + fields := make(map[string]string) + for _, line := range strings.Split(data, "\n") { + if strings.TrimSpace(line) == "" { + continue + } + key, value, ok := strings.Cut(line, "\t") + _, duplicate := fields[key] + if !ok || key == "" || duplicate { + return zooKeeperObservation{}, fmt.Errorf("invalid or unavailable ZooKeeper mntr response") + } + fields[key] = strings.TrimSpace(value) + } + if fields["zk_server_state"] == "" { + return zooKeeperObservation{}, fmt.Errorf("missing zk_server_state in mntr response") + } + observation := zooKeeperObservation{pod: pod, role: fields["zk_server_state"], peerState: fields["zk_peer_state"]} + voters, err := zooKeeperNumber(fields, "zk_quorum_size") + if err != nil { + return zooKeeperObservation{}, err + } + observation.voters = voters + if observation.role == zooKeeperLeader { + observation.synced, err = zooKeeperNumber(fields, "zk_synced_followers") + if err != nil { + return zooKeeperObservation{}, err + } + } + return observation, nil +} + +func zooKeeperNumber(fields map[string]string, key string) (int, error) { + value, err := strconv.Atoi(fields[key]) + if err != nil || value < 0 { + return 0, fmt.Errorf("missing or invalid %s in mntr response", key) + } + return value, nil +} + +func evaluateZooKeeper(observations []zooKeeperObservation) Result { + if len(observations) < minZooKeeperVoters { + return result("zookeeper", Degraded, "at least three voting members are required for fault-tolerant ZooKeeper") + } + leaders := 0 + seen := make(map[string]bool) + var problems []string + for _, observation := range observations { + if observation.pod == "" || seen[observation.pod] { + return result("zookeeper", Unknown, "missing or duplicate ZooKeeper member evidence") + } + seen[observation.pod] = true + if observation.voters != len(observations) { + return result("zookeeper", Unknown, fmt.Sprintf("%s reports %d configured voters; %d members discovered", observation.pod, observation.voters, len(observations))) + } + switch observation.role { + case zooKeeperLeader: + leaders++ + if observation.synced != len(observations)-1 { + problems = append(problems, fmt.Sprintf("%s reports %d/%d synchronized followers", observation.pod, observation.synced, len(observations)-1)) + } + case zooKeeperFollower: + default: + problems = append(problems, fmt.Sprintf("%s is %s; expected a voting leader or follower", observation.pod, observation.role)) + } + expectedPeerState := "following - broadcast" + if observation.role == zooKeeperLeader { + expectedPeerState = "leading - broadcast" + } + if observation.peerState != "" && observation.peerState != expectedPeerState { + problems = append(problems, fmt.Sprintf("%s is not in broadcast state: %s", observation.pod, observation.peerState)) + } + } + if leaders != 1 { + problems = append(problems, fmt.Sprintf("expected one ZooKeeper leader; observed %d", leaders)) + } + if len(problems) > 0 { + return Result{Component: "zookeeper", Status: Degraded, Messages: problems} + } + return result("zookeeper", Healthy, fmt.Sprintf("%d voting members available; one leader and %d synchronized followers", len(observations), len(observations)-1)) +} diff --git a/internal/orchestration/replication/zookeeper_test.go b/internal/orchestration/replication/zookeeper_test.go new file mode 100644 index 0000000..dfce640 --- /dev/null +++ b/internal/orchestration/replication/zookeeper_test.go @@ -0,0 +1,147 @@ +package replication + +import ( + "context" + "fmt" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "k8s.io/client-go/kubernetes/fake" +) + +func zooKeeperFixture(role string) string { + data := "zk_version\t3.9.5\nzk_server_state\t" + role + "\nzk_quorum_size\t3\nzk_avg_latency\t0.123\n" + if role == zooKeeperLeader { + return data + "zk_peer_state\tleading - broadcast\nzk_synced_followers\t2\n" + } + return data + "zk_peer_state\tfollowing - broadcast\nzk_synced_observers\tnull\n" +} + +func TestZooKeeperParsing(t *testing.T) { + tests := []struct { + name, data string + valid bool + }{ + {zooKeeperLeader, zooKeeperFixture(zooKeeperLeader), true}, + {zooKeeperFollower, zooKeeperFixture(zooKeeperFollower), true}, + {"optional peer state absent", strings.ReplaceAll(zooKeeperFixture(zooKeeperFollower), "zk_peer_state\tfollowing - broadcast\n", ""), true}, + {"disabled command", "mntr is not executed because it is not in the whitelist.\n", false}, + {"not serving", "This ZooKeeper instance is not currently serving requests\n", false}, + {"empty response", "", false}, + {"missing role", "zk_quorum_size\t3\n", false}, + {"missing membership", "zk_server_state\tfollower\n", false}, + {"missing synchronized count", "zk_server_state\tleader\nzk_quorum_size\t3\n", false}, + {"duplicate role", zooKeeperFixture(zooKeeperLeader) + "zk_server_state\tfollower\n", false}, + {"duplicate empty value", "zk_version\t\n" + zooKeeperFixture(zooKeeperLeader), false}, + {"negative synchronized count", strings.ReplaceAll(zooKeeperFixture(zooKeeperLeader), "zk_synced_followers\t2", "zk_synced_followers\t-1"), false}, + {"invalid membership", strings.ReplaceAll(zooKeeperFixture(zooKeeperFollower), "zk_quorum_size\t3", "zk_quorum_size\tnull"), false}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + observation, err := parseZooKeeper(test.data, "zk-0") + if !test.valid { + require.Error(t, err) + return + } + require.NoError(t, err) + assert.Equal(t, 3, observation.voters) + assert.Equal(t, "zk-0", observation.pod) + }) + } +} + +func TestZooKeeperRequiresFullVotingEnsemble(t *testing.T) { + tests := []struct { + name string + mutate func([]zooKeeperObservation) []zooKeeperObservation + status string + }{ + {"healthy", func(o []zooKeeperObservation) []zooKeeperObservation { return o }, Healthy}, + {"quorum without full recovery", func(o []zooKeeperObservation) []zooKeeperObservation { o[0].synced = 1; return o }, Degraded}, + {"no leader", func(o []zooKeeperObservation) []zooKeeperObservation { o[0].role = zooKeeperFollower; return o }, Degraded}, + {"two leaders", func(o []zooKeeperObservation) []zooKeeperObservation { o[1].role = zooKeeperLeader; return o }, Degraded}, + {"election", func(o []zooKeeperObservation) []zooKeeperObservation { o[1].role = "looking"; return o }, Degraded}, + {"observer", func(o []zooKeeperObservation) []zooKeeperObservation { o[1].role = "observer"; return o }, Degraded}, + {"standalone", func(o []zooKeeperObservation) []zooKeeperObservation { o[1].role = "standalone"; return o }, Degraded}, + {"synchronizing", func(o []zooKeeperObservation) []zooKeeperObservation { + o[1].peerState = "following - synchronization" + return o + }, Degraded}, + {"inconsistent role", func(o []zooKeeperObservation) []zooKeeperObservation { + o[0].peerState = "following - broadcast" + return o + }, Degraded}, + {"unexpected voter configuration", func(o []zooKeeperObservation) []zooKeeperObservation { o[1].voters = 5; return o }, Unknown}, + {"duplicate member", func(o []zooKeeperObservation) []zooKeeperObservation { o[1].pod = o[0].pod; return o }, Unknown}, + {"too few voters", func(o []zooKeeperObservation) []zooKeeperObservation { return o[:2] }, Degraded}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + observations := []zooKeeperObservation{ + {pod: "zk-0", role: zooKeeperLeader, voters: 3, synced: 2}, + {pod: "zk-1", role: zooKeeperFollower, voters: 3}, + {pod: "zk-2", role: zooKeeperFollower, voters: 3}, + } + assert.Equal(t, test.status, evaluateZooKeeper(test.mutate(observations)).Status) + }) + } +} + +func TestZooKeeperQueriesEveryMemberAndRechecksLeader(t *testing.T) { + tests := []struct { + name, status string + finalRole string + finalSynced string + queryError bool + }{ + {name: "healthy", status: Healthy, finalRole: zooKeeperLeader, finalSynced: "2"}, + {name: "election during observation", status: Unknown, finalRole: zooKeeperFollower}, + {name: "follower falls behind during observation", status: Degraded, finalRole: zooKeeperLeader, finalSynced: "1"}, + {name: "query denied", status: Unknown, queryError: true}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + var queried []string + kube := &fakeKubernetes{ + client: fake.NewSimpleClientset(databaseObjects("zookeeper", "zookeeper", "zookeeper", 3)...), + exec: func(_ context.Context, namespace, pod, container string, command []string) ([]byte, error) { + assert.Equal(t, "test", namespace) + assert.Equal(t, "zookeeper", container) + assert.Equal(t, []string{"bash", "-ec", zookeeperQuery}, command) + queried = append(queried, pod) + if test.queryError { + return nil, fmt.Errorf("query denied") + } + if len(queried) == 4 { + return []byte(strings.ReplaceAll(zooKeeperFixture(test.finalRole), "zk_synced_followers\t2", "zk_synced_followers\t"+test.finalSynced)), nil + } + if pod == "zookeeper-0" { + return []byte(zooKeeperFixture(zooKeeperLeader)), nil + } + return []byte(zooKeeperFixture(zooKeeperFollower)), nil + }, + } + options := testOptions() + options.Components = []string{"zookeeper"} + probe, err := New(kube, options) + require.NoError(t, err) + assert.Equal(t, test.status, probe.Check(context.Background()).Status) + if !test.queryError { + assert.Equal(t, []string{"zookeeper-0", "zookeeper-1", "zookeeper-2", "zookeeper-0"}, queried) + } + }) + } +} + +func TestZooKeeperMissingPodCannotPass(t *testing.T) { + objects := databaseObjects("zookeeper", "zookeeper", "zookeeper", 3) + kube := &fakeKubernetes{client: fake.NewSimpleClientset(objects[:len(objects)-1]...)} + options := testOptions() + options.Components = []string{"zookeeper"} + probe, err := New(kube, options) + require.NoError(t, err) + assert.Equal(t, Unknown, probe.Check(context.Background()).Status) + assert.Zero(t, kube.calls) +}