diff --git a/deploy/helm/nvca-operator/nvca-operator/README.md b/deploy/helm/nvca-operator/nvca-operator/README.md index 3292c2dd77..a823bf55dd 100644 --- a/deploy/helm/nvca-operator/nvca-operator/README.md +++ b/deploy/helm/nvca-operator/nvca-operator/README.md @@ -234,6 +234,9 @@ This release does not wire the catalog into backend selection. Runtime use requi | `clusterValidator.image.repository` | Cluster Validator container registry path, without tag | `""` | | `clusterValidator.image.tag` | Cluster Validator container image tag | `v2.0.0` | | `clusterValidator.image.pullPolicy` | K8s ImagePullPolicy for cluster-validator | `IfNotPresent` | +| `clusterValidator.role` | Check set: `control-plane`, or any other value for the GPU checks | `""` | +| `clusterValidator.openBaoNamespace` | Namespace holding OpenBao when it is not `vault-system` | `""` | +| `clusterValidator.envoyGatewayNamespace` | Namespace holding Envoy Gateway when it is not `envoy-gateway-system` | `""` | | `clusterValidator.schedule` | CronJob schedule (cron expression) | `0 */3 * * *` | | `clusterValidator.configMapName` | ConfigMap name for user-defined network checks | `cluster-validator-network-checks` | | `clusterValidator.networkChecks` | Network check configuration (creates the ConfigMap automatically when set) | `{}` | diff --git a/deploy/helm/nvca-operator/nvca-operator/templates/_helpers.tpl b/deploy/helm/nvca-operator/nvca-operator/templates/_helpers.tpl index f592907c6b..b453fa2297 100644 --- a/deploy/helm/nvca-operator/nvca-operator/templates/_helpers.tpl +++ b/deploy/helm/nvca-operator/nvca-operator/templates/_helpers.tpl @@ -418,6 +418,9 @@ Usage: {{- $cv := include "nvcaop.clusterValidatorConfig" . | fromYaml -}} "image" (dict "repository" "" "tag" "" "pullPolicy" "IfNotPresent") "schedule" "0 */3 * * *" "configMapName" "cluster-validator-network-checks" + "role" "" + "openBaoNamespace" "" + "envoyGatewayNamespace" "" "networkChecks" (dict) "resources" (dict "requests" (dict "cpu" "100m" "memory" "64Mi") diff --git a/deploy/helm/nvca-operator/nvca-operator/templates/cronjob.yaml b/deploy/helm/nvca-operator/nvca-operator/templates/cronjob.yaml index aff4ba0a5a..9e13796082 100644 --- a/deploy/helm/nvca-operator/nvca-operator/templates/cronjob.yaml +++ b/deploy/helm/nvca-operator/nvca-operator/templates/cronjob.yaml @@ -80,6 +80,24 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace + # Selects the check set: "control-plane" runs the gateway, + # storage, overlay and HA checks; anything else (including + # unset) runs the compute-plane GPU set. Without this the + # control-plane checks are unreachable from the chart. + - name: VALIDATOR_ROLE + value: {{ $cv.role | quote }} + {{- if $cv.openBaoNamespace }} + # Relocated OpenBao: without this the Tier-2 quorum check + # silently skips its StatefulSet. + - name: NVCF_OPENBAO_NAMESPACE + value: {{ $cv.openBaoNamespace | quote }} + {{- end }} + {{- if $cv.envoyGatewayNamespace }} + # Set when the stack's controllerNamespace differs from the + # Envoy Gateway chart default. + - name: NVCF_ENVOY_GATEWAY_NAMESPACE + value: {{ $cv.envoyGatewayNamespace | quote }} + {{- end }} resources: requests: cpu: {{ $cv.resources.requests.cpu | quote }} diff --git a/deploy/helm/nvca-operator/nvca-operator/templates/deployment.yaml b/deploy/helm/nvca-operator/nvca-operator/templates/deployment.yaml index 4f5f0d5514..2737a65a96 100644 --- a/deploy/helm/nvca-operator/nvca-operator/templates/deployment.yaml +++ b/deploy/helm/nvca-operator/nvca-operator/templates/deployment.yaml @@ -95,6 +95,21 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace + # Same role and namespace wiring as the CronJob. This init container + # writes the same summary ConfigMap, so without it every operator pod + # restart republishes a compute-plane summary over a control-plane + # one: the GPU keys reappear and all the control-plane keys are pruned + # until the next CronJob tick. + - name: VALIDATOR_ROLE + value: {{ $cv.role | quote }} + {{- if $cv.openBaoNamespace }} + - name: NVCF_OPENBAO_NAMESPACE + value: {{ $cv.openBaoNamespace | quote }} + {{- end }} + {{- if $cv.envoyGatewayNamespace }} + - name: NVCF_ENVOY_GATEWAY_NAMESPACE + value: {{ $cv.envoyGatewayNamespace | quote }} + {{- end }} resources: requests: cpu: {{ $cv.resources.requests.cpu | quote }} diff --git a/deploy/helm/nvca-operator/nvca-operator/templates/rbac.yaml b/deploy/helm/nvca-operator/nvca-operator/templates/rbac.yaml index 1aec060d85..5628190508 100644 --- a/deploy/helm/nvca-operator/nvca-operator/templates/rbac.yaml +++ b/deploy/helm/nvca-operator/nvca-operator/templates/rbac.yaml @@ -58,8 +58,15 @@ rules: resourceNames: ["cluster-validator-summary"] verbs: ["update"] - apiGroups: ["apps"] - resources: ["deployments", "daemonsets"] + resources: ["deployments", "statefulsets"] + # statefulsets: the Tier-2 quorum and placement check. Without it every + # control-plane namespace 403s and the critical row is permanently unknown. verbs: ["get", "list"] + - apiGroups: ["apps"] + resources: ["daemonsets"] + # create/delete: the node-to-node overlay probe runs a short-lived + # DaemonSet in a per-run namespace and deletes it in the same run. + verbs: ["get", "list", "create", "delete"] - apiGroups: ["admissionregistration.k8s.io"] resources: - mutatingwebhookconfigurations @@ -69,7 +76,9 @@ rules: resources: ["networkpolicies"] verbs: ["get", "list", "create", "update", "delete"] - apiGroups: ["storage.k8s.io"] - resources: ["csidrivers"] + # storageclasses: the default-StorageClass check. Without it the critical + # row is permanently unknown on every run. + resources: ["csidrivers", "storageclasses"] verbs: ["get", "list"] - apiGroups: ["nvidia.com"] resources: ["clusterpolicies"] diff --git a/deploy/helm/nvca-operator/nvca-operator/templates/role.yaml b/deploy/helm/nvca-operator/nvca-operator/templates/role.yaml index e565043fdd..0f139a05c1 100644 --- a/deploy/helm/nvca-operator/nvca-operator/templates/role.yaml +++ b/deploy/helm/nvca-operator/nvca-operator/templates/role.yaml @@ -114,7 +114,13 @@ rules: verbs: ["get", "list", "watch", "create", "update", "delete", "deletecollection", "patch"] - apiGroups: ["apps"] resources: ["daemonsets"] - verbs: ["get", "list", "watch"] + # create/delete: the cluster-validator init container runs under this + # ServiceAccount and the node-to-node overlay probe creates a short-lived + # DaemonSet in a per-run namespace. Without these the probe 403s, which is + # an unobserved critical check, which fails the verdict and CrashLoops the + # operator pod. Not an escalation: this role already has cluster-wide + # create/delete on namespaces, pods, deployments and statefulsets. + verbs: ["get", "list", "watch", "create", "delete"] - apiGroups: ["scheduling.run.ai"] resources: ["queues"] verbs: ["get", "list", "watch"] diff --git a/deploy/helm/nvca-operator/nvca-operator/values.yaml b/deploy/helm/nvca-operator/nvca-operator/values.yaml index c7b40c18be..9de93a9f07 100644 --- a/deploy/helm/nvca-operator/nvca-operator/values.yaml +++ b/deploy/helm/nvca-operator/nvca-operator/values.yaml @@ -497,6 +497,9 @@ networkPolicy: ## @param clusterValidator.image.repository Cluster Validator container registry path, without tag ## @param clusterValidator.image.tag Cluster Validator container image tag ## @param clusterValidator.image.pullPolicy K8s ImagePullPolicy for cluster-validator +## @param clusterValidator.role Check set to run: "control-plane" for gateway, storage, overlay and HA checks; any other value (including "") runs the compute-plane GPU checks +## @param clusterValidator.openBaoNamespace Namespace holding OpenBao, when it is not vault-system; without it the Tier-2 quorum check skips its StatefulSet +## @param clusterValidator.envoyGatewayNamespace Namespace holding Envoy Gateway, when it is not envoy-gateway-system ## @param clusterValidator.schedule CronJob schedule (cron expression) ## @param clusterValidator.configMapName ConfigMap name for user-defined network checks (reachability + network policy validation) ## @param clusterValidator.resources.limits.cpu CPU limit for the cluster-validator container @@ -506,6 +509,14 @@ networkPolicy: ## @param clusterValidator.networkChecks [object] Network check configuration (creates the ConfigMap automatically when set) clusterValidator: enabled: false + # Which check set the validator runs. "control-plane" enables the gateway, + # StorageClass, node-to-node overlay and Tier-1/Tier-2 HA checks and skips + # the GPU checks. Any other value (including "") runs the compute-plane set. + role: "" + # Set when OpenBao or Envoy Gateway are installed outside their default + # namespaces, so the control-plane checks look in the right place. + openBaoNamespace: "" + envoyGatewayNamespace: "" image: repository: "" tag: "" # defaults to .Chart.AppVersion (same as nvca-operator) diff --git a/src/compute-plane-services/nvca/.gitignore b/src/compute-plane-services/nvca/.gitignore index 3321e3c105..0a144c373a 100644 --- a/src/compute-plane-services/nvca/.gitignore +++ b/src/compute-plane-services/nvca/.gitignore @@ -15,3 +15,11 @@ clusterdump-* /bazel-* .bazel-cache/ .tls/ + +# Locally built command binaries. `go build -o ./cmd/` at the +# subtree root drops a ~60MB artifact next to the source, which is easy to +# sweep into a commit with `git add -A`. +/cluster-validator +/nvca +/nvca-operator +/webhook-server diff --git a/src/compute-plane-services/nvca/cmd/cluster-validator/BUILD.bazel b/src/compute-plane-services/nvca/cmd/cluster-validator/BUILD.bazel index ca27c79dcd..6d6af5482a 100644 --- a/src/compute-plane-services/nvca/cmd/cluster-validator/BUILD.bazel +++ b/src/compute-plane-services/nvca/cmd/cluster-validator/BUILD.bazel @@ -42,4 +42,7 @@ go_test( name = "cluster-validator_test", srcs = ["main_test.go"], embed = [":cluster-validator_lib"], + deps = [ + "//src/compute-plane-services/nvca/internal/clustervalidator", + ], ) diff --git a/src/compute-plane-services/nvca/cmd/cluster-validator/main.go b/src/compute-plane-services/nvca/cmd/cluster-validator/main.go index c479dfc767..34de6c77c7 100644 --- a/src/compute-plane-services/nvca/cmd/cluster-validator/main.go +++ b/src/compute-plane-services/nvca/cmd/cluster-validator/main.go @@ -72,11 +72,35 @@ func main() { clustervalidator.SummaryConfigMapNamespaceEnv) } - if err := clustervalidator.Run(ctx, client, configNS, configName, summaryNS, emitMetrics); err != nil { + // VALIDATOR_ROLE selects which check set runs: "control-plane" enables + // gateway and StorageClass checks and skips GPU/SMB; anything else (including + // unset) runs the compute-plane check set (backward-compatible default). + roleEnv := os.Getenv("VALIDATOR_ROLE") + role, roleKnown := parseRole(roleEnv) + if roleEnv != "" && !roleKnown { + log.Warnf("VALIDATOR_ROLE=%q is not recognized; defaulting to compute-plane", roleEnv) + } + + if err := clustervalidator.Run(ctx, client, configNS, configName, summaryNS, emitMetrics, role); err != nil { log.WithError(err).Fatal("Cluster validation failed") } } +// parseRole normalizes the VALIDATOR_ROLE env value. Returns the matching +// clustervalidator.Role constant and true for "control-plane" or +// "compute-plane"; returns the compute-plane default and false for any other +// value so unknown inputs are safe. +func parseRole(v string) (clustervalidator.Role, bool) { + switch strings.ToLower(strings.TrimSpace(v)) { + case string(clustervalidator.RoleControlPlane): + return clustervalidator.RoleControlPlane, true + case string(clustervalidator.RoleComputePlane): + return clustervalidator.RoleComputePlane, true + default: + return clustervalidator.RoleComputePlane, false + } +} + // preflightMode reports whether this is a one-shot preflight run (e.g. nvcf-cli, // before NVCA is installed), which skips the summary write. Read from an env // (not a flag) so an unknown value is ignored rather than crashing arg parsing. diff --git a/src/compute-plane-services/nvca/cmd/cluster-validator/main_test.go b/src/compute-plane-services/nvca/cmd/cluster-validator/main_test.go index 29c5a82130..c1a0394651 100644 --- a/src/compute-plane-services/nvca/cmd/cluster-validator/main_test.go +++ b/src/compute-plane-services/nvca/cmd/cluster-validator/main_test.go @@ -17,7 +17,37 @@ limitations under the License. package main -import "testing" +import ( + "testing" + + "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/internal/clustervalidator" +) + +func TestParseRole(t *testing.T) { + tests := []struct { + in string + want clustervalidator.Role + wantKnown bool + }{ + // Known roles are normalized and reported as known. + {"control-plane", clustervalidator.RoleControlPlane, true}, + {"CONTROL-PLANE", clustervalidator.RoleControlPlane, true}, + {" control-plane ", clustervalidator.RoleControlPlane, true}, + {"compute-plane", clustervalidator.RoleComputePlane, true}, + {"COMPUTE-PLANE", clustervalidator.RoleComputePlane, true}, + // Unknown values fall back to compute-plane and are reported as unknown. + {"", clustervalidator.RoleComputePlane, false}, + {"gpu", clustervalidator.RoleComputePlane, false}, + {"both", clustervalidator.RoleComputePlane, false}, + {"control_plane", clustervalidator.RoleComputePlane, false}, // underscore, not hyphen + } + for _, tt := range tests { + got, gotKnown := parseRole(tt.in) + if got != tt.want || gotKnown != tt.wantKnown { + t.Errorf("parseRole(%q) = (%q, %v), want (%q, %v)", tt.in, got, gotKnown, tt.want, tt.wantKnown) + } + } +} func TestPreflightMode(t *testing.T) { tests := []struct { diff --git a/src/compute-plane-services/nvca/deployments/nvca-operator/README.md b/src/compute-plane-services/nvca/deployments/nvca-operator/README.md index 2d4607bf2b..3651a3f07f 100644 --- a/src/compute-plane-services/nvca/deployments/nvca-operator/README.md +++ b/src/compute-plane-services/nvca/deployments/nvca-operator/README.md @@ -234,6 +234,9 @@ This release does not wire the catalog into backend selection. Runtime use requi | `clusterValidator.image.repository` | Cluster Validator container registry path, without tag | `""` | | `clusterValidator.image.tag` | Cluster Validator container image tag | `v2.0.0` | | `clusterValidator.image.pullPolicy` | K8s ImagePullPolicy for cluster-validator | `IfNotPresent` | +| `clusterValidator.role` | Check set: `control-plane`, or any other value for the GPU checks | `""` | +| `clusterValidator.openBaoNamespace` | Namespace holding OpenBao when it is not `vault-system` | `""` | +| `clusterValidator.envoyGatewayNamespace` | Namespace holding Envoy Gateway when it is not `envoy-gateway-system` | `""` | | `clusterValidator.schedule` | CronJob schedule (cron expression) | `0 */3 * * *` | | `clusterValidator.configMapName` | ConfigMap name for user-defined network checks | `cluster-validator-network-checks` | | `clusterValidator.networkChecks` | Network check configuration (creates the ConfigMap automatically when set) | `{}` | diff --git a/src/compute-plane-services/nvca/deployments/nvca-operator/templates/_helpers.tpl b/src/compute-plane-services/nvca/deployments/nvca-operator/templates/_helpers.tpl index f592907c6b..b453fa2297 100644 --- a/src/compute-plane-services/nvca/deployments/nvca-operator/templates/_helpers.tpl +++ b/src/compute-plane-services/nvca/deployments/nvca-operator/templates/_helpers.tpl @@ -418,6 +418,9 @@ Usage: {{- $cv := include "nvcaop.clusterValidatorConfig" . | fromYaml -}} "image" (dict "repository" "" "tag" "" "pullPolicy" "IfNotPresent") "schedule" "0 */3 * * *" "configMapName" "cluster-validator-network-checks" + "role" "" + "openBaoNamespace" "" + "envoyGatewayNamespace" "" "networkChecks" (dict) "resources" (dict "requests" (dict "cpu" "100m" "memory" "64Mi") diff --git a/src/compute-plane-services/nvca/deployments/nvca-operator/templates/cronjob.yaml b/src/compute-plane-services/nvca/deployments/nvca-operator/templates/cronjob.yaml index aff4ba0a5a..9e13796082 100644 --- a/src/compute-plane-services/nvca/deployments/nvca-operator/templates/cronjob.yaml +++ b/src/compute-plane-services/nvca/deployments/nvca-operator/templates/cronjob.yaml @@ -80,6 +80,24 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace + # Selects the check set: "control-plane" runs the gateway, + # storage, overlay and HA checks; anything else (including + # unset) runs the compute-plane GPU set. Without this the + # control-plane checks are unreachable from the chart. + - name: VALIDATOR_ROLE + value: {{ $cv.role | quote }} + {{- if $cv.openBaoNamespace }} + # Relocated OpenBao: without this the Tier-2 quorum check + # silently skips its StatefulSet. + - name: NVCF_OPENBAO_NAMESPACE + value: {{ $cv.openBaoNamespace | quote }} + {{- end }} + {{- if $cv.envoyGatewayNamespace }} + # Set when the stack's controllerNamespace differs from the + # Envoy Gateway chart default. + - name: NVCF_ENVOY_GATEWAY_NAMESPACE + value: {{ $cv.envoyGatewayNamespace | quote }} + {{- end }} resources: requests: cpu: {{ $cv.resources.requests.cpu | quote }} diff --git a/src/compute-plane-services/nvca/deployments/nvca-operator/templates/deployment.yaml b/src/compute-plane-services/nvca/deployments/nvca-operator/templates/deployment.yaml index 4f5f0d5514..2737a65a96 100644 --- a/src/compute-plane-services/nvca/deployments/nvca-operator/templates/deployment.yaml +++ b/src/compute-plane-services/nvca/deployments/nvca-operator/templates/deployment.yaml @@ -95,6 +95,21 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace + # Same role and namespace wiring as the CronJob. This init container + # writes the same summary ConfigMap, so without it every operator pod + # restart republishes a compute-plane summary over a control-plane + # one: the GPU keys reappear and all the control-plane keys are pruned + # until the next CronJob tick. + - name: VALIDATOR_ROLE + value: {{ $cv.role | quote }} + {{- if $cv.openBaoNamespace }} + - name: NVCF_OPENBAO_NAMESPACE + value: {{ $cv.openBaoNamespace | quote }} + {{- end }} + {{- if $cv.envoyGatewayNamespace }} + - name: NVCF_ENVOY_GATEWAY_NAMESPACE + value: {{ $cv.envoyGatewayNamespace | quote }} + {{- end }} resources: requests: cpu: {{ $cv.resources.requests.cpu | quote }} diff --git a/src/compute-plane-services/nvca/deployments/nvca-operator/templates/rbac.yaml b/src/compute-plane-services/nvca/deployments/nvca-operator/templates/rbac.yaml index 1aec060d85..5628190508 100644 --- a/src/compute-plane-services/nvca/deployments/nvca-operator/templates/rbac.yaml +++ b/src/compute-plane-services/nvca/deployments/nvca-operator/templates/rbac.yaml @@ -58,8 +58,15 @@ rules: resourceNames: ["cluster-validator-summary"] verbs: ["update"] - apiGroups: ["apps"] - resources: ["deployments", "daemonsets"] + resources: ["deployments", "statefulsets"] + # statefulsets: the Tier-2 quorum and placement check. Without it every + # control-plane namespace 403s and the critical row is permanently unknown. verbs: ["get", "list"] + - apiGroups: ["apps"] + resources: ["daemonsets"] + # create/delete: the node-to-node overlay probe runs a short-lived + # DaemonSet in a per-run namespace and deletes it in the same run. + verbs: ["get", "list", "create", "delete"] - apiGroups: ["admissionregistration.k8s.io"] resources: - mutatingwebhookconfigurations @@ -69,7 +76,9 @@ rules: resources: ["networkpolicies"] verbs: ["get", "list", "create", "update", "delete"] - apiGroups: ["storage.k8s.io"] - resources: ["csidrivers"] + # storageclasses: the default-StorageClass check. Without it the critical + # row is permanently unknown on every run. + resources: ["csidrivers", "storageclasses"] verbs: ["get", "list"] - apiGroups: ["nvidia.com"] resources: ["clusterpolicies"] diff --git a/src/compute-plane-services/nvca/deployments/nvca-operator/templates/role.yaml b/src/compute-plane-services/nvca/deployments/nvca-operator/templates/role.yaml index e565043fdd..0f139a05c1 100644 --- a/src/compute-plane-services/nvca/deployments/nvca-operator/templates/role.yaml +++ b/src/compute-plane-services/nvca/deployments/nvca-operator/templates/role.yaml @@ -114,7 +114,13 @@ rules: verbs: ["get", "list", "watch", "create", "update", "delete", "deletecollection", "patch"] - apiGroups: ["apps"] resources: ["daemonsets"] - verbs: ["get", "list", "watch"] + # create/delete: the cluster-validator init container runs under this + # ServiceAccount and the node-to-node overlay probe creates a short-lived + # DaemonSet in a per-run namespace. Without these the probe 403s, which is + # an unobserved critical check, which fails the verdict and CrashLoops the + # operator pod. Not an escalation: this role already has cluster-wide + # create/delete on namespaces, pods, deployments and statefulsets. + verbs: ["get", "list", "watch", "create", "delete"] - apiGroups: ["scheduling.run.ai"] resources: ["queues"] verbs: ["get", "list", "watch"] diff --git a/src/compute-plane-services/nvca/deployments/nvca-operator/values.yaml b/src/compute-plane-services/nvca/deployments/nvca-operator/values.yaml index 73a3c510fb..c13d3e3e61 100644 --- a/src/compute-plane-services/nvca/deployments/nvca-operator/values.yaml +++ b/src/compute-plane-services/nvca/deployments/nvca-operator/values.yaml @@ -528,6 +528,9 @@ networkPolicy: ## @param clusterValidator.image.repository Cluster Validator container registry path, without tag ## @param clusterValidator.image.tag Cluster Validator container image tag ## @param clusterValidator.image.pullPolicy K8s ImagePullPolicy for cluster-validator +## @param clusterValidator.role Check set to run: "control-plane" for gateway, storage, overlay and HA checks; any other value (including "") runs the compute-plane GPU checks +## @param clusterValidator.openBaoNamespace Namespace holding OpenBao, when it is not vault-system; without it the Tier-2 quorum check skips its StatefulSet +## @param clusterValidator.envoyGatewayNamespace Namespace holding Envoy Gateway, when it is not envoy-gateway-system ## @param clusterValidator.schedule CronJob schedule (cron expression) ## @param clusterValidator.configMapName ConfigMap name for user-defined network checks (reachability + network policy validation) ## @param clusterValidator.resources.limits.cpu CPU limit for the cluster-validator container @@ -537,6 +540,14 @@ networkPolicy: ## @param clusterValidator.networkChecks [object] Network check configuration (creates the ConfigMap automatically when set) clusterValidator: enabled: false + # Which check set the validator runs. "control-plane" enables the gateway, + # StorageClass, node-to-node overlay and Tier-1/Tier-2 HA checks and skips + # the GPU checks. Any other value (including "") runs the compute-plane set. + role: "" + # Set when OpenBao or Envoy Gateway are installed outside their default + # namespaces, so the control-plane checks look in the right place. + openBaoNamespace: "" + envoyGatewayNamespace: "" image: repository: "" tag: "" # defaults to .Chart.AppVersion (same as nvca-operator) diff --git a/src/compute-plane-services/nvca/internal/clustervalidator/BUILD.bazel b/src/compute-plane-services/nvca/internal/clustervalidator/BUILD.bazel index 92465815c7..b1c6b77722 100644 --- a/src/compute-plane-services/nvca/internal/clustervalidator/BUILD.bazel +++ b/src/compute-plane-services/nvca/internal/clustervalidator/BUILD.bazel @@ -20,12 +20,14 @@ go_library( deps = [ "//src/compute-plane-services/nvca/vendor/github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/core", "//src/compute-plane-services/nvca/vendor/github.com/sirupsen/logrus", + "//src/compute-plane-services/nvca/vendor/k8s.io/api/apps/v1:apps", "//src/compute-plane-services/nvca/vendor/k8s.io/api/core/v1:core", "//src/compute-plane-services/nvca/vendor/k8s.io/api/networking/v1:networking", "//src/compute-plane-services/nvca/vendor/k8s.io/apimachinery/pkg/api/errors", "//src/compute-plane-services/nvca/vendor/k8s.io/apimachinery/pkg/api/resource", "//src/compute-plane-services/nvca/vendor/k8s.io/apimachinery/pkg/apis/meta/v1:meta", "//src/compute-plane-services/nvca/vendor/k8s.io/apimachinery/pkg/util/intstr", + "//src/compute-plane-services/nvca/vendor/k8s.io/apimachinery/pkg/util/rand", "//src/compute-plane-services/nvca/vendor/k8s.io/client-go/discovery", "//src/compute-plane-services/nvca/vendor/k8s.io/client-go/kubernetes", "//src/compute-plane-services/nvca/vendor/sigs.k8s.io/yaml", @@ -41,8 +43,10 @@ alias( go_test( name = "clustervalidator_test", srcs = [ + "checks_controlplane_test.go", "checks_test.go", "config_test.go", + "connectivity_test.go", "enforcement_test.go", "netpol_test.go", "output_test.go", @@ -63,7 +67,10 @@ go_test( "//src/compute-plane-services/nvca/vendor/k8s.io/apimachinery/pkg/api/resource", "//src/compute-plane-services/nvca/vendor/k8s.io/apimachinery/pkg/apis/meta/v1:meta", "//src/compute-plane-services/nvca/vendor/k8s.io/apimachinery/pkg/runtime", + "//src/compute-plane-services/nvca/vendor/k8s.io/apimachinery/pkg/runtime/schema", + "//src/compute-plane-services/nvca/vendor/k8s.io/apimachinery/pkg/types", "//src/compute-plane-services/nvca/vendor/k8s.io/apimachinery/pkg/util/intstr", + "//src/compute-plane-services/nvca/vendor/k8s.io/client-go/discovery/fake", "//src/compute-plane-services/nvca/vendor/k8s.io/client-go/kubernetes", "//src/compute-plane-services/nvca/vendor/k8s.io/client-go/kubernetes/fake", "//src/compute-plane-services/nvca/vendor/k8s.io/client-go/rest", diff --git a/src/compute-plane-services/nvca/internal/clustervalidator/checks.go b/src/compute-plane-services/nvca/internal/clustervalidator/checks.go index 75c2cc031d..cd8fd8e508 100644 --- a/src/compute-plane-services/nvca/internal/clustervalidator/checks.go +++ b/src/compute-plane-services/nvca/internal/clustervalidator/checks.go @@ -21,14 +21,19 @@ import ( "context" "errors" "fmt" + "os" "regexp" "sort" "strconv" "strings" + "time" + "github.com/sirupsen/logrus" + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/rand" "k8s.io/client-go/discovery" "k8s.io/client-go/kubernetes" ) @@ -112,18 +117,8 @@ func summarizeContainerRuntimes(nodes []corev1.Node) string { return strings.Join(parts, ", ") } -// checkControlPlaneHealth verifies cluster health using three signals: -// 1. /readyz — canonical API-server health (works on every distribution). -// 2. Data-plane capabilities — DNS resolution of kubernetes.default.svc -// and HTTPS routing to kubernetes.default.svc/readyz via the in-cluster -// ClusterIP. Both must succeed; pod-presence detection (CoreDNS vs -// kube-dns, kube-proxy vs Cilium vs OVN-Kubernetes vs k3s-embedded) -// is diagnostic only and does not affect the verdict. -// 3. Control-plane pods (kube-apiserver, etcd, scheduler, controller-manager) -// — informational only. Visible on self-hosted, hidden on managed K8s -// (EKS, GKE, AKS) where the cloud provider runs them. /readyz already -// covers their health. -// +// checkControlPlaneHealth verifies /readyz, in-cluster DNS, and service routing. +// Control-plane pod presence is informational only; /readyz is authoritative. // NotReady worker nodes are Warning only (non-blocking). func checkControlPlaneHealth(ctx context.Context, client kubernetes.Interface, state *ValidationState) { log := state.Log @@ -310,16 +305,9 @@ var ( probeAPIServiceIPFn = probeKubernetesAPIServiceIP ) -// detectDNSProvider inspects kube-system pods and returns a short name for -// the cluster's DNS provider when recognised. Diagnostic only — the -// authoritative DNS health signal comes from probeInClusterDNS. -// -// Known providers: -// - CoreDNS: pod prefix "coredns" (vanilla, kubeadm, EKS, AKS, k3s) -// - kube-dns: pod prefix "kube-dns" (GKE's managed default) -// - OpenShift DNS: namespace openshift-dns hosts dns-default-*; this -// function only sees kube-system pods, so OpenShift returns "" here -// and the capability probe is authoritative. +// detectDNSProvider inspects kube-system pods and returns a short provider +// name (CoreDNS, kube-dns) when recognised. Diagnostic only; the authoritative +// DNS health signal comes from probeInClusterDNS. func detectDNSProvider(pods []corev1.Pod) string { switch { case countRunningPods(pods, "coredns") > 0: @@ -330,16 +318,9 @@ func detectDNSProvider(pods []corev1.Pod) string { return "" } -// detectServiceRoutingImpl inspects the K8s version and kube-system pods -// to identify the cluster's kube-proxy implementation. Diagnostic only — -// the authoritative routing health signal comes from -// probeKubernetesAPIServiceIP. -// -// Recognised implementations: -// - kube-proxy DaemonSet (vanilla / kubeadm / EKS / AKS / GKE classic) -// - kube-proxy embedded in the server binary (k3s / rke2) -// - Cilium with kubeProxyReplacement (GKE Dataplane V2, custom Cilium) -// - OVN-Kubernetes (OpenShift 4.x default) +// detectServiceRoutingImpl inspects K8s version and kube-system pods to +// identify the kube-proxy implementation (DaemonSet, k3s/rke2 embedded, +// Cilium, OVN-Kubernetes). Diagnostic only; probeKubernetesAPIServiceIP is authoritative. func detectServiceRoutingImpl(k8sVersion string, pods []corev1.Pod) string { switch { case isEmbeddedKubeProxyDistro(k8sVersion): @@ -833,6 +814,1476 @@ func checkGPUOperator(ctx context.Context, client kubernetes.Interface, state *V } } +// checkStorageClass verifies that a default StorageClass is present. NVCF +// workloads use PersistentVolumeClaims; without a default StorageClass those +// claims remain unbound and workloads fail to start. Critical for both +// control-plane (operator chart) and compute-plane (model cache), but surfaced +// here for the control-plane validator role. +func checkStorageClass(ctx context.Context, client kubernetes.Interface, state *ValidationState) { + log := state.Log + printHeader(log, "Default StorageClass") + + classes, err := client.StorageV1().StorageClasses().List(ctx, metav1.ListOptions{}) + if err != nil { + // Leave DefaultStorageClassOK nil (unknown) so the summary row is + // omitted rather than reported as "Not Found" — an API error is not + // confirmation that no default StorageClass exists. + printWarning(log, fmt.Sprintf("Could not list StorageClasses: %v", err)) + state.Warnings = append(state.Warnings, "Default StorageClass: status unknown (listing failed)") + return + } + + // Collect every default rather than stopping at the first. Two classes + // annotated is-default-class reject all PVCs on Kubernetes <1.26, and on + // >=1.26 the apiserver picks the newest, which need not be the one listed + // first here. + var defaults []string + for _, sc := range classes.Items { + if sc.Annotations["storageclass.kubernetes.io/is-default-class"] == "true" || + sc.Annotations["storageclass.beta.kubernetes.io/is-default-class"] == "true" { + defaults = append(defaults, sc.Name) + } + } + + if len(defaults) > 1 { + const multiDefaultTolerated = "1.26.0" + recommendation := "Exactly one StorageClass may be marked default. Clear the annotation on the extras with: " + + "kubectl patch storageclass -p " + + `'{"metadata":{"annotations":{"storageclass.kubernetes.io/is-default-class":"false"}}}'` + + // From 1.26 the apiserver resolves the ambiguity by picking the most + // recently created default, so PVCs still bind. Failing the critical + // check there would report NVCF-Not-Ready on a cluster that works, + // which mid-CSI-migration clusters (gp2 plus gp3) hit routinely. + if versionGTE(state.K8sVersion, multiDefaultTolerated) { + msg := fmt.Sprintf( + "Multiple default StorageClasses found (%s); Kubernetes >= %s binds PVCs with the newest, but the extras should be cleared", + strings.Join(defaults, ", "), multiDefaultTolerated) + printWarning(log, msg) + state.Warnings = append(state.Warnings, "Default StorageClass: "+msg) + state.Recommendations = append(state.Recommendations, recommendation) + ok := true + state.DefaultStorageClassOK = &ok + return + } + + printError(log, fmt.Sprintf("Multiple default StorageClasses found (%s); PVCs may fail to bind", + strings.Join(defaults, ", "))) + state.Recommendations = append(state.Recommendations, recommendation) + ok := false + state.DefaultStorageClassOK = &ok + return + } + + var defaultClass string + if len(defaults) == 1 { + defaultClass = defaults[0] + } + + if defaultClass == "" { + printError(log, fmt.Sprintf("No default StorageClass found (%d classes present, none marked as default)", len(classes.Items))) + state.Recommendations = append(state.Recommendations, + "Mark a StorageClass as default with: "+ + "kubectl patch storageclass -p '{\"metadata\":{\"annotations\":{\"storageclass.kubernetes.io/is-default-class\":\"true\"}}}'") + ok := false + state.DefaultStorageClassOK = &ok + return + } + + printSuccess(log, fmt.Sprintf("Default StorageClass: %s", defaultClass)) + ok := true + state.DefaultStorageClassOK = &ok +} + +const ( + gatewayAPIGroup = "gateway.networking.k8s.io" + // envoyGatewayNamespace is the namespace created by the Envoy Gateway Helm + // chart by default. The stack exposes controllerNamespace with no default, + // so an install can legitimately place it elsewhere: use + // envoyGatewayNamespaceName rather than this constant directly. + envoyGatewayNamespace = "envoy-gateway-system" + // envoyGatewayNamespaceEnv overrides the namespace for installs that set + // the stack's controllerNamespace to something other than the default. + // Without it, both Envoy checks probe a namespace that does not exist and + // report a live gateway as missing. + envoyGatewayNamespaceEnv = "NVCF_ENVOY_GATEWAY_NAMESPACE" + // envoyGatewayControllerSelector matches the controller Deployment's pods + // only, excluding the data-plane proxies and certgen Job in the same namespace. + envoyGatewayControllerSelector = "control-plane=envoy-gateway" +) + +// gatewayRouteRequirements are the route types this repo's own charts apply, +// each at the exact apiVersion the manifests declare (deploy/helm/gateway-routes). +// The version is part of the requirement: a CRD served only under some other +// version still fails the Helm apply, so checking the bare resource name would +// pass a cluster the stack cannot actually install on. +var gatewayRouteRequirements = []struct{ groupVersion, resource string }{ + {gatewayAPIGroup + "/v1", "httproutes"}, + {gatewayAPIGroup + "/v1", "grpcroutes"}, + // TCPRoute ships only in the Gateway API experimental channel, but the + // chart renders one by default (routes.grpc.enabled is true), so a + // standard-channel install genuinely cannot apply the stack. + {gatewayAPIGroup + "/v1alpha2", "tcproutes"}, + {gatewayAPIGroup + "/v1beta1", "referencegrants"}, +} + +// gatewayOptionalRouteRequirements are route types the charts render only when +// an opt-in feature is enabled, so their absence is not a reason to fail a +// cluster that never turns that feature on. Reported by the non-critical +// checkGatewayRoutes with the feature named, rather than by the critical CRD +// check: udproute-llm-worker.yaml and referencegrant-llm-worker.yaml render a +// UDPRoute whenever routes.llmWorker.enabled, which defaults to false. +var gatewayOptionalRouteRequirements = []struct{ groupVersion, resource, enabledBy string }{ + {gatewayAPIGroup + "/v1alpha2", "udproutes", "nvcfGatewayRoutes.routes.llmWorker.enabled"}, +} + +// gatewayControllerResources are created by the Envoy Gateway chart rather than +// by this repo, so which version it picks is not ours to pin. Presence under +// any served version is all this check can assert. +var gatewayControllerResources = []string{"gatewayclasses", "gateways"} + +// gatewayAPISurface is what the apiserver serves under gateway.networking.k8s.io. +type gatewayAPISurface struct { + // byGroupVersion is keyed "/", for example + // "gateway.networking.k8s.io/v1/httproutes". + byGroupVersion map[string]bool + // anyVersion holds resource names served under at least one version. + anyVersion map[string]bool +} + +func (s gatewayAPISurface) hasPair(groupVersion, resource string) bool { + return s.byGroupVersion[groupVersion+"/"+resource] +} + +// discoverGatewayAPIResources walks every served version of +// gateway.networking.k8s.io and records what it finds, keeping the version so +// callers can require an exact pair where the charts pin one. +func discoverGatewayAPIResources(client kubernetes.Interface) (gatewayAPISurface, error) { + surface := gatewayAPISurface{ + byGroupVersion: make(map[string]bool), + anyVersion: make(map[string]bool), + } + groups, err := client.Discovery().ServerGroups() + if err != nil { + return surface, err + } + for _, g := range groups.Groups { + if g.Name != gatewayAPIGroup { + continue + } + for _, v := range g.Versions { + resources, err := client.Discovery().ServerResourcesForGroupVersion(v.GroupVersion) + if err != nil { + // The group exists, so a failure here is an API problem, not an + // absent resource. Swallowing it would leave the surface + // incomplete and report the CRDs as missing on a healthy cluster. + return surface, fmt.Errorf("listing resources for %s: %w", v.GroupVersion, err) + } + for _, r := range resources.APIResources { + surface.byGroupVersion[v.GroupVersion+"/"+r.Name] = true + surface.anyVersion[r.Name] = true + } + } + } + return surface, nil +} + +// checkGatewayAPICRDs verifies that the Gateway API CRD set is installed and +// registers all four required resource types. Without these CRDs neither the +// Gateway controller nor nvcf-cli can create routing objects. +func checkGatewayAPICRDs(_ context.Context, client kubernetes.Interface, state *ValidationState) { + log := state.Log + printHeader(log, "Gateway API CRDs") + + surface, err := discoverGatewayAPIResources(client) + if err != nil { + // Leave the pointer nil: discovery failure is not evidence the CRDs + // are absent, and this row is critical. + printWarning(log, fmt.Sprintf("Could not discover Gateway API resources: %v", err)) + state.Warnings = append(state.Warnings, + "Gateway API CRDs: status unknown (API group discovery failed)") + return + } + + var missing []string + for _, r := range gatewayControllerResources { + if !surface.anyVersion[r] { + missing = append(missing, r) + } + } + for _, req := range gatewayRouteRequirements { + if !surface.hasPair(req.groupVersion, req.resource) { + missing = append(missing, req.groupVersion+"/"+req.resource) + } + } + if len(missing) > 0 { + printError(log, fmt.Sprintf("Gateway API CRDs missing resources: %s", strings.Join(missing, ", "))) + // Name the channel rather than pointing back at the installer: TCPRoute + // and UDPRoute exist only in the experimental channel, so an operator + // who ran the standard-channel manifest needs to know that is the + // difference, not to re-run the install that just failed. + state.Recommendations = append(state.Recommendations, + "Install the Gateway API CRDs from the experimental channel, which is the only one carrying "+ + "TCPRoute and UDPRoute: kubectl apply -f "+ + "https://github.com/kubernetes-sigs/gateway-api/releases/download//experimental-install.yaml") + ok := false + state.GatewayAPICRDsOK = &ok + return + } + + printSuccess(log, fmt.Sprintf("Gateway API CRDs installed: %s plus the route types the stack applies", + strings.Join(gatewayControllerResources, ", "))) + ok := true + state.GatewayAPICRDsOK = &ok +} + +// checkEnvoyGateway verifies the Envoy Gateway controller is installed and has +// at least one running pod in the envoy-gateway-system namespace. Without a +// running gateway controller, Gateway and HTTPRoute objects are never reconciled +// and no traffic reaches NVCF services. +func checkEnvoyGateway(ctx context.Context, client kubernetes.Interface, state *ValidationState) { + log := state.Log + printHeader(log, "Envoy Gateway") + + envoyNS := envoyGatewayNamespaceName() + _, err := client.CoreV1().Namespaces().Get(ctx, envoyNS, metav1.GetOptions{}) + if err != nil { + // Only NotFound is evidence that Envoy is absent. A 403 or an apiserver + // 500 means we never observed it, so leave the pointer nil and warn, + // matching every sibling control-plane check. + if !apierrors.IsNotFound(err) { + msg := fmt.Sprintf("Could not check Envoy Gateway namespace %s: %v", envoyNS, err) + printWarning(log, msg) + state.Warnings = append(state.Warnings, "Envoy Gateway: status unknown ("+msg+")") + return + } + printError(log, fmt.Sprintf("Envoy Gateway namespace %s not found", envoyNS)) + state.Recommendations = append(state.Recommendations, + "Install Envoy Gateway via the NVCF self-managed stack (nvcf-cli up) or "+ + "helm install eg oci://docker.io/envoyproxy/gateway-helm -n envoy-gateway-system --create-namespace") + ok := false + state.EnvoyGatewayOK = &ok + return + } + + // Select on the controller label: the same namespace also holds the + // envoy--- data-plane proxies and the certgen Job pod, and + // counting those lets a dead controller pass. + pods, err := client.CoreV1().Pods(envoyNS).List(ctx, metav1.ListOptions{ + LabelSelector: envoyGatewayControllerSelector, + }) + if err != nil { + // Same reasoning as the namespace Get above: a List failure is not + // evidence that no controller is running. + msg := fmt.Sprintf("Could not list Envoy Gateway pods in %s: %v", envoyNS, err) + printWarning(log, msg) + state.Warnings = append(state.Warnings, "Envoy Gateway: status unknown ("+msg+")") + return + } + + // Require Ready, not Running: .status.phase stays Running throughout + // CrashLoopBackOff, so a crash-looping controller counts as healthy. + ready := 0 + for i := range pods.Items { + if isPodReady(&pods.Items[i]) { + ready++ + } + } + log.Infof(" Controller pods in %s: %d total, %d ready", envoyNS, len(pods.Items), ready) + + if ready == 0 { + printError(log, fmt.Sprintf("No Ready Envoy Gateway controller pods in %s (%d found)", + envoyNS, len(pods.Items))) + ok := false + state.EnvoyGatewayOK = &ok + return + } + + printSuccess(log, fmt.Sprintf("Envoy Gateway: %d controller pod(s) Ready in %s", ready, envoyNS)) + ok := true + state.EnvoyGatewayOK = &ok +} + +// checkGatewayRoutes verifies that the route CR types NVCF creates are +// registered with the apiserver. It does discovery only: it does not list +// route objects, so it cannot tell whether any route actually exists. +// +// Non-critical: route CR types are installed by nvcf up and are expected to +// be absent on a fresh cluster before install. +func checkGatewayRoutes(_ context.Context, client kubernetes.Interface, state *ValidationState) { + log := state.Log + printHeader(log, "Gateway Route CR Types") + + surface, err := discoverGatewayAPIResources(client) + if err != nil { + // Leave the pointer nil: a discovery failure is not evidence that the + // route CR types are absent. + printWarning(log, fmt.Sprintf("Could not discover Gateway API resources: %v", err)) + state.Warnings = append(state.Warnings, + "Gateway Route CR Types: status unknown (API group discovery failed)") + return + } + + // Only the opt-in route types are assessed here. The mandatory set is the + // critical checkGatewayAPICRDs' job, and duplicating it would pay a second + // discovery walk to compute a row that can never differ from that one. + var missing, present []string + for _, req := range gatewayOptionalRouteRequirements { + pair := req.groupVersion + "/" + req.resource + if surface.hasPair(req.groupVersion, req.resource) { + present = append(present, pair) + continue + } + missing = append(missing, pair+" (needed when "+req.enabledBy+")") + } + + if len(missing) > 0 { + printWarning(log, fmt.Sprintf("Optional route CR types not registered: %s", + strings.Join(missing, ", "))) + state.Warnings = append(state.Warnings, + "Gateway Route CR Types: optional types absent ("+strings.Join(missing, ", ")+ + "); install the Gateway API experimental channel before enabling those routes") + ok := false + state.GatewayRoutesOK = &ok + return + } + + printSuccess(log, "Optional route CR types registered: "+strings.Join(present, ", ")) + ok := true + state.GatewayRoutesOK = &ok +} + +// checkExternalLoadBalancer performs a passive check: it lists Services of type +// LoadBalancer in the gateway namespace and looks for one with a populated +// .status.loadBalancer.ingress. A populated ingress means a load balancer +// controller (cloud LB, MetalLB, etc.) is active and assigned an IP or hostname. +// +// Non-critical: the passive form only detects an existing LB service; it does +// not create a probe service, so absence means either no LB service exists yet +// or no LB controller is installed. +func checkExternalLoadBalancer(ctx context.Context, client kubernetes.Interface, state *ValidationState) { + log := state.Log + printHeader(log, "External Load Balancer") + + // Scope to the gateway namespace. An unscoped list is satisfied by any + // LoadBalancer anywhere (ingress-nginx, a demo app), which masks the NVCF + // gateway's own Service sitting at on an exhausted address pool. + envoyNS := envoyGatewayNamespaceName() + services, err := client.CoreV1().Services(envoyNS).List(ctx, metav1.ListOptions{}) + if err != nil { + // Leave the pointer nil: a List failure is not evidence that no + // LoadBalancer has an address. + printWarning(log, fmt.Sprintf("Could not list services in %s: %v", envoyNS, err)) + state.Warnings = append(state.Warnings, + "External Load Balancer: status unknown (Service listing failed)") + return + } + + type lbResult struct { + name string + namespace string + addr string + } + var found []lbResult + // Envoy Gateway provisions one proxy Service per Gateway, and the stack + // defines several. Tracking the unassigned ones stops a partially + // satisfied address pool from passing on the strength of its siblings. + var pending []string + for i := range services.Items { + svc := &services.Items[i] + if svc.Spec.Type != corev1.ServiceTypeLoadBalancer { + continue + } + addr := "" + for _, ing := range svc.Status.LoadBalancer.Ingress { + if addr = ing.IP; addr == "" { + addr = ing.Hostname + } + if addr != "" { + break + } + } + if addr == "" { + pending = append(pending, svc.Namespace+"/"+svc.Name) + continue + } + found = append(found, lbResult{svc.Name, svc.Namespace, addr}) + } + + if len(pending) > 0 { + printWarning(log, fmt.Sprintf("LoadBalancer Service(s) awaiting an external address: %s", + strings.Join(pending, ", "))) + state.Warnings = append(state.Warnings, + "External Load Balancer: "+strings.Join(pending, ", ")+ + " have no external address. Check the load balancer controller and its address pool.") + ok := false + state.ExternalLBOK = &ok + return + } + + if len(found) == 0 { + printWarning(log, "No LoadBalancer Services with an assigned external address found") + printInfo(log, " This may indicate: no LB controller is installed (MetalLB, cloud LB), "+ + "or no LoadBalancer Service exists yet (normal before nvcf-cli up)") + state.Warnings = append(state.Warnings, + "External Load Balancer: no Service of type LoadBalancer has an assigned external IP or hostname. "+ + "Verify a load balancer controller is installed.") + ok := false + state.ExternalLBOK = &ok + return + } + + printSuccess(log, fmt.Sprintf("%d LoadBalancer Service(s) with external address:", len(found))) + for _, svc := range found { + printInfo(log, fmt.Sprintf(" %s/%s → %s", svc.namespace, svc.name, svc.addr)) + } + ok := true + state.ExternalLBOK = &ok +} + +const ( + nodeToNodeTestPort = 19999 + // nodeToNodeNSPrefix names a per-run probe namespace. Running in a + // dedicated namespace rather than "default" keeps a default-deny + // NetworkPolicy, istio-injection, an SCC rejecting the runAsUser, or a + // registry allowlist from surfacing as an overlay fault. The random suffix + // keeps concurrent runs from deleting each other's namespace. + nodeToNodeNSPrefix = "nvcf-n2n-validation-" + nodeToNodeDSName = "nvcf-n2n-server" + nodeToNodeCheckerName = "nvcf-n2n-checker" + nodeToNodeActiveDeadline = int64(180) + nodeToNodeDSTimeout = 2 * time.Minute + nodeToNodeStatusTimeout = 30 * time.Second + nodeToNodeCheckerTimeout = 90 * time.Second + // orphanN2NNamespaceTTL is the minimum age before a leftover + // nvcf-n2n-validation-* namespace is swept. Must exceed the sum of the + // DaemonSet and checker timeouts to avoid racing a concurrent run. + orphanN2NNamespaceTTL = 10 * time.Minute +) + +// nodeToNodeProbeImage resolves the probe image, honouring the same +// enforcement.testImage override the sibling NetworkPolicy probe uses. Without +// it, an air-gapped or registry-mirrored cluster ImagePullBackOffs on every +// DaemonSet pod and the timeout is reported as an overlay fault. +func nodeToNodeProbeImage(cfg *NetworkCheckConfig) string { + if cfg != nil && cfg.Enforcement != nil && cfg.Enforcement.TestImage != "" { + return cfg.Enforcement.TestImage + } + return enforcementDefaultImg +} + +// createNodeToNodeNamespace creates the per-run probe namespace. The labels +// are what sweepOrphanN2NNamespaces matches on, and are deliberately distinct +// from the netpol-validation labels so the two sweeps cannot cross-delete. +func createNodeToNodeNamespace(ctx context.Context, client kubernetes.Interface, ns string) error { + _, err := client.CoreV1().Namespaces().Create(ctx, &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: ns, + Labels: map[string]string{ + "app.kubernetes.io/managed-by": "nvcf-cluster-validator", + "app.kubernetes.io/component": "n2n-probe", + }, + }, + }, metav1.CreateOptions{}) + return err +} + +// legacyNodeToNodeNamespace is where validator versions before the per-run +// probe namespace created their DaemonSet. Kept so an orphan left by a +// currently deployed validator is still reclaimable; remove once those +// versions are out of service. +const legacyNodeToNodeNamespace = "default" + +// sweepLegacyOrphanN2NDaemonSets reclaims probe DaemonSets stranded in +// "default" by an older validator that was killed before its cleanup ran. +// The per-run namespace sweep cannot see those: they predate the namespace. +// Without this they persist indefinitely, one probe pod per node. +func sweepLegacyOrphanN2NDaemonSets(ctx context.Context, log *logrus.Entry, client kubernetes.Interface, ttl time.Duration) { + listCtx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + + dsList, err := client.AppsV1().DaemonSets(legacyNodeToNodeNamespace).List(listCtx, metav1.ListOptions{ + LabelSelector: "app.kubernetes.io/managed-by=nvcf-cluster-validator,app.kubernetes.io/component=n2n-server", + }) + if err != nil { + // Say so: a silent return here leaves legacy probe DaemonSets running + // one pod per node until some later sweep happens to succeed, with + // nothing in the log to explain why. Matches sweepOrphanN2NNamespaces. + log.Warnf("N2N legacy orphan sweep: failed to list DaemonSets in %s: %v", + legacyNodeToNodeNamespace, err) + return + } + if len(dsList.Items) == 0 { + return + } + + cutoff := time.Now().Add(-ttl) + grace := int64(0) + deleted := 0 + for i := range dsList.Items { + ds := &dsList.Items[i] + // Require the name too, as the namespace sweep does: these labels are + // public constants and this deletes objects in a shared namespace. + // Prefix, not equality: every version that created these named them + // nodeToNodeDSName + "-" + suffix, so an equality check matches nothing + // and the sweep silently reclaims none of the orphans it exists for. + if !strings.HasPrefix(ds.Name, nodeToNodeDSName+"-") { + continue + } + if ds.CreationTimestamp.After(cutoff) { + continue // still within TTL; might be a concurrent run + } + delCtx, delCancel := context.WithTimeout(ctx, 30*time.Second) + err := client.AppsV1().DaemonSets(legacyNodeToNodeNamespace).Delete(delCtx, ds.Name, + metav1.DeleteOptions{GracePeriodSeconds: &grace}) + delCancel() + if err != nil && !apierrors.IsNotFound(err) { + log.Warnf("N2N legacy orphan sweep: failed to delete DaemonSet %s: %v", ds.Name, err) + continue + } + deleted++ + } + if deleted > 0 { + printInfo(log, fmt.Sprintf("N2N legacy orphan sweep: deleted %d stale server DaemonSet(s) in %s older than %s", + deleted, legacyNodeToNodeNamespace, ttl)) + } +} + +// sweepOrphanN2NNamespaces deletes any nvcf-n2n-validation-* namespaces older +// than ttl, taking the DaemonSet and checker pod inside with them. These are +// left behind when the validator process is killed with SIGKILL (OOM, +// force-delete, node failure) before the deferred cleanup fires. Namespaces +// younger than ttl are skipped in case they belong to a concurrent run. +func sweepOrphanN2NNamespaces(ctx context.Context, log *logrus.Entry, client kubernetes.Interface, ttl time.Duration) { + listCtx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + + nsList, err := client.CoreV1().Namespaces().List(listCtx, metav1.ListOptions{ + LabelSelector: "app.kubernetes.io/managed-by=nvcf-cluster-validator,app.kubernetes.io/component=n2n-probe", + }) + if err != nil { + log.Warnf("N2N orphan sweep: failed to list namespaces: %v", err) + return + } + + cutoff := time.Now().Add(-ttl) + deleted := 0 + for i := range nsList.Items { + ns := &nsList.Items[i] + // Belt and braces: the label selector should be sufficient, but require + // the name prefix too so a mislabelled namespace is never deleted. + if !strings.HasPrefix(ns.Name, nodeToNodeNSPrefix) { + continue + } + if ns.CreationTimestamp.After(cutoff) { + continue // still within TTL; might be a concurrent run + } + delCtx, delCancel := context.WithTimeout(ctx, 30*time.Second) + err := client.CoreV1().Namespaces().Delete(delCtx, ns.Name, metav1.DeleteOptions{}) + delCancel() + if err != nil && !apierrors.IsNotFound(err) { + log.Warnf("N2N orphan sweep: failed to delete namespace %s: %v", ns.Name, err) + continue + } + deleted++ + } + if deleted > 0 { + printInfo(log, fmt.Sprintf("N2N orphan sweep: deleted %d stale probe namespace(s) older than %s", deleted, ttl)) + } +} + +// checkNodeToNode verifies overlay-network connectivity across all schedulable +// nodes using a DaemonSet-based probe. A server DaemonSet is deployed on every +// schedulable node; a checker pod on node[0] connects to each server pod IP on +// nodes[1..N-1]. +// +// The topology is a single-source star, not a full mesh: it proves node[0] +// reaches every other node, which catches a dead overlay and most single-node +// isolation. It does not prove node[i] reaches node[j] for i,j != 0, and it +// does not probe the reverse direction back toward node[0]. +// +// This check creates a namespace, a DaemonSet, and a pod, so the ServiceAccount +// must hold create/delete on all three. A denial on any of them leaves the +// result unknown rather than failing the overlay, because a missing grant is +// not evidence that node-to-node traffic is broken. +// +// Critical: broken overlay means NVCF services on different nodes cannot +// communicate, causing cascade failures across every API call. +func checkNodeToNode(ctx context.Context, client kubernetes.Interface, state *ValidationState, image string) { + log := state.Log + printHeader(log, "Node-to-Node Communication") + + // Reclaim DaemonSets orphaned by prior runs killed before their deferred + // cleanup fired (SIGKILL, OOM, node failure). + sweepOrphanN2NNamespaces(ctx, log, client, orphanN2NNamespaceTTL) + sweepLegacyOrphanN2NDaemonSets(ctx, log, client, orphanN2NNamespaceTTL) + + nodes, err := client.CoreV1().Nodes().List(ctx, metav1.ListOptions{}) + if err != nil { + printWarning(log, fmt.Sprintf("Could not list nodes: %v", err)) + state.Warnings = append(state.Warnings, "Node-to-Node: status unknown (node listing failed)") + return + } + + var schedulable []string + for i := range nodes.Items { + if !nodes.Items[i].Spec.Unschedulable { + schedulable = append(schedulable, nodes.Items[i].Name) + } + } + + // Leave the pointer nil rather than reporting Verified: there is no second + // node to reach, so the overlay was not exercised. The summary renders this + // as an explicit UNKNOWN row. + // Zero and one are different answers. No schedulable node at all means the + // cluster cannot place work and we observed nothing, so the result stays + // unknown. Exactly one means there is no cross-node path to exercise, so + // the requirement is vacuously met: reporting that as a critical UNKNOWN + // would leave every single-node or k3d control plane permanently + // NVCF-Not-Ready once an unobserved critical check fails the verdict. + if len(schedulable) == 0 { + printWarning(log, "No schedulable nodes; node-to-node overlay not observed") + state.Warnings = append(state.Warnings, + "Node-to-Node: status unknown (no schedulable nodes)") + return + } + if len(schedulable) == 1 { + printInfo(log, " 1 schedulable node; node-to-node check not applicable") + state.NodeToNodeNotApplicable = "single schedulable node, no cross-node path" + return + } + + suffix := rand.String(6) + dsName := nodeToNodeDSName + "-" + suffix + checkerName := nodeToNodeCheckerName + "-" + suffix + dsLabels := map[string]string{ + "app.kubernetes.io/managed-by": "nvcf-cluster-validator", + "app.kubernetes.io/component": "n2n-server", + "app.kubernetes.io/instance": suffix, + } + + ns := nodeToNodeNSPrefix + suffix + if err := createNodeToNodeNamespace(ctx, client, ns); err != nil && !apierrors.IsAlreadyExists(err) { + printWarning(log, fmt.Sprintf("Could not create probe namespace %s: %v", ns, err)) + state.Warnings = append(state.Warnings, + "Node-to-Node: status unknown (probe namespace could not be created)") + return + } + + // Deleting the namespace removes the DaemonSet and checker pod with it, but + // delete them first so a namespace stuck terminating does not strand the + // probe pods on every node. + defer func() { + grace := int64(0) + opts := metav1.DeleteOptions{GracePeriodSeconds: &grace} + _ = client.AppsV1().DaemonSets(ns).Delete(context.Background(), dsName, opts) + _ = client.CoreV1().Pods(ns).Delete(context.Background(), checkerName, opts) + delCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + if err := client.CoreV1().Namespaces().Delete(delCtx, ns, metav1.DeleteOptions{}); err != nil && + !apierrors.IsNotFound(err) { + log.Warnf("Failed to clean up probe namespace %s: %v", ns, err) + } + }() + + ds, err := client.AppsV1().DaemonSets(ns).Create( + ctx, buildNodeToNodeDaemonSet(dsName, ns, dsLabels, image), metav1.CreateOptions{}, + ) + if err != nil { + // A denial means we could not run the probe, not that the overlay is + // broken. Without this an operator chart missing the daemonsets + // create verb reports NVCF-Not-Ready on every CronJob tick of a + // healthy cluster. + if apierrors.IsForbidden(err) { + msg := fmt.Sprintf("RBAC denied creating the probe DaemonSet in %s: %v", ns, err) + printWarning(log, msg) + state.Warnings = append(state.Warnings, "Node-to-Node: status unknown ("+msg+")") + return + } + printError(log, fmt.Sprintf("Failed to create server DaemonSet: %v", err)) + ok := false + state.NodeToNodeOK = &ok + return + } + + // DesiredNumberScheduled is the only number that accounts for taints the + // DaemonSet has no toleration for. The Create response always carries a + // zeroed status because the DaemonSet controller populates it + // asynchronously, so poll for it instead of reading it off ds directly. + // Falling back to len(schedulable) would count NoSchedule-tainted + // control-plane nodes and fail a healthy cluster on timeout. + wantPods, err := waitForDaemonSetDesiredCount(ctx, client, ns, ds.Name, nodeToNodeStatusTimeout) + if err != nil { + printWarning(log, fmt.Sprintf("Could not determine DaemonSet scheduling target: %v", err)) + state.Warnings = append(state.Warnings, + "Node-to-Node: status unknown (DaemonSet status never reported a scheduling target)") + return + } + // wantPods is >= 1 here: waitForDaemonSetDesiredCount only returns a nil + // error once DesiredNumberScheduled is positive, so a zero target arrives + // as the error above rather than reaching this branch. + if wantPods < 2 { + printInfo(log, " Probe DaemonSet schedulable on 1 node; node-to-node check not applicable") + state.NodeToNodeNotApplicable = "probe DaemonSet schedulable on a single node" + return + } + + log.Infof(" Waiting for server DaemonSet pods on %d nodes...", wantPods) + selector := metav1.FormatLabelSelector(&metav1.LabelSelector{MatchLabels: dsLabels}) + // minNodes=2: two nodes is the smallest set that proves cross-node + // traffic, so a NotReady or cordoned node counted in wantPods degrades + // coverage rather than failing the check. + serverPods, err := waitForDaemonSetPods(ctx, client, ns, selector, wantPods, 2, nodeToNodeDSTimeout) + if err != nil { + printError(log, fmt.Sprintf("Server DaemonSet pods did not become ready: %v", err)) + ok := false + state.NodeToNodeOK = &ok + return + } + + // Partial coverage is a real result, but the operator has to be told the + // probe did not reach every node it was scheduled onto. + if len(serverPods) < wantPods { + msg := fmt.Sprintf("probe covered %d of %d scheduled node(s); the rest never reported a Running pod", + len(serverPods), wantPods) + printWarning(log, msg) + state.Warnings = append(state.Warnings, "Node-to-Node: "+msg) + } + + // Select checkerNode from a Running server pod so it is guaranteed to be + // a node where the DaemonSet actually scheduled. + checkerNode := serverPods[0].Spec.NodeName + var targetIPs []string + for i := range serverPods { + if serverPods[i].Spec.NodeName != checkerNode && serverPods[i].Status.PodIP != "" { + targetIPs = append(targetIPs, serverPods[i].Status.PodIP) + log.Infof(" Server pod on %s: %s", serverPods[i].Spec.NodeName, serverPods[i].Status.PodIP) + } + } + + if len(targetIPs) == 0 { + // Leave the pointer nil. Reporting a critical check as Verified having + // sent zero packets is worse than reporting it as not run. + printWarning(log, "No cross-node server pod IPs available; probe did not run") + state.Warnings = append(state.Warnings, + "Node-to-Node: status unknown (no cross-node probe targets were available)") + return + } + + if _, err := client.CoreV1().Pods(ns).Create( + ctx, buildNodeToNodeCheckerPod(checkerName, ns, checkerNode, targetIPs, image), metav1.CreateOptions{}, + ); err != nil { + // Same reasoning as the DaemonSet create above. + if apierrors.IsForbidden(err) { + msg := fmt.Sprintf("RBAC denied creating the checker pod in %s: %v", ns, err) + printWarning(log, msg) + state.Warnings = append(state.Warnings, "Node-to-Node: status unknown ("+msg+")") + return + } + printError(log, fmt.Sprintf("Failed to create checker pod: %v", err)) + ok := false + state.NodeToNodeOK = &ok + return + } + + succeeded, err := waitForPodDone(ctx, client, ns, checkerName, nodeToNodeCheckerTimeout) + if err != nil { + printError(log, fmt.Sprintf("Checker pod error: %v", err)) + ok := false + state.NodeToNodeOK = &ok + return + } + + if succeeded { + printSuccess(log, fmt.Sprintf("Node-to-node overlay verified: %s → %d node(s) reachable on port %d", + checkerNode, len(targetIPs), nodeToNodeTestPort)) + ok := true + state.NodeToNodeOK = &ok + } else { + printError(log, fmt.Sprintf("Checker on %s could not reach one or more server pods (port %d)", + checkerNode, nodeToNodeTestPort)) + printInfo(log, " Possible causes: CNI overlay misconfiguration, host firewall rules, "+ + "or cloud security group rules blocking inter-node pod traffic") + state.Recommendations = append(state.Recommendations, + "Check host firewall and security groups between nodes. "+ + "Verify the CNI overlay (VXLAN, Geneve, etc.) is not blocked across all nodes.") + ok := false + state.NodeToNodeOK = &ok + } +} + +// waitForDaemonSetDesiredCount polls until the DaemonSet controller has +// reconciled the object and published a scheduling target. The Create response +// always has a zeroed status, so reading DesiredNumberScheduled from it yields +// 0 on every real cluster. +func waitForDaemonSetDesiredCount( + ctx context.Context, client kubernetes.Interface, ns, name string, timeout time.Duration, +) (int, error) { + deadline := time.Now().Add(timeout) + var lastStatus string + for { + ds, err := client.AppsV1().DaemonSets(ns).Get(ctx, name, metav1.GetOptions{}) + switch { + case err != nil: + // Retry inside the deadline rather than aborting. client-go defaults + // to 5 QPS and this run issues ~22 namespaced LISTs, so a single 429 + // early in the window would otherwise fail a critical check with + // most of its budget unspent. Only a permission error is terminal. + if apierrors.IsForbidden(err) || apierrors.IsUnauthorized(err) { + return 0, err + } + lastStatus = err.Error() + case ds.Status.ObservedGeneration >= ds.Generation && ds.Status.DesiredNumberScheduled > 0: + return int(ds.Status.DesiredNumberScheduled), nil + default: + lastStatus = fmt.Sprintf("desired=%d, observedGeneration=%d, generation=%d", + ds.Status.DesiredNumberScheduled, ds.Status.ObservedGeneration, ds.Generation) + } + if time.Now().After(deadline) { + return 0, fmt.Errorf("timed out waiting for DaemonSet status (%s)", lastStatus) + } + select { + case <-ctx.Done(): + return 0, ctx.Err() + case <-time.After(2 * time.Second): + } + } +} + +// waitForDaemonSetPods waits for the DaemonSet's pods to come up. It returns as +// soon as wantCount pods are Running, and on timeout still returns whatever it +// has if that covers minNodes distinct nodes. +// +// The partial return matters because wantCount comes from +// DesiredNumberScheduled, which includes NotReady and cordoned nodes: the +// DaemonSet controller auto-tolerates those taints. Requiring every pod would +// fail this critical check on one NotReady node, contradicting the same run's +// non-blocking "Worker Nodes: N NotReady" policy. Two nodes are enough to +// prove the overlay carries cross-node traffic. +func waitForDaemonSetPods( + ctx context.Context, client kubernetes.Interface, ns, selector string, + wantCount, minNodes int, timeout time.Duration, +) ([]corev1.Pod, error) { + deadline := time.Now().Add(timeout) + var lastErr error + for { + pods, err := client.CoreV1().Pods(ns).List(ctx, metav1.ListOptions{LabelSelector: selector}) + if err != nil { + // Same reasoning as waitForDaemonSetDesiredCount: retry transient + // errors inside the deadline instead of failing the check outright. + if apierrors.IsForbidden(err) || apierrors.IsUnauthorized(err) { + return nil, err + } + lastErr = err + } + var running []corev1.Pod + if err == nil { + lastErr = nil + for i := range pods.Items { + if pods.Items[i].Status.Phase == corev1.PodRunning && pods.Items[i].Status.PodIP != "" { + running = append(running, pods.Items[i]) + } + } + if len(running) >= wantCount { + return running, nil + } + } + if time.Now().After(deadline) { + if lastErr != nil { + return nil, fmt.Errorf("listing DaemonSet pods: %w", lastErr) + } + if distinctNodeCount(running) >= minNodes { + return running, nil + } + return nil, fmt.Errorf("timed out waiting for %d Running pods (got %d on %d node(s))", + wantCount, len(running), distinctNodeCount(running)) + } + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-time.After(3 * time.Second): + } + } +} + +// distinctNodeCount counts how many different nodes a pod set covers. +func distinctNodeCount(pods []corev1.Pod) int { + nodes := make(map[string]struct{}, len(pods)) + for i := range pods { + if n := pods[i].Spec.NodeName; n != "" { + nodes[n] = struct{}{} + } + } + return len(nodes) +} + +// nodeToNodeTolerations mirrors the validator CronJob's own tolerations. The +// DaemonSet controller auto-tolerates the not-ready and unschedulable taints +// but not the control-plane one, so without these a dedicated control plane +// reports DesiredNumberScheduled=0 and the overlay is never probed at all. +func nodeToNodeTolerations() []corev1.Toleration { + return []corev1.Toleration{ + {Key: "node-role.kubernetes.io/control-plane", Operator: corev1.TolerationOpExists, Effect: corev1.TaintEffectNoSchedule}, + {Key: "node-role.kubernetes.io/master", Operator: corev1.TolerationOpExists, Effect: corev1.TaintEffectNoSchedule}, + } +} + +func nodeToNodeSecurityContext() *corev1.SecurityContext { + runAsNonRoot := true + allowPrivEsc := false + runAsUser := int64(65534) + return &corev1.SecurityContext{ + RunAsNonRoot: &runAsNonRoot, + RunAsUser: &runAsUser, + AllowPrivilegeEscalation: &allowPrivEsc, + Capabilities: &corev1.Capabilities{Drop: []corev1.Capability{"ALL"}}, + SeccompProfile: &corev1.SeccompProfile{Type: corev1.SeccompProfileTypeRuntimeDefault}, + } +} + +func buildNodeToNodeDaemonSet(name, namespace string, labels map[string]string, image string) *appsv1.DaemonSet { + return &appsv1.DaemonSet{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace, Labels: labels}, + Spec: appsv1.DaemonSetSpec{ + Selector: &metav1.LabelSelector{MatchLabels: labels}, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{Labels: labels}, + Spec: corev1.PodSpec{ + // ActiveDeadlineSeconds is forbidden on DaemonSet pod templates. + // Cleanup is handled by deleting the DaemonSet in the deferred sweep. + RestartPolicy: corev1.RestartPolicyAlways, + Tolerations: nodeToNodeTolerations(), + Containers: []corev1.Container{{ + Name: "server", + Image: image, + Command: []string{"sh", "-c", fmt.Sprintf("while true; do nc -l -p %d; done", nodeToNodeTestPort)}, + Resources: enforcementResources(), + SecurityContext: nodeToNodeSecurityContext(), + }}, + }, + }, + }, + } +} + +func buildNodeToNodeCheckerPod(name, namespace, nodeName string, targetIPs []string, image string) *corev1.Pod { + deadline := nodeToNodeActiveDeadline + var cmds []string + for _, ip := range targetIPs { + cmds = append(cmds, fmt.Sprintf("nc -z -w 5 %s %d || exit 1", ip, nodeToNodeTestPort)) + } + return &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + Labels: map[string]string{ + "app.kubernetes.io/managed-by": "nvcf-cluster-validator", + "app.kubernetes.io/component": "n2n-checker", + }, + }, + Spec: corev1.PodSpec{ + NodeName: nodeName, + RestartPolicy: corev1.RestartPolicyNever, + ActiveDeadlineSeconds: &deadline, + // NodeName bypasses the scheduler but not the NodeRestriction / + // taint admission plugin, so a control-plane node still rejects + // this pod without the same tolerations the DaemonSet carries. + Tolerations: nodeToNodeTolerations(), + Containers: []corev1.Container{{ + Name: "checker", + Image: image, + Command: []string{"sh", "-c", strings.Join(cmds, " && ")}, + Resources: enforcementResources(), + SecurityContext: nodeToNodeSecurityContext(), + }}, + }, + } +} + +// controlPlaneNamespaces is the set of namespaces scanned by Tier-1 and +// Tier-2 HA checks on the control-plane cluster. +// controlPlaneNamespaces lists the namespaces the self-managed stack deploys +// into, per deploy/stacks/self-managed/helmfile.d. Namespaces that are absent +// are skipped silently (a LIST against a missing namespace returns an empty +// 200), so listing one that a given install does not use is harmless. +// +// The OpenBao namespace is overridable via NVCF_OPENBAO_NAMESPACE, so its +// configured value is appended at runtime by controlPlaneNamespaceSet. +var controlPlaneNamespaces = []string{ + "nvcf", "sis", "api-keys", "ess", "nvcf-ui", + "nats-system", "vault-system", "cassandra-system", + "cert-manager", "envoy-gateway-system", +} + +// openBaoNamespaceEnv mirrors the nvcf-cli override so a cluster that relocates +// OpenBao does not silently drop its StatefulSet from the Tier-2 check. +const openBaoNamespaceEnv = "NVCF_OPENBAO_NAMESPACE" + +// controlPlaneNamespaceSet returns controlPlaneNamespaces plus any +// runtime-configured OpenBao and Envoy Gateway namespaces, de-duplicated. +func controlPlaneNamespaceSet() []string { + out := append([]string(nil), controlPlaneNamespaces...) + for _, env := range []string{openBaoNamespaceEnv, envoyGatewayNamespaceEnv} { + extra := strings.TrimSpace(os.Getenv(env)) + if extra == "" { + continue + } + seen := false + for _, ns := range out { + if ns == extra { + seen = true + break + } + } + if !seen { + out = append(out, extra) + } + } + return out +} + +// envoyGatewayNamespaceName is where the Envoy Gateway controller and its +// provisioned proxy Services live. +func envoyGatewayNamespaceName() string { + if ns := strings.TrimSpace(os.Getenv(envoyGatewayNamespaceEnv)); ns != "" { + return ns + } + return envoyGatewayNamespace +} + +// deploymentRolloutStalled reports whether the Deployment controller has given +// up on the current rollout. Kubernetes sets Progressing=False with reason +// ProgressDeadlineExceeded once progressDeadlineSeconds elapses without +// progress, which is what distinguishes a wedged rollout (bad image, no +// schedulable node) from one that is merely in flight. +func deploymentRolloutStalled(d *appsv1.Deployment) bool { + for i := range d.Status.Conditions { + c := &d.Status.Conditions[i] + if c.Type == appsv1.DeploymentProgressing && + c.Status == corev1.ConditionFalse && + c.Reason == "ProgressDeadlineExceeded" { + return true + } + } + return false +} + +// checkTier1Deployments verifies that every Deployment in the control-plane +// namespaces has readyReplicas >= spec.replicas. Any under-replicated Deployment +// means HA headroom is gone and a second failure causes a full outage. +// +// The check is generic; no hardcoded Deployment names. New services added to +// those namespaces are automatically covered. +// +// Critical: under-replication means a single additional failure causes a full +// service outage. +func checkTier1Deployments(ctx context.Context, client kubernetes.Interface, state *ValidationState) { + log := state.Log + printHeader(log, "Tier-1 Deployment Readiness") + + var underReplicated []string + var scaledToZero []string + checkedCount := 0 + deniedCount := 0 + rollingCount := 0 + // rollingUnderReplicated counts mid-rollout Deployments that are also + // below their target, which bounds the skip: a paused or sentinel-deadline + // Deployment can stay "rolling" forever, but if it is still serving its + // full replica count there is nothing to report. + rollingUnderReplicated := 0 + + for _, ns := range controlPlaneNamespaceSet() { + deploys, err := client.AppsV1().Deployments(ns).List(ctx, metav1.ListOptions{}) + if err != nil { + // A 403 means we could not observe the namespace, not that it is + // healthy. Track it separately so it cannot reach the trivial-pass + // exit below. A LIST against a missing namespace returns an empty + // 200, so IsNotFound is not a case here. + if apierrors.IsForbidden(err) { + deniedCount++ + continue + } + // Same shape as the 403 branch: keep going. Returning here throws + // away the under-replicated Deployments already collected from + // earlier namespaces and publishes the tier as unknown, even + // though a fully-down service was observed. + printWarning(log, fmt.Sprintf("Could not list Deployments in %s: %v", ns, err)) + state.Warnings = append(state.Warnings, + fmt.Sprintf("Tier-1 Deployments: status unknown (listing failed in %s)", ns)) + deniedCount++ + continue + } + for i := range deploys.Items { + d := &deploys.Items[i] + want := int32(1) + if d.Spec.Replicas != nil { + want = *d.Spec.Replicas + } + // A Deployment scaled to zero satisfies "ReadyReplicas >= want" + // with nothing running at all, so counting it as healthy lets a + // maintenance scale-down or a replicaCount:0 values error publish + // the critical row as All Ready. Report it instead of counting it. + if want == 0 { + scaledToZero = append(scaledToZero, ns+"/"+d.Name) + continue + } + // A rollout transiently drops readyReplicas below spec.replicas on + // a healthy cluster, so skip those. But UpdatedReplicas < want is + // not self-limiting: a bad image wedges there permanently with + // ObservedGeneration == Generation. ProgressDeadlineExceeded is the + // signal that separates "in flight" from "stuck", so a stalled + // rollout falls through to the under-replicated check below. + rollingOut := d.Status.ObservedGeneration < d.Generation || + d.Status.UpdatedReplicas < want + if rollingOut && !deploymentRolloutStalled(d) { + msg := fmt.Sprintf("%s/%s: rollout in progress (updated: %d/%d); re-run check after rollout completes", + ns, d.Name, d.Status.UpdatedReplicas, want) + printWarning(log, msg) + state.Warnings = append(state.Warnings, "Tier-1 Deployments: "+msg) + rollingCount++ + // ProgressDeadlineExceeded is never set for a paused rollout, + // for progressDeadlineSeconds=2147483647, or for a wedged + // controller, so rollingOut alone is not self-limiting. Only a + // skipped Deployment that is ALSO below its ready target can + // hide a problem, so only those make the tier unknown. + if d.Status.ReadyReplicas < want { + rollingUnderReplicated++ + } + continue + } + checkedCount++ + if d.Status.ReadyReplicas < want { + underReplicated = append(underReplicated, + fmt.Sprintf("%s/%s (ready: %d, want: %d)", ns, d.Name, d.Status.ReadyReplicas, want)) + } + } + } + + if checkedCount == 0 { + if deniedCount > 0 { + // Leave nil: every namespace was denied, so nothing was observed. + printWarning(log, fmt.Sprintf("Deployments not readable in %d control-plane namespace(s)", deniedCount)) + state.Warnings = append(state.Warnings, + "Tier-1 Deployments: status unknown (RBAC denied Deployment list in all control-plane namespaces)") + return + } + if rollingCount > 0 { + // Deployments exist but every one is mid-rollout, so readiness + // cannot be assessed yet. Reporting "pre-install" here would be wrong. + printWarning(log, fmt.Sprintf("All %d Deployment(s) are mid-rollout; readiness not assessed", rollingCount)) + return + } + if len(scaledToZero) > 0 { + // Every Deployment present is scaled to zero: the namespaces are + // populated but nothing is running, which is not a pass. + printError(log, fmt.Sprintf("All %d Deployment(s) are scaled to zero replicas: %s", + len(scaledToZero), strings.Join(scaledToZero, ", "))) + ok := false + state.Tier1DeploymentsOK = &ok + return + } + printInfo(log, " No Deployments found in control-plane namespaces (pre-install state)") + ok := true + state.Tier1DeploymentsOK = &ok + return + } + + // Surfaced as a warning rather than a failure: scaling a component down is + // a legitimate operator action, but it must not be invisible on a row that + // claims every Deployment is ready. + if len(scaledToZero) > 0 { + msg := fmt.Sprintf("%d Deployment(s) scaled to zero replicas: %s", + len(scaledToZero), strings.Join(scaledToZero, ", ")) + printWarning(log, msg) + state.Warnings = append(state.Warnings, "Tier-1 Deployments: "+msg) + } + + if len(underReplicated) > 0 { + printError(log, fmt.Sprintf("Under-replicated Deployments (%d):", len(underReplicated))) + for _, name := range underReplicated { + printInfo(log, " "+name) + } + state.Recommendations = append(state.Recommendations, + "Check for crashed, evicted, or unschedulable pods in the listed namespaces. "+ + "If a service is intentionally single-replica, raise its replicaCount in the "+ + "self-managed stack values to keep HA headroom.") + ok := false + state.Tier1DeploymentsOK = &ok + return + } + + if deniedCount > 0 { + // Some namespaces were never observed, so "all ready" is not a claim we + // can make even though every Deployment we could see passed. + printWarning(log, fmt.Sprintf("%d Deployment(s) ready, but %d namespace(s) were not readable", + checkedCount, deniedCount)) + state.Warnings = append(state.Warnings, + "Tier-1 Deployments: status unknown (RBAC denied Deployment list in one or more control-plane namespaces)") + return + } + + if rollingUnderReplicated > 0 { + // Only skipped Deployments that are also below their ready target make + // the tier unknown. A paused or sentinel-deadline Deployment serving + // its full replica count is skipped but hides nothing, so it must not + // pin this critical row to UNKNOWN indefinitely. + printWarning(log, fmt.Sprintf("%d Deployment(s) ready, but %d are mid-rollout and under-replicated; assessment is partial", + checkedCount, rollingUnderReplicated)) + state.Warnings = append(state.Warnings, + "Tier-1 Deployments: status unknown (one or more Deployments are mid-rollout and below their replica target)") + return + } + + if rollingCount > 0 { + printWarning(log, fmt.Sprintf("%d Deployment(s) ready, %d mid-rollout but still at their replica target", + checkedCount, rollingCount)) + } + + printSuccess(log, fmt.Sprintf("All %d assessed Deployment(s) in control-plane namespaces are fully ready", checkedCount)) + ok := true + state.Tier1DeploymentsOK = &ok +} + +// checkTier2StatefulSets verifies quorum membership and node placement for +// Tier-2 stateful components (NATS JetStream, OpenBao Raft, Cassandra). +// Any StatefulSet with an odd spec.replicas of 3 or more is treated as a quorum +// component and checked for: +// 1. readyReplicas == spec.replicas +// 2. all pods on distinct nodes +// +// StatefulSets mid-rolling-update are warned about, not failed: they roll one +// pod at a time, so a below-target ready count is the steady state for the +// duration of any upgrade. +// +// The check is generic; no hardcoded StatefulSet names. +// +// Critical: broken quorum or co-located peers leave the stack one failure +// away from a total control-plane outage. +func checkTier2StatefulSets(ctx context.Context, client kubernetes.Interface, state *ValidationState) { + log := state.Log + printHeader(log, "Tier-2 StatefulSet Quorum and Placement") + + const minQuorumSize = int32(3) + var failures []string + var skippedParity []string + checkedCount := 0 + deniedCount := 0 + rollingCount := 0 + // rollingUnderReplicated bounds the rollout skip, as in checkTier1Deployments. + rollingUnderReplicated := 0 + // placementUnknown counts StatefulSets whose pods could not be listed, so + // an unreadable namespace cannot masquerade as a clean placement result. + placementUnknown := 0 + + for _, ns := range controlPlaneNamespaceSet() { + stsList, err := client.AppsV1().StatefulSets(ns).List(ctx, metav1.ListOptions{}) + if err != nil { + // See checkTier1Deployments: a 403 must not reach the trivial-pass + // exit. This fires today, as the validator ClusterRole grants + // deployments and daemonsets but not statefulsets. + if apierrors.IsForbidden(err) { + deniedCount++ + continue + } + // See checkTier1Deployments: continue rather than return, so + // quorum failures already observed are not discarded. + printWarning(log, fmt.Sprintf("Could not list StatefulSets in %s: %v", ns, err)) + state.Warnings = append(state.Warnings, + fmt.Sprintf("Tier-2 StatefulSets: status unknown (listing failed in %s)", ns)) + deniedCount++ + continue + } + + for i := range stsList.Items { + sts := &stsList.Items[i] + // Any odd replica count of 3 or more is a quorum member. Requiring + // exactly 3 silently drops a Cassandra scaled to 5 from the check + // rather than failing it. + if sts.Spec.Replicas == nil { + continue + } + want := *sts.Spec.Replicas + if want < minQuorumSize { + continue + } + if want%2 == 0 { + // Even replica counts are not a quorum shape this check can + // reason about, but they are not nothing either: a 4-replica + // Cassandra with RF=3 can have lost quorum. Record it so an + // all-even cluster cannot reach the trivial-pass exit below + // having examined no StatefulSet at all. + skippedParity = append(skippedParity, fmt.Sprintf("%s/%s (replicas=%d)", ns, sts.Name, want)) + continue + } + + // StatefulSets roll one pod at a time, so readyReplicas == want-1 + // is the steady state for the whole duration of any image bump, + // PVC resize, or node drain. Warn rather than fail, unless the + // controller has not even observed the current generation. + // CurrentRevision only advances when a RollingUpdate completes, so + // a revision mismatch is permanent for updateStrategy OnDelete + // (which this repo's OpenBao uses), for a non-zero + // rollingUpdate.partition, and for a wedged rollout. There is no + // StatefulSet equivalent of ProgressDeadlineExceeded, so bound the + // tolerance by readiness instead: a StatefulSet at its full ready + // count is not hiding anything, and one below it is reported. + if sts.Status.UpdateRevision != "" && sts.Status.CurrentRevision != sts.Status.UpdateRevision { + msg := fmt.Sprintf("%s/%s: revision mismatch (ready: %d/%d)", + ns, sts.Name, sts.Status.ReadyReplicas, want) + printWarning(log, msg) + state.Warnings = append(state.Warnings, "Tier-2 StatefulSets: "+msg) + rollingCount++ + + switch { + case sts.Status.ReadyReplicas >= want: + // Full ready count despite the mismatch: nothing is hidden, + // so assess it normally and let the placement scan run. + // This is the steady state for OnDelete and for a non-zero + // rollingUpdate.partition, where the mismatch never clears. + case sts.Status.ReadyReplicas == want-1: + // Exactly one pod down is what rolling one at a time looks + // like, so tolerate it but do not claim the tier is clean. + rollingUnderReplicated++ + continue + default: + // More than one peer down is beyond what a rolling update + // explains, mismatch or not. + failures = append(failures, + fmt.Sprintf("%s/%s: readyReplicas=%d (need %d, revision mismatch)", + ns, sts.Name, sts.Status.ReadyReplicas, want)) + checkedCount++ + continue + } + } + checkedCount++ + + if sts.Status.ReadyReplicas < want { + failures = append(failures, + fmt.Sprintf("%s/%s: readyReplicas=%d (need %d)", + ns, sts.Name, sts.Status.ReadyReplicas, want)) + continue + } + + selector := metav1.FormatLabelSelector(sts.Spec.Selector) + pods, err := client.CoreV1().Pods(ns).List(ctx, metav1.ListOptions{LabelSelector: selector}) + if err != nil { + // Not a placement failure: we could not look. Recording it in + // failures would report a broken quorum for an RBAC gap on + // pods, while the identical gap on statefulsets above is + // correctly reported as unknown. + msg := fmt.Sprintf("%s/%s: could not list pods for placement check: %v", ns, sts.Name, err) + printWarning(log, msg) + state.Warnings = append(state.Warnings, "Tier-2 StatefulSets: "+msg) + placementUnknown++ + continue + } + + // Count only Ready pods owned by this StatefulSet. Phase stays + // Running through CrashLoopBackOff, and a surplus pod left over + // from a rollout would otherwise be reported as a co-location. + nodeOwner := make(map[string]string) + for j := range pods.Items { + p := &pods.Items[j] + if !metav1.IsControlledBy(p, sts) || !isPodReady(p) { + continue + } + if first, dup := nodeOwner[p.Spec.NodeName]; dup { + failures = append(failures, + fmt.Sprintf("%s/%s: pods %s and %s are co-located on node %s", + ns, sts.Name, first, p.Name, p.Spec.NodeName)) + } else { + nodeOwner[p.Spec.NodeName] = p.Name + } + } + } + } + + if checkedCount == 0 { + if deniedCount > 0 { + printWarning(log, fmt.Sprintf("StatefulSets not readable in %d control-plane namespace(s)", deniedCount)) + state.Warnings = append(state.Warnings, + "Tier-2 StatefulSets: status unknown (RBAC denied StatefulSet list in all control-plane namespaces)") + return + } + if rollingCount > 0 { + printWarning(log, fmt.Sprintf("All %d quorum StatefulSet(s) are mid-rollout; quorum not assessed", rollingCount)) + return + } + if len(skippedParity) > 0 { + // StatefulSets exist in the quorum namespaces but none has an odd + // replica count, so nothing was examined. Claiming "Quorum and + // Placement OK" here would certify a ring this check never looked at. + printWarning(log, fmt.Sprintf("No odd-replica quorum StatefulSets; %d even-replica StatefulSet(s) not assessed: %s", + len(skippedParity), strings.Join(skippedParity, ", "))) + state.Warnings = append(state.Warnings, + "Tier-2 StatefulSets: status unknown (only even-replica StatefulSets present: "+ + strings.Join(skippedParity, ", ")+")") + return + } + if placementUnknown > 0 { + printWarning(log, fmt.Sprintf("Placement not assessed for %d StatefulSet(s); pods were not readable", placementUnknown)) + return + } + printInfo(log, " No quorum StatefulSets (odd spec.replicas >= 3) found (pre-install or non-HA install)") + ok := true + state.Tier2StatefulSetsOK = &ok + return + } + + if len(failures) > 0 { + printError(log, fmt.Sprintf("Tier-2 quorum/placement findings (%d):", len(failures))) + for _, f := range failures { + printInfo(log, " "+f) + } + state.Recommendations = append(state.Recommendations, + "Ensure each Tier-2 StatefulSet (NATS, OpenBao, Cassandra) has all spec.replicas pods Ready "+ + "and spread across distinct nodes.") + ok := false + state.Tier2StatefulSetsOK = &ok + return + } + + if deniedCount > 0 { + printWarning(log, fmt.Sprintf("%d quorum StatefulSet(s) healthy, but %d namespace(s) were not readable", + checkedCount, deniedCount)) + state.Warnings = append(state.Warnings, + "Tier-2 StatefulSets: status unknown (RBAC denied StatefulSet list in one or more control-plane namespaces)") + return + } + + if placementUnknown > 0 { + printWarning(log, fmt.Sprintf("%d quorum StatefulSet(s) healthy, but placement was not assessed for %d", + checkedCount, placementUnknown)) + state.Warnings = append(state.Warnings, + "Tier-2 StatefulSets: status unknown (pod placement could not be read for one or more StatefulSets)") + return + } + + if rollingUnderReplicated > 0 { + printWarning(log, fmt.Sprintf("%d quorum StatefulSet(s) healthy, but %d are mid-rollout and below target; assessment is partial", + checkedCount, rollingUnderReplicated)) + state.Warnings = append(state.Warnings, + "Tier-2 StatefulSets: status unknown (one or more StatefulSets are mid-rollout and below their replica target)") + return + } + + if len(skippedParity) > 0 { + msg := fmt.Sprintf("%d even-replica StatefulSet(s) not assessed: %s", + len(skippedParity), strings.Join(skippedParity, ", ")) + printWarning(log, msg) + state.Warnings = append(state.Warnings, "Tier-2 StatefulSets: "+msg) + } + + if rollingCount > 0 { + printWarning(log, fmt.Sprintf("%d quorum StatefulSet(s) healthy, %d mid-rollout but at their replica target", + checkedCount, rollingCount)) + } + + printSuccess(log, fmt.Sprintf("All %d quorum StatefulSet(s) Ready on distinct nodes", checkedCount)) + ok := true + state.Tier2StatefulSetsOK = &ok +} + // checkConfigurableReachability probes user-defined endpoints loaded from the // cluster-validator ConfigMap. func checkConfigurableReachability(state *ValidationState, cfg *ReachabilityConfig) { diff --git a/src/compute-plane-services/nvca/internal/clustervalidator/checks_controlplane_test.go b/src/compute-plane-services/nvca/internal/clustervalidator/checks_controlplane_test.go new file mode 100644 index 0000000000..d4d6d2917e --- /dev/null +++ b/src/compute-plane-services/nvca/internal/clustervalidator/checks_controlplane_test.go @@ -0,0 +1,1477 @@ +/* +SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package clustervalidator + +import ( + "context" + "fmt" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + storagev1 "k8s.io/api/storage/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + fakediscovery "k8s.io/client-go/discovery/fake" + "k8s.io/client-go/kubernetes/fake" + ktesting "k8s.io/client-go/testing" +) + +// -- checkStorageClass -- + +func TestCheckStorageClass_DefaultPresent(t *testing.T) { + client := fake.NewSimpleClientset(&storagev1.StorageClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: "standard", + Annotations: map[string]string{ + "storageclass.kubernetes.io/is-default-class": "true", + }, + }, + }) + state := &ValidationState{Log: testLog()} + checkStorageClass(context.Background(), client, state) + + require.NotNil(t, state.DefaultStorageClassOK) + assert.True(t, *state.DefaultStorageClassOK, "a StorageClass with the default annotation must set DefaultStorageClassOK=true") + assert.Empty(t, state.Recommendations) +} + +func TestCheckStorageClass_BetaAnnotationAlsoAccepted(t *testing.T) { + client := fake.NewSimpleClientset(&storagev1.StorageClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: "local-path", + Annotations: map[string]string{ + "storageclass.beta.kubernetes.io/is-default-class": "true", + }, + }, + }) + state := &ValidationState{Log: testLog()} + checkStorageClass(context.Background(), client, state) + + require.NotNil(t, state.DefaultStorageClassOK) + assert.True(t, *state.DefaultStorageClassOK) +} + +func TestCheckStorageClass_NoDefault(t *testing.T) { + client := fake.NewSimpleClientset(&storagev1.StorageClass{ + ObjectMeta: metav1.ObjectMeta{Name: "no-annotation-class"}, + }) + state := &ValidationState{Log: testLog()} + checkStorageClass(context.Background(), client, state) + + require.NotNil(t, state.DefaultStorageClassOK) + assert.False(t, *state.DefaultStorageClassOK, "StorageClass without default annotation must set DefaultStorageClassOK=false") + assert.NotEmpty(t, state.Recommendations, "missing default StorageClass must add a recommendation") +} + +func TestCheckStorageClass_NoStorageClasses(t *testing.T) { + client := fake.NewSimpleClientset() + state := &ValidationState{Log: testLog()} + checkStorageClass(context.Background(), client, state) + + require.NotNil(t, state.DefaultStorageClassOK) + assert.False(t, *state.DefaultStorageClassOK) +} + +// -- checkGatewayAPICRDs -- +// The fake discovery client does not populate ServerResourcesForGroupVersion, +// so checkGatewayAPICRDs will always see the group as absent. +// We test that it runs without panic and sets GatewayAPICRDsOK=false. + +func TestCheckGatewayAPICRDs_AbsentOnFakeClient(t *testing.T) { + client := fake.NewSimpleClientset() + state := &ValidationState{Log: testLog()} + checkGatewayAPICRDs(context.Background(), client, state) + + require.NotNil(t, state.GatewayAPICRDsOK, + "GatewayAPICRDsOK must be set even when discovery returns an error") + assert.False(t, *state.GatewayAPICRDsOK, + "absent Gateway API CRDs must set GatewayAPICRDsOK=false") + assert.NotEmpty(t, state.Recommendations) +} + +// -- checkEnvoyGateway -- + +// makeEnvoyControllerPod builds a pod carrying the controller label the check +// selects on. ready=false yields a pod that is Running but not Ready, which is +// what a CrashLoopBackOff controller looks like via the API. +func makeEnvoyControllerPod(name string, ready bool) *corev1.Pod { + cond := corev1.ConditionFalse + if ready { + cond = corev1.ConditionTrue + } + return &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: envoyGatewayNamespaceName(), + Labels: map[string]string{"control-plane": "envoy-gateway"}, + }, + Status: corev1.PodStatus{ + Phase: corev1.PodRunning, + Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: cond}}, + }, + } +} + +func TestCheckEnvoyGateway_ReadyControllerPod(t *testing.T) { + client := fake.NewSimpleClientset( + &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: envoyGatewayNamespaceName()}}, + makeEnvoyControllerPod("envoy-gateway-abc", true), + ) + state := &ValidationState{Log: testLog()} + checkEnvoyGateway(context.Background(), client, state) + + require.NotNil(t, state.EnvoyGatewayOK) + assert.True(t, *state.EnvoyGatewayOK, "a Ready controller pod must set EnvoyGatewayOK=true") +} + +// A crash-looping controller keeps .status.phase == Running, so the check must +// key on the Ready condition or a dead gateway reports healthy. +func TestCheckEnvoyGateway_RunningButNotReadyFails(t *testing.T) { + client := fake.NewSimpleClientset( + &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: envoyGatewayNamespaceName()}}, + makeEnvoyControllerPod("envoy-gateway-abc", false), + ) + state := &ValidationState{Log: testLog()} + checkEnvoyGateway(context.Background(), client, state) + + require.NotNil(t, state.EnvoyGatewayOK) + assert.False(t, *state.EnvoyGatewayOK, + "Running-but-not-Ready controller (CrashLoopBackOff) must not pass") +} + +// The data-plane proxies and the certgen Job share this namespace. Counting +// them lets a dead controller pass, so they must be excluded by the selector. +func TestCheckEnvoyGateway_IgnoresNonControllerPods(t *testing.T) { + dataPlane := makePod("envoy-envoy-gateway-system-eg-abc123", envoyGatewayNamespaceName(), corev1.PodRunning) + dataPlane.Status.Conditions = []corev1.PodCondition{{Type: corev1.PodReady, Status: corev1.ConditionTrue}} + client := fake.NewSimpleClientset( + &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: envoyGatewayNamespaceName()}}, + dataPlane, + ) + state := &ValidationState{Log: testLog()} + checkEnvoyGateway(context.Background(), client, state) + + require.NotNil(t, state.EnvoyGatewayOK) + assert.False(t, *state.EnvoyGatewayOK, + "a Ready data-plane proxy must not satisfy the controller check") +} + +func TestCheckEnvoyGateway_NamespaceAbsent(t *testing.T) { + client := fake.NewSimpleClientset() + state := &ValidationState{Log: testLog()} + checkEnvoyGateway(context.Background(), client, state) + + require.NotNil(t, state.EnvoyGatewayOK) + assert.False(t, *state.EnvoyGatewayOK, "absent namespace must set EnvoyGatewayOK=false") + assert.NotEmpty(t, state.Recommendations) +} + +func TestCheckEnvoyGateway_NamespacePresentNoRunningPods(t *testing.T) { + client := fake.NewSimpleClientset( + &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: envoyGatewayNamespaceName()}}, + makePod("envoy-gateway-abc", envoyGatewayNamespaceName(), corev1.PodPending), + ) + state := &ValidationState{Log: testLog()} + checkEnvoyGateway(context.Background(), client, state) + + require.NotNil(t, state.EnvoyGatewayOK) + assert.False(t, *state.EnvoyGatewayOK, "no running pods must set EnvoyGatewayOK=false") +} + +// -- checkGatewayRoutes -- + +func TestCheckGatewayRoutes_MissingCRDs(t *testing.T) { + // Fake client with no gateway.networking.k8s.io group registered. + client := fake.NewSimpleClientset() + state := &ValidationState{Log: testLog()} + checkGatewayRoutes(context.Background(), client, state) + require.NotNil(t, state.GatewayRoutesOK) + assert.False(t, *state.GatewayRoutesOK, "missing route CR types must set GatewayRoutesOK=false") +} + +// -- checkExternalLoadBalancer -- + +func TestCheckExternalLoadBalancer_ServiceWithIP(t *testing.T) { + client := fake.NewSimpleClientset(&corev1.Service{ + ObjectMeta: metav1.ObjectMeta{Name: "envoy-gateway", Namespace: envoyGatewayNamespaceName()}, + Spec: corev1.ServiceSpec{Type: corev1.ServiceTypeLoadBalancer}, + Status: corev1.ServiceStatus{ + LoadBalancer: corev1.LoadBalancerStatus{ + Ingress: []corev1.LoadBalancerIngress{{IP: "203.0.113.1"}}, + }, + }, + }) + state := &ValidationState{Log: testLog()} + checkExternalLoadBalancer(context.Background(), client, state) + + require.NotNil(t, state.ExternalLBOK) + assert.True(t, *state.ExternalLBOK, "a LB service with an assigned IP must set ExternalLBOK=true") +} + +func TestCheckExternalLoadBalancer_ServiceWithHostname(t *testing.T) { + client := fake.NewSimpleClientset(&corev1.Service{ + ObjectMeta: metav1.ObjectMeta{Name: "envoy-gateway", Namespace: envoyGatewayNamespaceName()}, + Spec: corev1.ServiceSpec{Type: corev1.ServiceTypeLoadBalancer}, + Status: corev1.ServiceStatus{ + LoadBalancer: corev1.LoadBalancerStatus{ + Ingress: []corev1.LoadBalancerIngress{{Hostname: "lb.example.com"}}, + }, + }, + }) + state := &ValidationState{Log: testLog()} + checkExternalLoadBalancer(context.Background(), client, state) + + require.NotNil(t, state.ExternalLBOK) + assert.True(t, *state.ExternalLBOK, "a LB service with a hostname must set ExternalLBOK=true") +} + +func TestCheckExternalLoadBalancer_NoLBServices(t *testing.T) { + client := fake.NewSimpleClientset(&corev1.Service{ + ObjectMeta: metav1.ObjectMeta{Name: "cluster-ip-svc", Namespace: envoyGatewayNamespaceName()}, + Spec: corev1.ServiceSpec{Type: corev1.ServiceTypeClusterIP}, + }) + state := &ValidationState{Log: testLog()} + checkExternalLoadBalancer(context.Background(), client, state) + + require.NotNil(t, state.ExternalLBOK) + assert.False(t, *state.ExternalLBOK, "no LB service must set ExternalLBOK=false") + assert.NotEmpty(t, state.Warnings) +} + +func TestCheckExternalLoadBalancer_LBServicePendingNoIP(t *testing.T) { + // LB type but .status.loadBalancer.ingress is empty → no IP assigned yet. + client := fake.NewSimpleClientset(&corev1.Service{ + ObjectMeta: metav1.ObjectMeta{Name: "pending-lb", Namespace: envoyGatewayNamespaceName()}, + Spec: corev1.ServiceSpec{Type: corev1.ServiceTypeLoadBalancer}, + // No Status.LoadBalancer.Ingress + }) + state := &ValidationState{Log: testLog()} + checkExternalLoadBalancer(context.Background(), client, state) + + require.NotNil(t, state.ExternalLBOK) + assert.False(t, *state.ExternalLBOK, "LB service with no assigned IP must set ExternalLBOK=false") +} + +// -- checkNodeToNode -- + +func TestCheckNodeToNode_NoNodes(t *testing.T) { + client := fake.NewSimpleClientset() + state := &ValidationState{Log: testLog()} + checkNodeToNode(context.Background(), client, state, enforcementDefaultImg) + + assert.Nil(t, state.NodeToNodeOK, + "zero schedulable nodes exercised no overlay path, so the result must be unknown, not Verified") + assert.NotEmpty(t, state.Warnings, "skip must add a warning so the banner is qualified") +} + +func TestCheckNodeToNode_UnschedulableNodesSkipped(t *testing.T) { + // Two nodes but both unschedulable — should also skip. + n1 := makeNode("node-1", true, 0) + n1.Spec.Unschedulable = true + n2 := makeNode("node-2", true, 0) + n2.Spec.Unschedulable = true + + client := fake.NewSimpleClientset(n1, n2) + state := &ValidationState{Log: testLog()} + checkNodeToNode(context.Background(), client, state, enforcementDefaultImg) + + assert.Nil(t, state.NodeToNodeOK, "no schedulable nodes means the probe never ran") +} + +func TestCheckNodeToNode_TaintedNodeExcluded(t *testing.T) { + // Three nodes: two schedulable, one with a NoSchedule taint. + // DesiredNumberScheduled=2 (tainted node excluded by scheduler), so + // waitForDaemonSetPods must converge on 2 pods, not 3. If the old + // len(schedulable)=3 path were used the test would block until deadline. + n1 := makeNode("node-1", true, 0) + n2 := makeNode("node-2", true, 0) + n3 := makeNode("node-3", true, 0) + n3.Spec.Taints = []corev1.Taint{{ + Key: "dedicated", Value: "gpu", Effect: corev1.TaintEffectNoSchedule, + }} + + client := fake.NewSimpleClientset(n1, n2, n3) + + // Capture DaemonSet labels (which include a random suffix) so the pod-list + // reactor can return pods that survive FakePods.List label filtering. + // capturedLabels is set synchronously by the daemonset create reactor + // before any list call, so no synchronisation is needed. + var capturedLabels map[string]string + var capturedName, capturedNS string + // Return a ZEROED status, exactly as a real apiserver does: the DaemonSet + // controller populates status asynchronously, so the Create response never + // carries DesiredNumberScheduled. Reading it here would yield 0 and fall + // back to len(schedulable)=3, which is the regression this guards. + client.PrependReactor("create", "daemonsets", func(action ktesting.Action) (bool, runtime.Object, error) { + ds := action.(ktesting.CreateAction).GetObject().(*appsv1.DaemonSet) + capturedLabels = ds.Labels + capturedName = ds.Name + capturedNS = ds.Namespace + ds.Status = appsv1.DaemonSetStatus{} + return true, ds, nil + }) + + // The subsequent Get is where the reconciled status appears: 2, because the + // scheduler excludes the tainted node. + client.PrependReactor("get", "daemonsets", func(_ ktesting.Action) (bool, runtime.Object, error) { + return true, &appsv1.DaemonSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: capturedName, + Namespace: capturedNS, + Labels: capturedLabels, + Generation: 1, + }, + Status: appsv1.DaemonSetStatus{ + DesiredNumberScheduled: 2, + ObservedGeneration: 1, + }, + }, nil + }) + + // Return 2 Running pods whose labels match the DaemonSet selector. + // FakePods.List filters by label after the reactor returns, so pods must + // carry the full label set including the random instance suffix. + client.PrependReactor("list", "pods", func(_ ktesting.Action) (bool, runtime.Object, error) { + lbl := capturedLabels + return true, &corev1.PodList{Items: []corev1.Pod{ + { + ObjectMeta: metav1.ObjectMeta{Name: "s-1", Namespace: capturedNS, Labels: lbl}, + Spec: corev1.PodSpec{NodeName: "node-1"}, + Status: corev1.PodStatus{Phase: corev1.PodRunning, PodIP: "10.0.0.1"}, + }, + { + ObjectMeta: metav1.ObjectMeta{Name: "s-2", Namespace: capturedNS, Labels: lbl}, + Spec: corev1.PodSpec{NodeName: "node-2"}, + Status: corev1.PodStatus{Phase: corev1.PodRunning, PodIP: "10.0.0.2"}, + }, + }}, nil + }) + + // Fail checker pod creation so the test exits quickly without needing to + // simulate full pod lifecycle (no Get/poll needed). + var checkerPodCreateCalled bool + client.PrependReactor("create", "pods", func(_ ktesting.Action) (bool, runtime.Object, error) { + checkerPodCreateCalled = true + return true, nil, fmt.Errorf("no pods scheduled") + }) + + state := &ValidationState{Log: testLog()} + checkNodeToNode(context.Background(), client, state, enforcementDefaultImg) + + // checkerPodCreateCalled must be true: if the check had sized its wait from + // len(schedulable)=3 rather than polling for DesiredNumberScheduled=2, it + // would have timed out before reaching pod creation and this flag would + // stay false, catching the regression. + require.True(t, checkerPodCreateCalled, "check must reach checker pod creation step") + require.NotNil(t, state.NodeToNodeOK) + assert.False(t, *state.NodeToNodeOK, "NodeToNodeOK false because checker pod creation failed") +} + +func TestCheckNodeToNode_DaemonSetCreateFailure(t *testing.T) { + // Two schedulable nodes, but DaemonSet creation fails. + client := fake.NewSimpleClientset( + makeNode("node-1", true, 0), + makeNode("node-2", true, 0), + ) + client.PrependReactor("create", "daemonsets", func(_ ktesting.Action) (bool, runtime.Object, error) { + return true, nil, fmt.Errorf("quota exceeded") + }) + + state := &ValidationState{Log: testLog()} + checkNodeToNode(context.Background(), client, state, enforcementDefaultImg) + + require.NotNil(t, state.NodeToNodeOK) + assert.False(t, *state.NodeToNodeOK, "DaemonSet create failure must set NodeToNodeOK=false") +} + +// The probe creates a namespace, a DaemonSet, and a pod on every node. If the +// deferred cleanup regresses, every validator run leaks all three, so assert +// the deletes are actually issued rather than only that the verdict is right. +func TestCheckNodeToNode_CleansUpProbeResources(t *testing.T) { + client := fake.NewSimpleClientset( + makeNode("node-1", true, 0), + makeNode("node-2", true, 0), + ) + + var createdNS string + client.PrependReactor("create", "namespaces", func(action ktesting.Action) (bool, runtime.Object, error) { + ns := action.(ktesting.CreateAction).GetObject().(*corev1.Namespace) + createdNS = ns.Name + return false, nil, nil // fall through to the tracker + }) + + deleted := map[string]bool{} + for _, res := range []string{"namespaces", "daemonsets", "pods"} { + r := res + client.PrependReactor("delete", r, func(_ ktesting.Action) (bool, runtime.Object, error) { + deleted[r] = true + return false, nil, nil + }) + } + + // Fail the DaemonSet status poll fast so the test does not wait out the + // real timeout; cleanup must still run on this path. + client.PrependReactor("get", "daemonsets", func(_ ktesting.Action) (bool, runtime.Object, error) { + return true, nil, fmt.Errorf("simulated status read failure") + }) + + state := &ValidationState{Log: testLog()} + checkNodeToNode(context.Background(), client, state, enforcementDefaultImg) + + require.NotEmpty(t, createdNS, "probe must create its own namespace, not use default") + assert.True(t, strings.HasPrefix(createdNS, nodeToNodeNSPrefix), + "probe namespace %q must carry the sweepable prefix %q", createdNS, nodeToNodeNSPrefix) + assert.True(t, deleted["daemonsets"], "deferred cleanup must delete the server DaemonSet") + assert.True(t, deleted["pods"], "deferred cleanup must delete the checker pod") + assert.True(t, deleted["namespaces"], "deferred cleanup must delete the probe namespace") +} + +// An orphaned probe namespace older than the TTL must be reclaimed; one inside +// the TTL belongs to a possibly-concurrent run and must be left alone. +func TestSweepOrphanN2NNamespaces_TTL(t *testing.T) { + labels := map[string]string{ + "app.kubernetes.io/managed-by": "nvcf-cluster-validator", + "app.kubernetes.io/component": "n2n-probe", + } + stale := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{ + Name: nodeToNodeNSPrefix + "stale1", + Labels: labels, + CreationTimestamp: metav1.NewTime(time.Now().Add(-30 * time.Minute)), + }} + fresh := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{ + Name: nodeToNodeNSPrefix + "fresh1", + Labels: labels, + CreationTimestamp: metav1.NewTime(time.Now()), + }} + client := fake.NewSimpleClientset(stale, fresh) + + sweepOrphanN2NNamespaces(context.Background(), testLog(), client, orphanN2NNamespaceTTL) + + _, err := client.CoreV1().Namespaces().Get(context.Background(), stale.Name, metav1.GetOptions{}) + assert.True(t, apierrors.IsNotFound(err), "namespace older than the TTL must be swept") + + _, err = client.CoreV1().Namespaces().Get(context.Background(), fresh.Name, metav1.GetOptions{}) + assert.NoError(t, err, "namespace inside the TTL may belong to a concurrent run and must survive") +} + +// -- checkTier1Deployments -- + +func TestCheckTier1Deployments_AllReady(t *testing.T) { + replicas := int32(2) + client := fake.NewSimpleClientset(&appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{Name: "nvcf-api", Namespace: "nvcf"}, + Spec: appsv1.DeploymentSpec{Replicas: &replicas}, + Status: appsv1.DeploymentStatus{ + ObservedGeneration: 1, + UpdatedReplicas: 2, + ReadyReplicas: 2, + }, + }) + state := &ValidationState{Log: testLog()} + checkTier1Deployments(context.Background(), client, state) + + require.NotNil(t, state.Tier1DeploymentsOK) + assert.True(t, *state.Tier1DeploymentsOK) + assert.Empty(t, state.Warnings) +} + +func TestCheckTier1Deployments_UnderReplicated(t *testing.T) { + replicas := int32(2) + client := fake.NewSimpleClientset(&appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "nvcf-api", Namespace: "nvcf", + Generation: 1, + }, + Spec: appsv1.DeploymentSpec{Replicas: &replicas}, + Status: appsv1.DeploymentStatus{ + ObservedGeneration: 1, + UpdatedReplicas: 2, + ReadyReplicas: 1, // one pod crashed + }, + }) + state := &ValidationState{Log: testLog()} + checkTier1Deployments(context.Background(), client, state) + + require.NotNil(t, state.Tier1DeploymentsOK) + assert.False(t, *state.Tier1DeploymentsOK, "crashed pod must set Tier1DeploymentsOK=false") +} + +func TestCheckTier1Deployments_RollingOutEmitsWarningNotFailure(t *testing.T) { + replicas := int32(2) + client := fake.NewSimpleClientset(&appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "nvcf-api", Namespace: "nvcf", + Generation: 3, // new spec written + }, + Spec: appsv1.DeploymentSpec{Replicas: &replicas}, + Status: appsv1.DeploymentStatus{ + ObservedGeneration: 2, // controller hasn't caught up yet + UpdatedReplicas: 1, // only 1 of 2 pods updated + ReadyReplicas: 2, // old pods still serving (maxUnavailable=0) + }, + }) + state := &ValidationState{Log: testLog()} + checkTier1Deployments(context.Background(), client, state) + + assert.Nil(t, state.Tier1DeploymentsOK, + "the only Deployment is mid-rollout, so readiness is unknown, not a pass or a failure") + require.NotEmpty(t, state.Warnings, "rollout in progress must emit a warning") + assert.Contains(t, state.Warnings[0], "rollout in progress") +} + +// A stalled rollout is not transient: Kubernetes caps the new ReplicaSet at +// maxSurge and never progresses, so ProgressDeadlineExceeded must fall through +// to the replica check rather than being skipped forever as "in progress". +func TestCheckTier1Deployments_StalledRolloutFails(t *testing.T) { + replicas := int32(3) + client := fake.NewSimpleClientset(&appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{Name: "nvcf-api", Namespace: "nvcf", Generation: 2}, + Spec: appsv1.DeploymentSpec{Replicas: &replicas}, + Status: appsv1.DeploymentStatus{ + ObservedGeneration: 2, + UpdatedReplicas: 1, // wedged on a bad image, never reaches 3 + ReadyReplicas: 0, + Conditions: []appsv1.DeploymentCondition{{ + Type: appsv1.DeploymentProgressing, + Status: corev1.ConditionFalse, + Reason: "ProgressDeadlineExceeded", + }}, + }, + }) + state := &ValidationState{Log: testLog()} + checkTier1Deployments(context.Background(), client, state) + + require.NotNil(t, state.Tier1DeploymentsOK) + assert.False(t, *state.Tier1DeploymentsOK, + "a rollout that exceeded its progress deadline with 0 ready must fail, not be skipped") +} + +// With one Deployment mid-rollout and another fully ready, the ready one must +// still be evaluated: the rollout skip must not suppress the whole check. +func TestCheckTier1Deployments_MixedRollingAndUnderReplicated(t *testing.T) { + two := int32(2) + client := fake.NewSimpleClientset( + &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{Name: "rolling", Namespace: "nvcf", Generation: 3}, + Spec: appsv1.DeploymentSpec{Replicas: &two}, + Status: appsv1.DeploymentStatus{ + ObservedGeneration: 2, UpdatedReplicas: 1, ReadyReplicas: 2, + }, + }, + &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{Name: "degraded", Namespace: "sis", Generation: 1}, + Spec: appsv1.DeploymentSpec{Replicas: &two}, + Status: appsv1.DeploymentStatus{ + ObservedGeneration: 1, UpdatedReplicas: 2, ReadyReplicas: 1, + }, + }, + ) + state := &ValidationState{Log: testLog()} + checkTier1Deployments(context.Background(), client, state) + + require.NotNil(t, state.Tier1DeploymentsOK) + assert.False(t, *state.Tier1DeploymentsOK, + "the under-replicated Deployment must still fail the check alongside a rolling one") +} + +func TestCheckTier1Deployments_PreInstallPassesTrivially(t *testing.T) { + client := fake.NewSimpleClientset() // no namespaces, no deployments + state := &ValidationState{Log: testLog()} + checkTier1Deployments(context.Background(), client, state) + + require.NotNil(t, state.Tier1DeploymentsOK) + assert.True(t, *state.Tier1DeploymentsOK, "pre-install (no deployments) must pass trivially") +} + +// An RBAC denial means the namespace was never observed. Treating it as a pass +// publishes a green quorum for a control plane nobody looked at. +func TestCheckTier1Deployments_ForbiddenIsNotAPass(t *testing.T) { + client := fake.NewSimpleClientset() + client.PrependReactor("list", "deployments", func(_ ktesting.Action) (bool, runtime.Object, error) { + return true, nil, apierrors.NewForbidden( + schema.GroupResource{Group: "apps", Resource: "deployments"}, "", fmt.Errorf("denied")) + }) + + state := &ValidationState{Log: testLog()} + checkTier1Deployments(context.Background(), client, state) + + assert.Nil(t, state.Tier1DeploymentsOK, "a 403 in every namespace must not reach the trivial-pass exit") + assert.NotEmpty(t, state.Warnings, "an RBAC denial must be surfaced as a warning") +} + +// -- checkTier2StatefulSets -- + +// makeQuorumSTS builds a StatefulSet plus the pods its selector matches, so the +// co-location scan has something to walk. nodes gives one node name per pod. +func makeQuorumSTS(name, ns string, replicas, ready int32, nodes []string) []runtime.Object { + sel := map[string]string{"app": name} + // IsControlledBy compares the controller reference UID, so the fixture needs + // a real one on both sides. + uid := types.UID("uid-" + name) + controller := true + objs := []runtime.Object{&appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns, UID: uid}, + Spec: appsv1.StatefulSetSpec{ + Replicas: &replicas, + Selector: &metav1.LabelSelector{MatchLabels: sel}, + }, + Status: appsv1.StatefulSetStatus{ + ReadyReplicas: ready, + CurrentRevision: name + "-r1", + UpdateRevision: name + "-r1", + }, + }} + for i, node := range nodes { + objs = append(objs, &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-%d", name, i), Namespace: ns, Labels: sel, + OwnerReferences: []metav1.OwnerReference{{ + APIVersion: "apps/v1", Kind: "StatefulSet", + Name: name, UID: uid, Controller: &controller, + }}, + }, + Spec: corev1.PodSpec{NodeName: node}, + Status: corev1.PodStatus{ + Phase: corev1.PodRunning, + Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: corev1.ConditionTrue}}, + }, + }) + } + return objs +} + +func TestCheckTier2StatefulSets_HealthyQuorum(t *testing.T) { + objs := makeQuorumSTS("nats", "nats-system", 3, 3, []string{"node-1", "node-2", "node-3"}) + client := fake.NewSimpleClientset(objs...) + state := &ValidationState{Log: testLog()} + checkTier2StatefulSets(context.Background(), client, state) + + require.NotNil(t, state.Tier2StatefulSetsOK) + assert.True(t, *state.Tier2StatefulSetsOK, "3 Ready pods on distinct nodes must pass") +} + +func TestCheckTier2StatefulSets_BelowQuorumFails(t *testing.T) { + objs := makeQuorumSTS("openbao", "vault-system", 3, 2, []string{"node-1", "node-2"}) + client := fake.NewSimpleClientset(objs...) + state := &ValidationState{Log: testLog()} + checkTier2StatefulSets(context.Background(), client, state) + + require.NotNil(t, state.Tier2StatefulSetsOK) + assert.False(t, *state.Tier2StatefulSetsOK, "readyReplicas below spec.replicas must fail") +} + +// Two peers on one node means a single node loss takes out the quorum, which is +// the whole point of the placement half of this check. +func TestCheckTier2StatefulSets_CoLocatedPeersFail(t *testing.T) { + objs := makeQuorumSTS("cassandra", "cassandra-system", 3, 3, []string{"node-1", "node-1", "node-2"}) + client := fake.NewSimpleClientset(objs...) + state := &ValidationState{Log: testLog()} + checkTier2StatefulSets(context.Background(), client, state) + + require.NotNil(t, state.Tier2StatefulSetsOK) + assert.False(t, *state.Tier2StatefulSetsOK, "two peers on the same node must fail placement") +} + +// StatefulSets roll one pod at a time, so a below-target ready count is the +// steady state for the whole duration of any upgrade. That must warn, not fail. +func TestCheckTier2StatefulSets_RollingUpdateWarnsNotFails(t *testing.T) { + objs := makeQuorumSTS("nats", "nats-system", 3, 2, []string{"node-1", "node-2"}) + sts := objs[0].(*appsv1.StatefulSet) + sts.Status.UpdateRevision = "nats-r2" // differs from CurrentRevision + client := fake.NewSimpleClientset(objs...) + state := &ValidationState{Log: testLog()} + checkTier2StatefulSets(context.Background(), client, state) + + assert.Nil(t, state.Tier2StatefulSetsOK, + "the only quorum StatefulSet is mid-rollout, so quorum is unknown, not failed") + require.NotEmpty(t, state.Warnings) + assert.Contains(t, state.Warnings[0], "revision mismatch") +} + +// Requiring exactly 3 silently drops an operator-scaled 5-member Cassandra from +// the check instead of validating it. +func TestCheckTier2StatefulSets_FiveReplicasStillChecked(t *testing.T) { + objs := makeQuorumSTS("cassandra", "cassandra-system", 5, 4, + []string{"node-1", "node-2", "node-3", "node-4"}) + client := fake.NewSimpleClientset(objs...) + state := &ValidationState{Log: testLog()} + checkTier2StatefulSets(context.Background(), client, state) + + require.NotNil(t, state.Tier2StatefulSetsOK) + assert.False(t, *state.Tier2StatefulSetsOK, + "a 5-replica quorum with only 4 Ready must fail, not be skipped") +} + +// An even replica count cannot form a quorum majority, so it is not a Tier-2 +// component and must not be evaluated as one. +// An even-replica StatefulSet is not a quorum shape this check can reason +// about, but it is not nothing either: a 4-replica Cassandra ring with RF=3 can +// have lost quorum. When it is the only StatefulSet present, the tier must be +// unknown rather than a pass that certifies a ring nothing examined. +func TestCheckTier2StatefulSets_EvenReplicasNotAssessed(t *testing.T) { + objs := makeQuorumSTS("worker", "nvcf", 4, 2, []string{"node-1", "node-2"}) + client := fake.NewSimpleClientset(objs...) + state := &ValidationState{Log: testLog()} + checkTier2StatefulSets(context.Background(), client, state) + + assert.Nil(t, state.Tier2StatefulSetsOK, + "nothing was assessed, so the tier is unknown, not a pass") + require.NotEmpty(t, state.Warnings, "the skipped StatefulSet must be surfaced") + assert.Contains(t, state.Warnings[0], "nvcf/worker") +} + +// An odd-replica quorum member alongside an even-replica one still gets +// assessed; the even one is reported as a warning rather than silently dropped. +func TestCheckTier2StatefulSets_EvenReplicasWarnAlongsideQuorum(t *testing.T) { + objs := makeQuorumSTS("worker", "nvcf", 4, 2, []string{"node-1", "node-2"}) + objs = append(objs, makeQuorumSTS("nats", "nats-system", 3, 3, + []string{"node-1", "node-2", "node-3"})...) + client := fake.NewSimpleClientset(objs...) + state := &ValidationState{Log: testLog()} + checkTier2StatefulSets(context.Background(), client, state) + + require.NotNil(t, state.Tier2StatefulSetsOK) + assert.True(t, *state.Tier2StatefulSetsOK, "the odd-replica quorum member is healthy") + joined := strings.Join(state.Warnings, "; ") + assert.Contains(t, joined, "nvcf/worker", "the even-replica StatefulSet must still be reported") +} + +func TestCheckTier2StatefulSets_ForbiddenIsNotAPass(t *testing.T) { + client := fake.NewSimpleClientset() + client.PrependReactor("list", "statefulsets", func(_ ktesting.Action) (bool, runtime.Object, error) { + return true, nil, apierrors.NewForbidden( + schema.GroupResource{Group: "apps", Resource: "statefulsets"}, "", fmt.Errorf("denied")) + }) + + state := &ValidationState{Log: testLog()} + checkTier2StatefulSets(context.Background(), client, state) + + assert.Nil(t, state.Tier2StatefulSetsOK, + "a 403 in every namespace must not publish a green quorum") + assert.NotEmpty(t, state.Warnings) +} + +// A denial in only some namespaces still means part of the control plane was +// never observed, so the healthy remainder must not be published as a pass. +func TestCheckTier2StatefulSets_PartialDenialIsNotAPass(t *testing.T) { + objs := makeQuorumSTS("nats", "nats-system", 3, 3, []string{"node-1", "node-2", "node-3"}) + client := fake.NewSimpleClientset(objs...) + client.PrependReactor("list", "statefulsets", func(action ktesting.Action) (bool, runtime.Object, error) { + if action.GetNamespace() == "vault-system" { + return true, nil, apierrors.NewForbidden( + schema.GroupResource{Group: "apps", Resource: "statefulsets"}, "", fmt.Errorf("denied")) + } + return false, nil, nil + }) + + state := &ValidationState{Log: testLog()} + checkTier2StatefulSets(context.Background(), client, state) + + assert.Nil(t, state.Tier2StatefulSetsOK, + "a healthy StatefulSet elsewhere must not mask an unreadable namespace") + assert.NotEmpty(t, state.Warnings) +} + +func TestCheckTier1Deployments_PartialDenialIsNotAPass(t *testing.T) { + two := int32(2) + client := fake.NewSimpleClientset(&appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{Name: "nvcf-api", Namespace: "nvcf", Generation: 1}, + Spec: appsv1.DeploymentSpec{Replicas: &two}, + Status: appsv1.DeploymentStatus{ + ObservedGeneration: 1, UpdatedReplicas: 2, ReadyReplicas: 2, + }, + }) + client.PrependReactor("list", "deployments", func(action ktesting.Action) (bool, runtime.Object, error) { + if action.GetNamespace() == "sis" { + return true, nil, apierrors.NewForbidden( + schema.GroupResource{Group: "apps", Resource: "deployments"}, "", fmt.Errorf("denied")) + } + return false, nil, nil + }) + + state := &ValidationState{Log: testLog()} + checkTier1Deployments(context.Background(), client, state) + + assert.Nil(t, state.Tier1DeploymentsOK, + "a ready Deployment elsewhere must not mask an unreadable namespace") + assert.NotEmpty(t, state.Warnings) +} + +// A pod matching the selector but not owned by this StatefulSet, or not Ready, +// must not be counted: either would turn a healthy quorum into a false +// co-location failure. +func TestCheckTier2StatefulSets_IgnoresUnownedAndUnreadyPods(t *testing.T) { + objs := makeQuorumSTS("nats", "nats-system", 3, 3, []string{"node-1", "node-2", "node-3"}) + + // A surplus pod on an already-used node: matches the selector, but is + // neither owned by the StatefulSet nor Ready. + stray := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "stray", Namespace: "nats-system", Labels: map[string]string{"app": "nats"}, + }, + Spec: corev1.PodSpec{NodeName: "node-1"}, + Status: corev1.PodStatus{ + Phase: corev1.PodRunning, + Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: corev1.ConditionFalse}}, + }, + } + client := fake.NewSimpleClientset(append(objs, stray)...) + + state := &ValidationState{Log: testLog()} + checkTier2StatefulSets(context.Background(), client, state) + + require.NotNil(t, state.Tier2StatefulSetsOK) + assert.True(t, *state.Tier2StatefulSetsOK, + "an unowned, not-Ready pod on an occupied node must not be read as a co-located peer") +} + +// An owner reference naming the StatefulSet but belonging to another kind must +// not count: matching on name alone would let its pod cause a co-location +// failure on a healthy quorum. +func TestCheckTier2StatefulSets_IgnoresSameNamedOwnerOfAnotherKind(t *testing.T) { + objs := makeQuorumSTS("nats", "nats-system", 3, 3, []string{"node-1", "node-2", "node-3"}) + controller := true + impostor := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "impostor", Namespace: "nats-system", Labels: map[string]string{"app": "nats"}, + OwnerReferences: []metav1.OwnerReference{{ + APIVersion: "apps/v1", Kind: "Deployment", + Name: "nats", UID: types.UID("some-other-uid"), Controller: &controller, + }}, + }, + Spec: corev1.PodSpec{NodeName: "node-1"}, + Status: corev1.PodStatus{ + Phase: corev1.PodRunning, + Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: corev1.ConditionTrue}}, + }, + } + client := fake.NewSimpleClientset(append(objs, impostor)...) + + state := &ValidationState{Log: testLog()} + checkTier2StatefulSets(context.Background(), client, state) + + require.NotNil(t, state.Tier2StatefulSetsOK) + assert.True(t, *state.Tier2StatefulSetsOK, + "a Ready pod owned by a same-named Deployment must not be treated as a StatefulSet peer") +} + +// A healthy StatefulSet must not stand in for one that was skipped mid-rollout: +// its quorum was never assessed, so the tier result is partial, not a pass. +func TestCheckTier2StatefulSets_HealthyPeerDoesNotMaskRollingOne(t *testing.T) { + healthy := makeQuorumSTS("nats", "nats-system", 3, 3, []string{"node-1", "node-2", "node-3"}) + rolling := makeQuorumSTS("openbao", "vault-system", 3, 2, []string{"node-1", "node-2"}) + rolling[0].(*appsv1.StatefulSet).Status.UpdateRevision = "openbao-r2" + + client := fake.NewSimpleClientset(append(healthy, rolling...)...) + state := &ValidationState{Log: testLog()} + checkTier2StatefulSets(context.Background(), client, state) + + assert.Nil(t, state.Tier2StatefulSetsOK, + "one StatefulSet still rolling means the tier assessment is partial, not a pass") + assert.NotEmpty(t, state.Warnings) +} + +// Same shape for Tier-1: a ready Deployment does not certify one still rolling. +// A mid-rollout Deployment that is still serving its full replica count hides +// nothing, so it must not pin this critical row to UNKNOWN. rollingOut is not +// self-limiting: a paused rollout, progressDeadlineSeconds=2147483647, and a +// wedged controller all stay "rolling" forever without ever setting +// ProgressDeadlineExceeded. +func TestCheckTier1Deployments_RollingAtFullReplicasStillPasses(t *testing.T) { + two := int32(2) + client := fake.NewSimpleClientset( + &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{Name: "ready", Namespace: "nvcf", Generation: 1}, + Spec: appsv1.DeploymentSpec{Replicas: &two}, + Status: appsv1.DeploymentStatus{ + ObservedGeneration: 1, UpdatedReplicas: 2, ReadyReplicas: 2, + }, + }, + &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{Name: "rolling", Namespace: "sis", Generation: 3}, + Spec: appsv1.DeploymentSpec{Replicas: &two}, + Status: appsv1.DeploymentStatus{ + ObservedGeneration: 2, UpdatedReplicas: 1, ReadyReplicas: 2, + }, + }, + ) + state := &ValidationState{Log: testLog()} + checkTier1Deployments(context.Background(), client, state) + + require.NotNil(t, state.Tier1DeploymentsOK, + "a rollout at full ready count must not leave the tier permanently unknown") + assert.True(t, *state.Tier1DeploymentsOK) + assert.NotEmpty(t, state.Warnings, "the in-flight rollout is still reported") +} + +// A mid-rollout Deployment that is ALSO below its ready target is the case +// that can hide a real outage, so the tier assessment is partial. +func TestCheckTier1Deployments_RollingAndUnderReplicatedIsUnknown(t *testing.T) { + two := int32(2) + client := fake.NewSimpleClientset( + &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{Name: "ready", Namespace: "nvcf", Generation: 1}, + Spec: appsv1.DeploymentSpec{Replicas: &two}, + Status: appsv1.DeploymentStatus{ + ObservedGeneration: 1, UpdatedReplicas: 2, ReadyReplicas: 2, + }, + }, + &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{Name: "rolling", Namespace: "sis", Generation: 3}, + Spec: appsv1.DeploymentSpec{Replicas: &two}, + Status: appsv1.DeploymentStatus{ + ObservedGeneration: 2, UpdatedReplicas: 1, ReadyReplicas: 1, + }, + }, + ) + state := &ValidationState{Log: testLog()} + checkTier1Deployments(context.Background(), client, state) + + assert.Nil(t, state.Tier1DeploymentsOK, + "a rollout below its replica target leaves the tier assessment partial") + require.NotEmpty(t, state.Warnings) +} + +func TestControlPlaneNamespaceSet_HonoursOpenBaoOverride(t *testing.T) { + t.Setenv(openBaoNamespaceEnv, "vault-system-dev") + assert.Contains(t, controlPlaneNamespaceSet(), "vault-system-dev") + + t.Setenv(openBaoNamespaceEnv, "vault-system") // already in the base list + set := controlPlaneNamespaceSet() + count := 0 + for _, ns := range set { + if ns == "vault-system" { + count++ + } + } + assert.Equal(t, 1, count, "an override matching the default must not duplicate the entry") +} + +// A Deployment scaled to zero satisfies "ReadyReplicas >= spec.replicas" with +// nothing running, so counting it as healthy lets a maintenance scale-down or a +// replicaCount:0 values error publish the critical row as All Ready. +func TestCheckTier1Deployments_ScaledToZeroIsNotReady(t *testing.T) { + zero := int32(0) + client := fake.NewSimpleClientset( + &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{Name: "scaled-down", Namespace: "nvcf", Generation: 1}, + Spec: appsv1.DeploymentSpec{Replicas: &zero}, + Status: appsv1.DeploymentStatus{ObservedGeneration: 1}, + }, + ) + state := &ValidationState{Log: testLog()} + checkTier1Deployments(context.Background(), client, state) + + require.NotNil(t, state.Tier1DeploymentsOK) + assert.False(t, *state.Tier1DeploymentsOK, + "a control plane whose only Deployment is scaled to zero is not ready") +} + +// Alongside a healthy peer the scale-down is a warning, not a silent pass. +func TestCheckTier1Deployments_ScaledToZeroWarnsAlongsideHealthy(t *testing.T) { + zero, two := int32(0), int32(2) + client := fake.NewSimpleClientset( + &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{Name: "scaled-down", Namespace: "nvcf", Generation: 1}, + Spec: appsv1.DeploymentSpec{Replicas: &zero}, + Status: appsv1.DeploymentStatus{ObservedGeneration: 1}, + }, + &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{Name: "ready", Namespace: "sis", Generation: 1}, + Spec: appsv1.DeploymentSpec{Replicas: &two}, + Status: appsv1.DeploymentStatus{ + ObservedGeneration: 1, UpdatedReplicas: 2, ReadyReplicas: 2, + }, + }, + ) + state := &ValidationState{Log: testLog()} + checkTier1Deployments(context.Background(), client, state) + + require.NotNil(t, state.Tier1DeploymentsOK) + assert.True(t, *state.Tier1DeploymentsOK) + assert.Contains(t, strings.Join(state.Warnings, "; "), "nvcf/scaled-down") +} + +// An OnDelete StatefulSet never advances CurrentRevision, so a revision +// mismatch is permanent. At full ready count that must not hide the tier, and +// more than one pod down is beyond what rolling one at a time explains. +func TestCheckTier2StatefulSets_PermanentRevisionMismatchAtFullReadyPasses(t *testing.T) { + objs := makeQuorumSTS("openbao", "vault-system", 3, 3, + []string{"node-1", "node-2", "node-3"}) + sts := objs[0].(*appsv1.StatefulSet) + sts.Status.CurrentRevision = "rev-1" + sts.Status.UpdateRevision = "rev-2" + client := fake.NewSimpleClientset(objs...) + state := &ValidationState{Log: testLog()} + checkTier2StatefulSets(context.Background(), client, state) + + require.NotNil(t, state.Tier2StatefulSetsOK, + "an OnDelete StatefulSet at full ready count must not pin the tier to unknown") + assert.True(t, *state.Tier2StatefulSetsOK) +} + +func TestCheckTier2StatefulSets_RevisionMismatchTwoPodsDownFails(t *testing.T) { + objs := makeQuorumSTS("openbao", "vault-system", 3, 1, []string{"node-1"}) + sts := objs[0].(*appsv1.StatefulSet) + sts.Status.CurrentRevision = "rev-1" + sts.Status.UpdateRevision = "rev-2" + client := fake.NewSimpleClientset(objs...) + state := &ValidationState{Log: testLog()} + checkTier2StatefulSets(context.Background(), client, state) + + require.NotNil(t, state.Tier2StatefulSetsOK, + "losing two of three peers is a quorum finding, not an in-flight rollout") + assert.False(t, *state.Tier2StatefulSetsOK) +} + +// A DaemonSet stranded in "default" by a validator version that predates the +// per-run probe namespace is invisible to the namespace sweep, so it would +// otherwise persist forever as one probe pod per node. +func TestSweepLegacyOrphanN2NDaemonSets_DeletesStaleOnly(t *testing.T) { + labels := map[string]string{ + "app.kubernetes.io/managed-by": "nvcf-cluster-validator", + "app.kubernetes.io/component": "n2n-server", + } + old := metav1.NewTime(time.Now().Add(-time.Hour)) + fresh := metav1.NewTime(time.Now()) + client := fake.NewSimpleClientset( + &appsv1.DaemonSet{ObjectMeta: metav1.ObjectMeta{ + Name: nodeToNodeDSName + "-ab12cd", Namespace: "default", + Labels: labels, CreationTimestamp: old, + }}, + ) + sweepLegacyOrphanN2NDaemonSets(context.Background(), testLog(), client, orphanN2NNamespaceTTL) + _, err := client.AppsV1().DaemonSets("default").Get( + context.Background(), nodeToNodeDSName+"-ab12cd", metav1.GetOptions{}) + assert.True(t, apierrors.IsNotFound(err), "a stale legacy DaemonSet must be reclaimed") + + // A DaemonSet inside the TTL may belong to a concurrent run. + client2 := fake.NewSimpleClientset( + &appsv1.DaemonSet{ObjectMeta: metav1.ObjectMeta{ + Name: nodeToNodeDSName + "-ef34gh", Namespace: "default", + Labels: labels, CreationTimestamp: fresh, + }}, + ) + sweepLegacyOrphanN2NDaemonSets(context.Background(), testLog(), client2, orphanN2NNamespaceTTL) + _, err = client2.AppsV1().DaemonSets("default").Get( + context.Background(), nodeToNodeDSName+"-ef34gh", metav1.GetOptions{}) + assert.NoError(t, err, "a DaemonSet inside the TTL must be left alone") +} + +// The labels are three public constants, so the generated name is required too +// before deleting anything from a shared namespace. +func TestSweepLegacyOrphanN2NDaemonSets_RequiresTheGeneratedName(t *testing.T) { + client := fake.NewSimpleClientset( + &appsv1.DaemonSet{ObjectMeta: metav1.ObjectMeta{ + Name: "operator-owned", Namespace: "default", + Labels: map[string]string{ + "app.kubernetes.io/managed-by": "nvcf-cluster-validator", + "app.kubernetes.io/component": "n2n-server", + }, + CreationTimestamp: metav1.NewTime(time.Now().Add(-time.Hour)), + }}, + ) + sweepLegacyOrphanN2NDaemonSets(context.Background(), testLog(), client, orphanN2NNamespaceTTL) + _, err := client.AppsV1().DaemonSets("default").Get( + context.Background(), "operator-owned", metav1.GetOptions{}) + assert.NoError(t, err, "an object that does not carry our generated name must not be deleted") +} + +// From Kubernetes 1.26 the apiserver resolves multiple defaults by picking the +// newest, so PVCs bind and failing the critical row reports NVCF-Not-Ready on a +// working cluster. Mid-CSI-migration clusters (gp2 plus gp3) hit this. +func TestCheckStorageClass_MultipleDefaultsWarnOnModernKubernetes(t *testing.T) { + mk := func(name string) *storagev1.StorageClass { + return &storagev1.StorageClass{ObjectMeta: metav1.ObjectMeta{ + Name: name, + Annotations: map[string]string{"storageclass.kubernetes.io/is-default-class": "true"}, + }} + } + client := fake.NewSimpleClientset(mk("gp2"), mk("gp3")) + state := &ValidationState{Log: testLog(), K8sVersion: "v1.30.0"} + checkStorageClass(context.Background(), client, state) + + require.NotNil(t, state.DefaultStorageClassOK) + assert.True(t, *state.DefaultStorageClassOK, + "the apiserver picks the newest default, so PVCs still bind") + assert.Contains(t, strings.Join(state.Warnings, "; "), "Multiple default StorageClasses") +} + +func TestCheckStorageClass_MultipleDefaultsFailBefore126(t *testing.T) { + mk := func(name string) *storagev1.StorageClass { + return &storagev1.StorageClass{ObjectMeta: metav1.ObjectMeta{ + Name: name, + Annotations: map[string]string{"storageclass.kubernetes.io/is-default-class": "true"}, + }} + } + client := fake.NewSimpleClientset(mk("a"), mk("b")) + state := &ValidationState{Log: testLog(), K8sVersion: "v1.25.9"} + checkStorageClass(context.Background(), client, state) + + require.NotNil(t, state.DefaultStorageClassOK) + assert.False(t, *state.DefaultStorageClassOK, + "below 1.26 two defaults reject every PVC") +} + +// Only NotFound is evidence Envoy is absent. A 403 or an apiserver 500 means we +// never observed it, so the row must be unknown rather than a definite failure. +func TestCheckEnvoyGateway_APIErrorIsUnknownNotFailure(t *testing.T) { + client := fake.NewSimpleClientset() + client.PrependReactor("get", "namespaces", func(ktesting.Action) (bool, runtime.Object, error) { + return true, nil, apierrors.NewForbidden( + schema.GroupResource{Resource: "namespaces"}, envoyGatewayNamespace, fmt.Errorf("denied")) + }) + state := &ValidationState{Log: testLog()} + checkEnvoyGateway(context.Background(), client, state) + + assert.Nil(t, state.EnvoyGatewayOK, + "a denial is not evidence that Envoy Gateway is missing") + assert.NotEmpty(t, state.Warnings) +} + +func TestCheckEnvoyGateway_NotFoundStillFails(t *testing.T) { + client := fake.NewSimpleClientset() + state := &ValidationState{Log: testLog()} + checkEnvoyGateway(context.Background(), client, state) + + require.NotNil(t, state.EnvoyGatewayOK, + "an absent namespace is a real observation") + assert.False(t, *state.EnvoyGatewayOK) +} + +// Envoy Gateway provisions one proxy Service per Gateway and the stack defines +// several, so a partially satisfied address pool must not pass on the strength +// of its assigned siblings. +func TestCheckExternalLoadBalancer_PendingServiceIsNotAPass(t *testing.T) { + assigned := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{Name: "envoy-gateway-lb", Namespace: envoyGatewayNamespaceName()}, + Spec: corev1.ServiceSpec{Type: corev1.ServiceTypeLoadBalancer}, + Status: corev1.ServiceStatus{LoadBalancer: corev1.LoadBalancerStatus{ + Ingress: []corev1.LoadBalancerIngress{{IP: "10.0.0.1"}}, + }}, + } + pending := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{Name: "envoy-nats-gateway-lb", Namespace: envoyGatewayNamespaceName()}, + Spec: corev1.ServiceSpec{Type: corev1.ServiceTypeLoadBalancer}, + } + client := fake.NewSimpleClientset(assigned, pending) + state := &ValidationState{Log: testLog()} + checkExternalLoadBalancer(context.Background(), client, state) + + require.NotNil(t, state.ExternalLBOK) + assert.False(t, *state.ExternalLBOK, + "a Gateway still waiting on an address is the failure this check exists to catch") + assert.Contains(t, strings.Join(state.Warnings, "; "), "envoy-nats-gateway-lb") +} + +// The stack exposes controllerNamespace with no default, so an install can +// place Envoy outside envoy-gateway-system. Probing the wrong namespace +// reports a live gateway as missing. +func TestEnvoyGatewayNamespaceName_HonoursOverride(t *testing.T) { + // Pin the unset case explicitly: an override exported in the developer's + // shell would otherwise leak in and make this assert the wrong default. + t.Setenv(envoyGatewayNamespaceEnv, "") + assert.Equal(t, envoyGatewayNamespace, envoyGatewayNamespaceName()) + t.Setenv(envoyGatewayNamespaceEnv, "gateway") + assert.Equal(t, "gateway", envoyGatewayNamespaceName()) + assert.Contains(t, controlPlaneNamespaceSet(), "gateway", + "the relocated namespace must also be covered by the Tier checks") +} + +// The probe DaemonSet needs the same tolerations the validator CronJob carries: +// the DaemonSet controller auto-tolerates not-ready and unschedulable but not +// the control-plane taint, so a dedicated control plane schedules zero pods. +func TestBuildNodeToNodeDaemonSet_ToleratesControlPlaneTaint(t *testing.T) { + ds := buildNodeToNodeDaemonSet("n2n", "ns", map[string]string{"a": "b"}, "img") + var keys []string + for _, tol := range ds.Spec.Template.Spec.Tolerations { + keys = append(keys, tol.Key) + } + assert.Contains(t, keys, "node-role.kubernetes.io/control-plane") + assert.Contains(t, keys, "node-role.kubernetes.io/master") + + pod := buildNodeToNodeCheckerPod("checker", "ns", "node-1", []string{"10.0.0.1"}, "img") + assert.NotEmpty(t, pod.Spec.Tolerations, + "NodeName bypasses the scheduler but not taint admission") +} + +// gatewayDiscoveryClient returns a fake clientset whose discovery surface +// serves exactly the given "/" pairs. Until this +// existed no test in the package populated a discovery surface at all, so +// every pinned route pair and the hasPair lookup itself were uncovered: the +// Gateway tests passed identically against a bare fake, which serves nothing. +func gatewayDiscoveryClient(pairs ...string) *fake.Clientset { + client := fake.NewSimpleClientset() + byGV := map[string][]metav1.APIResource{} + for _, p := range pairs { + i := strings.LastIndex(p, "/") + gv, res := p[:i], p[i+1:] + byGV[gv] = append(byGV[gv], metav1.APIResource{Name: res}) + } + disco := client.Discovery().(*fakediscovery.FakeDiscovery) + for gv, resources := range byGV { + disco.Resources = append(disco.Resources, &metav1.APIResourceList{ + GroupVersion: gv, + APIResources: resources, + }) + } + return client +} + +// The full set the charts actually apply, at the versions the manifests pin. +func gatewayRequiredPairs() []string { + return []string{ + gatewayAPIGroup + "/v1/gatewayclasses", + gatewayAPIGroup + "/v1/gateways", + gatewayAPIGroup + "/v1/httproutes", + gatewayAPIGroup + "/v1/grpcroutes", + gatewayAPIGroup + "/v1alpha2/tcproutes", + gatewayAPIGroup + "/v1beta1/referencegrants", + } +} + +func TestCheckGatewayAPICRDs_AllRequiredPairsPresent(t *testing.T) { + client := gatewayDiscoveryClient(gatewayRequiredPairs()...) + state := &ValidationState{Log: testLog()} + checkGatewayAPICRDs(context.Background(), client, state) + + require.NotNil(t, state.GatewayAPICRDsOK) + assert.True(t, *state.GatewayAPICRDsOK) +} + +// TCPRoute ships only in the experimental channel, and the chart renders one by +// default, so a standard-channel cluster genuinely cannot apply the stack. +func TestCheckGatewayAPICRDs_StandardChannelMissingTCPRouteFails(t *testing.T) { + var pairs []string + for _, p := range gatewayRequiredPairs() { + if !strings.HasSuffix(p, "/tcproutes") { + pairs = append(pairs, p) + } + } + client := gatewayDiscoveryClient(pairs...) + state := &ValidationState{Log: testLog()} + checkGatewayAPICRDs(context.Background(), client, state) + + require.NotNil(t, state.GatewayAPICRDsOK) + assert.False(t, *state.GatewayAPICRDsOK) + assert.Contains(t, strings.Join(state.Recommendations, "; "), "experimental-install.yaml", + "the remediation must name the channel, not point back at the installer that just failed") +} + +// The version is part of the requirement: a CRD served only under some other +// version still fails the Helm apply. +func TestCheckGatewayAPICRDs_WrongVersionForPinnedRouteFails(t *testing.T) { + var pairs []string + for _, p := range gatewayRequiredPairs() { + if strings.HasSuffix(p, "/httproutes") { + p = gatewayAPIGroup + "/v1beta1/httproutes" // charts pin v1 + } + pairs = append(pairs, p) + } + client := gatewayDiscoveryClient(pairs...) + state := &ValidationState{Log: testLog()} + checkGatewayAPICRDs(context.Background(), client, state) + + require.NotNil(t, state.GatewayAPICRDsOK) + assert.False(t, *state.GatewayAPICRDsOK, + "httproutes served only at v1beta1 does not satisfy a v1 pin") +} + +// UDPRoute is rendered only when routes.llmWorker.enabled, which defaults to +// false, so its absence must not fail the critical CRD check. It still has to +// be reported somewhere, which is the non-critical route row's job. +func TestCheckGatewayRoutes_OptionalUDPRouteAbsentIsNonCritical(t *testing.T) { + client := gatewayDiscoveryClient(gatewayRequiredPairs()...) + + crdState := &ValidationState{Log: testLog()} + checkGatewayAPICRDs(context.Background(), client, crdState) + require.NotNil(t, crdState.GatewayAPICRDsOK) + assert.True(t, *crdState.GatewayAPICRDsOK, + "a missing opt-in route type must not fail the critical check") + + routeState := &ValidationState{Log: testLog()} + checkGatewayRoutes(context.Background(), client, routeState) + require.NotNil(t, routeState.GatewayRoutesOK) + assert.False(t, *routeState.GatewayRoutesOK) + assert.Contains(t, strings.Join(routeState.Warnings, "; "), "udproutes", + "UDPRoute is applied by udproute-llm-worker.yaml and must still be surfaced") +} + +func TestCheckGatewayRoutes_OptionalUDPRoutePresentPasses(t *testing.T) { + client := gatewayDiscoveryClient( + append(gatewayRequiredPairs(), gatewayAPIGroup+"/v1alpha2/udproutes")...) + state := &ValidationState{Log: testLog()} + checkGatewayRoutes(context.Background(), client, state) + + require.NotNil(t, state.GatewayRoutesOK) + assert.True(t, *state.GatewayRoutesOK) +} + +// An RBAC gap on pods must not be reported as a broken quorum, which is what +// appending it to failures did, while the identical gap on statefulsets is +// correctly reported as unknown. +func TestCheckTier2StatefulSets_PodListDenialIsUnknownNotFailure(t *testing.T) { + objs := makeQuorumSTS("nats", "nats-system", 3, 3, + []string{"node-1", "node-2", "node-3"}) + client := fake.NewSimpleClientset(objs...) + client.PrependReactor("list", "pods", func(ktesting.Action) (bool, runtime.Object, error) { + return true, nil, apierrors.NewForbidden( + schema.GroupResource{Resource: "pods"}, "", fmt.Errorf("denied")) + }) + state := &ValidationState{Log: testLog()} + checkTier2StatefulSets(context.Background(), client, state) + + assert.Nil(t, state.Tier2StatefulSetsOK, + "an unreadable pod list is not evidence of a broken quorum") + assert.Contains(t, strings.Join(state.Warnings, "; "), "placement check") +} + +// wantCount comes from DesiredNumberScheduled, which counts NotReady and +// cordoned nodes, so requiring every pod fails the check on one NotReady node. +// Two nodes is enough to prove the overlay carries cross-node traffic. +func TestWaitForDaemonSetPods_ReturnsPartialCoverageOnTimeout(t *testing.T) { + labels := map[string]string{"app": "n2n"} + mk := func(name, node string) *corev1.Pod { + return &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: "probe", Labels: labels}, + Spec: corev1.PodSpec{NodeName: node}, + Status: corev1.PodStatus{Phase: corev1.PodRunning, PodIP: "10.0.0." + node[len(node)-1:]}, + } + } + client := fake.NewSimpleClientset(mk("a", "node-1"), mk("b", "node-2")) + selector := metav1.FormatLabelSelector(&metav1.LabelSelector{MatchLabels: labels}) + + // Three scheduled (one node NotReady), two Running, minNodes=2. + pods, err := waitForDaemonSetPods(context.Background(), client, "probe", selector, 3, 2, time.Second) + require.NoError(t, err, "two nodes is enough to exercise the overlay") + assert.Len(t, pods, 2) + + // One node is not enough: there is no cross-node path to probe. + client2 := fake.NewSimpleClientset(mk("a", "node-1")) + _, err = waitForDaemonSetPods(context.Background(), client2, "probe", selector, 3, 2, time.Second) + assert.Error(t, err, "a single node cannot demonstrate cross-node connectivity") +} + +// A transient error must not burn the whole deadline budget: client-go defaults +// to 5 QPS and this run issues ~22 namespaced LISTs, so a 429 early in the +// window would otherwise fail a critical check with most of its budget unspent. +func TestWaitForDaemonSetPods_RetriesTransientErrors(t *testing.T) { + labels := map[string]string{"app": "n2n"} + client := fake.NewSimpleClientset( + &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: "a", Namespace: "probe", Labels: labels}, + Spec: corev1.PodSpec{NodeName: "node-1"}, + Status: corev1.PodStatus{Phase: corev1.PodRunning, PodIP: "10.0.0.1"}, + }, + &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: "b", Namespace: "probe", Labels: labels}, + Spec: corev1.PodSpec{NodeName: "node-2"}, + Status: corev1.PodStatus{Phase: corev1.PodRunning, PodIP: "10.0.0.2"}, + }, + ) + calls := 0 + client.PrependReactor("list", "pods", func(ktesting.Action) (bool, runtime.Object, error) { + calls++ + if calls == 1 { + return true, nil, apierrors.NewTooManyRequestsError("slow down") + } + return false, nil, nil + }) + selector := metav1.FormatLabelSelector(&metav1.LabelSelector{MatchLabels: labels}) + + pods, err := waitForDaemonSetPods(context.Background(), client, "probe", selector, 2, 2, 30*time.Second) + require.NoError(t, err, "a single 429 must not abort a 30s wait") + assert.Len(t, pods, 2) + assert.Greater(t, calls, 1, "the loop must have retried") +} + +// A permission error is terminal: retrying it just burns the deadline. +func TestWaitForDaemonSetPods_ForbiddenIsTerminal(t *testing.T) { + client := fake.NewSimpleClientset() + client.PrependReactor("list", "pods", func(ktesting.Action) (bool, runtime.Object, error) { + return true, nil, apierrors.NewForbidden( + schema.GroupResource{Resource: "pods"}, "", fmt.Errorf("denied")) + }) + start := time.Now() + _, err := waitForDaemonSetPods(context.Background(), client, "probe", "app=n2n", 2, 2, 30*time.Second) + require.Error(t, err) + assert.Less(t, time.Since(start), 5*time.Second, "a denial must return immediately") +} + +// Supersedes TestCheckNodeToNode_SingleNode_Skip, which asserted the opposite. +// A single-node control plane has no cross-node path to exercise, so the +// overlay requirement is vacuously met. Leaving the pointer nil would make it +// a critical UNKNOWN, which fails the verdict, so a k3d or single-node control +// plane would report NVCF-Not-Ready on every tick. +func TestCheckNodeToNode_SingleNodeIsNotApplicableNotUnknown(t *testing.T) { + client := fake.NewSimpleClientset( + &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "only-node"}}, + ) + state := &ValidationState{Log: testLog()} + checkNodeToNode(context.Background(), client, state, "busybox:1.36") + + // Not a pass: no cross-node packet was sent. Reporting Verified here is + // the regression Vaibhav's "sets NodeToNodeOK = true having sent zero + // packets" thread already closed once. + assert.Nil(t, state.NodeToNodeOK, "an unexercised check must not be reported as passed") + assert.NotEmpty(t, state.NodeToNodeNotApplicable, + "one schedulable node is not applicable, not unobserved") +} + +// A cordoned second node leaves one schedulable node: same reasoning. +func TestCheckNodeToNode_AllButOneCordonedIsNotApplicable(t *testing.T) { + client := fake.NewSimpleClientset( + &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "node-1"}}, + &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{Name: "node-2"}, + Spec: corev1.NodeSpec{Unschedulable: true}, + }, + ) + state := &ValidationState{Log: testLog()} + checkNodeToNode(context.Background(), client, state, "busybox:1.36") + + assert.Nil(t, state.NodeToNodeOK) + assert.NotEmpty(t, state.NodeToNodeNotApplicable) +} + +// An RBAC denial on the probe DaemonSet is genuinely unobserved, so it must +// stay UNKNOWN. This is the case the operator ClusterRole now grants for. +func TestCheckNodeToNode_DaemonSetDenialStaysUnknown(t *testing.T) { + client := fake.NewSimpleClientset( + &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "node-1"}}, + &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "node-2"}}, + ) + client.PrependReactor("create", "daemonsets", func(ktesting.Action) (bool, runtime.Object, error) { + return true, nil, apierrors.NewForbidden( + schema.GroupResource{Group: "apps", Resource: "daemonsets"}, "", fmt.Errorf("denied")) + }) + state := &ValidationState{Log: testLog()} + checkNodeToNode(context.Background(), client, state, "busybox:1.36") + + assert.Nil(t, state.NodeToNodeOK, + "a denial is not evidence the overlay works, so it must not pass") + assert.Contains(t, strings.Join(state.Warnings, "; "), "RBAC denied") +} diff --git a/src/compute-plane-services/nvca/internal/clustervalidator/summary.go b/src/compute-plane-services/nvca/internal/clustervalidator/summary.go index 27bd114707..3c64d17fd7 100644 --- a/src/compute-plane-services/nvca/internal/clustervalidator/summary.go +++ b/src/compute-plane-services/nvca/internal/clustervalidator/summary.go @@ -151,6 +151,17 @@ const ( CheckKeyGPUOperator = "gpu_operator" CheckKeyConfigurableNetpol = "configurable_netpol" CheckKeyNetpolEnforcement = "netpol_enforcement" + // Control-plane-specific check keys. Only written to the summary when the + // check ran (nil pointer = check was skipped for this role). + CheckKeyDefaultStorageClass = "default_storage_class" + CheckKeyGatewayAPICRDs = "gateway_api_crds" + CheckKeyEnvoyGateway = "envoy_gateway" + CheckKeyGatewayRoutes = "gateway_routes" + CheckKeyExternalLB = "external_lb" + CheckKeyNodeToNode = "node_to_node" + // Control-plane HA readiness checks. + CheckKeyTier1Deployments = "tier1_deployments" + CheckKeyTier2StatefulSets = "tier2_statefulsets" ) // AllCheckKeys is the canonical ordering used for documentation and @@ -160,17 +171,29 @@ var AllCheckKeys = []string{ CheckKeyWorkerNodesAllReady, CheckKeyWebhooks, CheckKeyNetworkPoliciesSupport, + // Compute-plane checks. CheckKeySMBCSI, CheckKeyEndpointReachability, CheckKeyGPUResources, CheckKeyGPUOperator, CheckKeyConfigurableNetpol, CheckKeyNetpolEnforcement, + // Control-plane checks (only present in summary when the role ran them). + CheckKeyDefaultStorageClass, + CheckKeyGatewayAPICRDs, + CheckKeyEnvoyGateway, + CheckKeyGatewayRoutes, + CheckKeyExternalLB, + CheckKeyNodeToNode, + CheckKeyTier1Deployments, + CheckKeyTier2StatefulSets, } // buildSummary projects a ValidationState into the wire format. Checks -// that were not run (their *bool is nil) are omitted from the Checks -// map so the agent can distinguish "not run" from "ran and failed". +// that were not run are omitted from the Checks map so the agent can +// distinguish "not run" from "ran and failed". For role-gated checks that +// means the pointer is nil; for the compute-plane bools it means the role +// is control-plane and they were never invoked. func buildSummary(state *ValidationState, startedAt time.Time, verdictReady bool, verdict string) *ValidatorSummary { now := time.Now().UTC() s := &ValidatorSummary{ @@ -191,9 +214,15 @@ func buildSummary(state *ValidationState, startedAt time.Time, verdictReady bool s.Checks[CheckKeyWorkerNodesAllReady] = state.NodesAllReady s.Checks[CheckKeyWebhooks] = state.WebhooksSupported s.Checks[CheckKeyNetworkPoliciesSupport] = state.NetworkPoliciesSupported - s.Checks[CheckKeySMBCSI] = state.SMBCSIDriverOK - s.Checks[CheckKeyGPUResources] = state.GPUAvailable - s.Checks[CheckKeyGPUOperator] = state.GPUOperatorInstalled + // Compute-plane checks are never invoked under the control-plane role, so + // their fields hold the zero value. Writing them unconditionally would + // publish gpu_resources=0 forever on a control plane that has no GPUs and + // was never checked for any, which METRICS.md documents as alertable. + if state.Role != RoleControlPlane { + s.Checks[CheckKeySMBCSI] = state.SMBCSIDriverOK + s.Checks[CheckKeyGPUResources] = state.GPUAvailable + s.Checks[CheckKeyGPUOperator] = state.GPUOperatorInstalled + } if state.ReachabilityOK != nil { s.Checks[CheckKeyEndpointReachability] = *state.ReachabilityOK @@ -204,6 +233,32 @@ func buildSummary(state *ValidationState, startedAt time.Time, verdictReady bool if state.EnforcementOK != nil { s.Checks[CheckKeyNetpolEnforcement] = *state.EnforcementOK } + // Control-plane checks are only written when the check ran (non-nil pointer). + // A nil pointer means the check was skipped because the role was compute-plane. + if state.DefaultStorageClassOK != nil { + s.Checks[CheckKeyDefaultStorageClass] = *state.DefaultStorageClassOK + } + if state.GatewayAPICRDsOK != nil { + s.Checks[CheckKeyGatewayAPICRDs] = *state.GatewayAPICRDsOK + } + if state.EnvoyGatewayOK != nil { + s.Checks[CheckKeyEnvoyGateway] = *state.EnvoyGatewayOK + } + if state.GatewayRoutesOK != nil { + s.Checks[CheckKeyGatewayRoutes] = *state.GatewayRoutesOK + } + if state.ExternalLBOK != nil { + s.Checks[CheckKeyExternalLB] = *state.ExternalLBOK + } + if state.NodeToNodeOK != nil { + s.Checks[CheckKeyNodeToNode] = *state.NodeToNodeOK + } + if state.Tier1DeploymentsOK != nil { + s.Checks[CheckKeyTier1Deployments] = *state.Tier1DeploymentsOK + } + if state.Tier2StatefulSetsOK != nil { + s.Checks[CheckKeyTier2StatefulSets] = *state.Tier2StatefulSetsOK + } if len(state.EndpointResults) > 0 { s.Endpoints = make(map[string]EndpointStatus, len(state.EndpointResults)) diff --git a/src/compute-plane-services/nvca/internal/clustervalidator/summary_test.go b/src/compute-plane-services/nvca/internal/clustervalidator/summary_test.go index 926d6856ef..ce9c34a5bd 100644 --- a/src/compute-plane-services/nvca/internal/clustervalidator/summary_test.go +++ b/src/compute-plane-services/nvca/internal/clustervalidator/summary_test.go @@ -282,8 +282,18 @@ func TestAllCheckKeysCoversEveryCheckKeyConst(t *testing.T) { CheckKeyGPUOperator, CheckKeyConfigurableNetpol, CheckKeyNetpolEnforcement, + // Control-plane-specific keys added with the role-aware validator. + CheckKeyDefaultStorageClass, + CheckKeyGatewayAPICRDs, + CheckKeyEnvoyGateway, + CheckKeyGatewayRoutes, + CheckKeyExternalLB, + CheckKeyNodeToNode, + // Control-plane HA readiness keys. + CheckKeyTier1Deployments, + CheckKeyTier2StatefulSets, } { assert.True(t, known[k], "%q is a CheckKey constant but missing from AllCheckKeys", k) } - assert.Len(t, AllCheckKeys, 10, "if you added a new CheckKey, also add it to AllCheckKeys AND to clusterValidatorCheckKeys() in internal/metrics/metrics.go") + assert.Len(t, AllCheckKeys, 18, "if you added a new CheckKey, also add it to AllCheckKeys AND to clusterValidatorCheckKeys() in internal/metrics/metrics.go") } diff --git a/src/compute-plane-services/nvca/internal/clustervalidator/validator.go b/src/compute-plane-services/nvca/internal/clustervalidator/validator.go index 0f41e2fdcc..fd67225b7a 100644 --- a/src/compute-plane-services/nvca/internal/clustervalidator/validator.go +++ b/src/compute-plane-services/nvca/internal/clustervalidator/validator.go @@ -27,9 +27,28 @@ import ( "k8s.io/client-go/kubernetes" ) +// Role is the check set selected by VALIDATOR_ROLE. +type Role string + +// Role values for VALIDATOR_ROLE. +const ( + RoleComputePlane Role = "compute-plane" + RoleControlPlane Role = "control-plane" +) + // ValidationState captures the results of every validation check. type ValidationState struct { - Log *logrus.Entry + Log *logrus.Entry + // NodeToNodeNotApplicable holds the reason the overlay probe could not + // apply (for example a single schedulable node). Non-empty means the check + // is reported as Not Applicable rather than Verified or Unknown, and is + // left out of the summary map so it is neither alerted on nor counted as + // a pass. + NodeToNodeNotApplicable string + + // Role is "control-plane" or "compute-plane" (empty = compute-plane default). + // printSummary uses it to include only the checks relevant to the role. + Role Role ControlPlaneHealthy bool // NodesAllReady tracks whether all worker nodes are Ready. False means at // least one NotReady node. Warning only — does not flip cluster readiness. @@ -67,6 +86,26 @@ type ValidationState struct { // critical: true, meaning enforcement failure blocks readiness. EnforcementCritical bool + // Control-plane-specific check outcomes. Nil means the check was not run + // (compute-plane role). Non-nil means the check ran and the bool holds + // the pass/fail result. + DefaultStorageClassOK *bool + GatewayAPICRDsOK *bool + EnvoyGatewayOK *bool + GatewayRoutesOK *bool + ExternalLBOK *bool + // NodeToNodeOK is nil when the check was skipped (single-node cluster or + // compute-plane role). true = overlay verified, false = failed. + NodeToNodeOK *bool + // Tier1DeploymentsOK is nil when the check did not run (compute-plane role) + // or when a Deployment list call fails. Pre-install (no Deployments found) + // sets this to true, not nil. + Tier1DeploymentsOK *bool + // Tier2StatefulSetsOK is nil when the check did not run (compute-plane role) + // or when a StatefulSet list call fails. No quorum StatefulSets found + // (pre-install or non-HA install) sets this to true, not nil. + Tier2StatefulSetsOK *bool + // EndpointResults captures per-endpoint reachability outcomes for the // summary ConfigMap / metrics pipeline. Keyed by the user-supplied // endpoint name (the same string Prometheus will use as the label @@ -96,25 +135,15 @@ type NetpolPairResult struct { Directions map[string]DirectionStatus } -// Run executes all cluster validation checks and prints a summary. -// It returns a non-nil error if the cluster is not ready, which the caller -// should use to set the process exit code. -// -// configNamespace and configName identify an optional ConfigMap that holds -// user-defined reachability and network-policy checks. When the ConfigMap -// does not exist the configurable checks are silently skipped. -// -// summaryNamespace is where the summary ConfigMap is written for the agent to -// read — kept separate from configNamespace so a config-namespace override -// can't redirect the summary away from the namespace the agent watches. -// -// emitMetrics gates that write. In-cluster runs emit by default; callers pass -// false for preflight (no agent to read it, no RBAC to write it). +// Run executes all cluster validation checks and returns a non-nil error when +// the cluster is not ready. role selects the check set; configNamespace/configName +// identify the optional ConfigMap; emitMetrics gates the summary write. func Run( ctx context.Context, client kubernetes.Interface, configNamespace, configName, summaryNamespace string, emitMetrics bool, + role Role, ) error { startedAt := time.Now() log := core.GetLogger(ctx) @@ -127,6 +156,7 @@ func Run( state := &ValidationState{ Log: log, + Role: role, ControlPlaneHealthy: true, NodesAllReady: true, } @@ -145,7 +175,6 @@ func Run( checkControlPlaneHealth(ctx, client, state) checkWebhookSupport(ctx, client, state) checkNetworkPolicies(ctx, client, state) - checkSMBCSIDriver(ctx, client, state) var netCfg *NetworkCheckConfig if configNamespace != "" && configName != "" { @@ -161,8 +190,24 @@ func Run( checkConfigurableReachability(state, netCfg.Reachability) } - checkGPUResources(ctx, client, state) - checkGPUOperator(ctx, client, state) + if role == RoleControlPlane { + // Control-plane cluster: check gateway infrastructure, storage, and + // inter-node overlay connectivity. GPU operator and SMB CSI are + // compute-plane concerns and are skipped. + checkStorageClass(ctx, client, state) + checkGatewayAPICRDs(ctx, client, state) + checkEnvoyGateway(ctx, client, state) + checkGatewayRoutes(ctx, client, state) + checkExternalLoadBalancer(ctx, client, state) + checkNodeToNode(ctx, client, state, nodeToNodeProbeImage(netCfg)) + checkTier1Deployments(ctx, client, state) + checkTier2StatefulSets(ctx, client, state) + } else { + // Compute-plane cluster (default): GPU operator, SMB CSI driver. + checkSMBCSIDriver(ctx, client, state) + checkGPUResources(ctx, client, state) + checkGPUOperator(ctx, client, state) + } if netCfg != nil { if netCfg.NetworkPolicies != nil && len(netCfg.NetworkPolicies.Pairs) > 0 { @@ -205,6 +250,18 @@ func printSummary(state *ValidationState) error { PassMsg string FailMsg string Critical bool + // Unknown marks a check that did not run. Rendered as its own row so a + // critical check cannot silently vanish from the verdict, which would + // otherwise make a throttled API call look better than a clean run. + Unknown bool + UnknownMsg string + // NotApplicable marks a check this cluster's shape cannot exercise, as + // distinct from one we failed to observe. Both are non-passes, but + // only Unknown means something is hidden, so only Unknown blocks the + // verdict. Reporting a not-applicable check as Passed would claim a + // result the run never produced. + NotApplicable bool + NAMsg string } // Distinguish "we listed nodes and found N not-ready" (NotReadyNodes>0) @@ -219,69 +276,134 @@ func printSummary(state *ValidationState) error { } checks := []check{ - {state.ControlPlaneHealthy, "Control Plane: Healthy", "Control Plane: Unhealthy", true}, - {state.NodesAllReady, - "Worker Nodes: All Ready", - nodesFailMsg, - false}, - {state.WebhooksSupported, "Admission Webhooks: Mutating & Validating Supported", "Admission Webhooks: Not Supported", true}, - {state.NetworkPoliciesSupported, "Network Policies: Supported", "Network Policies: Not Confirmed", false}, - // SMB CSI Driver missing is non-blocking: it is required only when - // the HelmSharedStorage feature flag is enabled (NVCA model-cache). - // pkg/storage/smbcsidriver.go's runtime health check itself flags - // this at StatusLevelWarn, not StatusLevelError — block install - // only when the customer has explicitly opted in to a feature that - // needs SMB CSI, not for every operator install. - {state.SMBCSIDriverOK, "SMB CSI Driver: v1.16.0+ Installed", "SMB CSI Driver: Not Installed or Below v1.16.0", false}, + {Passed: state.ControlPlaneHealthy, PassMsg: "Control Plane: Healthy", + FailMsg: "Control Plane: Unhealthy", Critical: true}, + {Passed: state.NodesAllReady, PassMsg: "Worker Nodes: All Ready", + FailMsg: nodesFailMsg, Critical: false}, + {Passed: state.WebhooksSupported, PassMsg: "Admission Webhooks: Mutating & Validating Supported", + FailMsg: "Admission Webhooks: Not Supported", Critical: true}, + {Passed: state.NetworkPoliciesSupported, PassMsg: "Network Policies: Supported", + FailMsg: "Network Policies: Not Confirmed", Critical: false}, } if state.ReachabilityOK != nil { isCritical := state.ReachabilityCriticalOK != nil && !*state.ReachabilityCriticalOK checks = append(checks, check{ - *state.ReachabilityOK, - "Endpoint Reachability: All Endpoints Reachable", - "Endpoint Reachability: One or more endpoints not reachable", - isCritical, + Passed: *state.ReachabilityOK, + PassMsg: "Endpoint Reachability: All Endpoints Reachable", + FailMsg: "Endpoint Reachability: One or more endpoints not reachable", + Critical: isCritical, }) } - checks = append(checks, - check{state.GPUAvailable, "GPU Resources: Available", "GPU Resources: Not Available", true}, - // GPU Operator missing is non-blocking: clusters registered with - // Manual Instance Configuration expose GPUs via an alternative - // mechanism (pre-labeled nodes, DaemonSet, etc.) and do not require - // GPU Operator. GPU Resources above is the load-bearing signal — - // if GPUs aren't usable that fails Critical separately. - check{state.GPUOperatorInstalled, "GPU Operator: Installed", "GPU Operator: Not Installed", false}, - ) + if state.Role == RoleControlPlane { + // Control-plane checks: gateway infrastructure and storage. GPU and + // SMB checks are compute-plane concerns and are excluded here. + // + // addCP renders a nil pointer as an explicit UNKNOWN row for critical + // checks, so an API error during the run cannot quietly drop a critical + // row and leave a cleaner-looking summary than a successful run. + addCP := func(ptr *bool, label, passDetail, failDetail string, critical bool) { + if ptr != nil { + checks = append(checks, check{ + Passed: *ptr, + PassMsg: label + ": " + passDetail, + FailMsg: label + ": " + failDetail, + Critical: critical, + }) + return + } + if critical { + checks = append(checks, check{ + Critical: critical, + Unknown: true, + UnknownMsg: label + ": Status Unknown (check did not run)", + }) + } + } + + addCP(state.DefaultStorageClassOK, "Default StorageClass", "Present", "Not Found", true) + addCP(state.GatewayAPICRDsOK, "Gateway API CRDs", "Installed", "Not Installed", true) + // Non-critical: Envoy Gateway is installed by nvcf-cli up, so it is + // expected to be absent on a fresh cluster before the first install. + // A missing Envoy is informative (tells the operator the stack is not + // yet deployed) but must not block a pre-install readiness check. + addCP(state.EnvoyGatewayOK, "Envoy Gateway", "Installed and Running", "Not Found or Not Running", false) + addCP(state.GatewayRoutesOK, "Gateway Route CR Types", "Registered", "Not Registered", false) + addCP(state.ExternalLBOK, "External Load Balancer", "IP Assigned", "No IP Assigned", false) + if state.NodeToNodeNotApplicable != "" { + // Non-blocking, but not "Verified": no cross-node packet was sent. + checks = append(checks, check{ + NotApplicable: true, + NAMsg: "Node-to-Node Communication: Not Applicable (" + state.NodeToNodeNotApplicable + ")", + }) + } else { + addCP(state.NodeToNodeOK, "Node-to-Node Communication", "Verified", "Failed", true) + } + addCP(state.Tier1DeploymentsOK, "Tier-1 Deployments", "All Ready", "Under-replicated", true) + addCP(state.Tier2StatefulSetsOK, "Tier-2 StatefulSets", + "Quorum and Placement OK", "Quorum or Placement Failed", true) + } else { + // Compute-plane checks: GPU resources, GPU operator, SMB CSI driver. + // SMB CSI Driver missing is non-blocking: it is required only when + // the HelmSharedStorage feature flag is enabled (NVCA model-cache). + checks = append(checks, + check{Passed: state.SMBCSIDriverOK, PassMsg: "SMB CSI Driver: v1.16.0+ Installed", + FailMsg: "SMB CSI Driver: Not Installed or Below v1.16.0", Critical: false}, + check{Passed: state.GPUAvailable, PassMsg: "GPU Resources: Available", + FailMsg: "GPU Resources: Not Available", Critical: true}, + // GPU Operator missing is non-blocking: clusters registered with + // Manual Instance Configuration expose GPUs via an alternative + // mechanism (pre-labeled nodes, DaemonSet, etc.) and do not require + // GPU Operator. GPU Resources above is the load-bearing signal. + check{Passed: state.GPUOperatorInstalled, PassMsg: "GPU Operator: Installed", + FailMsg: "GPU Operator: Not Installed", Critical: false}, + ) + } if state.ConfigurableNetPolOK != nil { isCritical := state.ConfigurableNetPolCriticalOK != nil && !*state.ConfigurableNetPolCriticalOK checks = append(checks, check{ - *state.ConfigurableNetPolOK, - "Configurable Network Policies: All Checks Passed", - "Configurable Network Policies: One or more checks failed", - isCritical, + Passed: *state.ConfigurableNetPolOK, + PassMsg: "Configurable Network Policies: All Checks Passed", + FailMsg: "Configurable Network Policies: One or more checks failed", + Critical: isCritical, }) } if state.EnforcementOK != nil { checks = append(checks, check{ - *state.EnforcementOK, - "Network Policy Enforcement: Active Validation Passed", - "Network Policy Enforcement: Active Validation Failed", - state.EnforcementCritical, + Passed: *state.EnforcementOK, + PassMsg: "Network Policy Enforcement: Active Validation Passed", + FailMsg: "Network Policy Enforcement: Active Validation Failed", + Critical: state.EnforcementCritical, }) } + var unknownCritical []string for _, c := range checks { - if c.Passed { + switch { + case c.NotApplicable: + // Neither pass nor failure: the cluster shape made the check moot. + printInfo(log, fmt.Sprintf(" %s", c.NAMsg)) + case c.Unknown: + // A critical check we could not observe cannot be certified as + // ready. Logging it while still publishing verdict=NVCF-Ready and + // VerdictReady=true would export a perfect green SLI for a + // precondition nothing looked at, and the check key is pruned from + // the metric, so there is no series left to alert on either. + printWarning(log, fmt.Sprintf(" %s", c.UnknownMsg)) + if c.Critical { + unknownCritical = append(unknownCritical, c.UnknownMsg) + isReady = false + } + case c.Passed: printSuccess(log, fmt.Sprintf(" %s", c.PassMsg)) - } else if c.Critical { + case c.Critical: printError(log, fmt.Sprintf(" %s", c.FailMsg)) isReady = false - } else { + default: printWarning(log, fmt.Sprintf(" %s", c.FailMsg)) } } @@ -316,6 +438,16 @@ func printSummary(state *ValidationState) error { log.Infof("%s║ %s Cluster is NVCF-Not-Ready %s ║%s", colorRed, iconCross, iconCross, colorReset) log.Infof("%s╚═══════════════════════════════════════════════════════════╝%s", colorRed, colorReset) log.Info("") + if len(unknownCritical) > 0 { + // Distinguish "could not check" from "checked and broken": the + // operator's next step is to fix access or re-run, not to go + // looking for a fault that was never observed. + printError(log, fmt.Sprintf( + "%d critical check(s) could not be observed, so readiness cannot be confirmed", len(unknownCritical))) + for _, m := range unknownCritical { + printInfo(log, " "+m) + } + } printError(log, "Your cluster does not meet all requirements for NVCF workloads") } diff --git a/src/compute-plane-services/nvca/internal/clustervalidator/validator_test.go b/src/compute-plane-services/nvca/internal/clustervalidator/validator_test.go index 1c102e81c4..d09c252bfa 100644 --- a/src/compute-plane-services/nvca/internal/clustervalidator/validator_test.go +++ b/src/compute-plane-services/nvca/internal/clustervalidator/validator_test.go @@ -25,6 +25,7 @@ import ( "crypto/rand" "crypto/tls" "crypto/x509" + "encoding/json" "fmt" "math/big" "net" @@ -42,6 +43,7 @@ import ( "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/fake" ) @@ -64,7 +66,7 @@ func TestRun_EmitMetricsGatesSummaryWrite(t *testing.T) { t.Run("preflight (emitMetrics=false) does not write the summary", func(t *testing.T) { client := fake.NewSimpleClientset() - _ = Run(context.Background(), client, ns, "cluster-validator-network-checks", ns, false) + _ = Run(context.Background(), client, ns, "cluster-validator-network-checks", ns, false, "") _, err := client.CoreV1().ConfigMaps(ns).Get( context.Background(), SummaryConfigMapName, metav1.GetOptions{}) assert.True(t, apierrors.IsNotFound(err), @@ -73,7 +75,7 @@ func TestRun_EmitMetricsGatesSummaryWrite(t *testing.T) { t.Run("post-install (emitMetrics=true) writes the summary", func(t *testing.T) { client := fake.NewSimpleClientset() - _ = Run(context.Background(), client, ns, "cluster-validator-network-checks", ns, true) + _ = Run(context.Background(), client, ns, "cluster-validator-network-checks", ns, true, "") cm, err := client.CoreV1().ConfigMaps(ns).Get( context.Background(), SummaryConfigMapName, metav1.GetOptions{}) require.NoError(t, err, "summary ConfigMap must be written when emitMetrics=true") @@ -85,7 +87,7 @@ func TestRun_EmitMetricsGatesSummaryWrite(t *testing.T) { // Guards the decoupling: a non-operator config namespace must NOT // redirect the summary away from the namespace the agent watches. client := fake.NewSimpleClientset() - _ = Run(context.Background(), client, "some-config-ns", "cluster-validator-network-checks", ns, true) + _ = Run(context.Background(), client, "some-config-ns", "cluster-validator-network-checks", ns, true, "") _, err := client.CoreV1().ConfigMaps(ns).Get( context.Background(), SummaryConfigMapName, metav1.GetOptions{}) @@ -98,6 +100,194 @@ func TestRun_EmitMetricsGatesSummaryWrite(t *testing.T) { }) } +// runAndReadSummary drives the real Run dispatch for a role and reads back the +// summary ConfigMap it publishes. Asserting on the published summary rather +// than on the returned error is what makes the role dispatch observable: the +// error string is the same constant for every role, so it cannot distinguish +// them, and the ConfigMap is what the agent actually turns into metrics. +func runAndReadSummary(t *testing.T, client kubernetes.Interface, role Role) *ValidatorSummary { + t.Helper() + const ns = "nvca-system" + _ = Run(context.Background(), client, "", "", ns, true, role) + + cm, err := client.CoreV1().ConfigMaps(ns).Get( + context.Background(), SummaryConfigMapName, metav1.GetOptions{}) + require.NoError(t, err, "Run must publish the summary ConfigMap") + + var s ValidatorSummary + require.NoError(t, json.Unmarshal([]byte(cm.Data[SummaryConfigMapKey]), &s)) + return &s +} + +// The control-plane role must dispatch the control-plane check set and none of +// the compute-plane ones. Both halves are asserted on the published summary: +// a GPU key present at all means either the check ran or its zero value leaked +// onto the wire, and both are bugs under this role. +func TestRun_ControlPlaneRoleDispatch(t *testing.T) { + client := fake.NewSimpleClientset(makeNode("node-1", true, 0)) + s := runAndReadSummary(t, client, RoleControlPlane) + + assert.Contains(t, s.Checks, CheckKeyDefaultStorageClass, + "control-plane role must run and publish the StorageClass check") + + for _, k := range []string{CheckKeyGPUResources, CheckKeyGPUOperator, CheckKeySMBCSI} { + assert.NotContains(t, s.Checks, k, + "compute-plane check %q must not be published under the control-plane role", k) + } +} + +// The mirror of the above: the compute-plane role publishes the GPU keys and +// none of the control-plane ones. +func TestRun_ComputePlaneRoleDispatch(t *testing.T) { + client := fake.NewSimpleClientset(makeNode("node-1", true, 0)) + s := runAndReadSummary(t, client, RoleComputePlane) + + for _, k := range []string{CheckKeyGPUResources, CheckKeyGPUOperator, CheckKeySMBCSI} { + assert.Contains(t, s.Checks, k, + "compute-plane check %q must be published under the compute-plane role", k) + } + assert.NotContains(t, s.Checks, CheckKeyDefaultStorageClass, + "control-plane check must not run under the compute-plane role") +} + +// TestPrintSummary_ControlPlaneRole verifies that with Role=RoleControlPlane +// the summary omits GPU rows and includes control-plane check rows. +func TestPrintSummary_ControlPlaneRole(t *testing.T) { + t.Run("control-plane role excludes GPU rows", func(t *testing.T) { + ok := true + buf := &bytes.Buffer{} + l := logrus.New() + l.SetOutput(buf) + state := &ValidationState{ + Log: logrus.NewEntry(l), + Role: RoleControlPlane, + ControlPlaneHealthy: true, + NodesAllReady: true, + WebhooksSupported: true, + NetworkPoliciesSupported: true, + // Control-plane checks all pass + DefaultStorageClassOK: &ok, + GatewayAPICRDsOK: &ok, + EnvoyGatewayOK: &ok, + GatewayRoutesOK: &ok, + ExternalLBOK: &ok, + // The remaining critical control-plane rows have to be set too: + // leaving them nil is UNKNOWN, which is not "all checks passing". + NodeToNodeOK: &ok, + Tier1DeploymentsOK: &ok, + Tier2StatefulSetsOK: &ok, + K8sVersion: "v1.30.0", + TotalNodes: "2", + } + err := printSummary(state) + assert.NoError(t, err, "all control-plane checks passing must yield NVCF-Ready") + out := buf.String() + assert.NotContains(t, out, "GPU Resources", "GPU row must not appear for control-plane role") + assert.NotContains(t, out, "GPU Operator", "GPU Operator row must not appear for control-plane role") + assert.Contains(t, out, "Default StorageClass", "StorageClass row must appear for control-plane role") + assert.Contains(t, out, "Gateway API CRDs", "Gateway CRD row must appear for control-plane role") + assert.Contains(t, out, "Envoy Gateway", "Envoy Gateway row must appear for control-plane role") + }) + + t.Run("control-plane role critical failure blocks readiness", func(t *testing.T) { + fail := false + ok := true + state := &ValidationState{ + Log: testLog(), + Role: RoleControlPlane, + ControlPlaneHealthy: true, + NodesAllReady: true, + WebhooksSupported: true, + NetworkPoliciesSupported: true, + DefaultStorageClassOK: &fail, // critical: no default StorageClass + GatewayAPICRDsOK: &ok, + EnvoyGatewayOK: &ok, + K8sVersion: "v1.30.0", + TotalNodes: "2", + } + err := printSummary(state) + assert.Error(t, err, "missing default StorageClass must block control-plane readiness") + }) + + t.Run("unknown critical check blocks readiness", func(t *testing.T) { + // A critical check nothing could observe must not publish a green + // verdict: that exports a perfect SLI for a precondition that was + // never looked at, and the pruned metric leaves nothing to alert on. + ok := true + buf := &bytes.Buffer{} + l := logrus.New() + l.SetOutput(buf) + state := &ValidationState{ + Log: logrus.NewEntry(l), + Role: RoleControlPlane, + ControlPlaneHealthy: true, + NodesAllReady: true, + WebhooksSupported: true, + NetworkPoliciesSupported: true, + DefaultStorageClassOK: &ok, + GatewayAPICRDsOK: &ok, + EnvoyGatewayOK: &ok, + GatewayRoutesOK: &ok, + ExternalLBOK: &ok, + NodeToNodeOK: &ok, + Tier1DeploymentsOK: &ok, + // Tier2StatefulSetsOK left nil: RBAC denied the StatefulSet list. + K8sVersion: "v1.30.0", + TotalNodes: "2", + } + err := printSummary(state) + assert.Error(t, err, "an unobserved critical check cannot be certified ready") + assert.Contains(t, buf.String(), "could not be observed", + "the operator must be told this is unobserved, not broken") + }) + + t.Run("unknown non-critical check does not block readiness", func(t *testing.T) { + ok := true + state := &ValidationState{ + Log: testLog(), + Role: RoleControlPlane, + ControlPlaneHealthy: true, + NodesAllReady: true, + WebhooksSupported: true, + NetworkPoliciesSupported: true, + DefaultStorageClassOK: &ok, + GatewayAPICRDsOK: &ok, + NodeToNodeOK: &ok, + Tier1DeploymentsOK: &ok, + Tier2StatefulSetsOK: &ok, + // EnvoyGatewayOK / GatewayRoutesOK / ExternalLBOK nil: all + // non-critical, so they stay out of the verdict entirely. + K8sVersion: "v1.30.0", + TotalNodes: "2", + } + assert.NoError(t, printSummary(state)) + }) + + t.Run("compute-plane role (default) still includes GPU rows", func(t *testing.T) { + buf := &bytes.Buffer{} + l := logrus.New() + l.SetOutput(buf) + state := &ValidationState{ + Log: logrus.NewEntry(l), + Role: "", + ControlPlaneHealthy: true, + NodesAllReady: true, + WebhooksSupported: true, + NetworkPoliciesSupported: true, + SMBCSIDriverOK: true, + GPUAvailable: true, + GPUOperatorInstalled: true, + K8sVersion: "v1.30.0", + TotalNodes: "2", + } + err := printSummary(state) + assert.NoError(t, err) + out := buf.String() + assert.Contains(t, out, "GPU Resources", "GPU row must appear for compute-plane role") + assert.NotContains(t, out, "Default StorageClass", "StorageClass row must not appear for compute-plane role") + }) +} + func TestVersionGTE(t *testing.T) { tests := []struct { name string @@ -1023,3 +1213,56 @@ func init() { &corev1.Namespace{}, } } + +// A check the cluster shape cannot exercise is neither a pass nor a hidden +// failure. It must not block the verdict, and it must not be rendered as +// Verified or exported as a passing check. +func TestPrintSummary_NotApplicableIsNeitherPassNorUnknown(t *testing.T) { + ok := true + buf := &bytes.Buffer{} + l := logrus.New() + l.SetOutput(buf) + state := &ValidationState{ + Log: logrus.NewEntry(l), + Role: RoleControlPlane, + ControlPlaneHealthy: true, + NodesAllReady: true, + WebhooksSupported: true, + NetworkPoliciesSupported: true, + DefaultStorageClassOK: &ok, + GatewayAPICRDsOK: &ok, + EnvoyGatewayOK: &ok, + GatewayRoutesOK: &ok, + ExternalLBOK: &ok, + Tier1DeploymentsOK: &ok, + Tier2StatefulSetsOK: &ok, + // NodeToNodeOK deliberately nil, with a not-applicable reason. + NodeToNodeNotApplicable: "single schedulable node, no cross-node path", + K8sVersion: "v1.30.0", + TotalNodes: "1", + } + assert.NoError(t, printSummary(state), + "a not-applicable check must not block readiness") + + out := buf.String() + assert.Contains(t, out, "Node-to-Node Communication: Not Applicable") + assert.NotContains(t, out, "Node-to-Node Communication: Verified", + "nothing was probed, so it cannot be reported as Verified") + assert.NotContains(t, out, "Node-to-Node Communication: Status Unknown", + "the cluster shape is known; it is not an unobserved check") +} + +// The metrics pipeline must not see a not-applicable check as a pass. +func TestBuildSummary_OmitsNotApplicableCheck(t *testing.T) { + ok := true + state := &ValidationState{ + Log: testLog(), + Role: RoleControlPlane, + DefaultStorageClassOK: &ok, + NodeToNodeNotApplicable: "single schedulable node, no cross-node path", + } + s := buildSummary(state, time.Now(), true, "NVCF-Ready") + _, present := s.Checks[CheckKeyNodeToNode] + assert.False(t, present, + "an unexercised check must be absent, not exported as node_to_node=1") +} diff --git a/src/compute-plane-services/nvca/internal/metrics/METRICS.md b/src/compute-plane-services/nvca/internal/metrics/METRICS.md index 9d7a26fc31..76241b1e2d 100644 --- a/src/compute-plane-services/nvca/internal/metrics/METRICS.md +++ b/src/compute-plane-services/nvca/internal/metrics/METRICS.md @@ -1413,10 +1413,20 @@ Overall verdict for the latest cluster-validator run. **This is the load-bearing ### `nvca_cluster_validator_check_status` -Per-check status from the latest run. The check set is fixed (~10 entries; see `CheckKey*` constants in `internal/clustervalidator/summary.go`). +Per-check status from the latest run. The check set is fixed (18 entries; see `CheckKey*` constants in `internal/clustervalidator/summary.go`). Which subset appears depends on the validator role and on which conditional checks ran; see the caveat below. - **Type**: Gauge -- **Value**: 1 = passed, 0 = failed (or not-run; the `check` label is omitted entirely when a check was skipped) +- **Value**: 1 = passed, 0 = failed. Two other outcomes are reported as + absence rather than a number, so `absent()` and `== 0` mean different + things: a check that could not be observed (an RBAC denial or an apiserver + error), and one the cluster's shape made moot (the node-to-node overlay on a + single-node cluster). Neither is exported as 1, because no result was + produced; the run's log and the `warnings` list say which applies. + + One exception to the "0 means failed" rule: at process start, and after + `ResetClusterValidatorMetrics`, all 18 keys are emitted at 0 as an + init-to-zero baseline, before any run has happened. A 0 in that window means + "no result yet", not "failed"; it is replaced or pruned by the first summary. - **Labels**: default labels + `check` > **Alerting caveat — absent vs. zero for optional checks.** Three checks are @@ -1429,9 +1439,24 @@ Per-check status from the latest run. The check set is fixed (~10 entries; see ` > distinguishable from "ran and failed"). Write alerts on these three with an > `absent()` guard, not a bare `== 0`, e.g. > `absent(nvca_cluster_validator_check_status{check="endpoint_reachability"}) or nvca_cluster_validator_check_status{check="endpoint_reachability"} == 0`. -> The seven always-run checks (control_plane, worker_nodes_all_ready, webhooks, -> network_policies_supported, smb_csi, gpu_resources, gpu_operator) are always +> Four checks (control_plane, worker_nodes_all_ready, webhooks, +> network_policies_supported) run under both validator roles, so they are always > present and safe to alert on with `== 0`. +> +> The three GPU and storage checks (smb_csi, gpu_resources, gpu_operator) run +> only under the compute-plane role. A control-plane run omits them, so they are +> pruned after its first summary and go absent. They are still pre-initialized +> to `0` in the init-to-zero baseline, which means a control-plane cluster +> reports `gpu_resources 0` from process start until its first summary lands. +> Alert on these three with an `absent()` guard, as for the conditional checks +> above, and scope the alert to compute-plane clusters. +> +> The control-plane-only checks (default_storage_class, gateway_api_crds, +> envoy_gateway, gateway_routes, external_lb, node_to_node, tier1_deployments, +> tier2_statefulsets) are the mirror image: present only on a control-plane run, +> and absent when the check could not be observed at all (an RBAC denial or an +> apiserver error). Absent means "not observed", which is not the same as `0` +> ("observed and failing"), so these also need an `absent()` guard. ### `nvca_cluster_validator_endpoint_reachable` diff --git a/src/compute-plane-services/nvca/internal/metrics/metrics.go b/src/compute-plane-services/nvca/internal/metrics/metrics.go index 1271e7834d..3e50838386 100644 --- a/src/compute-plane-services/nvca/internal/metrics/metrics.go +++ b/src/compute-plane-services/nvca/internal/metrics/metrics.go @@ -1310,6 +1310,16 @@ func clusterValidatorCheckKeys() []string { "gpu_operator", "configurable_netpol", "netpol_enforcement", + // Control-plane-specific keys (only populated when VALIDATOR_ROLE=control-plane). + "default_storage_class", + "gateway_api_crds", + "envoy_gateway", + "gateway_routes", + "external_lb", + "node_to_node", + // Control-plane HA readiness keys. + "tier1_deployments", + "tier2_statefulsets", } }