feat(nvca): extend control-plane validator with HA checks, DaemonSet n2n, and route CR type check

deploy/helm/nvca-operator/nvca-operator/README.md

@@ -234,6 +234,9 @@ This release does not wire the catalog into backend selection. Runtime use requi
 | `clusterValidator.image.repository`                   | Cluster Validator container registry path, without tag                            | `""`                                   |
 | `clusterValidator.image.tag`                          | Cluster Validator container image tag                                             | `v2.0.0`                               |
 | `clusterValidator.image.pullPolicy`                   | K8s ImagePullPolicy for cluster-validator                                         | `IfNotPresent`                         |
+| `clusterValidator.role`                               | Check set: `control-plane`, or any other value for the GPU checks                 | `""`                                   |
+| `clusterValidator.openBaoNamespace`                   | Namespace holding OpenBao when it is not `vault-system`                           | `""`                                   |
+| `clusterValidator.envoyGatewayNamespace`              | Namespace holding Envoy Gateway when it is not `envoy-gateway-system`             | `""`                                   |
 | `clusterValidator.schedule`                           | CronJob schedule (cron expression)                                                | `0 */3 * * *`                          |
 | `clusterValidator.configMapName`                      | ConfigMap name for user-defined network checks                                    | `cluster-validator-network-checks`     |
 | `clusterValidator.networkChecks`                      | Network check configuration (creates the ConfigMap automatically when set)        | `{}`                                   |

deploy/helm/nvca-operator/nvca-operator/templates/_helpers.tpl

@@ -418,6 +418,9 @@ Usage: {{- $cv := include "nvcaop.clusterValidatorConfig" . | fromYaml -}}
     "image" (dict "repository" "" "tag" "" "pullPolicy" "IfNotPresent")
     "schedule" "0 */3 * * *"
     "configMapName" "cluster-validator-network-checks"
+    "role" ""
+    "openBaoNamespace" ""
+    "envoyGatewayNamespace" ""
     "networkChecks" (dict)
     "resources" (dict
       "requests" (dict "cpu" "100m" "memory" "64Mi")

deploy/helm/nvca-operator/nvca-operator/templates/cronjob.yaml

@@ -80,6 +80,24 @@ spec:
                   valueFrom:
                     fieldRef:
                       fieldPath: metadata.namespace
+                # Selects the check set: "control-plane" runs the gateway,
+                # storage, overlay and HA checks; anything else (including
+                # unset) runs the compute-plane GPU set. Without this the
+                # control-plane checks are unreachable from the chart.
+                - name: VALIDATOR_ROLE
+                  value: {{ $cv.role | quote }}
+                {{- if $cv.openBaoNamespace }}
+                # Relocated OpenBao: without this the Tier-2 quorum check
+                # silently skips its StatefulSet.
+                - name: NVCF_OPENBAO_NAMESPACE
+                  value: {{ $cv.openBaoNamespace | quote }}
+                {{- end }}
+                {{- if $cv.envoyGatewayNamespace }}
+                # Set when the stack's controllerNamespace differs from the
+                # Envoy Gateway chart default.
+                - name: NVCF_ENVOY_GATEWAY_NAMESPACE
+                  value: {{ $cv.envoyGatewayNamespace | quote }}
+                {{- end }}
               resources:
                 requests:
                   cpu: {{ $cv.resources.requests.cpu | quote }}

deploy/helm/nvca-operator/nvca-operator/templates/deployment.yaml

@@ -95,6 +95,21 @@ spec:
           valueFrom:
             fieldRef:
               fieldPath: metadata.namespace
+        # Same role and namespace wiring as the CronJob. This init container
+        # writes the same summary ConfigMap, so without it every operator pod
+        # restart republishes a compute-plane summary over a control-plane
+        # one: the GPU keys reappear and all the control-plane keys are pruned
+        # until the next CronJob tick.
+        - name: VALIDATOR_ROLE
+          value: {{ $cv.role | quote }}
+        {{- if $cv.openBaoNamespace }}
+        - name: NVCF_OPENBAO_NAMESPACE
+          value: {{ $cv.openBaoNamespace | quote }}
+        {{- end }}
+        {{- if $cv.envoyGatewayNamespace }}
+        - name: NVCF_ENVOY_GATEWAY_NAMESPACE
+          value: {{ $cv.envoyGatewayNamespace | quote }}
+        {{- end }}
         resources:
           requests:
             cpu: {{ $cv.resources.requests.cpu | quote }}

deploy/helm/nvca-operator/nvca-operator/templates/rbac.yaml

@@ -58,8 +58,15 @@ rules:
     resourceNames: ["cluster-validator-summary"]
     verbs: ["update"]
   - apiGroups: ["apps"]
-    resources: ["deployments", "daemonsets"]
+    resources: ["deployments", "statefulsets"]
+    # statefulsets: the Tier-2 quorum and placement check. Without it every
+    # control-plane namespace 403s and the critical row is permanently unknown.
     verbs: ["get", "list"]
+  - apiGroups: ["apps"]
+    resources: ["daemonsets"]
+    # create/delete: the node-to-node overlay probe runs a short-lived
+    # DaemonSet in a per-run namespace and deletes it in the same run.
+    verbs: ["get", "list", "create", "delete"]
   - apiGroups: ["admissionregistration.k8s.io"]
     resources:
       - mutatingwebhookconfigurations
@@ -69,7 +76,9 @@ rules:
     resources: ["networkpolicies"]
     verbs: ["get", "list", "create", "update", "delete"]
   - apiGroups: ["storage.k8s.io"]
-    resources: ["csidrivers"]
+    # storageclasses: the default-StorageClass check. Without it the critical
+    # row is permanently unknown on every run.
+    resources: ["csidrivers", "storageclasses"]
     verbs: ["get", "list"]
   - apiGroups: ["nvidia.com"]
     resources: ["clusterpolicies"]

deploy/helm/nvca-operator/nvca-operator/templates/role.yaml

@@ -114,7 +114,13 @@ rules:
   verbs: ["get", "list", "watch", "create", "update", "delete", "deletecollection", "patch"]
 - apiGroups: ["apps"]
   resources: ["daemonsets"]
-  verbs: ["get", "list", "watch"]
+  # create/delete: the cluster-validator init container runs under this
+  # ServiceAccount and the node-to-node overlay probe creates a short-lived
+  # DaemonSet in a per-run namespace. Without these the probe 403s, which is
+  # an unobserved critical check, which fails the verdict and CrashLoops the
+  # operator pod. Not an escalation: this role already has cluster-wide
+  # create/delete on namespaces, pods, deployments and statefulsets.
+  verbs: ["get", "list", "watch", "create", "delete"]
 - apiGroups: ["scheduling.run.ai"]
   resources: ["queues"]
   verbs: ["get", "list", "watch"]

deploy/helm/nvca-operator/nvca-operator/values.yaml

@@ -497,6 +497,9 @@ networkPolicy:
 ## @param clusterValidator.image.repository Cluster Validator container registry path, without tag
 ## @param clusterValidator.image.tag Cluster Validator container image tag
 ## @param clusterValidator.image.pullPolicy K8s ImagePullPolicy for cluster-validator
+## @param clusterValidator.role Check set to run: "control-plane" for gateway, storage, overlay and HA checks; any other value (including "") runs the compute-plane GPU checks
+## @param clusterValidator.openBaoNamespace Namespace holding OpenBao, when it is not vault-system; without it the Tier-2 quorum check skips its StatefulSet
+## @param clusterValidator.envoyGatewayNamespace Namespace holding Envoy Gateway, when it is not envoy-gateway-system
 ## @param clusterValidator.schedule CronJob schedule (cron expression)
 ## @param clusterValidator.configMapName ConfigMap name for user-defined network checks (reachability + network policy validation)
 ## @param clusterValidator.resources.limits.cpu CPU limit for the cluster-validator container
@@ -506,6 +509,14 @@ networkPolicy:
 ## @param clusterValidator.networkChecks [object] Network check configuration (creates the ConfigMap automatically when set)
 clusterValidator:
   enabled: false
+  # Which check set the validator runs. "control-plane" enables the gateway,
+  # StorageClass, node-to-node overlay and Tier-1/Tier-2 HA checks and skips
+  # the GPU checks. Any other value (including "") runs the compute-plane set.
+  role: ""
+  # Set when OpenBao or Envoy Gateway are installed outside their default
+  # namespaces, so the control-plane checks look in the right place.
+  openBaoNamespace: ""
+  envoyGatewayNamespace: ""
   image:
     repository: ""
     tag: "" # defaults to .Chart.AppVersion (same as nvca-operator)

src/compute-plane-services/nvca/.gitignore

@@ -15,3 +15,11 @@ clusterdump-*
 /bazel-*
 .bazel-cache/
 .tls/
+
+# Locally built command binaries. `go build -o <name> ./cmd/<name>` at the
+# subtree root drops a ~60MB artifact next to the source, which is easy to
+# sweep into a commit with `git add -A`.
+/cluster-validator
+/nvca
+/nvca-operator
+/webhook-server

src/compute-plane-services/nvca/cmd/cluster-validator/BUILD.bazel

@@ -42,4 +42,7 @@ go_test(
     name = "cluster-validator_test",
     srcs = ["main_test.go"],
     embed = [":cluster-validator_lib"],
+    deps = [
+        "//src/compute-plane-services/nvca/internal/clustervalidator",
+    ],
 )

src/compute-plane-services/nvca/cmd/cluster-validator/main.go

@@ -72,11 +72,35 @@ func main() {
 			clustervalidator.SummaryConfigMapNamespaceEnv)
 	}
 
-	if err := clustervalidator.Run(ctx, client, configNS, configName, summaryNS, emitMetrics); err != nil {
+	// VALIDATOR_ROLE selects which check set runs: "control-plane" enables
+	// gateway and StorageClass checks and skips GPU/SMB; anything else (including
+	// unset) runs the compute-plane check set (backward-compatible default).
+	roleEnv := os.Getenv("VALIDATOR_ROLE")
+	role, roleKnown := parseRole(roleEnv)
+	if roleEnv != "" && !roleKnown {
+		log.Warnf("VALIDATOR_ROLE=%q is not recognized; defaulting to compute-plane", roleEnv)
+	}
+
+	if err := clustervalidator.Run(ctx, client, configNS, configName, summaryNS, emitMetrics, role); err != nil {
 		log.WithError(err).Fatal("Cluster validation failed")
 	}
 }
 
+// parseRole normalizes the VALIDATOR_ROLE env value. Returns the matching
+// clustervalidator.Role constant and true for "control-plane" or
+// "compute-plane"; returns the compute-plane default and false for any other
+// value so unknown inputs are safe.
+func parseRole(v string) (clustervalidator.Role, bool) {
+	switch strings.ToLower(strings.TrimSpace(v)) {
+	case string(clustervalidator.RoleControlPlane):
+		return clustervalidator.RoleControlPlane, true
+	case string(clustervalidator.RoleComputePlane):
+		return clustervalidator.RoleComputePlane, true
+	default:
+		return clustervalidator.RoleComputePlane, false
+	}
+}
+
 // preflightMode reports whether this is a one-shot preflight run (e.g. nvcf-cli,
 // before NVCA is installed), which skips the summary write. Read from an env
 // (not a flag) so an unknown value is ignored rather than crashing arg parsing.

src/compute-plane-services/nvca/cmd/cluster-validator/main_test.go

@@ -17,7 +17,37 @@ limitations under the License.
 
 package main
 
-import "testing"
+import (
+	"testing"
+
+	"github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/internal/clustervalidator"
+)
+
+func TestParseRole(t *testing.T) {
+	tests := []struct {
+		in        string
+		want      clustervalidator.Role
+		wantKnown bool
+	}{
+		// Known roles are normalized and reported as known.
+		{"control-plane", clustervalidator.RoleControlPlane, true},
+		{"CONTROL-PLANE", clustervalidator.RoleControlPlane, true},
+		{" control-plane ", clustervalidator.RoleControlPlane, true},
+		{"compute-plane", clustervalidator.RoleComputePlane, true},
+		{"COMPUTE-PLANE", clustervalidator.RoleComputePlane, true},
+		// Unknown values fall back to compute-plane and are reported as unknown.
+		{"", clustervalidator.RoleComputePlane, false},
+		{"gpu", clustervalidator.RoleComputePlane, false},
+		{"both", clustervalidator.RoleComputePlane, false},
+		{"control_plane", clustervalidator.RoleComputePlane, false}, // underscore, not hyphen
+	}
+	for _, tt := range tests {
+		got, gotKnown := parseRole(tt.in)
+		if got != tt.want || gotKnown != tt.wantKnown {
+			t.Errorf("parseRole(%q) = (%q, %v), want (%q, %v)", tt.in, got, gotKnown, tt.want, tt.wantKnown)
+		}
+	}
+}
 
 func TestPreflightMode(t *testing.T) {
 	tests := []struct {

src/compute-plane-services/nvca/deployments/nvca-operator/README.md

@@ -234,6 +234,9 @@ This release does not wire the catalog into backend selection. Runtime use requi
 | `clusterValidator.image.repository`                   | Cluster Validator container registry path, without tag                            | `""`                                   |
 | `clusterValidator.image.tag`                          | Cluster Validator container image tag                                             | `v2.0.0`                               |
 | `clusterValidator.image.pullPolicy`                   | K8s ImagePullPolicy for cluster-validator                                         | `IfNotPresent`                         |
+| `clusterValidator.role`                               | Check set: `control-plane`, or any other value for the GPU checks                 | `""`                                   |
+| `clusterValidator.openBaoNamespace`                   | Namespace holding OpenBao when it is not `vault-system`                           | `""`                                   |
+| `clusterValidator.envoyGatewayNamespace`              | Namespace holding Envoy Gateway when it is not `envoy-gateway-system`             | `""`                                   |
 | `clusterValidator.schedule`                           | CronJob schedule (cron expression)                                                | `0 */3 * * *`                          |
 | `clusterValidator.configMapName`                      | ConfigMap name for user-defined network checks                                    | `cluster-validator-network-checks`     |
 | `clusterValidator.networkChecks`                      | Network check configuration (creates the ConfigMap automatically when set)        | `{}`                                   |

src/compute-plane-services/nvca/deployments/nvca-operator/templates/_helpers.tpl

@@ -418,6 +418,9 @@ Usage: {{- $cv := include "nvcaop.clusterValidatorConfig" . | fromYaml -}}
     "image" (dict "repository" "" "tag" "" "pullPolicy" "IfNotPresent")
     "schedule" "0 */3 * * *"
     "configMapName" "cluster-validator-network-checks"
+    "role" ""
+    "openBaoNamespace" ""
+    "envoyGatewayNamespace" ""
     "networkChecks" (dict)
     "resources" (dict
       "requests" (dict "cpu" "100m" "memory" "64Mi")

src/compute-plane-services/nvca/deployments/nvca-operator/templates/cronjob.yaml

@@ -80,6 +80,24 @@ spec:
                   valueFrom:
                     fieldRef:
                       fieldPath: metadata.namespace
+                # Selects the check set: "control-plane" runs the gateway,
+                # storage, overlay and HA checks; anything else (including
+                # unset) runs the compute-plane GPU set. Without this the
+                # control-plane checks are unreachable from the chart.
+                - name: VALIDATOR_ROLE
+                  value: {{ $cv.role | quote }}
+                {{- if $cv.openBaoNamespace }}
+                # Relocated OpenBao: without this the Tier-2 quorum check
+                # silently skips its StatefulSet.
+                - name: NVCF_OPENBAO_NAMESPACE
+                  value: {{ $cv.openBaoNamespace | quote }}
+                {{- end }}
+                {{- if $cv.envoyGatewayNamespace }}
+                # Set when the stack's controllerNamespace differs from the
+                # Envoy Gateway chart default.
+                - name: NVCF_ENVOY_GATEWAY_NAMESPACE
+                  value: {{ $cv.envoyGatewayNamespace | quote }}
+                {{- end }}
               resources:
                 requests:
                   cpu: {{ $cv.resources.requests.cpu | quote }}

src/compute-plane-services/nvca/deployments/nvca-operator/templates/deployment.yaml

@@ -95,6 +95,21 @@ spec:
           valueFrom:
             fieldRef:
               fieldPath: metadata.namespace
+        # Same role and namespace wiring as the CronJob. This init container
+        # writes the same summary ConfigMap, so without it every operator pod
+        # restart republishes a compute-plane summary over a control-plane
+        # one: the GPU keys reappear and all the control-plane keys are pruned
+        # until the next CronJob tick.
+        - name: VALIDATOR_ROLE
+          value: {{ $cv.role | quote }}
+        {{- if $cv.openBaoNamespace }}
+        - name: NVCF_OPENBAO_NAMESPACE
+          value: {{ $cv.openBaoNamespace | quote }}
+        {{- end }}
+        {{- if $cv.envoyGatewayNamespace }}
+        - name: NVCF_ENVOY_GATEWAY_NAMESPACE
+          value: {{ $cv.envoyGatewayNamespace | quote }}
+        {{- end }}
         resources:
           requests:
             cpu: {{ $cv.resources.requests.cpu | quote }}

src/compute-plane-services/nvca/deployments/nvca-operator/templates/rbac.yaml

@@ -58,8 +58,15 @@ rules:
     resourceNames: ["cluster-validator-summary"]
     verbs: ["update"]
   - apiGroups: ["apps"]
-    resources: ["deployments", "daemonsets"]
+    resources: ["deployments", "statefulsets"]
+    # statefulsets: the Tier-2 quorum and placement check. Without it every
+    # control-plane namespace 403s and the critical row is permanently unknown.
     verbs: ["get", "list"]
+  - apiGroups: ["apps"]
+    resources: ["daemonsets"]
+    # create/delete: the node-to-node overlay probe runs a short-lived
+    # DaemonSet in a per-run namespace and deletes it in the same run.
+    verbs: ["get", "list", "create", "delete"]
   - apiGroups: ["admissionregistration.k8s.io"]
     resources:
       - mutatingwebhookconfigurations
@@ -69,7 +76,9 @@ rules:
     resources: ["networkpolicies"]
     verbs: ["get", "list", "create", "update", "delete"]
   - apiGroups: ["storage.k8s.io"]
-    resources: ["csidrivers"]
+    # storageclasses: the default-StorageClass check. Without it the critical
+    # row is permanently unknown on every run.
+    resources: ["csidrivers", "storageclasses"]
     verbs: ["get", "list"]
   - apiGroups: ["nvidia.com"]
     resources: ["clusterpolicies"]

src/compute-plane-services/nvca/deployments/nvca-operator/templates/role.yaml

@@ -114,7 +114,13 @@ rules:
   verbs: ["get", "list", "watch", "create", "update", "delete", "deletecollection", "patch"]
 - apiGroups: ["apps"]
   resources: ["daemonsets"]
-  verbs: ["get", "list", "watch"]
+  # create/delete: the cluster-validator init container runs under this
+  # ServiceAccount and the node-to-node overlay probe creates a short-lived
+  # DaemonSet in a per-run namespace. Without these the probe 403s, which is
+  # an unobserved critical check, which fails the verdict and CrashLoops the
+  # operator pod. Not an escalation: this role already has cluster-wide
+  # create/delete on namespaces, pods, deployments and statefulsets.
+  verbs: ["get", "list", "watch", "create", "delete"]
 - apiGroups: ["scheduling.run.ai"]
   resources: ["queues"]
   verbs: ["get", "list", "watch"]

src/compute-plane-services/nvca/deployments/nvca-operator/values.yaml

@@ -528,6 +528,9 @@ networkPolicy:
 ## @param clusterValidator.image.repository Cluster Validator container registry path, without tag
 ## @param clusterValidator.image.tag Cluster Validator container image tag
 ## @param clusterValidator.image.pullPolicy K8s ImagePullPolicy for cluster-validator
+## @param clusterValidator.role Check set to run: "control-plane" for gateway, storage, overlay and HA checks; any other value (including "") runs the compute-plane GPU checks
+## @param clusterValidator.openBaoNamespace Namespace holding OpenBao, when it is not vault-system; without it the Tier-2 quorum check skips its StatefulSet
+## @param clusterValidator.envoyGatewayNamespace Namespace holding Envoy Gateway, when it is not envoy-gateway-system
 ## @param clusterValidator.schedule CronJob schedule (cron expression)
 ## @param clusterValidator.configMapName ConfigMap name for user-defined network checks (reachability + network policy validation)
 ## @param clusterValidator.resources.limits.cpu CPU limit for the cluster-validator container
@@ -537,6 +540,14 @@ networkPolicy:
 ## @param clusterValidator.networkChecks [object] Network check configuration (creates the ConfigMap automatically when set)
 clusterValidator:
   enabled: false
+  # Which check set the validator runs. "control-plane" enables the gateway,
+  # StorageClass, node-to-node overlay and Tier-1/Tier-2 HA checks and skips
+  # the GPU checks. Any other value (including "") runs the compute-plane set.
+  role: ""
+  # Set when OpenBao or Envoy Gateway are installed outside their default
+  # namespaces, so the control-plane checks look in the right place.
+  openBaoNamespace: ""
+  envoyGatewayNamespace: ""
   image:
     repository: ""
     tag: ""  # defaults to .Chart.AppVersion (same as nvca-operator)