feat(check): add control-plane validator, stale namespace detection, and registry credential checks

ai-tooling/user/skills/nvcf-self-managed-cli/examples/ci-pipelines.md

@@ -12,15 +12,18 @@ deploy_nvcf:
   variables:
     KUBECONFIG: $KUBECONFIG_FILE
   script:
-    - nvcf-cli self-hosted check --pre --json | jq -e '.event != "phase_failed"' || exit 2
+    - nvcf-cli self-hosted check --pre --json 2>&1 >/dev/null | grep '^{' | jq -se 'length > 0 and all(.[]; .event != "phase_failed")' || exit 2
     - nvcf-cli self-hosted up --cluster-name=$CLUSTER_NAME --token=$NVCF_ADMIN_JWT --non-interactive --json
-    - nvcf-cli self-hosted status --json | jq -e '.verdict == "healthy"'
+    - nvcf-cli self-hosted status --json 2>&1 >/dev/null | grep '^{' | jq -se 'any(.[]; .verdict == "healthy")'
 ```
 
 Notes:
 
 - `--non-interactive --token=$JWT` is required in CI; never use interactive `init`.
 - Always `--json` for machine-parsing.
+- `--json` writes JSONL to stderr, not stdout, so redirect with `2>&1 >/dev/null` before a parser.
+- Slurp with `jq -s` before testing a condition. Without it `jq -e` takes its exit status from the last event alone, so a `phase_failed` followed by any other event still exits 0.
+- stderr also carries plain-text notices, so filter to JSON lines. Do not add `--show-logs` here: it appends a non-JSON transcript to the same stream.
 - Final status check gates downstream stages on `verdict == "healthy"`.
 
 ## GitOps (Argo / Flux) pattern

ai-tooling/user/skills/nvcf-self-managed-cli/reference/flags.md

@@ -26,6 +26,35 @@
 | `--icms-url URL` | Public ICMS URL; required when contexts differ | derived from `base_http_url` |
 | `--local-only` | `check --pre` only; skip all kubectl contact | `false` |
 
+## `check`-specific
+
+At least one of `--pre`, `--control-plane`, `--compute-plane`, or `--all` is
+required. Each selects a role; only the selected roles contact a cluster.
+
+| Flag | Purpose | Default |
+|---|---|---|
+| `--pre` | Pre-flight: local-host tools plus cluster readiness | `false` |
+| `--control-plane` | Control-plane checks | `false` |
+| `--compute-plane` | Compute-plane checks | `false` |
+| `--all` | Every category | `false` |
+| `--cluster-name NAME` | Cluster name for compute-plane checks | - |
+| `--skip-inotify-check` | Skip the per-node inotify-limits probe. Needed when the kubeconfig user cannot create pods in `default`. Env: `NVCF_CLI_SELFHOSTED_SKIP_INOTIFY` | `false` |
+
+### Cluster-validator flags
+
+The validator runs as a Job in the cluster being checked. The CLI creates a
+ServiceAccount, ClusterRole, and ClusterRoleBinding for it and removes them
+after the run, so the kubeconfig context needs permission to manage those.
+
+| Flag | Purpose | Default |
+|---|---|---|
+| `--cluster-validator-image REF` | Validator image. Resolution order: flag, `NVCF_CLI_CLUSTER_VALIDATOR_IMAGE`, config key `cluster_validator_image`. A ref with no tag discovers the latest stable tag from the registry. Unset everywhere skips the probe with a warning | - |
+| `--cluster-validator-registries host:port,...` | Extra registries the control-plane validator probes for reachability. `nvcr.io` is always probed. Repeatable or comma-separated. Env: `NVCF_CLI_CLUSTER_VALIDATOR_REGISTRIES`; config key `cluster_validator_registries` | `nvcr.io` only |
+| `--cluster-validator-pull-secret NAME` | docker-registry Secret in `default` used to pull the validator image. When empty, the CLI looks for one in the NVCF namespaces and falls back to minting one from `NGC_API_KEY` | auto-detect |
+| `--skip-cluster-validation` | Skip the in-cluster validator probe entirely. Env: `NVCF_CLI_SELFHOSTED_SKIP_CLUSTER_VALIDATION` | `false` |
+| `--no-cleanup` | Keep the validator Job (and its RBAC) after the run for debugging. The next run still sweeps prior Jobs | `false` |
+| `--show-logs` | Print the validator transcript to stderr after the check events. The transcript is not JSON, and `--json` also writes to stderr, so leave this off when a parser is reading the stream | `false` |
+
 ## `up`-specific
 
 | Flag | Purpose | Default |

src/clis/nvcf-cli/.nvcf-cli.yaml.template

@@ -142,7 +142,13 @@ api_keys_owner_id: svc@nvcf-api.local
 
 # Image reference for the cluster-validator pod used by
 # `nvcf-cli self-hosted check`. The CLI runs this image as a Job in the
-# compute-plane cluster to verify NVCA prerequisites before install.
+# cluster being checked to verify prerequisites before install. One image
+# serves both roles; the Job environment selects the check set.
+#
+# The Job needs a ServiceAccount, ClusterRole, and ClusterRoleBinding, which
+# the CLI creates and removes per run. The kubeconfig context therefore needs
+# permission to manage those objects; without it the probe reports a warning
+# instead of a result.
 #
 # Config key: cluster_validator_image
 # Environment variable: NVCF_CLI_CLUSTER_VALIDATOR_IMAGE
@@ -156,6 +162,18 @@ api_keys_owner_id: svc@nvcf-api.local
 # Prod:    nvcr.io/nvidia/nvcf-byoc/cluster-validator
 # cluster_validator_image: nvcr.io/nvidia/nvcf-byoc/cluster-validator
 
+# Extra container registries the control-plane validator probes for
+# reachability, as host:port. nvcr.io is always probed, so list only the
+# registries your stack pulls from in addition to it.
+#
+# Config key: cluster_validator_registries (list)
+# Environment variable: NVCF_CLI_CLUSTER_VALIDATOR_REGISTRIES (comma-separated)
+# Command-line flag: --cluster-validator-registries (repeatable or comma-separated)
+#
+# cluster_validator_registries:
+#   - harbor.example.com:443
+#   - ghcr.io:443
+
 # ==============================================================================
 # General Settings
 # ==============================================================================

src/clis/nvcf-cli/cmd/BUILD.bazel

@@ -141,6 +141,7 @@ go_test(
         "//src/clis/nvcf-cli/internal/selfhosted",
         "//src/clis/nvcf-cli/internal/selfhosted/auth",
         "//src/clis/nvcf-cli/internal/selfhosted/controlplaneprofile",
+        "//src/clis/nvcf-cli/internal/selfhosted/kubectx",
         "//src/clis/nvcf-cli/internal/selfhosted/progress",
         "//src/clis/nvcf-cli/internal/selfhosted/reachability",
         "//src/clis/nvcf-cli/internal/selfhosted/teardown",

src/clis/nvcf-cli/cmd/main_test.go

@@ -21,14 +21,35 @@ import (
 	"context"
 	"os"
 	"testing"
+
+	"nvcf-cli/internal/selfhosted"
 )
 
-// Default-stub out the validator-tag registry probe so cmd tests don't make
-// real network calls or pollute the on-disk cache. Individual tests can
-// reassign the variable if they explicitly want to exercise discovery.
+// Default-stub every seam that would otherwise reach the developer's cluster or
+// the network. Individual tests reassign a variable when they explicitly want
+// to exercise that path.
+//
+// These are not conveniences. Without them `go test ./cmd/` creates a hostPath
+// busybox pod per node in `default` (the inotify probe), lists Secrets across
+// the stack namespaces of whatever kubeconfig happens to be current, and makes
+// an outbound request to nvcr.io per configured registry. That mutates a real
+// cluster from a unit test, and it is why the package took minutes and failed
+// on a proxied kubeconfig rather than seconds and deterministically.
 func TestMain(m *testing.M) {
 	resolveLatestValidatorTagForSelfHosted = func(_ context.Context, _ string) (string, bool) {
 		return "", false
 	}
+	// Nil prober: the inotify check is skipped rather than creating pods.
+	newInotifyProberForSelfHosted = func() selfhosted.NodeInotifyProber { return nil }
+	// No cluster contact, and a clean result so the category still renders.
+	newStaleNamespaceProberForSelfHosted = func() selfhosted.StaleNamespaceProber {
+		return func(context.Context, string, []string) ([]selfhosted.StaleNamespace, error) {
+			return nil, nil
+		}
+	}
+	// No outbound registry request.
+	newRegistryCredentialCheckerForSelfHosted = func() selfhosted.RegistryCredentialChecker {
+		return func(context.Context, string, string, bool) error { return nil }
+	}
 	os.Exit(m.Run())
 }

src/clis/nvcf-cli/cmd/root.go

@@ -285,6 +285,7 @@ func initConfig() {
 	// swapping --config files. The env name matches the historical
 	// NVCF_CLI_CLUSTER_VALIDATOR_IMAGE override.
 	viper.BindEnv("cluster_validator_image", "NVCF_CLI_CLUSTER_VALIDATOR_IMAGE")
+	viper.BindEnv("cluster_validator_registries", "NVCF_CLI_CLUSTER_VALIDATOR_REGISTRIES")
 
 	// If a config file is found, read it in.
 	if err := viper.ReadInConfig(); err == nil && viper.GetBool("debug") {