feat(self-managed): record the installed stack version, and check migrations against the proposed bump

.github/workflows/build-test.yml

@@ -187,6 +187,54 @@ jobs:
       - name: Build, vet, and test the Go modules under tools/
         run: tools/ci/check-go-tools
 
+  stack-upgrade-policy:
+    name: stack upgrade policy
+    # Pull requests only. The check weighs the migrations a candidate
+    # introduces against the version bump its commits imply, which needs two
+    # sides: a baseline to measure from, and a proposal to measure. A pull
+    # request has both -- origin/main is the baseline and the branch is the
+    # proposal. A push to main has already merged, so origin/main..HEAD is
+    # empty and there is no proposal left to judge.
+    #
+    # Concretely: a branch that adds
+    # migrations/cassandra/keyspaces/nvct_api/05_drop_health_info.up.sql and
+    # carries only `fix:` commits fails this job, because dropping a column
+    # cannot be skipped past and so needs a major, while `fix` asks for a
+    # patch. Once those same commits are on main, running here would compare
+    # main against itself and find nothing to report.
+    if: github.event_name == 'pull_request'
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        stack:
+          - nvcf-self-managed-stack
+          - nvcf-compute-plane-stack
+          - nvcf-observability-stack
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+          # The check diffs against the stack's last published release tag, so
+          # it needs the tags and the history behind them.
+          fetch-depth: 0
+          fetch-tags: true
+
+      - uses: actions/setup-go@v5
+        with:
+          go-version-file: tools/go-toolchain/go.mod
+
+      # Interpolations go through env, never into the script body: a ref name
+      # expanded inline is a shell injection.
+      - name: Check migrations against the proposed version bump
+        env:
+          BASE_REF: ${{ github.base_ref }}
+          STACK: ${{ matrix.stack }}
+        run: |
+          bump="$(tools/ci/release-bump-type for-branch "origin/${BASE_REF}")"
+          echo "proposed bump: ${bump}"
+          tools/ci/check-stack-upgrade-policy --stack "${STACK}" --bump "${bump}"
+
   github-release-helper:
     name: GitHub release helper
     runs-on: ubuntu-latest

deploy/stacks/self-managed/Makefile

@@ -15,6 +15,7 @@ test:
 	@tests/grpc-proxy-nats-endpoint.sh
 	@tests/llm-pki-openbao-migration.sh
 	@tests/api-keys-startup-probe.sh
+	@tests/upgrade-receipt-wiring.sh
 	@tests/cassandra-openbao-credential-wiring.sh
 	@tests/llm-pki-release.sh
 	@tests/check-llm-pki-issuer.sh

deploy/stacks/self-managed/charts/nvcf-upgrade-receipt/Chart.yaml

@@ -0,0 +1,8 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+apiVersion: v2
+name: nvcf-upgrade-receipt
+description: Records the installed NVCF stack version in-cluster so an upgrade can tell where it is starting from.
+type: application
+version: 0.1.0
+appVersion: "0.1.0"

deploy/stacks/self-managed/charts/nvcf-upgrade-receipt/templates/job.yaml

@@ -0,0 +1,56 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+{{- $version := required "stackVersion is required: a receipt that names the wrong version is worse than no receipt" .Values.stackVersion }}
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: {{ .Release.Name }}
+  namespace: {{ .Release.Namespace }}
+  annotations:
+    # post-* rather than pre-*: the receipt states what the cluster is running,
+    # so it must not be written until the release it describes has been applied.
+    #
+    # Both install and upgrade, because the first cluster to receive this chart
+    # has no prior release of it, and Helm runs post-install there rather than
+    # post-upgrade. Omitting post-install would leave exactly the clusters this
+    # exists for without a receipt.
+    "helm.sh/hook": post-install,post-upgrade
+    "helm.sh/hook-weight": "0"
+    "helm.sh/hook-delete-policy": before-hook-creation
+spec:
+  backoffLimit: 3
+  ttlSecondsAfterFinished: 600
+  template:
+    metadata:
+      name: {{ .Release.Name }}
+    spec:
+      restartPolicy: Never
+      serviceAccountName: {{ .Release.Name }}
+      {{- with .Values.imagePullSecrets }}
+      imagePullSecrets:
+{{ toYaml . | indent 8 }}
+      {{- end }}
+      containers:
+        - name: receipt
+          image: "{{ with .Values.image.registry }}{{ . }}/{{ end }}{{ .Values.image.repository }}:{{ .Values.image.tag }}"
+          imagePullPolicy: {{ .Values.image.pullPolicy }}
+          env:
+            - name: RECEIPT_CONFIGMAP
+              value: {{ .Values.configMapName | quote }}
+            - name: INSTALLED_STACK_VERSION
+              value: {{ $version | quote }}
+          command:
+            - /bin/sh
+            - -c
+            - |
+              set -eu
+              # Rendered through apply rather than create so that the first
+              # install and every later upgrade take the same path. A receipt
+              # that only appears on a fresh install would be absent from
+              # exactly the clusters that are upgrading.
+              kubectl create configmap "${RECEIPT_CONFIGMAP}" \
+                --from-literal=installed_stack_version="${INSTALLED_STACK_VERSION}" \
+                --from-literal=recorded_at="$(date -u +%Y-%m-%dT%H:%M:%SZ)" \
+                --dry-run=client -o yaml \
+                | kubectl apply -f -
+              echo "recorded installed_stack_version=${INSTALLED_STACK_VERSION} in ${RECEIPT_CONFIGMAP}"

deploy/stacks/self-managed/charts/nvcf-upgrade-receipt/templates/role.yaml

@@ -0,0 +1,26 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: {{ .Release.Name }}
+  namespace: {{ .Release.Namespace }}
+  annotations:
+    "helm.sh/hook": post-install,post-upgrade
+    "helm.sh/hook-weight": "-5"
+    "helm.sh/hook-delete-policy": before-hook-creation
+rules:
+  # Scoped to the receipt itself. get and patch cover the upgrade case where a
+  # receipt already exists, and naming the resource keeps this identity from
+  # reaching any other ConfigMap in the namespace.
+  - apiGroups: [""]
+    resources: ["configmaps"]
+    resourceNames: [{{ .Values.configMapName | quote }}]
+    verbs: ["get", "patch"]
+  # create cannot be scoped: RBAC matches resourceNames against an object that
+  # does not exist yet, so a create rule naming one is never satisfied. It is
+  # kept in its own rule so the unscoped verb is visible rather than buried
+  # alongside the scoped ones, and it is only reachable on a first install.
+  - apiGroups: [""]
+    resources: ["configmaps"]
+    verbs: ["create"]

deploy/stacks/self-managed/charts/nvcf-upgrade-receipt/templates/rolebinding.yaml

@@ -0,0 +1,19 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: {{ .Release.Name }}
+  namespace: {{ .Release.Namespace }}
+  annotations:
+    "helm.sh/hook": post-install,post-upgrade
+    "helm.sh/hook-weight": "-5"
+    "helm.sh/hook-delete-policy": before-hook-creation
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: {{ .Release.Name }}
+subjects:
+  - kind: ServiceAccount
+    name: {{ .Release.Name }}
+    namespace: {{ .Release.Namespace }}

deploy/stacks/self-managed/charts/nvcf-upgrade-receipt/templates/serviceaccount.yaml

@@ -0,0 +1,17 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ .Release.Name }}
+  namespace: {{ .Release.Namespace }}
+  annotations:
+    # Must exist before the Job that uses it, and survive long enough to be
+    # bound. A lower weight than the Job is what orders them.
+    "helm.sh/hook": post-install,post-upgrade
+    "helm.sh/hook-weight": "-5"
+    "helm.sh/hook-delete-policy": before-hook-creation
+{{- with .Values.imagePullSecrets }}
+imagePullSecrets:
+{{ toYaml . | indent 2 }}
+{{- end }}

deploy/stacks/self-managed/charts/nvcf-upgrade-receipt/values.yaml

@@ -0,0 +1,19 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+# The stack version this bundle installs. The Helmfile supplies it from the
+# bundle's own VERSION file; there is no sensible default, so rendering fails
+# rather than recording a version the cluster is not running.
+stackVersion: ""
+
+image:
+  registry: ""
+  repository: alpine-k8s
+  tag: "1.33.1"
+  pullPolicy: IfNotPresent
+
+imagePullSecrets: []
+
+# Name of the ConfigMap holding the receipt. An upgrade reads this to decide
+# whether the jump it has been asked to make is one it can make safely.
+configMapName: nvcf-upgrade-receipt

deploy/stacks/self-managed/helmfile.d/04-upgrade-receipt.yaml.gotmpl

@@ -0,0 +1,30 @@
+environments:
+  default:
+    values:
+      - ../environments/base.yaml
+      - ../environments/{{ requiredEnv "HELMFILE_ENV" }}.yaml
+
+---
+
+{{- /*
+  A stage of its own, and the last one, so the receipt is written only after
+  every other release has been applied. Ordering is a stage boundary rather
+  than a needs: edge on purpose: under the helmfile version this stack pins,
+  needs: places a release in a later DAG layer where it waits on every peer in
+  the previous one, and a single slow or failed peer silently skips it. See the
+  admin-issuer-proxy comment in 02-core.yaml.gotmpl.
+*/}}
+
+releases:
+  - name: upgrade-receipt
+    chart: ../charts/nvcf-upgrade-receipt
+    namespace: nvcf
+    values:
+      - stackVersion: {{ readFile "../VERSION" | trim | quote }}
+        image:
+          registry: {{ .Values.global.image.registry | quote }}
+          repository: {{ .Values.global.image.repository }}/alpine-k8s
+        {{- with .Values.global.imagePullSecrets }}
+        imagePullSecrets:
+          {{- toYaml . | nindent 10 }}
+        {{- end }}

deploy/stacks/self-managed/tests/upgrade-receipt-wiring.sh

@@ -0,0 +1,52 @@
+#!/usr/bin/env bash
+# Test that the stack records the version it installed.
+#
+# An upgrade has to know where it is starting from, and nothing else in a
+# cluster carries that: Helm tracks chart versions per release, and helmfile has
+# no concept of the bundle's own version. Without this receipt every cluster
+# looks identical to every other one at upgrade time.
+#
+# The assertions that matter are the hook kinds and the recorded version. A
+# pre-* hook would claim a version before it was applied, and a post-upgrade
+# hook alone would skip the first install of this chart, which is precisely the
+# set of clusters that need a receipt written.
+set -euo pipefail
+
+stack_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+expected_version="$(tr -d '[:space:]' < "$stack_dir/VERSION")"
+
+rendered="$(cd "$stack_dir" && HELMFILE_ENV=base helmfile \
+  --file helmfile.d/04-upgrade-receipt.yaml.gotmpl template)"
+
+fail() { echo "FAIL: $1" >&2; exit 1; }
+
+grep -q 'kind: Job' <<<"$rendered" || fail "no Job rendered"
+grep -q '"helm.sh/hook": post-install,post-upgrade' <<<"$rendered" \
+  || fail "receipt must run on both install and upgrade, after the release it describes"
+grep -q "value: \"${expected_version}\"" <<<"$rendered" \
+  || fail "recorded version does not match VERSION (${expected_version})"
+# The ConfigMap is created by the Job at run time, not rendered, so its name
+# reaches the cluster as the env var the script reads.
+grep -q 'value: "nvcf-upgrade-receipt"' <<<"$rendered" \
+  || fail "receipt ConfigMap name is not the one an upgrade will read"
+
+for kind in ServiceAccount Role RoleBinding; do
+  grep -q "kind: ${kind}" <<<"$rendered" || fail "missing ${kind}; the Job cannot write the ConfigMap without it"
+done
+# get and patch are scoped to the receipt by name so this identity cannot
+# reach any other ConfigMap. create cannot be scoped -- RBAC matches
+# resourceNames against an object that does not exist yet.
+grep -qE '^\s+resourceNames: \["nvcf-upgrade-receipt"\]' <<<"$rendered" \
+  || fail "get/patch are not scoped to the receipt ConfigMap by name"
+grep -qE '^\s+verbs: \["get", "patch"\]' <<<"$rendered" \
+  || fail "scoped rule should carry only get and patch"
+grep -qE '^\s+verbs: \["create"\]' <<<"$rendered" \
+  || fail "create must remain, in its own rule, for the first install"
+
+# The stage number is the ordering guarantee. needs: is deliberately not used
+# here; see the comment in the stage file.
+last_stage="$(ls "$stack_dir"/helmfile.d/*.gotmpl | sort | tail -1)"
+[[ "$(basename "$last_stage")" == "04-upgrade-receipt.yaml.gotmpl" ]] \
+  || fail "receipt is not the last stage; it would record a version before the stack finished applying"
+
+echo "PASS: upgrade-receipt-wiring"

tools/ci/check-stack-upgrade-policy

@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+# SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Stable CI entrypoint for the Go tool in tools/stack-upgrade-policy.
+#
+# The wrapper exists for the same two reasons tools/ci/chart-service-edge does.
+#
+# The repository root. `go run -C <dir>` leaves the process running with that
+# directory as its working directory, so the tool cannot find the release
+# metadata or the git history on its own. Resolving the root from this script's
+# own location means callers do not have to pass it.
+#
+# The exit code. `go run` does NOT propagate the program's status: it prints
+# "exit status N" and exits 1. This tool distinguishes 1 (policy violation or
+# error) from 2 (bad invocation), so collapsing them would be a trap.
+#
+# Run the tests with: go test -C tools/stack-upgrade-policy ./...
+set -euo pipefail
+
+repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+bin_dir="$(mktemp -d)"
+trap 'rm -rf "${bin_dir}"' EXIT
+
+go build -C "${repo_root}/tools/stack-upgrade-policy" -o "${bin_dir}/stack-upgrade-policy" .
+
+# Not exec, so the trap above still runs, and not under errexit, so the exit
+# code reaches the caller rather than aborting the shell first.
+set +e
+"${bin_dir}/stack-upgrade-policy" --root "${repo_root}" "$@"
+status=$?
+set -e
+exit "${status}"

tools/ci/github-release-subprojects.json

@@ -42,6 +42,7 @@
     {
       "id": "nvcf-self-managed-stack",
       "path": "deploy/stacks/self-managed",
+      "migration_paths": ["migrations/cassandra", "migrations/openbao"],
       "service_name": "nvcf-self-managed-stack",
       "tag_format": "deploy/stacks/self-managed/v${version}",
       "version_file": "VERSION",

tools/stack-upgrade-policy/.gitignore

@@ -0,0 +1,2 @@
+# go build ./... drops the binary here; it must never be committed.
+/stack-upgrade-policy