From 34d08c97269d9215c7e564f6081864bf7e8e9294 Mon Sep 17 00:00:00 2001 From: Gianluca Mardente Date: Thu, 17 Sep 2026 09:53:13 +0200 Subject: [PATCH] fix: don't wait on sibling profiles when the cluster is being deleted allMatchingProfilesProcessed serializes Helm chart hand-off between sibling ClusterProfile/Profiles targeting the same cluster. Before a ClusterSummary uninstalls a chart it no longer references, it waits for every other matching profile to finish registering. In case one of them is about to take over management of that same release there will be an upgrade instead of uninstall-then-reinstall. That wait assumed there was always something to eventually finish. When the underlying cluster itself is being deleted, every ClusterSummary targeting it starts undeploying at roughly the same time. There is no other profile that could "take over" a cluster that's going away. This caused a ClusterSummary delete process to wait till cluster is gone without removing the helm chart. This surfaced with SveltosCluster.Spec.CleanupGracePeriod, which keeps a deleting cluster present for a while instead of vanishing instantly, long enough for two sibling ClusterSummarys undeploying in parallel to actually hit this wait. It was always a latent bug; instant cluster deletion just never exposed it before. This PR also advances calico version used in fv to v3.32.0 --- Makefile | 4 +- config/default/manager_auth_proxy_patch.yaml | 2 +- config/default/manager_image_patch.yaml | 4 +- controllers/handlers_helm.go | 7 ++++ controllers/handlers_helm_test.go | 37 +++++++++++++++++++ go.mod | 2 +- go.sum | 8 +--- manifest/deployment-agentless.yaml | 6 +-- manifest/deployment-shard.yaml | 6 +-- manifest/manifest.yaml | 6 +-- ...drift-detection-manager-in-mgmt-cluster.go | 4 +- ...ift-detection-manager-in-mgmt-cluster.yaml | 4 +- .../drift-detection-manager.go | 4 +- .../drift-detection-manager.yaml | 4 +- 14 files changed, 69 insertions(+), 29 deletions(-) diff --git a/Makefile b/Makefile index 1756c82d..2890e895 100644 --- a/Makefile +++ b/Makefile @@ -34,7 +34,7 @@ OS ?= $(shell uname -s) OS := $(shell echo $(OS) | tr '[:upper:]' '[:lower:]') K8S_LATEST_VER ?= $(shell curl -s https://dl.k8s.io/release/stable.txt) export CONTROLLER_IMG ?= $(REGISTRY)/$(IMAGE_NAME) -TAG ?= v1.15.0 +TAG ?= main .PHONY: all all: build @@ -419,7 +419,7 @@ create-workload-cluster: $(KIND) $(KUBECTL) $(KIND) get kubeconfig --name $(WORKLOAD_CLUSTER_NAME) > test/fv/workload_kubeconfig @echo "install calico on workload cluster" - $(KUBECTL) --kubeconfig=./test/fv/workload_kubeconfig apply -f https://raw.githubusercontent.com/projectcalico/calico/v3.29.0/manifests/calico.yaml + $(KUBECTL) --kubeconfig=./test/fv/workload_kubeconfig apply -f https://raw.githubusercontent.com/projectcalico/calico/v3.32.0/manifests/calico.yaml @echo wait for calico pod $(KUBECTL) --kubeconfig=./test/fv/workload_kubeconfig wait --for=condition=Available deployment/calico-kube-controllers -n kube-system --timeout=$(TIMEOUT) diff --git a/config/default/manager_auth_proxy_patch.yaml b/config/default/manager_auth_proxy_patch.yaml index 84dbcc74..8f8dea58 100644 --- a/config/default/manager_auth_proxy_patch.yaml +++ b/config/default/manager_auth_proxy_patch.yaml @@ -35,7 +35,7 @@ spec: - --shard-key= - --capi-onboard-annotation= - "--v=5" - - "--version=v1.15.0" + - "--version=main" - "--agent-in-mgmt-cluster=false" env: - name: GOMEMLIMIT diff --git a/config/default/manager_image_patch.yaml b/config/default/manager_image_patch.yaml index 4f426b49..22ae3140 100644 --- a/config/default/manager_image_patch.yaml +++ b/config/default/manager_image_patch.yaml @@ -7,8 +7,8 @@ spec: template: spec: initContainers: - - image: docker.io/projectsveltos/addon-controller:v1.15.0 + - image: docker.io/projectsveltos/addon-controller:main name: initialization containers: - - image: docker.io/projectsveltos/addon-controller:v1.15.0 + - image: docker.io/projectsveltos/addon-controller:main name: controller diff --git a/controllers/handlers_helm.go b/controllers/handlers_helm.go index d79d1e6a..60158159 100644 --- a/controllers/handlers_helm.go +++ b/controllers/handlers_helm.go @@ -760,6 +760,13 @@ func allMatchingProfilesProcessed(ctx context.Context, c client.Client, } return false, err } + if !cluster.GetDeletionTimestamp().IsZero() { + // Cluster is being removed (possibly still present for a while, e.g. + // SveltosCluster.Spec.CleanupGracePeriod). No other ClusterSummary can take + // over management of a chart on a cluster that is going away, so there is + // nothing to wait for. + return true, nil + } clusterLabels := labels.Set(cluster.GetLabels()) hasDeletionTimestamp := clusterSummary.DeletionTimestamp != nil && !clusterSummary.DeletionTimestamp.IsZero() diff --git a/controllers/handlers_helm_test.go b/controllers/handlers_helm_test.go index 7011a0f8..f2478806 100644 --- a/controllers/handlers_helm_test.go +++ b/controllers/handlers_helm_test.go @@ -2617,6 +2617,43 @@ var _ = Describe("allMatchingProfilesProcessed", func() { Expect(processed).To(BeTrue()) }) + It("returns true when the cluster is marked for deletion even if a matching ClusterProfile's ClusterSummary has not been registered yet", func() { + s, err := setupScheme() + Expect(err).To(BeNil()) + + now := metav1.Now() + cluster.DeletionTimestamp = &now + // A deleting-but-still-present cluster always has at least one finalizer holding + // it open (e.g. an infrastructure provider's, or SveltosCluster.Spec.CleanupGracePeriod); + // the fake client rejects an object with a DeletionTimestamp and no finalizers. + cluster.Finalizers = []string{randomString()} + + // cpB matches the cluster and has a HelmChart but its ClusterSummary does not exist yet. + // It must not be waited on: the cluster itself is going away, so there is no other + // ClusterSummary that could take over management of a chart on it. + cpB := &configv1beta1.ClusterProfile{ + ObjectMeta: metav1.ObjectMeta{Name: randomString()}, + Spec: configv1beta1.Spec{ + ClusterSelector: libsveltosv1beta1.Selector{ + LabelSelector: metav1.LabelSelector{ + MatchLabels: map[string]string{testEnvLabelKey: testProductionValue}, + }, + }, + HelmCharts: []configv1beta1.HelmChart{ + {ReleaseName: randomString(), ReleaseNamespace: randomString(), + RepositoryURL: randomString(), ChartName: randomString(), ChartVersion: randomString()}, + }, + }, + } + + c := fake.NewClientBuilder().WithScheme(s).WithObjects(cluster, cpB).Build() + + processed, err := controllers.AllMatchingProfilesProcessed(context.TODO(), c, + clusterSummary, textlogger.NewLogger(textlogger.NewConfig())) + Expect(err).To(BeNil()) + Expect(processed).To(BeTrue()) + }) + It("returns false when a matching ClusterProfile's ClusterSummary has not been registered yet", func() { s, err := setupScheme() Expect(err).To(BeNil()) diff --git a/go.mod b/go.mod index 868a7316..5456b49b 100644 --- a/go.mod +++ b/go.mod @@ -20,7 +20,7 @@ require ( github.com/onsi/gomega v1.43.0 github.com/opencontainers/image-spec v1.1.1 github.com/pkg/errors v0.9.1 - github.com/projectsveltos/libsveltos v1.15.0 + github.com/projectsveltos/libsveltos v1.15.1-0.20260917070258-2cea5f03d976 github.com/prometheus/client_golang v1.24.1 github.com/sigstore/cosign/v3 v3.1.3 github.com/sigstore/sigstore v1.10.10 diff --git a/go.sum b/go.sum index 2921da97..7a9935c5 100644 --- a/go.sum +++ b/go.sum @@ -248,8 +248,6 @@ github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5Qvfr github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI= github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= -github.com/docker/cli v29.8.0+incompatible h1:ih0c2jq/nN7QfES8zIfwSzIhRScjs4ehER+kZ60aeSk= -github.com/docker/cli v29.8.0+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= github.com/docker/cli v29.8.1+incompatible h1:qYL1bCp6cRw2SB1xmLlIOPyV171dilw9W2Jew38vy9c= github.com/docker/cli v29.8.1+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= github.com/docker/docker-credential-helpers v0.9.5 h1:EFNN8DHvaiK8zVqFA2DT6BjXE0GzfLOZ38ggPTKePkY= @@ -624,8 +622,6 @@ github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAl github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro= github.com/onsi/gomega v1.43.0 h1:VlG/1FxqNxhSO+lq/OHBNaaqwiBK/mO8JbVkX9Y+FeU= github.com/onsi/gomega v1.43.0/go.mod h1:REff/hsDsodHoKlWsP2mAPhu1+5/6hVYNf9rIEBpeSg= -github.com/opencontainers/go-digest v1.0.1-0.20260807152313-538f27dd2513 h1:Q1NgZeNZiTPyhTMf7RcUSIN8Rs6QIb5cG2zWsqthXzQ= -github.com/opencontainers/go-digest v1.0.1-0.20260807152313-538f27dd2513/go.mod h1:RqnyioA3pIEZMkSbOIcrw32YSgETfn/VrLuEikEdPNU= github.com/opencontainers/go-digest v1.0.1-0.20260914153615-3650f26df2a1 h1:hvMBFNaB1bXiFje/fK5rW4iE0DZKKXdlatsFw39VINA= github.com/opencontainers/go-digest v1.0.1-0.20260914153615-3650f26df2a1/go.mod h1:RqnyioA3pIEZMkSbOIcrw32YSgETfn/VrLuEikEdPNU= github.com/opencontainers/go-digest/blake3 v0.0.0-20260615172202-b50d36f46dea h1:OedKvkUaI3EfCAmzWaRjdP72olcfvLwTMsq5ztDXLR0= @@ -647,8 +643,8 @@ github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRI github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/poy/onpar v1.1.2 h1:QaNrNiZx0+Nar5dLgTVp5mXkyoVFIbepjyEoGSnhbAY= github.com/poy/onpar v1.1.2/go.mod h1:6X8FLNoxyr9kkmnlqpK6LSoiOtrO6MICtWwEuWkLjzg= -github.com/projectsveltos/libsveltos v1.15.0 h1:O0iE6ZCnft4Dy9P5bgZUG384FdoQGtkE5wWk1g7L2Qk= -github.com/projectsveltos/libsveltos v1.15.0/go.mod h1:YKMqwREDkoQH84WTmixxsB8tN5V0cwOnWQ5hNjrg7xM= +github.com/projectsveltos/libsveltos v1.15.1-0.20260917070258-2cea5f03d976 h1:4jFJzp9lqf7bGXvekBoffXU/gtzPqaLk2RNpRI5E8wY= +github.com/projectsveltos/libsveltos v1.15.1-0.20260917070258-2cea5f03d976/go.mod h1:YKMqwREDkoQH84WTmixxsB8tN5V0cwOnWQ5hNjrg7xM= github.com/projectsveltos/lua-utils/glua-json v0.0.0-20251212200258-2b3cdcb7c0f5 h1:khnc+994UszxZYu69J+R5FKiLA/Nk1JQj0EYAkwTWz0= github.com/projectsveltos/lua-utils/glua-json v0.0.0-20251212200258-2b3cdcb7c0f5/go.mod h1:yVL8KQFa9tmcxgwl9nwIMtKgtmIVC1zaFRSCfOwYvPY= github.com/projectsveltos/lua-utils/glua-runes v0.0.0-20251212200258-2b3cdcb7c0f5 h1:YbsebwRwTRhV8QacvEAdFqxcxHdeu7JTVtsBovbkgos= diff --git a/manifest/deployment-agentless.yaml b/manifest/deployment-agentless.yaml index 25429de9..5504187b 100644 --- a/manifest/deployment-agentless.yaml +++ b/manifest/deployment-agentless.yaml @@ -26,7 +26,7 @@ spec: - --shard-key= - --capi-onboard-annotation= - --v=5 - - --version=v1.15.0 + - --version=main - --agent-in-mgmt-cluster=true command: - /manager @@ -43,7 +43,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - image: docker.io/projectsveltos/addon-controller:v1.15.0 + image: docker.io/projectsveltos/addon-controller:main livenessProbe: failureThreshold: 3 httpGet: @@ -101,7 +101,7 @@ spec: fieldPath: metadata.namespace - name: IS_INITIALIZATION value: "true" - image: docker.io/projectsveltos/addon-controller:v1.15.0 + image: docker.io/projectsveltos/addon-controller:main name: initialization securityContext: allowPrivilegeEscalation: false diff --git a/manifest/deployment-shard.yaml b/manifest/deployment-shard.yaml index e86b023b..c17ef18d 100644 --- a/manifest/deployment-shard.yaml +++ b/manifest/deployment-shard.yaml @@ -26,7 +26,7 @@ spec: - --shard-key={{.SHARD}} - --capi-onboard-annotation= - --v=5 - - --version=v1.15.0 + - --version=main - --agent-in-mgmt-cluster=false command: - /manager @@ -43,7 +43,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - image: docker.io/projectsveltos/addon-controller:v1.15.0 + image: docker.io/projectsveltos/addon-controller:main livenessProbe: failureThreshold: 3 httpGet: @@ -101,7 +101,7 @@ spec: fieldPath: metadata.namespace - name: IS_INITIALIZATION value: "true" - image: docker.io/projectsveltos/addon-controller:v1.15.0 + image: docker.io/projectsveltos/addon-controller:main name: initialization securityContext: allowPrivilegeEscalation: false diff --git a/manifest/manifest.yaml b/manifest/manifest.yaml index 4697eb86..ee05cb9a 100644 --- a/manifest/manifest.yaml +++ b/manifest/manifest.yaml @@ -12044,7 +12044,7 @@ spec: - --shard-key= - --capi-onboard-annotation= - --v=5 - - --version=v1.15.0 + - --version=main - --agent-in-mgmt-cluster=false command: - /manager @@ -12061,7 +12061,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - image: docker.io/projectsveltos/addon-controller:v1.15.0 + image: docker.io/projectsveltos/addon-controller:main livenessProbe: failureThreshold: 3 httpGet: @@ -12119,7 +12119,7 @@ spec: fieldPath: metadata.namespace - name: IS_INITIALIZATION value: "true" - image: docker.io/projectsveltos/addon-controller:v1.15.0 + image: docker.io/projectsveltos/addon-controller:main name: initialization securityContext: allowPrivilegeEscalation: false diff --git a/pkg/drift-detection/drift-detection-manager-in-mgmt-cluster.go b/pkg/drift-detection/drift-detection-manager-in-mgmt-cluster.go index b0593623..471556e2 100644 --- a/pkg/drift-detection/drift-detection-manager-in-mgmt-cluster.go +++ b/pkg/drift-detection/drift-detection-manager-in-mgmt-cluster.go @@ -45,7 +45,7 @@ spec: - --watch-namespaces= - --current-cluster=management-cluster - --run-mode=do-not-send-updates - - --version=v1.15.0 + - --version=main command: - /manager env: @@ -61,7 +61,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - image: docker.io/projectsveltos/drift-detection-manager@sha256:42c0d65cde4f5e1fb51e5a69ee9bff18137859841dc49ad0c987b51da60b4a47 + image: docker.io/projectsveltos/drift-detection-manager@sha256:62e399c5250b14278d8a769817b2dd2b6c58e0f3717ed3117713f812f521122a livenessProbe: failureThreshold: 3 httpGet: diff --git a/pkg/drift-detection/drift-detection-manager-in-mgmt-cluster.yaml b/pkg/drift-detection/drift-detection-manager-in-mgmt-cluster.yaml index c00acb67..cb37518c 100644 --- a/pkg/drift-detection/drift-detection-manager-in-mgmt-cluster.yaml +++ b/pkg/drift-detection/drift-detection-manager-in-mgmt-cluster.yaml @@ -27,7 +27,7 @@ spec: - --watch-namespaces= - --current-cluster=management-cluster - --run-mode=do-not-send-updates - - --version=v1.15.0 + - --version=main command: - /manager env: @@ -43,7 +43,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - image: docker.io/projectsveltos/drift-detection-manager@sha256:42c0d65cde4f5e1fb51e5a69ee9bff18137859841dc49ad0c987b51da60b4a47 + image: docker.io/projectsveltos/drift-detection-manager@sha256:62e399c5250b14278d8a769817b2dd2b6c58e0f3717ed3117713f812f521122a livenessProbe: failureThreshold: 3 httpGet: diff --git a/pkg/drift-detection/drift-detection-manager.go b/pkg/drift-detection/drift-detection-manager.go index ae87f23f..0d23ffd1 100644 --- a/pkg/drift-detection/drift-detection-manager.go +++ b/pkg/drift-detection/drift-detection-manager.go @@ -146,7 +146,7 @@ spec: - --cluster-type= - --current-cluster=managed-cluster - --run-mode=do-not-send-updates - - --version=v1.15.0 + - --version=main - --watch-namespaces= command: - /manager @@ -163,7 +163,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - image: docker.io/projectsveltos/drift-detection-manager@sha256:42c0d65cde4f5e1fb51e5a69ee9bff18137859841dc49ad0c987b51da60b4a47 + image: docker.io/projectsveltos/drift-detection-manager@sha256:62e399c5250b14278d8a769817b2dd2b6c58e0f3717ed3117713f812f521122a livenessProbe: failureThreshold: 3 httpGet: diff --git a/pkg/drift-detection/drift-detection-manager.yaml b/pkg/drift-detection/drift-detection-manager.yaml index f6eb7754..32e49ab1 100644 --- a/pkg/drift-detection/drift-detection-manager.yaml +++ b/pkg/drift-detection/drift-detection-manager.yaml @@ -128,7 +128,7 @@ spec: - --cluster-type= - --current-cluster=managed-cluster - --run-mode=do-not-send-updates - - --version=v1.15.0 + - --version=main - --watch-namespaces= command: - /manager @@ -145,7 +145,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - image: docker.io/projectsveltos/drift-detection-manager@sha256:42c0d65cde4f5e1fb51e5a69ee9bff18137859841dc49ad0c987b51da60b4a47 + image: docker.io/projectsveltos/drift-detection-manager@sha256:62e399c5250b14278d8a769817b2dd2b6c58e0f3717ed3117713f812f521122a livenessProbe: failureThreshold: 3 httpGet: