diff --git a/controllers/classifier_transformations.go b/controllers/classifier_transformations.go index 13445f1..b64b023 100644 --- a/controllers/classifier_transformations.go +++ b/controllers/classifier_transformations.go @@ -28,6 +28,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" libsveltosv1beta1 "github.com/projectsveltos/libsveltos/api/v1beta1" + "github.com/projectsveltos/libsveltos/lib/clustercache" logs "github.com/projectsveltos/libsveltos/lib/logsettings" ) @@ -95,6 +96,19 @@ func (r *ClassifierReconciler) requeueClassifierForSecret( logger.V(logs.LogDebug).Info("reacting to Secret change") + // A Secret change might be a managed cluster's kubeconfig being rotated/pointed at a new + // endpoint. clustercache has no other way to learn that (see #1954): evict whatever it may + // have cached under this Secret so the next read rebuilds from current content. This must + // run regardless of the AccessRequest-specific check below, which only gates whether any + // Classifier gets requeued, not whether the kubeconfig cache is still valid. + key := corev1.ObjectReference{ + APIVersion: corev1.SchemeGroupVersion.String(), + Kind: string(libsveltosv1beta1.SecretReferencedResourceKind), + Namespace: secret.Namespace, + Name: secret.Name, + } + clustercache.GetManager().RemoveSecret(&key) + r.Mux.Lock() defer r.Mux.Unlock() diff --git a/controllers/classifier_transformations_test.go b/controllers/classifier_transformations_test.go index eeeb9ca..c21453e 100644 --- a/controllers/classifier_transformations_test.go +++ b/controllers/classifier_transformations_test.go @@ -18,6 +18,7 @@ package controllers_test import ( "context" + "fmt" "sync" . "github.com/onsi/ginkgo/v2" @@ -34,11 +35,36 @@ import ( "github.com/projectsveltos/classifier/controllers" libsveltosv1beta1 "github.com/projectsveltos/libsveltos/api/v1beta1" + "github.com/projectsveltos/libsveltos/lib/clustercache" libsveltosset "github.com/projectsveltos/libsveltos/lib/set" ) +// buildFakeKubeconfig returns a minimal, syntactically valid kubeconfig pointing at server. +// clientcmd only needs to parse this, never dial it, so no real cert/token data is needed. +func buildFakeKubeconfig(server string) []byte { + return []byte(fmt.Sprintf(`apiVersion: v1 +kind: Config +clusters: +- cluster: + server: %s + insecure-skip-tls-verify: true + name: test +contexts: +- context: + cluster: test + user: test + name: test +current-context: test +users: +- name: test + user: + token: fake-token +`, server)) +} + const ( testKubeVersion124 = "1.24.0" + value = "value" ) var _ = Describe("ClassifierTransformations map functions", func() { @@ -164,3 +190,54 @@ var _ = Describe("ClassifierTransformations map functions", func() { Expect(requests).To(ContainElement(reconcile.Request{NamespacedName: types.NamespacedName{Name: classifierName}})) }) }) + +var _ = Describe("requeueClassifierForSecret", func() { + It("evicts clustercache when a cluster's kubeconfig Secret changes, regardless of AccessRequest labels", func() { + // A kubeconfig Secret's content can change (endpoint, credentials) with no auth error + // and no cluster deletion - clustercache's other eviction paths never fire for that. + // requeueClassifierForSecret otherwise only reacts to AccessRequest-labeled Secrets, so + // this must not be gated on that label. See #1954. + clusterNamespace := randomString() + clusterName := randomString() + secretName := clusterName + "-kubeconfig" + secretKey := types.NamespacedName{Namespace: clusterNamespace, Name: secretName} + + cluster := &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{Namespace: clusterNamespace, Name: clusterName}, + } + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Namespace: clusterNamespace, Name: secretName}, + Data: map[string][]byte{ + value: buildFakeKubeconfig("https://10.0.0.1:6443"), + }, + } + + c := fake.NewClientBuilder().WithScheme(scheme).WithObjects(cluster, secret).Build() + logger := textlogger.NewLogger(textlogger.NewConfig(textlogger.Verbosity(1))) + + cacheMgr := clustercache.GetManager() + config, err := cacheMgr.GetKubernetesRestConfig(context.TODO(), c, clusterNamespace, clusterName, + "", "", libsveltosv1beta1.ClusterTypeCapi, logger) + Expect(err).To(BeNil()) + Expect(config.Host).To(Equal("https://10.0.0.1:6443")) + + // Point the Secret at a different endpoint - no AccessRequest label, no deletion, no + // auth error. + Expect(c.Get(context.TODO(), secretKey, secret)).To(Succeed()) + secret.Data[value] = buildFakeKubeconfig("https://10.0.0.2:6443") + Expect(c.Update(context.TODO(), secret)).To(Succeed()) + + reconciler := &controllers.ClassifierReconciler{ + Client: c, + Scheme: scheme, + Mux: sync.Mutex{}, + Logger: logger, + } + controllers.RequeueClassifierForSecret(reconciler, context.TODO(), secret) + + config, err = cacheMgr.GetKubernetesRestConfig(context.TODO(), c, clusterNamespace, clusterName, + "", "", libsveltosv1beta1.ClusterTypeCapi, logger) + Expect(err).To(BeNil()) + Expect(config.Host).To(Equal("https://10.0.0.2:6443")) + }) +}) diff --git a/controllers/export_test.go b/controllers/export_test.go index 220709a..ad625e4 100644 --- a/controllers/export_test.go +++ b/controllers/export_test.go @@ -76,6 +76,7 @@ var ( ProcessClassifier = (*ClassifierReconciler).processClassifier RemoveClassifier = (*ClassifierReconciler).removeClassifier RequeueClassifierForCluster = (*ClassifierReconciler).requeueClassifierForCluster + RequeueClassifierForSecret = (*ClassifierReconciler).requeueClassifierForSecret RequeueClassifierForClassifierReport = (*ClassifierReconciler).requeueClassifierForClassifierReport UpdateMatchingClustersAndRegistrations = (*ClassifierReconciler).updateMatchingClustersAndRegistrations UpdateLabelsOnMatchingClusters = (*ClassifierReconciler).updateLabelsOnMatchingClusters @@ -87,7 +88,8 @@ var ( ) var ( - CreatFeatureHandlerMaps = creatFeatureHandlerMaps + CreatFeatureHandlerMaps = creatFeatureHandlerMaps + CleanClusterStaleResources = cleanClusterStaleResources ) const ( diff --git a/controllers/sveltoscluster_controller.go b/controllers/sveltoscluster_controller.go index acd36c4..1082c28 100644 --- a/controllers/sveltoscluster_controller.go +++ b/controllers/sveltoscluster_controller.go @@ -32,6 +32,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" libsveltosv1beta1 "github.com/projectsveltos/libsveltos/api/v1beta1" + "github.com/projectsveltos/libsveltos/lib/clustercache" "github.com/projectsveltos/libsveltos/lib/clusterproxy" logs "github.com/projectsveltos/libsveltos/lib/logsettings" ) @@ -97,6 +98,8 @@ func cleanClusterStaleResources(ctx context.Context, c client.Client, clusterNamespace, clusterName string, clusterType libsveltosv1beta1.ClusterType, logger logr.Logger) (ctrl.Result, error) { + clustercache.GetManager().RemoveCluster(clusterNamespace, clusterName, clusterType) + err := removeClusterClassifierReports(ctx, c, clusterNamespace, clusterName, clusterType, logger) if err != nil { logger.V(logs.LogInfo).Info( diff --git a/controllers/sveltoscluster_controller_test.go b/controllers/sveltoscluster_controller_test.go new file mode 100644 index 0000000..f74cc9d --- /dev/null +++ b/controllers/sveltoscluster_controller_test.go @@ -0,0 +1,81 @@ +/* +Copyright 2026. projectsveltos.io. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers_test + +import ( + "context" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/klog/v2/textlogger" + clusterv1 "sigs.k8s.io/cluster-api/api/core/v1beta2" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + "github.com/projectsveltos/classifier/controllers" + libsveltosv1beta1 "github.com/projectsveltos/libsveltos/api/v1beta1" + "github.com/projectsveltos/libsveltos/lib/clustercache" +) + +var _ = Describe("cleanClusterStaleResources", func() { + It("evicts clustercache when the cluster is deleted", func() { + // A cluster that is deleted and immediately replaced (same namespace/name, new + // kubeconfig) must not leave the old rest.Config cached. Deletion is the one case + // InvalidateOnAuthError never sees. cleanClusterStaleResources is shared by both the + // CAPI Cluster and SveltosCluster deletion paths. + clusterNamespace := randomString() + clusterName := randomString() + secretName := clusterName + "-kubeconfig" + secretKey := types.NamespacedName{Namespace: clusterNamespace, Name: secretName} + + cluster := &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{Namespace: clusterNamespace, Name: clusterName}, + } + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Namespace: clusterNamespace, Name: secretName}, + Data: map[string][]byte{ + value: buildFakeKubeconfig("https://10.0.0.1:6443"), + }, + } + + c := fake.NewClientBuilder().WithScheme(scheme).WithObjects(cluster, secret).Build() + logger := textlogger.NewLogger(textlogger.NewConfig()) + + cacheMgr := clustercache.GetManager() + config, err := cacheMgr.GetKubernetesRestConfig(context.TODO(), c, clusterNamespace, clusterName, + "", "", libsveltosv1beta1.ClusterTypeCapi, logger) + Expect(err).To(BeNil()) + Expect(config.Host).To(Equal("https://10.0.0.1:6443")) + + // Same namespace/name comes back as a brand new cluster with a different endpoint - + // exactly what cleanClusterStaleResources must not leave stale. + Expect(c.Get(context.TODO(), secretKey, secret)).To(Succeed()) + secret.Data[value] = buildFakeKubeconfig("https://10.0.0.2:6443") + Expect(c.Update(context.TODO(), secret)).To(Succeed()) + + _, err = controllers.CleanClusterStaleResources(context.TODO(), c, clusterNamespace, clusterName, + libsveltosv1beta1.ClusterTypeCapi, logger) + Expect(err).To(BeNil()) + + config, err = cacheMgr.GetKubernetesRestConfig(context.TODO(), c, clusterNamespace, clusterName, + "", "", libsveltosv1beta1.ClusterTypeCapi, logger) + Expect(err).To(BeNil()) + Expect(config.Host).To(Equal("https://10.0.0.2:6443")) + }) +})