patch pod annotations to force volume sync

apis/proxy/v1alpha1/instance_types.go

@@ -30,9 +30,6 @@ type InstanceSpec struct {
 	Network Network `json:"network"`
 	// Configuration is used to bootstrap the global and defaults section of the HAProxy configuration.
 	Configuration Configuration `json:"configuration"`
-	// RolloutOnConfigChange enable rollout on config changes
-	// +optional
-	RolloutOnConfigChange bool `json:"rolloutOnConfigChange"`
 	// Image specifies the HaProxy image including th tag.
 	// +kubebuilder:default="haproxy:latest"
 	Image string `json:"image"`

controllers/instance/config.go

@@ -107,13 +107,15 @@ func (r *Reconciler) reconcileConfig(ctx context.Context, instance *proxyv1alpha
 	if err != nil {
 		return "", err
 	}
+
+	var checksum string
+
 	if result != controllerutil.OperationResultNone {
 		logger.Info(fmt.Sprintf("Object %s", result), "secret", configSecret.Name)
+		checksum = generateChecksum(configSecret)
 	}
 
-	cs := generateChecksum(configSecret)
-
-	return cs, nil
+	return checksum, nil
 }
 
 // #nosec

controllers/instance/instance_controller.go

@@ -5,7 +5,9 @@ import (
 
 	configv1alpha1 "github.com/six-group/haproxy-operator/apis/config/v1alpha1"
 	proxyv1alpha1 "github.com/six-group/haproxy-operator/apis/proxy/v1alpha1"
+	"github.com/six-group/haproxy-operator/pkg/utils"
 	"go.uber.org/multierr"
+	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
@@ -94,7 +96,7 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu
 		}
 	}
 
-	if err := r.reconcileStatefulSet(ctx, instance, checksum); err != nil {
+	if err := r.reconcileStatefulSet(ctx, instance); err != nil {
 		return reconcile.Result{}, r.handleError(ctx, instance, err)
 	}
 
@@ -111,6 +113,12 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu
 
 	r.updateConfig(ctx, instance, listens, frontends, backends, resolvers)
 
+	if checksum != "" {
+		if err = r.patchPods(ctx, instance, checksum); err != nil {
+			return ctrl.Result{}, err
+		}
+	}
+
 	return ctrl.Result{}, nil
 }
 
@@ -123,6 +131,35 @@ func (r *Reconciler) handleError(ctx context.Context, instance *proxyv1alpha1.In
 	return multierr.Combine(err, r.Status().Update(ctx, instance))
 }
 
+func (r *Reconciler) patchPods(ctx context.Context, instance *proxyv1alpha1.Instance, checksum string) error {
+	ls := client.MatchingLabels{
+		corev1.LabelMetadataName: utils.GetServiceAndStatefulsetName(instance),
+	}
+
+	l := &corev1.PodList{}
+	err := r.List(ctx, l, client.InNamespace(instance.Namespace), ls)
+	if err != nil {
+		return err
+	}
+
+	for i := range l.Items {
+		pod := &l.Items[i]
+		original := pod.DeepCopy()
+
+		if pod.Annotations == nil {
+			pod.Annotations = map[string]string{}
+		}
+		pod.Annotations["haproxy.operator/checksum"] = checksum
+
+		err = r.Patch(ctx, pod, client.MergeFrom(original))
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
 func (r *Reconciler) updateConfig(ctx context.Context, instance *proxyv1alpha1.Instance, listens *configv1alpha1.ListenList, frontends *configv1alpha1.FrontendList, backends *configv1alpha1.BackendList, resolvers *configv1alpha1.ResolverList) {
 	for i := range listens.Items {
 		listen := listens.Items[i]

controllers/instance/instance_controller_test.go

@@ -37,6 +37,7 @@ var _ = Describe("Reconcile", Label("controller"), func() {
 			resolver          *configv1alpha1.Resolver
 			secret            *corev1.Secret
 			initObjs          []client.Object
+			haPod1, haPod2    *corev1.Pod
 
 			frontend, frontendCustomCerts, frontendCustomCerts2,
 			frontendCustomCertsEmpty, frontendWithBackendSwitching *configv1alpha1.Frontend
@@ -467,7 +468,27 @@ var _ = Describe("Reconcile", Label("controller"), func() {
 				},
 			}
 
-			initObjs = []client.Object{proxy, frontend, frontendCustomCerts, frontendCustomCerts2, frontendCustomCertsEmpty, backend, backend2, resolver, secret}
+			haPod1 = &corev1.Pod{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "haproxy-0",
+					Namespace: "foo",
+					Labels: map[string]string{
+						corev1.LabelMetadataName: utils.GetServiceAndStatefulsetName(proxy),
+					},
+				},
+			}
+
+			haPod2 = &corev1.Pod{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "haproxy-1",
+					Namespace: "foo",
+					Labels: map[string]string{
+						corev1.LabelMetadataName: utils.GetServiceAndStatefulsetName(proxy),
+					},
+				},
+			}
+
+			initObjs = []client.Object{proxy, frontend, frontendCustomCerts, frontendCustomCerts2, frontendCustomCertsEmpty, backend, backend2, resolver, secret, haPod1, haPod2}
 		})
 
 		It("should deploy haproxy instance", func() {
@@ -485,18 +506,18 @@ var _ = Describe("Reconcile", Label("controller"), func() {
 			Ω(proxy.Status.Error).Should(BeEmpty())
 
 			service := &corev1.Service{}
-			Ω(cli.Get(ctx, client.ObjectKey{Namespace: proxy.Namespace, Name: utils.GetServiceName(proxy)}, service)).ShouldNot(HaveOccurred())
+			Ω(cli.Get(ctx, client.ObjectKey{Namespace: proxy.Namespace, Name: utils.GetServiceAndStatefulsetName(proxy)}, service)).ShouldNot(HaveOccurred())
 			Ω(service.Spec.Type).Should(Equal(corev1.ServiceTypeLoadBalancer))
 			Ω(service.Annotations["service.beta.kubernetes.io/aws-load-balancer-scheme"]).Should(Equal("internet-facing"))
-			Ω(service.Spec.Selector["app.kubernetes.io/name"]).Should(Equal(proxy.Name + "-haproxy"))
+			Ω(service.Spec.Selector[corev1.LabelMetadataName]).Should(Equal(proxy.Name + "-haproxy"))
 
 			secret := &corev1.Secret{}
 			Ω(cli.Get(ctx, client.ObjectKey{Namespace: proxy.Namespace, Name: "bar-foo-haproxy-config"}, secret)).ShouldNot(HaveOccurred())
 			Ω(string(secret.Data["haproxy.cfg"])).Should(Equal(haproxyConfig))
 
 			statefulSet := &appsv1.StatefulSet{}
 			Ω(cli.Get(ctx, client.ObjectKey{Namespace: proxy.Namespace, Name: "bar-foo-haproxy"}, statefulSet)).ShouldNot(HaveOccurred())
-			Ω(statefulSet.Spec.Template.ObjectMeta.Labels["app.kubernetes.io/name"]).Should(Equal(proxy.Name + "-haproxy"))
+			Ω(statefulSet.Spec.Template.ObjectMeta.Labels[corev1.LabelMetadataName]).Should(Equal(proxy.Name + "-haproxy"))
 			Ω(statefulSet.Spec.Template.ObjectMeta.Labels["label-test"]).Should(Equal("ok"))
 			Ω(statefulSet.Spec.Template.Spec.InitContainers).Should(HaveLen(1))
 			Ω(statefulSet.Spec.Template.Spec.InitContainers[0].Name).Should(Equal(proxy.Spec.InitContainers[0].Name))
@@ -642,9 +663,7 @@ var _ = Describe("Reconcile", Label("controller"), func() {
 			Ω(statefulSet.Spec.Template.Spec.Containers[0].ReadinessProbe.HTTPGet.Path).Should(Equal("/health"))
 			Ω(statefulSet.Spec.Template.Spec.Containers[0].LivenessProbe.Exec).ShouldNot(BeNil())
 		})
-		It("add checksum", func() {
-			proxy.Spec.RolloutOnConfigChange = true
-
+		It("patch pod annotation", func() {
 			cli := fake.NewClientBuilder().WithScheme(scheme).WithObjects(initObjs...).WithStatusSubresource(initObjs...).Build()
 			r := instance.Reconciler{
 				Client: cli,
@@ -661,7 +680,11 @@ var _ = Describe("Reconcile", Label("controller"), func() {
 			statefulSet := &appsv1.StatefulSet{}
 			Ω(cli.Get(ctx, client.ObjectKey{Namespace: proxy.Namespace, Name: "bar-foo-haproxy"}, statefulSet)).ShouldNot(HaveOccurred())
 			Ω(statefulSet.Annotations).Should(BeEmpty())
-			Ω(statefulSet.Spec.Template.ObjectMeta.Annotations).Should(HaveKey("checksum/config"))
+
+			pods := &corev1.PodList{}
+			Ω(cli.List(ctx, pods)).ShouldNot(HaveOccurred())
+			Ω(pods.Items).ShouldNot(BeEmpty())
+			Ω(pods.Items[0].Annotations).ShouldNot(BeEmpty())
 		})
 		It("add pdb", func() {
 			proxy.Spec.PodDisruptionBudget.MaxUnavailable = &intstr.IntOrString{IntVal: 2}
@@ -703,7 +726,7 @@ var _ = Describe("Reconcile", Label("controller"), func() {
 			Ω(result).ShouldNot(BeNil())
 
 			service := &corev1.Service{}
-			Ω(cli.Get(ctx, client.ObjectKey{Namespace: proxy.Namespace, Name: utils.GetServiceName(proxy)}, service)).ShouldNot(HaveOccurred())
+			Ω(cli.Get(ctx, client.ObjectKey{Namespace: proxy.Namespace, Name: utils.GetServiceAndStatefulsetName(proxy)}, service)).ShouldNot(HaveOccurred())
 			Ω(service.Spec.Ports).Should(HaveLen(1))
 			Ω(service.Annotations["service.beta.kubernetes.io/aws-load-balancer-scheme"]).Should(Equal("internet-facing"))
 		})

controllers/instance/pod_disruption_budget.go

@@ -18,7 +18,7 @@ func (r *Reconciler) reconcilePDB(ctx context.Context, instance *proxyv1alpha1.I
 
 	pdb := &policyv1.PodDisruptionBudget{
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      fmt.Sprintf("%s-haproxy", instance.Name),
+			Name:      utils.GetServiceAndStatefulsetName(instance),
 			Namespace: instance.Namespace,
 		},
 	}

controllers/instance/prometheus.go

@@ -27,7 +27,7 @@ func (r *Reconciler) reconcileServiceMonitor(ctx context.Context, instance *prox
 
 	monitor := &monitoringv1.ServiceMonitor{
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      utils.GetServiceName(instance),
+			Name:      utils.GetServiceAndStatefulsetName(instance),
 			Namespace: instance.Namespace,
 		},
 	}

controllers/instance/route.go

@@ -59,7 +59,7 @@ func (r *Reconciler) createOrUpdateRouteForFrontend(ctx context.Context, instanc
 
 			route.Spec.To = routev1.RouteTargetReference{
 				Kind: "Service",
-				Name: utils.GetServiceName(instance),
+				Name: utils.GetServiceAndStatefulsetName(instance),
 			}
 
 			route.Spec.Port = &routev1.RoutePort{

controllers/instance/service.go

@@ -23,7 +23,7 @@ func (r *Reconciler) reconcileService(ctx context.Context, instance *proxyv1alph
 
 	service := &corev1.Service{
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      utils.GetServiceName(instance),
+			Name:      utils.GetServiceAndStatefulsetName(instance),
 			Namespace: instance.Namespace,
 		},
 	}
@@ -117,7 +117,7 @@ func (r *Reconciler) reconcileServiceEndpoints(ctx context.Context, instance *pr
 
 	endpointSlice := &discoveryv1.EndpointSlice{
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      utils.GetServiceName(instance),
+			Name:      utils.GetServiceAndStatefulsetName(instance),
 			Namespace: instance.Namespace,
 		},
 	}

controllers/instance/statefulset.go

@@ -3,7 +3,6 @@ package instance
 import (
 	"bytes"
 	"context"
-	"fmt"
 	"net"
 	"path/filepath"
 	"sort"
@@ -62,12 +61,12 @@ type initScriptData struct {
 	File string
 }
 
-func (r *Reconciler) reconcileStatefulSet(ctx context.Context, instance *proxyv1alpha1.Instance, checksum string) error {
+func (r *Reconciler) reconcileStatefulSet(ctx context.Context, instance *proxyv1alpha1.Instance) error {
 	logger := log.FromContext(ctx)
 
 	statefulset := &appsv1.StatefulSet{
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      fmt.Sprintf("%s-haproxy", instance.Name),
+			Name:      utils.GetServiceAndStatefulsetName(instance),
 			Namespace: instance.Namespace,
 		},
 	}
@@ -150,10 +149,6 @@ func (r *Reconciler) reconcileStatefulSet(ctx context.Context, instance *proxyv1
 		},
 	}
 
-	if instance.Spec.RolloutOnConfigChange {
-		statefulset.Spec.Template.Annotations["checksum/config"] = checksum
-	}
-
 	if hasLocalLoggingTarget(instance) {
 		volumes := []corev1.Volume{
 			{

controllers/instance/statefulset_test.go

@@ -8,6 +8,7 @@ import (
 	configv1alpha1 "github.com/six-group/haproxy-operator/apis/config/v1alpha1"
 	proxyv1alpha1 "github.com/six-group/haproxy-operator/apis/proxy/v1alpha1"
 	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/util/uuid"
@@ -76,12 +77,12 @@ var _ = Describe("Reconcile", Label("controller"), func() {
 				Client: cli,
 				Scheme: scheme,
 			}
-			err := r.reconcileStatefulSet(ctx, proxy, "checksumtest")
+			err := r.reconcileStatefulSet(ctx, proxy)
 			Ω(err).ShouldNot(HaveOccurred())
 
 			statefulSet := &appsv1.StatefulSet{}
 			Ω(cli.Get(ctx, client.ObjectKey{Namespace: proxy.Namespace, Name: "bar-foo-haproxy"}, statefulSet)).ShouldNot(HaveOccurred())
-			Ω(statefulSet.Spec.Template.ObjectMeta.Labels["app.kubernetes.io/name"]).Should(Equal(proxy.Name + "-haproxy"))
+			Ω(statefulSet.Spec.Template.ObjectMeta.Labels[corev1.LabelMetadataName]).Should(Equal(proxy.Name + "-haproxy"))
 			Ω(statefulSet.Spec.Template.ObjectMeta.Labels["label-test"]).Should(Equal("ok"))
 			Ω(statefulSet.Spec.Template.Spec.InitContainers).Should(HaveLen(1))
 			Ω(statefulSet.Spec.Template.Spec.InitContainers[0].Args[0]).Should(ContainSubstring("10.158.182.27"))
@@ -92,39 +93,5 @@ var _ = Describe("Reconcile", Label("controller"), func() {
 				"  sleep 5\n\n  echo -n \"BIND_ADDRESS=10.158.182.27\" > /var/lib/haproxy/run/env\n  cat /var/lib/haproxy/run/env\n  exit 0\nfi\n\nexit 1\n"))
 			Ω(statefulSet.Spec.Template.Spec.Containers[0].Env).Should(HaveLen(4))
 		})
-
-		It("update only on spec change", func() {
-			proxy.Spec.RolloutOnConfigChange = true
-
-			cli := fake.NewClientBuilder().WithScheme(scheme).WithObjects(initObjs...).WithStatusSubresource(initObjs...).Build()
-			r := Reconciler{
-				Client: cli,
-				Scheme: scheme,
-			}
-			err := r.reconcileStatefulSet(ctx, proxy, "checksum1")
-			Ω(err).ShouldNot(HaveOccurred())
-
-			statefulSet := &appsv1.StatefulSet{}
-
-			Ω(cli.Get(ctx, client.ObjectKey{Namespace: proxy.Namespace, Name: "bar-foo-haproxy"}, statefulSet)).ShouldNot(HaveOccurred())
-			Ω(statefulSet.Spec.Template.ObjectMeta.Annotations["checksum/config"]).Should(Equal("checksum1"))
-			rv1 := statefulSet.ResourceVersion
-
-			err = r.reconcileStatefulSet(ctx, proxy, "checksum2")
-			Ω(err).ShouldNot(HaveOccurred())
-
-			Ω(cli.Get(ctx, client.ObjectKey{Namespace: proxy.Namespace, Name: "bar-foo-haproxy"}, statefulSet)).ShouldNot(HaveOccurred())
-			Ω(statefulSet.Spec.Template.ObjectMeta.Annotations["checksum/config"]).Should(Equal("checksum2"))
-			rv2 := statefulSet.ResourceVersion
-			Ω(rv2).ShouldNot(Equal(rv1))
-
-			err = r.reconcileStatefulSet(ctx, proxy, "checksum2")
-			Ω(err).ShouldNot(HaveOccurred())
-
-			Ω(cli.Get(ctx, client.ObjectKey{Namespace: proxy.Namespace, Name: "bar-foo-haproxy"}, statefulSet)).ShouldNot(HaveOccurred())
-			Ω(statefulSet.Spec.Template.ObjectMeta.Annotations["checksum/config"]).Should(Equal("checksum2"))
-			rv3 := statefulSet.ResourceVersion
-			Ω(rv3).Should(Equal(rv2))
-		})
 	})
 })

docs/api-reference.md

@@ -1416,7 +1416,6 @@ _Appears in:_
 | `replicas` _integer_ | Replicas is the desired number of replicas of the HAProxy Instance. | 1 |  |
 | `network` _[Network](#network)_ | Network contains the configuration of Route, Services and other network related configuration. |  |  |
 | `configuration` _[Configuration](#configuration)_ | Configuration is used to bootstrap the global and defaults section of the HAProxy configuration. |  |  |
-| `rolloutOnConfigChange` _boolean_ | RolloutOnConfigChange enable rollout on config changes |  | Optional: \{\} <br /> |
 | `image` _string_ | Image specifies the HaProxy image including th tag. | haproxy:latest |  |
 | `resources` _[ResourceRequirements](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#resourcerequirements-v1-core)_ | Resources defines the resource requirements for the HAProxy pods. |  | Optional: \{\} <br /> |
 | `initContainers` _[Container](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#container-v1-core) array_ | InitContainers additional init containers |  | Optional: \{\} <br /> |

helm/haproxy-operator/crds/proxy.haproxy.com_instances.yaml

@@ -2980,9 +2980,6 @@ spec:
                       More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/
                     type: object
                 type: object
-              rolloutOnConfigChange:
-                description: RolloutOnConfigChange enable rollout on config changes
-                type: boolean
               serviceAccountName:
                 description: ServiceAccountName is the name of the ServiceAccount
                   to use to run this Instance.

pkg/utils/labels.go

@@ -1,14 +1,13 @@
 package utils
 
 import (
-	"fmt"
-
 	"github.com/six-group/haproxy-operator/apis/proxy/v1alpha1"
+	corev1 "k8s.io/api/core/v1"
 )
 
 func GetAppSelectorLabels(instance *v1alpha1.Instance) map[string]string {
 	return map[string]string{
-		"app.kubernetes.io/name": fmt.Sprintf("%s-haproxy", instance.Name),
+		corev1.LabelMetadataName: GetServiceAndStatefulsetName(instance),
 	}
 }
 

pkg/utils/names.go

@@ -11,7 +11,7 @@ func GetConfigSecretName(instance *proxyv1alpha1.Instance) string {
 	return fmt.Sprintf("%s-haproxy-config", instance.Name)
 }
 
-func GetServiceName(instance *proxyv1alpha1.Instance) string {
+func GetServiceAndStatefulsetName(instance *proxyv1alpha1.Instance) string {
 	return fmt.Sprintf("%s-haproxy", instance.Name)
 }