From 67e49f6c7c981becf9181a186d1b719ed2f139a5 Mon Sep 17 00:00:00 2001 From: m-terra Date: Wed, 27 May 2026 09:52:55 +0200 Subject: [PATCH] patch pod annotations to force volume sync --- apis/proxy/v1alpha1/instance_types.go | 3 -- controllers/instance/config.go | 8 ++-- controllers/instance/instance_controller.go | 39 +++++++++++++++++- .../instance/instance_controller_test.go | 41 +++++++++++++++---- controllers/instance/pod_disruption_budget.go | 2 +- controllers/instance/prometheus.go | 2 +- controllers/instance/route.go | 2 +- controllers/instance/service.go | 4 +- controllers/instance/statefulset.go | 9 +--- controllers/instance/statefulset_test.go | 39 ++---------------- docs/api-reference.md | 1 - .../crds/proxy.haproxy.com_instances.yaml | 3 -- pkg/utils/labels.go | 5 +-- pkg/utils/names.go | 2 +- 14 files changed, 88 insertions(+), 72 deletions(-) diff --git a/apis/proxy/v1alpha1/instance_types.go b/apis/proxy/v1alpha1/instance_types.go index 7b4ccac..8f257ef 100644 --- a/apis/proxy/v1alpha1/instance_types.go +++ b/apis/proxy/v1alpha1/instance_types.go @@ -30,9 +30,6 @@ type InstanceSpec struct { Network Network `json:"network"` // Configuration is used to bootstrap the global and defaults section of the HAProxy configuration. Configuration Configuration `json:"configuration"` - // RolloutOnConfigChange enable rollout on config changes - // +optional - RolloutOnConfigChange bool `json:"rolloutOnConfigChange"` // Image specifies the HaProxy image including th tag. // +kubebuilder:default="haproxy:latest" Image string `json:"image"` diff --git a/controllers/instance/config.go b/controllers/instance/config.go index 7f06066..b56609d 100644 --- a/controllers/instance/config.go +++ b/controllers/instance/config.go @@ -107,13 +107,15 @@ func (r *Reconciler) reconcileConfig(ctx context.Context, instance *proxyv1alpha if err != nil { return "", err } + + var checksum string + if result != controllerutil.OperationResultNone { logger.Info(fmt.Sprintf("Object %s", result), "secret", configSecret.Name) + checksum = generateChecksum(configSecret) } - cs := generateChecksum(configSecret) - - return cs, nil + return checksum, nil } // #nosec diff --git a/controllers/instance/instance_controller.go b/controllers/instance/instance_controller.go index c24ceee..b2ad75d 100644 --- a/controllers/instance/instance_controller.go +++ b/controllers/instance/instance_controller.go @@ -5,7 +5,9 @@ import ( configv1alpha1 "github.com/six-group/haproxy-operator/apis/config/v1alpha1" proxyv1alpha1 "github.com/six-group/haproxy-operator/apis/proxy/v1alpha1" + "github.com/six-group/haproxy-operator/pkg/utils" "go.uber.org/multierr" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -94,7 +96,7 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu } } - if err := r.reconcileStatefulSet(ctx, instance, checksum); err != nil { + if err := r.reconcileStatefulSet(ctx, instance); err != nil { return reconcile.Result{}, r.handleError(ctx, instance, err) } @@ -111,6 +113,12 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu r.updateConfig(ctx, instance, listens, frontends, backends, resolvers) + if checksum != "" { + if err = r.patchPods(ctx, instance, checksum); err != nil { + return ctrl.Result{}, err + } + } + return ctrl.Result{}, nil } @@ -123,6 +131,35 @@ func (r *Reconciler) handleError(ctx context.Context, instance *proxyv1alpha1.In return multierr.Combine(err, r.Status().Update(ctx, instance)) } +func (r *Reconciler) patchPods(ctx context.Context, instance *proxyv1alpha1.Instance, checksum string) error { + ls := client.MatchingLabels{ + corev1.LabelMetadataName: utils.GetServiceAndStatefulsetName(instance), + } + + l := &corev1.PodList{} + err := r.List(ctx, l, client.InNamespace(instance.Namespace), ls) + if err != nil { + return err + } + + for i := range l.Items { + pod := &l.Items[i] + original := pod.DeepCopy() + + if pod.Annotations == nil { + pod.Annotations = map[string]string{} + } + pod.Annotations["haproxy.operator/checksum"] = checksum + + err = r.Patch(ctx, pod, client.MergeFrom(original)) + if err != nil { + return err + } + } + + return nil +} + func (r *Reconciler) updateConfig(ctx context.Context, instance *proxyv1alpha1.Instance, listens *configv1alpha1.ListenList, frontends *configv1alpha1.FrontendList, backends *configv1alpha1.BackendList, resolvers *configv1alpha1.ResolverList) { for i := range listens.Items { listen := listens.Items[i] diff --git a/controllers/instance/instance_controller_test.go b/controllers/instance/instance_controller_test.go index 32b2099..a96c601 100644 --- a/controllers/instance/instance_controller_test.go +++ b/controllers/instance/instance_controller_test.go @@ -37,6 +37,7 @@ var _ = Describe("Reconcile", Label("controller"), func() { resolver *configv1alpha1.Resolver secret *corev1.Secret initObjs []client.Object + haPod1, haPod2 *corev1.Pod frontend, frontendCustomCerts, frontendCustomCerts2, frontendCustomCertsEmpty, frontendWithBackendSwitching *configv1alpha1.Frontend @@ -467,7 +468,27 @@ var _ = Describe("Reconcile", Label("controller"), func() { }, } - initObjs = []client.Object{proxy, frontend, frontendCustomCerts, frontendCustomCerts2, frontendCustomCertsEmpty, backend, backend2, resolver, secret} + haPod1 = &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "haproxy-0", + Namespace: "foo", + Labels: map[string]string{ + corev1.LabelMetadataName: utils.GetServiceAndStatefulsetName(proxy), + }, + }, + } + + haPod2 = &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "haproxy-1", + Namespace: "foo", + Labels: map[string]string{ + corev1.LabelMetadataName: utils.GetServiceAndStatefulsetName(proxy), + }, + }, + } + + initObjs = []client.Object{proxy, frontend, frontendCustomCerts, frontendCustomCerts2, frontendCustomCertsEmpty, backend, backend2, resolver, secret, haPod1, haPod2} }) It("should deploy haproxy instance", func() { @@ -485,10 +506,10 @@ var _ = Describe("Reconcile", Label("controller"), func() { Ω(proxy.Status.Error).Should(BeEmpty()) service := &corev1.Service{} - Ω(cli.Get(ctx, client.ObjectKey{Namespace: proxy.Namespace, Name: utils.GetServiceName(proxy)}, service)).ShouldNot(HaveOccurred()) + Ω(cli.Get(ctx, client.ObjectKey{Namespace: proxy.Namespace, Name: utils.GetServiceAndStatefulsetName(proxy)}, service)).ShouldNot(HaveOccurred()) Ω(service.Spec.Type).Should(Equal(corev1.ServiceTypeLoadBalancer)) Ω(service.Annotations["service.beta.kubernetes.io/aws-load-balancer-scheme"]).Should(Equal("internet-facing")) - Ω(service.Spec.Selector["app.kubernetes.io/name"]).Should(Equal(proxy.Name + "-haproxy")) + Ω(service.Spec.Selector[corev1.LabelMetadataName]).Should(Equal(proxy.Name + "-haproxy")) secret := &corev1.Secret{} Ω(cli.Get(ctx, client.ObjectKey{Namespace: proxy.Namespace, Name: "bar-foo-haproxy-config"}, secret)).ShouldNot(HaveOccurred()) @@ -496,7 +517,7 @@ var _ = Describe("Reconcile", Label("controller"), func() { statefulSet := &appsv1.StatefulSet{} Ω(cli.Get(ctx, client.ObjectKey{Namespace: proxy.Namespace, Name: "bar-foo-haproxy"}, statefulSet)).ShouldNot(HaveOccurred()) - Ω(statefulSet.Spec.Template.ObjectMeta.Labels["app.kubernetes.io/name"]).Should(Equal(proxy.Name + "-haproxy")) + Ω(statefulSet.Spec.Template.ObjectMeta.Labels[corev1.LabelMetadataName]).Should(Equal(proxy.Name + "-haproxy")) Ω(statefulSet.Spec.Template.ObjectMeta.Labels["label-test"]).Should(Equal("ok")) Ω(statefulSet.Spec.Template.Spec.InitContainers).Should(HaveLen(1)) Ω(statefulSet.Spec.Template.Spec.InitContainers[0].Name).Should(Equal(proxy.Spec.InitContainers[0].Name)) @@ -642,9 +663,7 @@ var _ = Describe("Reconcile", Label("controller"), func() { Ω(statefulSet.Spec.Template.Spec.Containers[0].ReadinessProbe.HTTPGet.Path).Should(Equal("/health")) Ω(statefulSet.Spec.Template.Spec.Containers[0].LivenessProbe.Exec).ShouldNot(BeNil()) }) - It("add checksum", func() { - proxy.Spec.RolloutOnConfigChange = true - + It("patch pod annotation", func() { cli := fake.NewClientBuilder().WithScheme(scheme).WithObjects(initObjs...).WithStatusSubresource(initObjs...).Build() r := instance.Reconciler{ Client: cli, @@ -661,7 +680,11 @@ var _ = Describe("Reconcile", Label("controller"), func() { statefulSet := &appsv1.StatefulSet{} Ω(cli.Get(ctx, client.ObjectKey{Namespace: proxy.Namespace, Name: "bar-foo-haproxy"}, statefulSet)).ShouldNot(HaveOccurred()) Ω(statefulSet.Annotations).Should(BeEmpty()) - Ω(statefulSet.Spec.Template.ObjectMeta.Annotations).Should(HaveKey("checksum/config")) + + pods := &corev1.PodList{} + Ω(cli.List(ctx, pods)).ShouldNot(HaveOccurred()) + Ω(pods.Items).ShouldNot(BeEmpty()) + Ω(pods.Items[0].Annotations).ShouldNot(BeEmpty()) }) It("add pdb", func() { proxy.Spec.PodDisruptionBudget.MaxUnavailable = &intstr.IntOrString{IntVal: 2} @@ -703,7 +726,7 @@ var _ = Describe("Reconcile", Label("controller"), func() { Ω(result).ShouldNot(BeNil()) service := &corev1.Service{} - Ω(cli.Get(ctx, client.ObjectKey{Namespace: proxy.Namespace, Name: utils.GetServiceName(proxy)}, service)).ShouldNot(HaveOccurred()) + Ω(cli.Get(ctx, client.ObjectKey{Namespace: proxy.Namespace, Name: utils.GetServiceAndStatefulsetName(proxy)}, service)).ShouldNot(HaveOccurred()) Ω(service.Spec.Ports).Should(HaveLen(1)) Ω(service.Annotations["service.beta.kubernetes.io/aws-load-balancer-scheme"]).Should(Equal("internet-facing")) }) diff --git a/controllers/instance/pod_disruption_budget.go b/controllers/instance/pod_disruption_budget.go index 1c5adda..f89a97a 100644 --- a/controllers/instance/pod_disruption_budget.go +++ b/controllers/instance/pod_disruption_budget.go @@ -18,7 +18,7 @@ func (r *Reconciler) reconcilePDB(ctx context.Context, instance *proxyv1alpha1.I pdb := &policyv1.PodDisruptionBudget{ ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("%s-haproxy", instance.Name), + Name: utils.GetServiceAndStatefulsetName(instance), Namespace: instance.Namespace, }, } diff --git a/controllers/instance/prometheus.go b/controllers/instance/prometheus.go index ac057af..b320d1a 100644 --- a/controllers/instance/prometheus.go +++ b/controllers/instance/prometheus.go @@ -27,7 +27,7 @@ func (r *Reconciler) reconcileServiceMonitor(ctx context.Context, instance *prox monitor := &monitoringv1.ServiceMonitor{ ObjectMeta: metav1.ObjectMeta{ - Name: utils.GetServiceName(instance), + Name: utils.GetServiceAndStatefulsetName(instance), Namespace: instance.Namespace, }, } diff --git a/controllers/instance/route.go b/controllers/instance/route.go index a346e1b..9d626b3 100644 --- a/controllers/instance/route.go +++ b/controllers/instance/route.go @@ -59,7 +59,7 @@ func (r *Reconciler) createOrUpdateRouteForFrontend(ctx context.Context, instanc route.Spec.To = routev1.RouteTargetReference{ Kind: "Service", - Name: utils.GetServiceName(instance), + Name: utils.GetServiceAndStatefulsetName(instance), } route.Spec.Port = &routev1.RoutePort{ diff --git a/controllers/instance/service.go b/controllers/instance/service.go index 56ce688..052e905 100644 --- a/controllers/instance/service.go +++ b/controllers/instance/service.go @@ -23,7 +23,7 @@ func (r *Reconciler) reconcileService(ctx context.Context, instance *proxyv1alph service := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ - Name: utils.GetServiceName(instance), + Name: utils.GetServiceAndStatefulsetName(instance), Namespace: instance.Namespace, }, } @@ -117,7 +117,7 @@ func (r *Reconciler) reconcileServiceEndpoints(ctx context.Context, instance *pr endpointSlice := &discoveryv1.EndpointSlice{ ObjectMeta: metav1.ObjectMeta{ - Name: utils.GetServiceName(instance), + Name: utils.GetServiceAndStatefulsetName(instance), Namespace: instance.Namespace, }, } diff --git a/controllers/instance/statefulset.go b/controllers/instance/statefulset.go index bcd88ec..dfbcf17 100644 --- a/controllers/instance/statefulset.go +++ b/controllers/instance/statefulset.go @@ -3,7 +3,6 @@ package instance import ( "bytes" "context" - "fmt" "net" "path/filepath" "sort" @@ -62,12 +61,12 @@ type initScriptData struct { File string } -func (r *Reconciler) reconcileStatefulSet(ctx context.Context, instance *proxyv1alpha1.Instance, checksum string) error { +func (r *Reconciler) reconcileStatefulSet(ctx context.Context, instance *proxyv1alpha1.Instance) error { logger := log.FromContext(ctx) statefulset := &appsv1.StatefulSet{ ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("%s-haproxy", instance.Name), + Name: utils.GetServiceAndStatefulsetName(instance), Namespace: instance.Namespace, }, } @@ -150,10 +149,6 @@ func (r *Reconciler) reconcileStatefulSet(ctx context.Context, instance *proxyv1 }, } - if instance.Spec.RolloutOnConfigChange { - statefulset.Spec.Template.Annotations["checksum/config"] = checksum - } - if hasLocalLoggingTarget(instance) { volumes := []corev1.Volume{ { diff --git a/controllers/instance/statefulset_test.go b/controllers/instance/statefulset_test.go index 7469141..fe038cf 100644 --- a/controllers/instance/statefulset_test.go +++ b/controllers/instance/statefulset_test.go @@ -8,6 +8,7 @@ import ( configv1alpha1 "github.com/six-group/haproxy-operator/apis/config/v1alpha1" proxyv1alpha1 "github.com/six-group/haproxy-operator/apis/proxy/v1alpha1" appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/uuid" @@ -76,12 +77,12 @@ var _ = Describe("Reconcile", Label("controller"), func() { Client: cli, Scheme: scheme, } - err := r.reconcileStatefulSet(ctx, proxy, "checksumtest") + err := r.reconcileStatefulSet(ctx, proxy) Ω(err).ShouldNot(HaveOccurred()) statefulSet := &appsv1.StatefulSet{} Ω(cli.Get(ctx, client.ObjectKey{Namespace: proxy.Namespace, Name: "bar-foo-haproxy"}, statefulSet)).ShouldNot(HaveOccurred()) - Ω(statefulSet.Spec.Template.ObjectMeta.Labels["app.kubernetes.io/name"]).Should(Equal(proxy.Name + "-haproxy")) + Ω(statefulSet.Spec.Template.ObjectMeta.Labels[corev1.LabelMetadataName]).Should(Equal(proxy.Name + "-haproxy")) Ω(statefulSet.Spec.Template.ObjectMeta.Labels["label-test"]).Should(Equal("ok")) Ω(statefulSet.Spec.Template.Spec.InitContainers).Should(HaveLen(1)) Ω(statefulSet.Spec.Template.Spec.InitContainers[0].Args[0]).Should(ContainSubstring("10.158.182.27")) @@ -92,39 +93,5 @@ var _ = Describe("Reconcile", Label("controller"), func() { " sleep 5\n\n echo -n \"BIND_ADDRESS=10.158.182.27\" > /var/lib/haproxy/run/env\n cat /var/lib/haproxy/run/env\n exit 0\nfi\n\nexit 1\n")) Ω(statefulSet.Spec.Template.Spec.Containers[0].Env).Should(HaveLen(4)) }) - - It("update only on spec change", func() { - proxy.Spec.RolloutOnConfigChange = true - - cli := fake.NewClientBuilder().WithScheme(scheme).WithObjects(initObjs...).WithStatusSubresource(initObjs...).Build() - r := Reconciler{ - Client: cli, - Scheme: scheme, - } - err := r.reconcileStatefulSet(ctx, proxy, "checksum1") - Ω(err).ShouldNot(HaveOccurred()) - - statefulSet := &appsv1.StatefulSet{} - - Ω(cli.Get(ctx, client.ObjectKey{Namespace: proxy.Namespace, Name: "bar-foo-haproxy"}, statefulSet)).ShouldNot(HaveOccurred()) - Ω(statefulSet.Spec.Template.ObjectMeta.Annotations["checksum/config"]).Should(Equal("checksum1")) - rv1 := statefulSet.ResourceVersion - - err = r.reconcileStatefulSet(ctx, proxy, "checksum2") - Ω(err).ShouldNot(HaveOccurred()) - - Ω(cli.Get(ctx, client.ObjectKey{Namespace: proxy.Namespace, Name: "bar-foo-haproxy"}, statefulSet)).ShouldNot(HaveOccurred()) - Ω(statefulSet.Spec.Template.ObjectMeta.Annotations["checksum/config"]).Should(Equal("checksum2")) - rv2 := statefulSet.ResourceVersion - Ω(rv2).ShouldNot(Equal(rv1)) - - err = r.reconcileStatefulSet(ctx, proxy, "checksum2") - Ω(err).ShouldNot(HaveOccurred()) - - Ω(cli.Get(ctx, client.ObjectKey{Namespace: proxy.Namespace, Name: "bar-foo-haproxy"}, statefulSet)).ShouldNot(HaveOccurred()) - Ω(statefulSet.Spec.Template.ObjectMeta.Annotations["checksum/config"]).Should(Equal("checksum2")) - rv3 := statefulSet.ResourceVersion - Ω(rv3).Should(Equal(rv2)) - }) }) }) diff --git a/docs/api-reference.md b/docs/api-reference.md index 201261b..98f231f 100644 --- a/docs/api-reference.md +++ b/docs/api-reference.md @@ -1416,7 +1416,6 @@ _Appears in:_ | `replicas` _integer_ | Replicas is the desired number of replicas of the HAProxy Instance. | 1 | | | `network` _[Network](#network)_ | Network contains the configuration of Route, Services and other network related configuration. | | | | `configuration` _[Configuration](#configuration)_ | Configuration is used to bootstrap the global and defaults section of the HAProxy configuration. | | | -| `rolloutOnConfigChange` _boolean_ | RolloutOnConfigChange enable rollout on config changes | | Optional: \{\}
| | `image` _string_ | Image specifies the HaProxy image including th tag. | haproxy:latest | | | `resources` _[ResourceRequirements](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#resourcerequirements-v1-core)_ | Resources defines the resource requirements for the HAProxy pods. | | Optional: \{\}
| | `initContainers` _[Container](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#container-v1-core) array_ | InitContainers additional init containers | | Optional: \{\}
| diff --git a/helm/haproxy-operator/crds/proxy.haproxy.com_instances.yaml b/helm/haproxy-operator/crds/proxy.haproxy.com_instances.yaml index a48e312..4515bac 100644 --- a/helm/haproxy-operator/crds/proxy.haproxy.com_instances.yaml +++ b/helm/haproxy-operator/crds/proxy.haproxy.com_instances.yaml @@ -2980,9 +2980,6 @@ spec: More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ type: object type: object - rolloutOnConfigChange: - description: RolloutOnConfigChange enable rollout on config changes - type: boolean serviceAccountName: description: ServiceAccountName is the name of the ServiceAccount to use to run this Instance. diff --git a/pkg/utils/labels.go b/pkg/utils/labels.go index ba2758b..169a73e 100644 --- a/pkg/utils/labels.go +++ b/pkg/utils/labels.go @@ -1,14 +1,13 @@ package utils import ( - "fmt" - "github.com/six-group/haproxy-operator/apis/proxy/v1alpha1" + corev1 "k8s.io/api/core/v1" ) func GetAppSelectorLabels(instance *v1alpha1.Instance) map[string]string { return map[string]string{ - "app.kubernetes.io/name": fmt.Sprintf("%s-haproxy", instance.Name), + corev1.LabelMetadataName: GetServiceAndStatefulsetName(instance), } } diff --git a/pkg/utils/names.go b/pkg/utils/names.go index e63d1a2..3e76f6f 100644 --- a/pkg/utils/names.go +++ b/pkg/utils/names.go @@ -11,7 +11,7 @@ func GetConfigSecretName(instance *proxyv1alpha1.Instance) string { return fmt.Sprintf("%s-haproxy-config", instance.Name) } -func GetServiceName(instance *proxyv1alpha1.Instance) string { +func GetServiceAndStatefulsetName(instance *proxyv1alpha1.Instance) string { return fmt.Sprintf("%s-haproxy", instance.Name) }