From 9499353f851212976ef0e34f34200b54a154d697 Mon Sep 17 00:00:00 2001 From: Casey Davenport Date: Thu, 28 May 2026 16:08:16 -0700 Subject: [PATCH 01/54] operator: add extension Context type --- pkg/operator/context.go | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 pkg/operator/context.go diff --git a/pkg/operator/context.go b/pkg/operator/context.go new file mode 100644 index 0000000000..cb84bbd052 --- /dev/null +++ b/pkg/operator/context.go @@ -0,0 +1,41 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package operator + +import ( + v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/tls/certificatemanagement" +) + +// Context carries reconcile-derived inputs from controllers into render +// modifiers. OSS code never reads these fields - only registered modifiers do. +// Two kinds of value live here: +// - raw cluster state gathered generically (Installation, FelixConfiguration, +// ClusterDomain) that modifiers derive their own values from, and +// - controller-produced artifacts (TrustedBundle, NodePrometheusTLS) that can +// only be created controller-side because they have cluster side effects. +type Context struct { + Installation *operatorv1.InstallationSpec + FelixConfiguration *v3.FelixConfiguration + ClusterDomain string + + // TrustedBundle is the shared CA bundle for the calico-system namespace. + TrustedBundle certificatemanagement.TrustedBundle + + // NodePrometheusTLS is produced by the installation controller's enterprise + // extension and mounted by the node modifier. + NodePrometheusTLS certificatemanagement.KeyPairInterface +} From a5767f2913c0db85c3ca059cbf6657dc47a1dbf8 Mon Sep 17 00:00:00 2001 From: Casey Davenport Date: Thu, 28 May 2026 16:11:25 -0700 Subject: [PATCH 02/54] operator: add patch registry --- pkg/operator/operator_suite_test.go | 27 +++++++++++++++ pkg/operator/patch.go | 54 +++++++++++++++++++++++++++++ pkg/operator/patch_test.go | 54 +++++++++++++++++++++++++++++ 3 files changed, 135 insertions(+) create mode 100644 pkg/operator/operator_suite_test.go create mode 100644 pkg/operator/patch.go create mode 100644 pkg/operator/patch_test.go diff --git a/pkg/operator/operator_suite_test.go b/pkg/operator/operator_suite_test.go new file mode 100644 index 0000000000..42d3e32f63 --- /dev/null +++ b/pkg/operator/operator_suite_test.go @@ -0,0 +1,27 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package operator_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestOperator(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "pkg/operator Suite") +} diff --git a/pkg/operator/patch.go b/pkg/operator/patch.go new file mode 100644 index 0000000000..29d84dbc09 --- /dev/null +++ b/pkg/operator/patch.go @@ -0,0 +1,54 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package operator + +import "sigs.k8s.io/controller-runtime/pkg/client" + +// PatchFunc post-processes the objects a render component produced. It may +// mutate matched objects and/or append additional objects, and must return the +// (possibly extended) slice. Implementations self-gate on ctx.Installation.Variant. +type PatchFunc func(ctx Context, objs []client.Object) []client.Object + +var patches = map[string][]PatchFunc{} + +// Patch registers fn to run against the named component's objects. Multiple +// patches may be registered for the same component; they run in registration order. +func Patch(component string, fn PatchFunc) { + patches[component] = append(patches[component], fn) +} + +// ApplyPatches runs every patch registered for the named component over objs. +func ApplyPatches(component string, ctx Context, objs []client.Object) []client.Object { + for _, fn := range patches[component] { + objs = fn(ctx, objs) + } + return objs +} + +// FindObject returns the first object of type T with the given name. +func FindObject[T client.Object](objs []client.Object, name string) (T, bool) { + var zero T + for _, o := range objs { + if t, ok := o.(T); ok && o.GetName() == name { + return t, true + } + } + return zero, false +} + +// ResetForTest clears all registries. Test-only. +func ResetForTest() { + patches = map[string][]PatchFunc{} +} diff --git a/pkg/operator/patch_test.go b/pkg/operator/patch_test.go new file mode 100644 index 0000000000..d9cd4de108 --- /dev/null +++ b/pkg/operator/patch_test.go @@ -0,0 +1,54 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package operator_test + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/tigera/operator/pkg/operator" +) + +var _ = Describe("patch registry", func() { + AfterEach(func() { + operator.ResetForTest() + }) + + It("applies a registered patch to the matching component", func() { + operator.Patch("test", func(ctx operator.Context, objs []client.Object) []client.Object { + cm, ok := operator.FindObject[*corev1.ConfigMap](objs, "cm") + Expect(ok).To(BeTrue()) + cm.Data = map[string]string{"k": "v"} + return append(objs, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "extra"}}) + }) + + in := []client.Object{&corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "cm"}}} + out := operator.ApplyPatches("test", operator.Context{}, in) + + Expect(out).To(HaveLen(2)) + cm := out[0].(*corev1.ConfigMap) + Expect(cm.Data).To(HaveKeyWithValue("k", "v")) + Expect(out[1].GetName()).To(Equal("extra")) + }) + + It("returns objects unchanged when no patch is registered", func() { + in := []client.Object{&corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "cm"}}} + out := operator.ApplyPatches("unregistered", operator.Context{}, in) + Expect(out).To(Equal(in)) + }) +}) From b4482276533440d16711c7b5d85127367170aab1 Mon Sep 17 00:00:00 2001 From: Casey Davenport Date: Thu, 28 May 2026 16:13:59 -0700 Subject: [PATCH 03/54] operator: add image-override registry --- pkg/operator/image.go | 44 +++++++++++++++++++++++++++++++++++ pkg/operator/image_test.go | 47 ++++++++++++++++++++++++++++++++++++++ pkg/operator/patch.go | 1 + 3 files changed, 92 insertions(+) create mode 100644 pkg/operator/image.go create mode 100644 pkg/operator/image_test.go diff --git a/pkg/operator/image.go b/pkg/operator/image.go new file mode 100644 index 0000000000..b253611254 --- /dev/null +++ b/pkg/operator/image.go @@ -0,0 +1,44 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package operator + +import ( + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/components" +) + +// ImageOverride returns the component image to use for an installation, and +// false to decline (leaving the default in place). Implementations self-gate +// on in.Variant. +type ImageOverride func(in *operatorv1.InstallationSpec) (components.Component, bool) + +var imageOverrides = map[string]ImageOverride{} + +// OverrideImage registers an image override under key. The key is the render +// component's image identifier (e.g. "node"). +func OverrideImage(key string, fn ImageOverride) { + imageOverrides[key] = fn +} + +// ResolveImage returns the override registered for key if it applies to in, +// otherwise def. Render components call this inside ResolveImages. +func ResolveImage(key string, def components.Component, in *operatorv1.InstallationSpec) components.Component { + if fn, ok := imageOverrides[key]; ok { + if c, override := fn(in); override { + return c + } + } + return def +} diff --git a/pkg/operator/image_test.go b/pkg/operator/image_test.go new file mode 100644 index 0000000000..93e9beb67d --- /dev/null +++ b/pkg/operator/image_test.go @@ -0,0 +1,47 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package operator_test + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/components" + "github.com/tigera/operator/pkg/operator" +) + +var _ = Describe("image overrides", func() { + AfterEach(func() { + operator.ResetForTest() + }) + + It("uses the override when one matches", func() { + operator.OverrideImage("node", func(in *operatorv1.InstallationSpec) (components.Component, bool) { + if !in.Variant.IsEnterprise() { + return components.Component{}, false + } + return components.ComponentTigeraNode, true + }) + + ent := &operatorv1.InstallationSpec{Variant: operatorv1.TigeraSecureEnterprise} + Expect(operator.ResolveImage("node", components.ComponentCalicoNode, ent)).To(Equal(components.ComponentTigeraNode)) + }) + + It("falls back to the default when no override matches", func() { + oss := &operatorv1.InstallationSpec{Variant: operatorv1.Calico} + Expect(operator.ResolveImage("node", components.ComponentCalicoNode, oss)).To(Equal(components.ComponentCalicoNode)) + }) +}) diff --git a/pkg/operator/patch.go b/pkg/operator/patch.go index 29d84dbc09..94721a76bd 100644 --- a/pkg/operator/patch.go +++ b/pkg/operator/patch.go @@ -51,4 +51,5 @@ func FindObject[T client.Object](objs []client.Object, name string) (T, bool) { // ResetForTest clears all registries. Test-only. func ResetForTest() { patches = map[string][]PatchFunc{} + imageOverrides = map[string]ImageOverride{} } From e651d7db4479239becea914109c2c06be7a9eb61 Mon Sep 17 00:00:00 2001 From: Casey Davenport Date: Thu, 28 May 2026 16:14:55 -0700 Subject: [PATCH 04/54] render: add Named interface and component name constants for extension keying --- pkg/render/component.go | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/pkg/render/component.go b/pkg/render/component.go index eea911fbe3..585355438d 100644 --- a/pkg/render/component.go +++ b/pkg/render/component.go @@ -1,4 +1,4 @@ -// Copyright (c) 2021-2024 Tigera, Inc. All rights reserved. +// Copyright (c) 2021-2026 Tigera, Inc. All rights reserved. // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -39,3 +39,18 @@ type Component interface { // that create pods. Return OSTypeAny means that no node selector should be set for the "kubernetes.io/os" label. SupportedOSType() rmeta.OSType } + +// Named is implemented by components that expose enterprise extension points. +// The componentHandler uses Name() to look up registered patches. Components +// without enterprise extensions need not implement it. +type Named interface { + Name() string +} + +// Component names used as keys into the operator patch registry. Keep these in +// sync with the Name() methods that return them. +const ( + ComponentNameTypha = "typha" + ComponentNameNode = "node" + ComponentNameAPIServer = "apiserver" +) From c1cd817a8f1d16cbe2573dd416afa2ee5d82b64f Mon Sep 17 00:00:00 2001 From: Casey Davenport Date: Thu, 28 May 2026 16:42:07 -0700 Subject: [PATCH 05/54] utils: apply registered render patches in componentHandler Add WithContext/ComponentHandlerOption to NewComponentHandler (variadic, backward-compatible) and call operator.ApplyPatches in CreateOrUpdateOrDelete for components implementing render.Named. --- .../gatewayapi/gatewayapi_controller.go | 2 +- .../gatewayapi/gatewayapi_controller_test.go | 2 +- .../installation/core_controller.go | 2 +- .../installation/core_controller_test.go | 18 ++++---- pkg/controller/utils/component.go | 21 ++++++++- pkg/controller/utils/component_test.go | 43 +++++++++++++++++++ 6 files changed, 74 insertions(+), 14 deletions(-) diff --git a/pkg/controller/gatewayapi/gatewayapi_controller.go b/pkg/controller/gatewayapi/gatewayapi_controller.go index cdbcd38237..d949c8c959 100644 --- a/pkg/controller/gatewayapi/gatewayapi_controller.go +++ b/pkg/controller/gatewayapi/gatewayapi_controller.go @@ -147,7 +147,7 @@ type ReconcileGatewayAPI struct { status status.StatusManager clusterDomain string multiTenant bool - newComponentHandler func(log logr.Logger, client client.Client, scheme *runtime.Scheme, cr metav1.Object) utils.ComponentHandler + newComponentHandler func(log logr.Logger, client client.Client, scheme *runtime.Scheme, cr metav1.Object, opts ...utils.ComponentHandlerOption) utils.ComponentHandler watchEnvoyProxy func(namespacedName operatorv1.NamespacedName) error watchEnvoyGateway func(namespacedName operatorv1.NamespacedName) error } diff --git a/pkg/controller/gatewayapi/gatewayapi_controller_test.go b/pkg/controller/gatewayapi/gatewayapi_controller_test.go index cf918699a0..9a968e880a 100644 --- a/pkg/controller/gatewayapi/gatewayapi_controller_test.go +++ b/pkg/controller/gatewayapi/gatewayapi_controller_test.go @@ -702,7 +702,7 @@ var _ = Describe("Gateway API controller tests", func() { var fakeComponentHandlers []*fakeComponentHandler -func FakeComponentHandler(log logr.Logger, client client.Client, scheme *runtime.Scheme, cr metav1.Object) utils.ComponentHandler { +func FakeComponentHandler(log logr.Logger, client client.Client, scheme *runtime.Scheme, cr metav1.Object, opts ...utils.ComponentHandlerOption) utils.ComponentHandler { h := &fakeComponentHandler{ client: client, scheme: scheme, diff --git a/pkg/controller/installation/core_controller.go b/pkg/controller/installation/core_controller.go index a018c16364..89acae7416 100644 --- a/pkg/controller/installation/core_controller.go +++ b/pkg/controller/installation/core_controller.go @@ -405,7 +405,7 @@ type ReconcileInstallation struct { kubernetesVersion *common.VersionInfo // newComponentHandler returns a new component handler. Useful stub for unit testing. - newComponentHandler func(log logr.Logger, client client.Client, scheme *runtime.Scheme, cr metav1.Object) utils.ComponentHandler + newComponentHandler func(log logr.Logger, client client.Client, scheme *runtime.Scheme, cr metav1.Object, opts ...utils.ComponentHandlerOption) utils.ComponentHandler } // GetActivePools returns the full set of enabled IP pools in the cluster. diff --git a/pkg/controller/installation/core_controller_test.go b/pkg/controller/installation/core_controller_test.go index aa113b34e3..5676d0ebe9 100644 --- a/pkg/controller/installation/core_controller_test.go +++ b/pkg/controller/installation/core_controller_test.go @@ -2458,7 +2458,7 @@ var _ = Describe("Testing core-controller installation", func() { migrationChecked: true, tierWatchReady: ready, migrationWatchReady: &utils.ReadyFlag{}, - newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object) utils.ComponentHandler { + newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object, ...utils.ComponentHandlerOption) utils.ComponentHandler { return componentHandler }, } @@ -2605,7 +2605,7 @@ var _ = Describe("updateMutatingAdmissionPolicies", func() { manageCRDs: true, v3CRDs: true, kubernetesVersion: &common.VersionInfo{Major: 1, Minor: 32}, - newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object) utils.ComponentHandler { + newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object, ...utils.ComponentHandlerOption) utils.ComponentHandler { return componentHandler }, } @@ -2638,7 +2638,7 @@ var _ = Describe("updateMutatingAdmissionPolicies", func() { manageCRDs: true, v3CRDs: true, kubernetesVersion: &common.VersionInfo{Major: 1, Minor: 31}, - newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object) utils.ComponentHandler { + newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object, ...utils.ComponentHandlerOption) utils.ComponentHandler { return componentHandler }, } @@ -2657,7 +2657,7 @@ var _ = Describe("updateMutatingAdmissionPolicies", func() { manageCRDs: true, v3CRDs: false, kubernetesVersion: &common.VersionInfo{Major: 1, Minor: 32}, - newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object) utils.ComponentHandler { + newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object, ...utils.ComponentHandlerOption) utils.ComponentHandler { return componentHandler }, } @@ -2675,7 +2675,7 @@ var _ = Describe("updateMutatingAdmissionPolicies", func() { manageCRDs: false, v3CRDs: true, kubernetesVersion: &common.VersionInfo{Major: 1, Minor: 32}, - newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object) utils.ComponentHandler { + newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object, ...utils.ComponentHandlerOption) utils.ComponentHandler { return componentHandler }, } @@ -2693,7 +2693,7 @@ var _ = Describe("updateMutatingAdmissionPolicies", func() { manageCRDs: true, v3CRDs: true, kubernetesVersion: nil, - newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object) utils.ComponentHandler { + newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object, ...utils.ComponentHandlerOption) utils.ComponentHandler { return componentHandler }, } @@ -2728,7 +2728,7 @@ var _ = Describe("updateMutatingAdmissionPolicies", func() { manageCRDs: true, v3CRDs: true, kubernetesVersion: &common.VersionInfo{Major: 1, Minor: 32}, - newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object) utils.ComponentHandler { + newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object, ...utils.ComponentHandlerOption) utils.ComponentHandler { return componentHandler }, } @@ -2787,7 +2787,7 @@ var _ = Describe("updateMutatingAdmissionPolicies", func() { manageCRDs: true, v3CRDs: true, kubernetesVersion: &common.VersionInfo{Major: 1, Minor: 32}, - newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object) utils.ComponentHandler { + newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object, ...utils.ComponentHandlerOption) utils.ComponentHandler { return componentHandler }, } @@ -2810,7 +2810,7 @@ var _ = Describe("updateMutatingAdmissionPolicies", func() { manageCRDs: true, v3CRDs: true, kubernetesVersion: &common.VersionInfo{Major: 1, Minor: 32}, - newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object) utils.ComponentHandler { + newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object, ...utils.ComponentHandlerOption) utils.ComponentHandler { return componentHandler }, } diff --git a/pkg/controller/utils/component.go b/pkg/controller/utils/component.go index b6679a2647..292aa99479 100644 --- a/pkg/controller/utils/component.go +++ b/pkg/controller/utils/component.go @@ -46,6 +46,7 @@ import ( "github.com/tigera/operator/pkg/apigroup" "github.com/tigera/operator/pkg/common" "github.com/tigera/operator/pkg/controller/status" + "github.com/tigera/operator/pkg/operator" "github.com/tigera/operator/pkg/render" rmeta "github.com/tigera/operator/pkg/render/common/meta" ) @@ -73,16 +74,28 @@ type ComponentHandler interface { SetCreateOnly() } +// ComponentHandlerOption configures a componentHandler. +type ComponentHandlerOption func(*componentHandler) + +// WithContext supplies the operator.Context passed to registered render patches. +func WithContext(ctx operator.Context) ComponentHandlerOption { + return func(c *componentHandler) { c.modCtx = ctx } +} + // cr is allowed to be nil in the case we don't want to put ownership on a resource, // this is useful for CRD management so that they are not removed automatically. -func NewComponentHandler(log logr.Logger, cli client.Client, scheme *runtime.Scheme, cr metav1.Object) ComponentHandler { - return &componentHandler{ +func NewComponentHandler(log logr.Logger, cli client.Client, scheme *runtime.Scheme, cr metav1.Object, opts ...ComponentHandlerOption) ComponentHandler { + h := &componentHandler{ client: cli, scheme: scheme, cr: cr, log: log, apiGroupEnvs: apigroup.EnvVars(), } + for _, o := range opts { + o(h) + } + return h } type componentHandler struct { @@ -92,6 +105,7 @@ type componentHandler struct { log logr.Logger createOnly bool apiGroupEnvs []v1.EnvVar + modCtx operator.Context } func (c *componentHandler) SetCreateOnly() { @@ -459,6 +473,9 @@ func (c *componentHandler) CreateOrUpdateOrDelete(ctx context.Context, component var cronJobs []types.NamespacedName objsToCreate, objsToDelete := component.Objects() + if named, ok := component.(render.Named); ok { + objsToCreate = operator.ApplyPatches(named.Name(), c.modCtx, objsToCreate) + } osType := component.SupportedOSType() if len(c.apiGroupEnvs) > 0 { diff --git a/pkg/controller/utils/component_test.go b/pkg/controller/utils/component_test.go index d2e18c8482..d4b34cb871 100644 --- a/pkg/controller/utils/component_test.go +++ b/pkg/controller/utils/component_test.go @@ -46,6 +46,7 @@ import ( "github.com/tigera/operator/pkg/common" "github.com/tigera/operator/pkg/controller/status" ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" + "github.com/tigera/operator/pkg/operator" "github.com/tigera/operator/pkg/render" rmeta "github.com/tigera/operator/pkg/render/common/meta" ) @@ -2457,3 +2458,45 @@ func (mc *mockClient) RESTMapper() restMeta.RESTMapper { func (mc *mockClient) SubResource(subResource string) client.SubResourceClient { panic("SubResource not implemented in mockClient") } + +var _ = Describe("componentHandler patch application", func() { + AfterEach(func() { + operator.ResetForTest() + }) + + It("applies registered patches to a named component before create", func() { + operator.Patch("fake", func(ctx operator.Context, objs []client.Object) []client.Object { + cm := objs[0].(*corev1.ConfigMap) + cm.Data = map[string]string{"patched": "yes"} + return objs + }) + + s := runtime.NewScheme() + Expect(apis.AddToScheme(s, false)).NotTo(HaveOccurred()) + Expect(corev1.SchemeBuilder.AddToScheme(s)).NotTo(HaveOccurred()) + + c := ctrlrfake.DefaultFakeClientBuilder(s).Build() + handler := NewComponentHandler(logf.Log, c, s, nil) + comp := &namedFakeComponent{name: "fake", obj: &corev1.ConfigMap{ + TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"}, + ObjectMeta: metav1.ObjectMeta{Name: "cm", Namespace: "default"}, + }} + + Expect(handler.CreateOrUpdateOrDelete(context.Background(), comp, nil)).NotTo(HaveOccurred()) + + got := &corev1.ConfigMap{} + Expect(c.Get(context.Background(), client.ObjectKey{Name: "cm", Namespace: "default"}, got)).NotTo(HaveOccurred()) + Expect(got.Data).To(HaveKeyWithValue("patched", "yes")) + }) +}) + +type namedFakeComponent struct { + name string + obj client.Object +} + +func (f *namedFakeComponent) Name() string { return f.name } +func (f *namedFakeComponent) ResolveImages(*operatorv1.ImageSet) error { return nil } +func (f *namedFakeComponent) Objects() ([]client.Object, []client.Object) { return []client.Object{f.obj}, nil } +func (f *namedFakeComponent) Ready() bool { return true } +func (f *namedFakeComponent) SupportedOSType() rmeta.OSType { return rmeta.OSTypeLinux } From cc62d3cb3c39b098799f234d2fd88b52aaa3962f Mon Sep 17 00:00:00 2001 From: Casey Davenport Date: Thu, 28 May 2026 16:49:29 -0700 Subject: [PATCH 06/54] operator: add installation controller extension registry --- pkg/operator/extension.go | 59 ++++++++++++++++++++++++++++++++++ pkg/operator/extension_test.go | 42 ++++++++++++++++++++++++ 2 files changed, 101 insertions(+) create mode 100644 pkg/operator/extension.go create mode 100644 pkg/operator/extension_test.go diff --git a/pkg/operator/extension.go b/pkg/operator/extension.go new file mode 100644 index 0000000000..de5080c45d --- /dev/null +++ b/pkg/operator/extension.go @@ -0,0 +1,59 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package operator + +import ( + "context" + + v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/controller/certificatemanager" + "github.com/tigera/operator/pkg/tls/certificatemanagement" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// InstallationPrep is the input to an InstallationExtension's Prepare. It holds +// the generically-gathered reconcile state the extension needs to do its +// side-effecting work (create certs, assemble the trusted bundle). +type InstallationPrep struct { + Ctx context.Context + Client client.Client + Installation *operatorv1.InstallationSpec + FelixConfiguration *v3.FelixConfiguration + CertificateManager certificatemanager.CertificateManager + TrustedBundle certificatemanagement.TrustedBundle + ClusterDomain string +} + +// InstallationExtension is the enterprise hook for the installation controller. +// Prepare runs controller-side before rendering. It performs work modifiers +// can't (cluster side effects, fetching/creating certificates) and may abort +// the reconcile by returning an error. It returns the Context handed to the +// render patches; on the Calico variant it should return an empty Context and +// nil error. +type InstallationExtension interface { + Prepare(p InstallationPrep) (Context, error) +} + +var installationExtension InstallationExtension + +// RegisterInstallationExtension registers the installation controller extension. +func RegisterInstallationExtension(e InstallationExtension) { installationExtension = e } + +// GetInstallationExtension returns the registered extension, or nil. +func GetInstallationExtension() InstallationExtension { return installationExtension } + +// ResetExtensionsForTest clears registered extensions. Test-only. +func ResetExtensionsForTest() { installationExtension = nil } diff --git a/pkg/operator/extension_test.go b/pkg/operator/extension_test.go new file mode 100644 index 0000000000..e3ecd95592 --- /dev/null +++ b/pkg/operator/extension_test.go @@ -0,0 +1,42 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package operator_test + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/tigera/operator/pkg/operator" +) + +var _ = Describe("controller extensions", func() { + AfterEach(func() { operator.ResetExtensionsForTest() }) + + It("returns the registered installation extension", func() { + ext := &fakeInstallationExtension{} + operator.RegisterInstallationExtension(ext) + Expect(operator.GetInstallationExtension()).To(BeIdenticalTo(ext)) + }) + + It("returns nil when none is registered", func() { + Expect(operator.GetInstallationExtension()).To(BeNil()) + }) +}) + +type fakeInstallationExtension struct{} + +func (f *fakeInstallationExtension) Prepare(_ operator.InstallationPrep) (operator.Context, error) { + return operator.Context{}, nil +} From e8c588f4aaf3dc4d2df914d13b4bb2a16c1a72ca Mon Sep 17 00:00:00 2001 From: Casey Davenport Date: Thu, 28 May 2026 16:50:25 -0700 Subject: [PATCH 07/54] render: typha implements Named --- pkg/render/typha.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/render/typha.go b/pkg/render/typha.go index cc352771dc..49c3f13917 100644 --- a/pkg/render/typha.go +++ b/pkg/render/typha.go @@ -113,6 +113,8 @@ func (c *typhaComponent) SupportedOSType() rmeta.OSType { return rmeta.OSTypeLinux } +func (c *typhaComponent) Name() string { return ComponentNameTypha } + func (c *typhaComponent) Objects() ([]client.Object, []client.Object) { objs := []client.Object{ c.typhaServiceAccount(), From f7b28f35adefeaf72c1b385565ba087996f33bda Mon Sep 17 00:00:00 2001 From: Casey Davenport Date: Thu, 28 May 2026 23:17:12 -0700 Subject: [PATCH 08/54] enterprise: move typha variant branches into a modifier Pulls the enterprise RBAC extra-rules and MULTI_INTERFACE_MODE env branches out of pkg/render/typha.go into a new pkg/enterprise package. The enterprise package registers a patch via operator.Patch on startup; pkg/render/typha.go now has zero IsEnterprise branches. --- pkg/enterprise/enterprise_suite_test.go | 27 ++++++++ pkg/enterprise/register.go | 22 +++++++ pkg/enterprise/typha.go | 66 +++++++++++++++++++ pkg/enterprise/typha_test.go | 88 +++++++++++++++++++++++++ pkg/render/typha.go | 29 -------- 5 files changed, 203 insertions(+), 29 deletions(-) create mode 100644 pkg/enterprise/enterprise_suite_test.go create mode 100644 pkg/enterprise/register.go create mode 100644 pkg/enterprise/typha.go create mode 100644 pkg/enterprise/typha_test.go diff --git a/pkg/enterprise/enterprise_suite_test.go b/pkg/enterprise/enterprise_suite_test.go new file mode 100644 index 0000000000..e7cfab281b --- /dev/null +++ b/pkg/enterprise/enterprise_suite_test.go @@ -0,0 +1,27 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package enterprise_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestEnterprise(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "pkg/enterprise Suite") +} diff --git a/pkg/enterprise/register.go b/pkg/enterprise/register.go new file mode 100644 index 0000000000..88315d93ba --- /dev/null +++ b/pkg/enterprise/register.go @@ -0,0 +1,22 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package enterprise + +// Register wires all in-repo enterprise modifiers and controller extensions +// into the operator registries. Called once at process startup. After the +// monorepo split this is what calico-private's main will do instead. +func Register() { + registerTypha() +} diff --git a/pkg/enterprise/typha.go b/pkg/enterprise/typha.go new file mode 100644 index 0000000000..8deef34b28 --- /dev/null +++ b/pkg/enterprise/typha.go @@ -0,0 +1,66 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package enterprise + +import ( + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/tigera/operator/pkg/operator" + "github.com/tigera/operator/pkg/render" +) + +func registerTypha() { + operator.Patch(render.ComponentNameTypha, patchTypha) +} + +func patchTypha(ctx operator.Context, objs []client.Object) []client.Object { + if ctx.Installation == nil || !ctx.Installation.Variant.IsEnterprise() { + return objs + } + + if role, ok := operator.FindObject[*rbacv1.ClusterRole](objs, "calico-typha"); ok { + role.Rules = append(role.Rules, rbacv1.PolicyRule{ + APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, + Resources: []string{ + "bfdconfigurations", + "deeppacketinspections", + "egressgatewaypolicies", + "externalnetworks", + "licensekeys", + "networks", + "packetcaptures", + "remoteclusterconfigurations", + }, + Verbs: []string{"get", "list", "watch"}, + }) + } + + if dep, ok := operator.FindObject[*appsv1.Deployment](objs, "calico-typha"); ok { + net := ctx.Installation.CalicoNetwork + if net != nil && net.MultiInterfaceMode != nil { + for i := range dep.Spec.Template.Spec.Containers { + if dep.Spec.Template.Spec.Containers[i].Name == render.TyphaContainerName { + c := &dep.Spec.Template.Spec.Containers[i] + c.Env = append(c.Env, corev1.EnvVar{Name: "MULTI_INTERFACE_MODE", Value: net.MultiInterfaceMode.Value()}) + } + } + } + } + + return objs +} diff --git a/pkg/enterprise/typha_test.go b/pkg/enterprise/typha_test.go new file mode 100644 index 0000000000..67330b7539 --- /dev/null +++ b/pkg/enterprise/typha_test.go @@ -0,0 +1,88 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package enterprise_test + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/enterprise" + "github.com/tigera/operator/pkg/operator" + "github.com/tigera/operator/pkg/render" +) + +var _ = Describe("typha enterprise modifier", func() { + BeforeEach(func() { enterprise.Register() }) + AfterEach(func() { + operator.ResetForTest() + operator.ResetExtensionsForTest() + }) + + multiMode := operatorv1.MultiInterfaceModeMultus + + newObjs := func() []client.Object { + return []client.Object{ + &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-typha"}}, + &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{Name: "calico-typha"}, + Spec: appsv1.DeploymentSpec{Template: corev1.PodTemplateSpec{Spec: corev1.PodSpec{ + Containers: []corev1.Container{{Name: render.TyphaContainerName}}, + }}}, + }, + } + } + + It("adds enterprise RBAC and MULTI_INTERFACE_MODE for the enterprise variant", func() { + ctx := operator.Context{Installation: &operatorv1.InstallationSpec{ + Variant: operatorv1.TigeraSecureEnterprise, + CalicoNetwork: &operatorv1.CalicoNetworkSpec{MultiInterfaceMode: &multiMode}, + }} + out := operator.ApplyPatches(render.ComponentNameTypha, ctx, newObjs()) + + role := out[0].(*rbacv1.ClusterRole) + Expect(role.Rules).To(ContainElement(HaveField("Resources", ContainElement("licensekeys")))) + + dep := out[1].(*appsv1.Deployment) + var c *corev1.Container + for i := range dep.Spec.Template.Spec.Containers { + if dep.Spec.Template.Spec.Containers[i].Name == render.TyphaContainerName { + c = &dep.Spec.Template.Spec.Containers[i] + } + } + Expect(c.Env).To(ContainElement(corev1.EnvVar{Name: "MULTI_INTERFACE_MODE", Value: multiMode.Value()})) + }) + + It("is a no-op for the Calico variant", func() { + ctx := operator.Context{Installation: &operatorv1.InstallationSpec{ + Variant: operatorv1.Calico, + CalicoNetwork: &operatorv1.CalicoNetworkSpec{MultiInterfaceMode: &multiMode}, + }} + out := operator.ApplyPatches(render.ComponentNameTypha, ctx, newObjs()) + Expect(out[0].(*rbacv1.ClusterRole).Rules).To(BeEmpty()) + dep := out[1].(*appsv1.Deployment) + Expect(dep.Spec.Template.Spec.Containers[0].Env).To(BeEmpty()) + }) + + It("does not panic on a zero Context (nil Installation)", func() { + out := operator.ApplyPatches(render.ComponentNameTypha, operator.Context{}, newObjs()) + Expect(out[0].(*rbacv1.ClusterRole).Rules).To(BeEmpty()) + }) +}) diff --git a/pkg/render/typha.go b/pkg/render/typha.go index 49c3f13917..84cc8dab9f 100644 --- a/pkg/render/typha.go +++ b/pkg/render/typha.go @@ -353,26 +353,6 @@ func (c *typhaComponent) typhaRole() *rbacv1.ClusterRole { }, }, } - if c.cfg.Installation.Variant.IsEnterprise() { - extraRules := []rbacv1.PolicyRule{ - { - // Tigera Secure needs to be able to read licenses, and config. - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{ - "bfdconfigurations", - "deeppacketinspections", - "egressgatewaypolicies", - "externalnetworks", - "licensekeys", - "networks", - "packetcaptures", - "remoteclusterconfigurations", - }, - Verbs: []string{"get", "list", "watch"}, - }, - } - role.Rules = append(role.Rules, extraRules...) - } if c.cfg.Installation.KubernetesProvider.IsOpenShift() { role.Rules = append(role.Rules, rbacv1.PolicyRule{ APIGroups: []string{"security.openshift.io"}, @@ -630,15 +610,6 @@ func (c *typhaComponent) typhaEnvVars(typhaSecret certificatemanagement.KeyPairI typhaEnv = append(typhaEnv, corev1.EnvVar{Name: "FELIX_INTERFACEPREFIX", Value: "azv"}) } - if c.cfg.Installation.Variant.IsEnterprise() { - if c.cfg.Installation.CalicoNetwork != nil && c.cfg.Installation.CalicoNetwork.MultiInterfaceMode != nil { - typhaEnv = append(typhaEnv, corev1.EnvVar{ - Name: "MULTI_INTERFACE_MODE", - Value: c.cfg.Installation.CalicoNetwork.MultiInterfaceMode.Value(), - }) - } - } - // If host-local IPAM is in use, we need to configure typha to use the Kubernetes pod CIDR. cni := c.cfg.Installation.CNI if cni != nil && cni.IPAM != nil && cni.IPAM.Type == operatorv1.IPAMPluginHostLocal { From e78d6336b3eb93d984c683522a967c4bd10d1c80 Mon Sep 17 00:00:00 2001 From: Casey Davenport Date: Fri, 29 May 2026 08:09:13 -0700 Subject: [PATCH 09/54] installation: register enterprise modifiers and pass render Context Calls enterprise.Register() at startup so the typha modifier is wired in. Builds an operator.Context in the installation reconciler and passes it to the component handler so registered modifiers receive reconcile-derived state. --- cmd/main.go | 6 ++++++ pkg/controller/installation/core_controller.go | 12 +++++++++++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/cmd/main.go b/cmd/main.go index 7b1117b355..b0aee8fb59 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -41,6 +41,7 @@ import ( "github.com/tigera/operator/pkg/controller/options" "github.com/tigera/operator/pkg/controller/utils" "github.com/tigera/operator/pkg/dns" + "github.com/tigera/operator/pkg/enterprise" "github.com/tigera/operator/pkg/imports/admission" "github.com/tigera/operator/pkg/imports/crds" "github.com/tigera/operator/pkg/render" @@ -517,6 +518,11 @@ If a value other than 'all' is specified, the first CRD with a prefix of the spe os.Exit(1) } + // Wire in-repo enterprise modifiers and extensions into the operator + // registries. After the monorepo split this call moves to calico-private's + // main. + enterprise.Register() + err = controller.AddToManager(mgr, options) if err != nil { setupLog.Error(err, "unable to create controllers") diff --git a/pkg/controller/installation/core_controller.go b/pkg/controller/installation/core_controller.go index 89acae7416..a966a02a0e 100644 --- a/pkg/controller/installation/core_controller.go +++ b/pkg/controller/installation/core_controller.go @@ -63,6 +63,7 @@ import ( "github.com/tigera/operator/pkg/active" "github.com/tigera/operator/pkg/common" "github.com/tigera/operator/pkg/components" + "github.com/tigera/operator/pkg/operator" "github.com/tigera/operator/pkg/controller/certificatemanager" "github.com/tigera/operator/pkg/controller/ippool" "github.com/tigera/operator/pkg/controller/k8sapi" @@ -1294,8 +1295,17 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile nodeAppArmorProfile = val } + // Build the render Context carrying reconcile-derived state for any + // registered modifiers (e.g. enterprise typha branches). + modCtx := operator.Context{ + Installation: &instance.Spec, + FelixConfiguration: felixConfiguration, + ClusterDomain: r.clusterDomain, + TrustedBundle: typhaNodeTLS.TrustedBundle, + } + // Create a component handler to create or update the rendered components. - handler := r.newComponentHandler(log, r.client, r.scheme, instance) + handler := r.newComponentHandler(log, r.client, r.scheme, instance, utils.WithContext(modCtx)) // Render namespaces first - this ensures that any other controllers blocked on namespace existence can proceed. namespaceCfg := &render.NamespaceConfiguration{ From 2e48ab3ab8eb84ed57473c9f80a395e367b35ff0 Mon Sep 17 00:00:00 2001 From: Casey Davenport Date: Fri, 29 May 2026 09:03:04 -0700 Subject: [PATCH 10/54] enterprise: route node image selection through ResolveImage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extracts the image override registry into a leaf pkg/imageoverride package (no render/operator transitive deps) to avoid the render→operator import cycle. operator.OverrideImage/ResolveImage now delegate there. Registers the enterprise node image override in pkg/enterprise. Removes the IsEnterprise image switch from render/node.go; FIPS handling is preserved via a post-resolve check. --- pkg/enterprise/node.go | 31 ++++++++++++++++++ pkg/enterprise/node_test.go | 43 ++++++++++++++++++++++++ pkg/enterprise/register.go | 1 + pkg/imageoverride/imageoverride.go | 51 +++++++++++++++++++++++++++++ pkg/operator/image.go | 14 +++----- pkg/operator/patch.go | 8 +++-- pkg/render/enterprise_setup_test.go | 29 ++++++++++++++++ pkg/render/node.go | 20 +++++++---- 8 files changed, 178 insertions(+), 19 deletions(-) create mode 100644 pkg/enterprise/node.go create mode 100644 pkg/enterprise/node_test.go create mode 100644 pkg/imageoverride/imageoverride.go create mode 100644 pkg/render/enterprise_setup_test.go diff --git a/pkg/enterprise/node.go b/pkg/enterprise/node.go new file mode 100644 index 0000000000..039303681f --- /dev/null +++ b/pkg/enterprise/node.go @@ -0,0 +1,31 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package enterprise + +import ( + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/components" + "github.com/tigera/operator/pkg/operator" + "github.com/tigera/operator/pkg/render" +) + +func registerNode() { + operator.OverrideImage(render.ComponentNameNode, func(in *operatorv1.InstallationSpec) (components.Component, bool) { + if !in.Variant.IsEnterprise() { + return components.Component{}, false + } + return components.ComponentTigeraNode, true + }) +} diff --git a/pkg/enterprise/node_test.go b/pkg/enterprise/node_test.go new file mode 100644 index 0000000000..d12d9812f2 --- /dev/null +++ b/pkg/enterprise/node_test.go @@ -0,0 +1,43 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package enterprise_test + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/components" + "github.com/tigera/operator/pkg/enterprise" + "github.com/tigera/operator/pkg/operator" +) + +var _ = Describe("node enterprise image override", func() { + BeforeEach(func() { enterprise.Register() }) + AfterEach(func() { + operator.ResetForTest() + operator.ResetExtensionsForTest() + }) + + It("selects the enterprise node image for the enterprise variant", func() { + ent := &operatorv1.InstallationSpec{Variant: operatorv1.TigeraSecureEnterprise} + Expect(operator.ResolveImage("node", components.ComponentCalicoNode, ent)).To(Equal(components.ComponentTigeraNode)) + }) + + It("leaves the default in place for the Calico variant", func() { + oss := &operatorv1.InstallationSpec{Variant: operatorv1.Calico} + Expect(operator.ResolveImage("node", components.ComponentCalicoNode, oss)).To(Equal(components.ComponentCalicoNode)) + }) +}) diff --git a/pkg/enterprise/register.go b/pkg/enterprise/register.go index 88315d93ba..d32e8c66b1 100644 --- a/pkg/enterprise/register.go +++ b/pkg/enterprise/register.go @@ -19,4 +19,5 @@ package enterprise // monorepo split this is what calico-private's main will do instead. func Register() { registerTypha() + registerNode() } diff --git a/pkg/imageoverride/imageoverride.go b/pkg/imageoverride/imageoverride.go new file mode 100644 index 0000000000..bf8950cf84 --- /dev/null +++ b/pkg/imageoverride/imageoverride.go @@ -0,0 +1,51 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package imageoverride is a leaf package (no render/operator dependencies) +// that holds the image override registry. Both pkg/operator and pkg/render +// import it to avoid the render→operator→render import cycle. +package imageoverride + +import ( + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/components" +) + +// Override selects the component image to use for an installation, returning +// false to decline (leaving the default in place). +type Override func(in *operatorv1.InstallationSpec) (components.Component, bool) + +var registry = map[string]Override{} + +// Register stores fn under key. The key is the render component's image +// identifier (e.g. "node"). +func Register(key string, fn Override) { + registry[key] = fn +} + +// Resolve returns the override registered for key if it applies to in, +// otherwise def. +func Resolve(key string, def components.Component, in *operatorv1.InstallationSpec) components.Component { + if fn, ok := registry[key]; ok { + if c, override := fn(in); override { + return c + } + } + return def +} + +// ResetForTest clears the registry. Test-only. +func ResetForTest() { + registry = map[string]Override{} +} diff --git a/pkg/operator/image.go b/pkg/operator/image.go index b253611254..c6de3b917a 100644 --- a/pkg/operator/image.go +++ b/pkg/operator/image.go @@ -17,28 +17,22 @@ package operator import ( operatorv1 "github.com/tigera/operator/api/v1" "github.com/tigera/operator/pkg/components" + "github.com/tigera/operator/pkg/imageoverride" ) // ImageOverride returns the component image to use for an installation, and // false to decline (leaving the default in place). Implementations self-gate // on in.Variant. -type ImageOverride func(in *operatorv1.InstallationSpec) (components.Component, bool) - -var imageOverrides = map[string]ImageOverride{} +type ImageOverride = imageoverride.Override // OverrideImage registers an image override under key. The key is the render // component's image identifier (e.g. "node"). func OverrideImage(key string, fn ImageOverride) { - imageOverrides[key] = fn + imageoverride.Register(key, fn) } // ResolveImage returns the override registered for key if it applies to in, // otherwise def. Render components call this inside ResolveImages. func ResolveImage(key string, def components.Component, in *operatorv1.InstallationSpec) components.Component { - if fn, ok := imageOverrides[key]; ok { - if c, override := fn(in); override { - return c - } - } - return def + return imageoverride.Resolve(key, def, in) } diff --git a/pkg/operator/patch.go b/pkg/operator/patch.go index 94721a76bd..bce2ff783c 100644 --- a/pkg/operator/patch.go +++ b/pkg/operator/patch.go @@ -14,7 +14,11 @@ package operator -import "sigs.k8s.io/controller-runtime/pkg/client" +import ( + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/tigera/operator/pkg/imageoverride" +) // PatchFunc post-processes the objects a render component produced. It may // mutate matched objects and/or append additional objects, and must return the @@ -51,5 +55,5 @@ func FindObject[T client.Object](objs []client.Object, name string) (T, bool) { // ResetForTest clears all registries. Test-only. func ResetForTest() { patches = map[string][]PatchFunc{} - imageOverrides = map[string]ImageOverride{} + imageoverride.ResetForTest() } diff --git a/pkg/render/enterprise_setup_test.go b/pkg/render/enterprise_setup_test.go new file mode 100644 index 0000000000..5f464c950b --- /dev/null +++ b/pkg/render/enterprise_setup_test.go @@ -0,0 +1,29 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package render_test + +import ( + . "github.com/onsi/ginkgo/v2" + + "github.com/tigera/operator/pkg/enterprise" +) + +// The render suite asserts enterprise-variant output for components whose +// variant-specific behavior now lives in registered modifiers/overrides +// (e.g. the node image). Register them once so the suite exercises the same +// integrated behavior the operator binary produces. +var _ = BeforeSuite(func() { + enterprise.Register() +}) diff --git a/pkg/render/node.go b/pkg/render/node.go index 638c95b351..07e1be94db 100644 --- a/pkg/render/node.go +++ b/pkg/render/node.go @@ -34,6 +34,7 @@ import ( "github.com/tigera/operator/pkg/common" "github.com/tigera/operator/pkg/components" "github.com/tigera/operator/pkg/controller/k8sapi" + "github.com/tigera/operator/pkg/imageoverride" "github.com/tigera/operator/pkg/controller/migration" rcomp "github.com/tigera/operator/pkg/render/common/components" "github.com/tigera/operator/pkg/render/common/configmap" @@ -184,14 +185,15 @@ func (c *nodeComponent) ResolveImages(is *operatorv1.ImageSet) error { } c.calicoImage = appendIfErr(components.GetReference(components.CombinedCalicoImage(c.cfg.Installation), reg, path, prefix, is)) - switch { - case c.cfg.Installation.Variant.IsEnterprise(): - c.nodeImage = appendIfErr(components.GetReference(components.ComponentTigeraNode, reg, path, prefix, is)) - case operatorv1.IsFIPSModeEnabled(c.cfg.Installation.FIPSMode): - c.nodeImage = appendIfErr(components.GetReference(components.ComponentCalicoNodeFIPS, reg, path, prefix, is)) - default: - c.nodeImage = appendIfErr(components.GetReference(components.ComponentCalicoNode, reg, path, prefix, is)) + nodeImage := imageoverride.Resolve(ComponentNameNode, components.ComponentCalicoNode, c.cfg.Installation) + if operatorv1.IsFIPSModeEnabled(c.cfg.Installation.FIPSMode) { + // FIPS only applies to the Calico variant; the enterprise override (if + // registered) has already replaced nodeImage for the enterprise variant. + if nodeImage == components.ComponentCalicoNode { + nodeImage = components.ComponentCalicoNodeFIPS + } } + c.nodeImage = appendIfErr(components.GetReference(nodeImage, reg, path, prefix, is)) if len(errMsgs) != 0 { return fmt.Errorf("%s", strings.Join(errMsgs, ",")) @@ -203,6 +205,10 @@ func (c *nodeComponent) SupportedOSType() rmeta.OSType { return rmeta.OSTypeLinux } +func (c *nodeComponent) Name() string { + return ComponentNameNode +} + func (c *nodeComponent) Objects() ([]client.Object, []client.Object) { objs := []client.Object{ c.nodeServiceAccount(), From 0506f4c1c127d3e31b2d1041aed985520a1db0fb Mon Sep 17 00:00:00 2001 From: Casey Davenport Date: Fri, 29 May 2026 09:05:12 -0700 Subject: [PATCH 11/54] render: fix node.go import ordering --- pkg/render/node.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/render/node.go b/pkg/render/node.go index 07e1be94db..9c335d9ea4 100644 --- a/pkg/render/node.go +++ b/pkg/render/node.go @@ -34,8 +34,8 @@ import ( "github.com/tigera/operator/pkg/common" "github.com/tigera/operator/pkg/components" "github.com/tigera/operator/pkg/controller/k8sapi" - "github.com/tigera/operator/pkg/imageoverride" "github.com/tigera/operator/pkg/controller/migration" + "github.com/tigera/operator/pkg/imageoverride" rcomp "github.com/tigera/operator/pkg/render/common/components" "github.com/tigera/operator/pkg/render/common/configmap" rmeta "github.com/tigera/operator/pkg/render/common/meta" From f240c2d1518c29c33a400111b9bc875dbc42f230 Mon Sep 17 00:00:00 2001 From: Casey Davenport Date: Fri, 29 May 2026 09:28:24 -0700 Subject: [PATCH 12/54] enterprise: move node-prometheus cert setup into an InstallationExtension The OSS installation controller no longer directly creates the node-prometheus keypair or fetches the prometheus/esgw certs. Those are now handled by a registered InstallationExtension in pkg/enterprise. Port value derivation and the kube-controller TLS block remain in the OSS controller unchanged. --- .../installation/core_controller.go | 79 ++++++----------- .../installation_controller_suite_test.go | 6 ++ pkg/enterprise/installation.go | 80 +++++++++++++++++ pkg/enterprise/installation_test.go | 85 +++++++++++++++++++ pkg/enterprise/register.go | 1 + 5 files changed, 199 insertions(+), 52 deletions(-) create mode 100644 pkg/enterprise/installation.go create mode 100644 pkg/enterprise/installation_test.go diff --git a/pkg/controller/installation/core_controller.go b/pkg/controller/installation/core_controller.go index a966a02a0e..bf92bf3e2f 100644 --- a/pkg/controller/installation/core_controller.go +++ b/pkg/controller/installation/core_controller.go @@ -80,8 +80,7 @@ import ( "github.com/tigera/operator/pkg/imports/crds" "github.com/tigera/operator/pkg/render" rcertificatemanagement "github.com/tigera/operator/pkg/render/certificatemanagement" - relasticsearch "github.com/tigera/operator/pkg/render/common/elasticsearch" - "github.com/tigera/operator/pkg/render/common/networkpolicy" +"github.com/tigera/operator/pkg/render/common/networkpolicy" "github.com/tigera/operator/pkg/render/common/resourcequota" "github.com/tigera/operator/pkg/render/goldmane" "github.com/tigera/operator/pkg/render/kubecontrollers" @@ -1202,62 +1201,47 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile // nodeReporterMetricsPort is a port used in Enterprise to host internal metrics. // Operator is responsible for creating a service which maps to that port. - // Here, we'll check the default felixconfiguration to see if the user is specifying - // a non-default port, and use that value if they are. nodeReporterMetricsPort := defaultNodeReporterPort - var nodePrometheusTLS certificatemanagement.KeyPairInterface - calicoVersion := components.CalicoRelease - felixPrometheusMetricsPort := defaultFelixMetricsDefaultPort - + calicoVersion := components.CalicoRelease if instance.Spec.Variant.IsEnterprise() { - - // Determine the port to use for nodeReporter metrics. + calicoVersion = components.EnterpriseRelease if felixConfiguration.Spec.PrometheusReporterPort != nil { nodeReporterMetricsPort = *felixConfiguration.Spec.PrometheusReporterPort } - if nodeReporterMetricsPort == 0 { - err := errors.New("felixConfiguration prometheusReporterPort=0 not supported") - r.status.SetDegraded(operatorv1.InvalidConfigurationError, "invalid metrics port", err, reqLogger) - return reconcile.Result{}, err - } - if felixConfiguration.Spec.PrometheusMetricsPort != nil { felixPrometheusMetricsPort = *felixConfiguration.Spec.PrometheusMetricsPort } + } - nodePrometheusTLS, err = certificateManager.GetOrCreateKeyPair(r.client, render.NodePrometheusTLSServerSecret, common.OperatorNamespace(), dns.GetServiceDNSNames(render.CalicoNodeMetricsService, common.CalicoNamespace, r.clusterDomain)) - if err != nil { - r.status.SetDegraded(operatorv1.ResourceCreateError, "Error creating TLS certificate", err, reqLogger) - return reconcile.Result{}, err - } - if nodePrometheusTLS != nil { - typhaNodeTLS.TrustedBundle.AddCertificates(nodePrometheusTLS) - } - prometheusClientCert, err := certificateManager.GetCertificate(r.client, monitor.PrometheusClientTLSSecretName, common.OperatorNamespace()) - if err != nil { - r.status.SetDegraded(operatorv1.CertificateError, "Unable to fetch prometheus certificate", err, reqLogger) - return reconcile.Result{}, err - } - if prometheusClientCert != nil { - typhaNodeTLS.TrustedBundle.AddCertificates(prometheusClientCert) - } - - // es-kube-controllers needs to trust the ESGW certificate. We'll fetch it here and add it to the trusted bundle. - // Note that although we're adding this to the typhaNodeTLS trusted bundle, it will be used by es-kube-controllers. This is because - // all components within this namespace share a trusted CA bundle. This is necessary because prior to v3.13 secrets were not signed by - // a single CA so we need to include each individually. - esgwCertificate, err := certificateManager.GetCertificate(r.client, relasticsearch.PublicCertSecret, common.OperatorNamespace()) + // Run the registered installation extension. For the enterprise variant this + // validates config and creates the node-prometheus certificate, adding it (and + // the prometheus/esgw certs) to the trusted bundle. Returns the render Context + // consumed by registered modifiers. + var modCtx operator.Context + if ext := operator.GetInstallationExtension(); ext != nil { + modCtx, err = ext.Prepare(operator.InstallationPrep{ + Ctx: ctx, + Client: r.client, + Installation: &instance.Spec, + FelixConfiguration: felixConfiguration, + CertificateManager: certificateManager, + TrustedBundle: typhaNodeTLS.TrustedBundle, + ClusterDomain: r.clusterDomain, + }) if err != nil { - r.status.SetDegraded(operatorv1.CertificateError, fmt.Sprintf("Failed to retrieve / validate %s", relasticsearch.PublicCertSecret), err, reqLogger) + r.status.SetDegraded(operatorv1.ResourceCreateError, "Error preparing enterprise installation", err, reqLogger) return reconcile.Result{}, err } - if esgwCertificate != nil { - typhaNodeTLS.TrustedBundle.AddCertificates(esgwCertificate) + } else { + modCtx = operator.Context{ + Installation: &instance.Spec, + FelixConfiguration: felixConfiguration, + ClusterDomain: r.clusterDomain, + TrustedBundle: typhaNodeTLS.TrustedBundle, } - - calicoVersion = components.EnterpriseRelease } + nodePrometheusTLS := modCtx.NodePrometheusTLS kubeControllersMetricsPort, err := utils.GetKubeControllerMetricsPort(ctx, r.client) if err != nil { @@ -1295,15 +1279,6 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile nodeAppArmorProfile = val } - // Build the render Context carrying reconcile-derived state for any - // registered modifiers (e.g. enterprise typha branches). - modCtx := operator.Context{ - Installation: &instance.Spec, - FelixConfiguration: felixConfiguration, - ClusterDomain: r.clusterDomain, - TrustedBundle: typhaNodeTLS.TrustedBundle, - } - // Create a component handler to create or update the rendered components. handler := r.newComponentHandler(log, r.client, r.scheme, instance, utils.WithContext(modCtx)) diff --git a/pkg/controller/installation/installation_controller_suite_test.go b/pkg/controller/installation/installation_controller_suite_test.go index 4924f3468b..778ee91f8f 100644 --- a/pkg/controller/installation/installation_controller_suite_test.go +++ b/pkg/controller/installation/installation_controller_suite_test.go @@ -25,8 +25,14 @@ import ( clientfeaturestesting "k8s.io/client-go/features/testing" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" + + "github.com/tigera/operator/pkg/enterprise" ) +var _ = ginkgo.BeforeSuite(func() { + enterprise.Register() +}) + func TestInstallation(t *testing.T) { // Disable WatchListClient for tests. In client-go v0.35+, this feature defaults to true and // causes informers to wait for bookmark events that fake clients never send, leading to timeouts. diff --git a/pkg/enterprise/installation.go b/pkg/enterprise/installation.go new file mode 100644 index 0000000000..7b0abaf4b5 --- /dev/null +++ b/pkg/enterprise/installation.go @@ -0,0 +1,80 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package enterprise + +import ( + "errors" + "fmt" + + "github.com/tigera/operator/pkg/common" + "github.com/tigera/operator/pkg/dns" + "github.com/tigera/operator/pkg/render/monitor" + "github.com/tigera/operator/pkg/operator" + "github.com/tigera/operator/pkg/render" + relasticsearch "github.com/tigera/operator/pkg/render/common/elasticsearch" +) + +type installationExtension struct{} + +func registerInstallation() { + operator.RegisterInstallationExtension(&installationExtension{}) +} + +func (e *installationExtension) Prepare(p operator.InstallationPrep) (operator.Context, error) { + ctx := operator.Context{ + Installation: p.Installation, + FelixConfiguration: p.FelixConfiguration, + ClusterDomain: p.ClusterDomain, + TrustedBundle: p.TrustedBundle, + } + if !p.Installation.Variant.IsEnterprise() { + return ctx, nil + } + + // Reject the unsupported zero reporter port. (Port value derivation stays in + // the OSS controller; only validation moves here.) + if p.FelixConfiguration.Spec.PrometheusReporterPort != nil && *p.FelixConfiguration.Spec.PrometheusReporterPort == 0 { + return ctx, errors.New("felixConfiguration prometheusReporterPort=0 not supported") + } + + nodePrometheusTLS, err := p.CertificateManager.GetOrCreateKeyPair( + p.Client, render.NodePrometheusTLSServerSecret, common.OperatorNamespace(), + dns.GetServiceDNSNames(render.CalicoNodeMetricsService, common.CalicoNamespace, p.ClusterDomain)) + if err != nil { + return ctx, fmt.Errorf("error creating node prometheus TLS certificate: %w", err) + } + if nodePrometheusTLS != nil { + p.TrustedBundle.AddCertificates(nodePrometheusTLS) + } + ctx.NodePrometheusTLS = nodePrometheusTLS + + prometheusClientCert, err := p.CertificateManager.GetCertificate(p.Client, monitor.PrometheusClientTLSSecretName, common.OperatorNamespace()) + if err != nil { + return ctx, fmt.Errorf("unable to fetch prometheus certificate: %w", err) + } + if prometheusClientCert != nil { + p.TrustedBundle.AddCertificates(prometheusClientCert) + } + + esgwCertificate, err := p.CertificateManager.GetCertificate(p.Client, relasticsearch.PublicCertSecret, common.OperatorNamespace()) + if err != nil { + return ctx, fmt.Errorf("failed to retrieve / validate %s: %w", relasticsearch.PublicCertSecret, err) + } + if esgwCertificate != nil { + p.TrustedBundle.AddCertificates(esgwCertificate) + } + + return ctx, nil +} diff --git a/pkg/enterprise/installation_test.go b/pkg/enterprise/installation_test.go new file mode 100644 index 0000000000..dffe6a7470 --- /dev/null +++ b/pkg/enterprise/installation_test.go @@ -0,0 +1,85 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package enterprise_test + +import ( + "context" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/apis" + "github.com/tigera/operator/pkg/common" + "github.com/tigera/operator/pkg/controller/certificatemanager" + ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" + "github.com/tigera/operator/pkg/enterprise" + "github.com/tigera/operator/pkg/operator" + "k8s.io/apimachinery/pkg/runtime" +) + +var _ = Describe("installation enterprise extension", func() { + BeforeEach(func() { enterprise.Register() }) + AfterEach(func() { + operator.ResetForTest() + operator.ResetExtensionsForTest() + }) + + It("rejects a zero prometheus reporter port", func() { + port := 0 + p := newPrep(operatorv1.TigeraSecureEnterprise) + p.FelixConfiguration = &v3.FelixConfiguration{Spec: v3.FelixConfigurationSpec{PrometheusReporterPort: &port}} + _, err := operator.GetInstallationExtension().Prepare(p) + Expect(err).To(HaveOccurred()) + }) + + It("creates the node prometheus keypair for the enterprise variant", func() { + p := newPrep(operatorv1.TigeraSecureEnterprise) + p.FelixConfiguration = &v3.FelixConfiguration{} + ctx, err := operator.GetInstallationExtension().Prepare(p) + Expect(err).NotTo(HaveOccurred()) + Expect(ctx.NodePrometheusTLS).NotTo(BeNil()) + }) + + It("is a no-op for the Calico variant", func() { + p := newPrep(operatorv1.Calico) + ctx, err := operator.GetInstallationExtension().Prepare(p) + Expect(err).NotTo(HaveOccurred()) + Expect(ctx.NodePrometheusTLS).To(BeNil()) + }) +}) + +func newPrep(variant operatorv1.ProductVariant) operator.InstallationPrep { + scheme := runtime.NewScheme() + Expect(apis.AddToScheme(scheme, false)).NotTo(HaveOccurred()) + c := ctrlrfake.DefaultFakeClientBuilder(scheme).Build() + + certManager, err := certificatemanager.Create(c, nil, "", common.OperatorNamespace(), certificatemanager.AllowCACreation()) + Expect(err).NotTo(HaveOccurred()) + trustedBundle := certManager.CreateTrustedBundle() + + return operator.InstallationPrep{ + Ctx: context.Background(), + Client: c, + Installation: &operatorv1.InstallationSpec{ + Variant: variant, + }, + FelixConfiguration: &v3.FelixConfiguration{}, + CertificateManager: certManager, + TrustedBundle: trustedBundle, + ClusterDomain: "cluster.local", + } +} diff --git a/pkg/enterprise/register.go b/pkg/enterprise/register.go index d32e8c66b1..a01c9b7cfa 100644 --- a/pkg/enterprise/register.go +++ b/pkg/enterprise/register.go @@ -20,4 +20,5 @@ package enterprise func Register() { registerTypha() registerNode() + registerInstallation() } From 6e426cb3cf2e1ad331f0f265f0f45b695d3e2ed8 Mon Sep 17 00:00:00 2001 From: Casey Davenport Date: Fri, 29 May 2026 10:04:45 -0700 Subject: [PATCH 13/54] enterprise: render node metrics service via a modifier Moves the calico-node-metrics Service out of OSS node render and into the enterprise node modifier, where it derives ports from ctx.FelixConfiguration. Also exports NodeBGPReporterPort so the modifier can reference it. --- pkg/enterprise/node.go | 74 +++++++++++++++++++++++++++++++++++++ pkg/enterprise/node_test.go | 53 ++++++++++++++++++++++++++ pkg/render/node.go | 58 +---------------------------- pkg/render/node_test.go | 49 +----------------------- pkg/render/render_test.go | 3 +- pkg/render/windows.go | 4 +- 6 files changed, 134 insertions(+), 107 deletions(-) diff --git a/pkg/enterprise/node.go b/pkg/enterprise/node.go index 039303681f..e47c593076 100644 --- a/pkg/enterprise/node.go +++ b/pkg/enterprise/node.go @@ -15,12 +15,24 @@ package enterprise import ( + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + client "sigs.k8s.io/controller-runtime/pkg/client" + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/common" "github.com/tigera/operator/pkg/components" + "github.com/tigera/operator/pkg/controller/utils" "github.com/tigera/operator/pkg/operator" "github.com/tigera/operator/pkg/render" ) +const ( + defaultNodeReporterPort = 9081 + defaultFelixMetricsPort = 9091 +) + func registerNode() { operator.OverrideImage(render.ComponentNameNode, func(in *operatorv1.InstallationSpec) (components.Component, bool) { if !in.Variant.IsEnterprise() { @@ -28,4 +40,66 @@ func registerNode() { } return components.ComponentTigeraNode, true }) + operator.Patch(render.ComponentNameNode, patchNode) +} + +func patchNode(ctx operator.Context, objs []client.Object) []client.Object { + if ctx.Installation == nil || !ctx.Installation.Variant.IsEnterprise() { + return objs + } + return append(objs, nodeMetricsService(ctx)) +} + +// nodeMetricsService builds the enterprise-only calico-node-metrics Service. +// Ports are derived from FelixConfiguration exactly as the installation controller does. +func nodeMetricsService(ctx operator.Context) *corev1.Service { + reporterPort := defaultNodeReporterPort + felixPort := defaultFelixMetricsPort + felixEnabled := false + if fc := ctx.FelixConfiguration; fc != nil { + if fc.Spec.PrometheusReporterPort != nil { + reporterPort = *fc.Spec.PrometheusReporterPort + } + if fc.Spec.PrometheusMetricsPort != nil { + felixPort = *fc.Spec.PrometheusMetricsPort + } + felixEnabled = utils.IsFelixPrometheusMetricsEnabled(fc) + } + + ports := []corev1.ServicePort{ + { + Name: "calico-metrics-port", + Port: int32(reporterPort), + TargetPort: intstr.FromInt(reporterPort), + Protocol: corev1.ProtocolTCP, + }, + { + Name: "calico-bgp-metrics-port", + Port: render.NodeBGPReporterPort, + TargetPort: intstr.FromInt(int(render.NodeBGPReporterPort)), + Protocol: corev1.ProtocolTCP, + }, + } + if felixEnabled { + ports = append(ports, corev1.ServicePort{ + Name: "felix-metrics-port", + Port: int32(felixPort), + TargetPort: intstr.FromInt(felixPort), + Protocol: corev1.ProtocolTCP, + }) + } + + return &corev1.Service{ + TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: render.CalicoNodeMetricsService, + Namespace: common.CalicoNamespace, + Labels: map[string]string{"k8s-app": render.CalicoNodeObjectName}, + }, + Spec: corev1.ServiceSpec{ + Selector: map[string]string{"k8s-app": render.CalicoNodeObjectName}, + ClusterIP: "None", + Ports: ports, + }, + } } diff --git a/pkg/enterprise/node_test.go b/pkg/enterprise/node_test.go index d12d9812f2..509fd6b18f 100644 --- a/pkg/enterprise/node_test.go +++ b/pkg/enterprise/node_test.go @@ -18,10 +18,15 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + client "sigs.k8s.io/controller-runtime/pkg/client" + + v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" operatorv1 "github.com/tigera/operator/api/v1" "github.com/tigera/operator/pkg/components" "github.com/tigera/operator/pkg/enterprise" "github.com/tigera/operator/pkg/operator" + "github.com/tigera/operator/pkg/render" ) var _ = Describe("node enterprise image override", func() { @@ -41,3 +46,51 @@ var _ = Describe("node enterprise image override", func() { Expect(operator.ResolveImage("node", components.ComponentCalicoNode, oss)).To(Equal(components.ComponentCalicoNode)) }) }) + +var _ = Describe("node metrics service modifier", func() { + BeforeEach(func() { enterprise.Register() }) + AfterEach(func() { + operator.ResetForTest() + operator.ResetExtensionsForTest() + }) + + It("appends the node metrics service for the enterprise variant", func() { + ctx := operator.Context{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.TigeraSecureEnterprise}} + out := operator.ApplyPatches(render.ComponentNameNode, ctx, []client.Object{}) + svc, ok := operator.FindObject[*corev1.Service](out, render.CalicoNodeMetricsService) + Expect(ok).To(BeTrue()) + // default ports when FelixConfiguration is nil: 9081 + 9900, felix-metrics-port absent + Expect(svc.Spec.Ports).To(HaveLen(2)) + Expect(svc.Spec.Ports[0].Port).To(Equal(int32(9081))) + Expect(svc.Spec.Ports[1].Port).To(Equal(int32(9900))) + }) + + It("derives ports and felix-metrics-port from FelixConfiguration", func() { + reporter := 7081 + metrics := 7091 + enabled := true + ctx := operator.Context{ + Installation: &operatorv1.InstallationSpec{Variant: operatorv1.TigeraSecureEnterprise}, + FelixConfiguration: &v3.FelixConfiguration{Spec: v3.FelixConfigurationSpec{ + PrometheusReporterPort: &reporter, + PrometheusMetricsPort: &metrics, + PrometheusMetricsEnabled: &enabled, + }}, + } + out := operator.ApplyPatches(render.ComponentNameNode, ctx, []client.Object{}) + svc, ok := operator.FindObject[*corev1.Service](out, render.CalicoNodeMetricsService) + Expect(ok).To(BeTrue()) + Expect(svc.Spec.Ports).To(HaveLen(3)) + Expect(svc.Spec.Ports[0].Port).To(Equal(int32(7081))) + Expect(svc.Spec.Ports[2].Name).To(Equal("felix-metrics-port")) + Expect(svc.Spec.Ports[2].Port).To(Equal(int32(7091))) + }) + + It("does not append it for the Calico variant", func() { + ctx := operator.Context{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.Calico}} + out := operator.ApplyPatches(render.ComponentNameNode, ctx, []client.Object{}) + _, ok := operator.FindObject[*corev1.Service](out, render.CalicoNodeMetricsService) + Expect(ok).To(BeFalse()) + }) +}) + diff --git a/pkg/render/node.go b/pkg/render/node.go index 9c335d9ea4..2128d55021 100644 --- a/pkg/render/node.go +++ b/pkg/render/node.go @@ -73,9 +73,9 @@ const ( ) var ( - // The port used by calico/node to report Calico Enterprise BGP metrics. + // NodeBGPReporterPort is the port used by calico/node to report Calico Enterprise BGP metrics. // This is currently not intended to be user configurable. - nodeBGPReporterPort int32 = 9900 + NodeBGPReporterPort int32 = 9900 NodeTLSSecretName = "node-certs" NodeTLSSecretNameNonClusterHost = NodeTLSSecretName + TyphaNonClusterHostSuffix @@ -234,11 +234,6 @@ func (c *nodeComponent) Objects() ([]client.Object, []client.Object) { var objsToDelete []client.Object - if c.cfg.Installation.Variant.IsEnterprise() { - // Include Service for exposing node metrics. - objs = append(objs, c.nodeMetricsService()) - } - cniConfig := c.nodeCNIConfigMap() if cniConfig != nil { objs = append(objs, cniConfig) @@ -1777,55 +1772,6 @@ func (c *nodeComponent) nodeLivenessReadinessProbes() (*corev1.Probe, *corev1.Pr return lp, rp } -// nodeMetricsService creates a Service which exposes two endpoints on calico/node for -// reporting Prometheus metrics (for policy enforcement activity and BGP stats). -// This service is used internally by Calico Enterprise and is separate from general -// Prometheus metrics which are user-configurable. -func (c *nodeComponent) nodeMetricsService() *corev1.Service { - ports := []corev1.ServicePort{ - { - Name: "calico-metrics-port", - Port: int32(c.cfg.NodeReporterMetricsPort), - TargetPort: intstr.FromInt(c.cfg.NodeReporterMetricsPort), - Protocol: corev1.ProtocolTCP, - }, - { - Name: "calico-bgp-metrics-port", - Port: nodeBGPReporterPort, - TargetPort: intstr.FromInt(int(nodeBGPReporterPort)), - Protocol: corev1.ProtocolTCP, - }, - } - - if c.cfg.FelixPrometheusMetricsEnabled { - felixMetricsPort := int32(c.cfg.FelixPrometheusMetricsPort) - - ports = append(ports, corev1.ServicePort{ - Name: "felix-metrics-port", - Port: felixMetricsPort, - TargetPort: intstr.FromInt(int(felixMetricsPort)), - Protocol: corev1.ProtocolTCP, - }) - } - - return &corev1.Service{ - TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}, - ObjectMeta: metav1.ObjectMeta{ - Name: CalicoNodeMetricsService, - Namespace: common.CalicoNamespace, - Labels: map[string]string{"k8s-app": CalicoNodeObjectName}, - }, - Spec: corev1.ServiceSpec{ - Selector: map[string]string{"k8s-app": CalicoNodeObjectName}, - // Important: "None" tells Kubernetes that we want a headless service with - // no kube-proxy load balancer. If we omit this then kube-proxy will render - // a huge set of iptables rules for this service since there's an instance - // on every node. - ClusterIP: "None", - Ports: ports, - }, - } -} // getAutodetectionMethod returns the IP auto detection method in a form understandable by the calico/node // startup processing. It returns an empty string if IP auto detection should not be enabled. diff --git a/pkg/render/node_test.go b/pkg/render/node_test.go index e7a4ba37d4..30c772a91d 100644 --- a/pkg/render/node_test.go +++ b/pkg/render/node_test.go @@ -651,7 +651,6 @@ var _ = Describe("Node rendering tests", func() { {name: "calico-cni-plugin", ns: common.CalicoNamespace, group: "", version: "v1", kind: "ServiceAccount"}, {name: "calico-cni-plugin", ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRole"}, {name: "calico-cni-plugin", ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRoleBinding"}, - {name: "calico-node-metrics", ns: "calico-system", group: "", version: "v1", kind: "Service"}, {name: "cni-config", ns: common.CalicoNamespace, group: "", version: "v1", kind: "ConfigMap"}, {name: common.NodeDaemonSetName, ns: common.CalicoNamespace, group: "apps", version: "v1", kind: "DaemonSet"}, } @@ -725,10 +724,6 @@ var _ = Describe("Node rendering tests", func() { Expect(ds.Spec.Template.Spec.Containers[0].Env).To(ConsistOf(expectedNodeEnv)) Expect(len(ds.Spec.Template.Spec.Containers[0].Env)).To(Equal(len(expectedNodeEnv))) - // Expect 2 Ports when FelixPrometheusMetricsEnabled is false - ms := rtest.GetResource(resources, "calico-node-metrics", "calico-system", "", "v1", "Service").(*corev1.Service) - Expect(len(ms.Spec.Ports)).To(Equal(2)) - dirMustExist := corev1.HostPathDirectory bpfVol := corev1.Volume{Name: "bpffs", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/sys/fs/bpf", Type: &dirMustExist}}} Expect(ds.Spec.Template.Spec.Volumes).To(ContainElement(bpfVol)) @@ -739,40 +734,6 @@ var _ = Describe("Node rendering tests", func() { verifyProbesAndLifecycle(ds, false, true) }) - It("should render felix service metric with FelixPrometheusMetricPort when FelixPrometheusMetricsEnabled is true", func() { - defaultInstance.Variant = operatorv1.CalicoEnterprise - cfg.NodeReporterMetricsPort = 9081 - cfg.FelixPrometheusMetricsEnabled = true - - component := render.Node(&cfg) - Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() - - expectedServicePorts := []corev1.ServicePort{ - { - Name: "calico-metrics-port", - Port: int32(cfg.NodeReporterMetricsPort), - TargetPort: intstr.FromInt(cfg.NodeReporterMetricsPort), - Protocol: corev1.ProtocolTCP, - }, - { - Name: "calico-bgp-metrics-port", - Port: 9900, - TargetPort: intstr.FromInt(int(9900)), - Protocol: corev1.ProtocolTCP, - }, - { - Name: "felix-metrics-port", - Port: 9098, - TargetPort: intstr.FromInt(int(9098)), - Protocol: corev1.ProtocolTCP, - }, - } - - // Expect 3 Ports when FelixPrometheusMetricsEnabled is true - ms := rtest.GetResource(resources, "calico-node-metrics", "calico-system", "", "v1", "Service").(*corev1.Service) - Expect(ms.Spec.Ports).To(Equal(expectedServicePorts)) - }) It("should render all resources when using Calico CNI on EKS", func() { expectedResources := []struct { @@ -1639,7 +1600,6 @@ var _ = Describe("Node rendering tests", func() { {name: "calico-cni-plugin", ns: common.CalicoNamespace, group: "", version: "v1", kind: "ServiceAccount"}, {name: "calico-cni-plugin", ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRole"}, {name: "calico-cni-plugin", ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRoleBinding"}, - {name: "calico-node-metrics", ns: "calico-system", group: "", version: "v1", kind: "Service"}, {name: "cni-config", ns: common.CalicoNamespace, group: "", version: "v1", kind: "ConfigMap"}, {name: common.NodeDaemonSetName, ns: common.CalicoNamespace, group: "apps", version: "v1", kind: "DaemonSet"}, } @@ -1744,7 +1704,6 @@ var _ = Describe("Node rendering tests", func() { {name: "calico-cni-plugin", ns: common.CalicoNamespace, group: "", version: "v1", kind: "ServiceAccount"}, {name: "calico-cni-plugin", ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRole"}, {name: "calico-cni-plugin", ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRoleBinding"}, - {name: "calico-node-metrics", ns: "calico-system", group: "", version: "v1", kind: "Service"}, {name: "cni-config", ns: common.CalicoNamespace, group: "", version: "v1", kind: "ConfigMap"}, {name: common.NodeDaemonSetName, ns: common.CalicoNamespace, group: "apps", version: "v1", kind: "DaemonSet"}, } @@ -1827,10 +1786,6 @@ var _ = Describe("Node rendering tests", func() { Expect(len(ds.Spec.Template.Spec.Containers[0].Env)).To(Equal(len(expectedNodeEnv))) verifyProbesAndLifecycle(ds, true, true) - - // The metrics service should have the correct configuration. - ms := rtest.GetResource(resources, "calico-node-metrics", "calico-system", "", "v1", "Service").(*corev1.Service) - Expect(ms.Spec.ClusterIP).To(Equal("None"), "metrics service should be headless to prevent kube-proxy from rendering too many iptables rules") }) It("should render volumes and node volumemounts when bird templates are provided", func() { @@ -2094,7 +2049,7 @@ var _ = Describe("Node rendering tests", func() { component := render.Node(&cfg) Expect(component.ResolveImages(nil)).To(BeNil()) resources, _ := component.Objects() - Expect(len(resources)).To(Equal(defaultNumExpectedResources + 1)) + Expect(len(resources)).To(Equal(defaultNumExpectedResources)) dsResource := rtest.GetResource(resources, "calico-node", "calico-system", "apps", "v1", "DaemonSet") Expect(dsResource).ToNot(BeNil()) @@ -2115,7 +2070,7 @@ var _ = Describe("Node rendering tests", func() { component := render.Node(&cfg) Expect(component.ResolveImages(nil)).To(BeNil()) resources, _ := component.Objects() - Expect(len(resources)).To(Equal(defaultNumExpectedResources + 1)) + Expect(len(resources)).To(Equal(defaultNumExpectedResources)) dsResource := rtest.GetResource(resources, "calico-node", "calico-system", "apps", "v1", "DaemonSet") Expect(dsResource).ToNot(BeNil()) diff --git a/pkg/render/render_test.go b/pkg/render/render_test.go index 2b97cfaeb9..42afa04570 100644 --- a/pkg/render/render_test.go +++ b/pkg/render/render_test.go @@ -234,7 +234,7 @@ var _ = Describe("Rendering tests", func() { instance.NodeMetricsPort = &nodeMetricsPort c, err := allCalicoComponents(k8sServiceEp, instance, nil, nil, nil, typhaNodeTLS, nil, nil, false, "", dns.DefaultClusterDomain, 9094, 0, nil, nil) Expect(err).To(BeNil(), "Expected Calico to create successfully %s", err) - Expect(componentCount(c)).To(Equal((5 + 3 + 4 + 1 + 6 + 6 + 1 + 2) + 1 + 1)) + Expect(componentCount(c)).To(Equal((5 + 3 + 4 + 1 + 6 + 6 + 1 + 2) + 1)) }) It("should render all resources when variant is Tigera Secure and Management Cluster", func() { @@ -267,7 +267,6 @@ var _ = Describe("Rendering tests", func() { &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: "calico-cni-plugin", Namespace: common.CalicoNamespace}, TypeMeta: metav1.TypeMeta{Kind: "ServiceAccount", APIVersion: "v1"}}, &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-cni-plugin"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-cni-plugin"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: "calico-node-metrics", Namespace: common.CalicoNamespace}, TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}}, &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "cni-config", Namespace: common.CalicoNamespace}, TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"}}, &appsv1.DaemonSet{ObjectMeta: metav1.ObjectMeta{Name: common.NodeDaemonSetName, Namespace: common.CalicoNamespace}, TypeMeta: metav1.TypeMeta{Kind: "DaemonSet", APIVersion: "apps/v1"}}, diff --git a/pkg/render/windows.go b/pkg/render/windows.go index d642178133..a796bd5d0d 100644 --- a/pkg/render/windows.go +++ b/pkg/render/windows.go @@ -163,8 +163,8 @@ func (c *windowsComponent) nodeMetricsService() *corev1.Service { }, { Name: "calico-bgp-metrics-port", - Port: nodeBGPReporterPort, - TargetPort: intstr.FromInt(int(nodeBGPReporterPort)), + Port: NodeBGPReporterPort, + TargetPort: intstr.FromInt(int(NodeBGPReporterPort)), Protocol: corev1.ProtocolTCP, }, }, From c74b106d2dc10b2da8685658bb176bfad50d3b2f Mon Sep 17 00:00:00 2001 From: Casey Davenport Date: Fri, 29 May 2026 13:56:08 -0700 Subject: [PATCH 14/54] variant-extensions: gofmt and replace deprecated TigeraSecureEnterprise in tests --- pkg/controller/installation/core_controller.go | 4 ++-- pkg/controller/utils/component_test.go | 12 +++++++----- pkg/enterprise/installation.go | 2 +- pkg/enterprise/installation_test.go | 4 ++-- pkg/enterprise/node.go | 4 ++-- pkg/enterprise/node_test.go | 7 +++---- pkg/enterprise/typha_test.go | 2 +- pkg/operator/image_test.go | 2 +- pkg/render/node.go | 1 - pkg/render/node_test.go | 1 - 10 files changed, 19 insertions(+), 20 deletions(-) diff --git a/pkg/controller/installation/core_controller.go b/pkg/controller/installation/core_controller.go index bf92bf3e2f..af2fcd6735 100644 --- a/pkg/controller/installation/core_controller.go +++ b/pkg/controller/installation/core_controller.go @@ -63,7 +63,6 @@ import ( "github.com/tigera/operator/pkg/active" "github.com/tigera/operator/pkg/common" "github.com/tigera/operator/pkg/components" - "github.com/tigera/operator/pkg/operator" "github.com/tigera/operator/pkg/controller/certificatemanager" "github.com/tigera/operator/pkg/controller/ippool" "github.com/tigera/operator/pkg/controller/k8sapi" @@ -78,9 +77,10 @@ import ( "github.com/tigera/operator/pkg/dns" "github.com/tigera/operator/pkg/imports/admission" "github.com/tigera/operator/pkg/imports/crds" + "github.com/tigera/operator/pkg/operator" "github.com/tigera/operator/pkg/render" rcertificatemanagement "github.com/tigera/operator/pkg/render/certificatemanagement" -"github.com/tigera/operator/pkg/render/common/networkpolicy" + "github.com/tigera/operator/pkg/render/common/networkpolicy" "github.com/tigera/operator/pkg/render/common/resourcequota" "github.com/tigera/operator/pkg/render/goldmane" "github.com/tigera/operator/pkg/render/kubecontrollers" diff --git a/pkg/controller/utils/component_test.go b/pkg/controller/utils/component_test.go index d4b34cb871..88da64d06d 100644 --- a/pkg/controller/utils/component_test.go +++ b/pkg/controller/utils/component_test.go @@ -2495,8 +2495,10 @@ type namedFakeComponent struct { obj client.Object } -func (f *namedFakeComponent) Name() string { return f.name } -func (f *namedFakeComponent) ResolveImages(*operatorv1.ImageSet) error { return nil } -func (f *namedFakeComponent) Objects() ([]client.Object, []client.Object) { return []client.Object{f.obj}, nil } -func (f *namedFakeComponent) Ready() bool { return true } -func (f *namedFakeComponent) SupportedOSType() rmeta.OSType { return rmeta.OSTypeLinux } +func (f *namedFakeComponent) Name() string { return f.name } +func (f *namedFakeComponent) ResolveImages(*operatorv1.ImageSet) error { return nil } +func (f *namedFakeComponent) Objects() ([]client.Object, []client.Object) { + return []client.Object{f.obj}, nil +} +func (f *namedFakeComponent) Ready() bool { return true } +func (f *namedFakeComponent) SupportedOSType() rmeta.OSType { return rmeta.OSTypeLinux } diff --git a/pkg/enterprise/installation.go b/pkg/enterprise/installation.go index 7b0abaf4b5..dc99e70eaf 100644 --- a/pkg/enterprise/installation.go +++ b/pkg/enterprise/installation.go @@ -20,10 +20,10 @@ import ( "github.com/tigera/operator/pkg/common" "github.com/tigera/operator/pkg/dns" - "github.com/tigera/operator/pkg/render/monitor" "github.com/tigera/operator/pkg/operator" "github.com/tigera/operator/pkg/render" relasticsearch "github.com/tigera/operator/pkg/render/common/elasticsearch" + "github.com/tigera/operator/pkg/render/monitor" ) type installationExtension struct{} diff --git a/pkg/enterprise/installation_test.go b/pkg/enterprise/installation_test.go index dffe6a7470..bf64a2ad7d 100644 --- a/pkg/enterprise/installation_test.go +++ b/pkg/enterprise/installation_test.go @@ -40,14 +40,14 @@ var _ = Describe("installation enterprise extension", func() { It("rejects a zero prometheus reporter port", func() { port := 0 - p := newPrep(operatorv1.TigeraSecureEnterprise) + p := newPrep(operatorv1.CalicoEnterprise) p.FelixConfiguration = &v3.FelixConfiguration{Spec: v3.FelixConfigurationSpec{PrometheusReporterPort: &port}} _, err := operator.GetInstallationExtension().Prepare(p) Expect(err).To(HaveOccurred()) }) It("creates the node prometheus keypair for the enterprise variant", func() { - p := newPrep(operatorv1.TigeraSecureEnterprise) + p := newPrep(operatorv1.CalicoEnterprise) p.FelixConfiguration = &v3.FelixConfiguration{} ctx, err := operator.GetInstallationExtension().Prepare(p) Expect(err).NotTo(HaveOccurred()) diff --git a/pkg/enterprise/node.go b/pkg/enterprise/node.go index e47c593076..f1ea2bcce2 100644 --- a/pkg/enterprise/node.go +++ b/pkg/enterprise/node.go @@ -29,8 +29,8 @@ import ( ) const ( - defaultNodeReporterPort = 9081 - defaultFelixMetricsPort = 9091 + defaultNodeReporterPort = 9081 + defaultFelixMetricsPort = 9091 ) func registerNode() { diff --git a/pkg/enterprise/node_test.go b/pkg/enterprise/node_test.go index 509fd6b18f..d2370fda6e 100644 --- a/pkg/enterprise/node_test.go +++ b/pkg/enterprise/node_test.go @@ -37,7 +37,7 @@ var _ = Describe("node enterprise image override", func() { }) It("selects the enterprise node image for the enterprise variant", func() { - ent := &operatorv1.InstallationSpec{Variant: operatorv1.TigeraSecureEnterprise} + ent := &operatorv1.InstallationSpec{Variant: operatorv1.CalicoEnterprise} Expect(operator.ResolveImage("node", components.ComponentCalicoNode, ent)).To(Equal(components.ComponentTigeraNode)) }) @@ -55,7 +55,7 @@ var _ = Describe("node metrics service modifier", func() { }) It("appends the node metrics service for the enterprise variant", func() { - ctx := operator.Context{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.TigeraSecureEnterprise}} + ctx := operator.Context{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.CalicoEnterprise}} out := operator.ApplyPatches(render.ComponentNameNode, ctx, []client.Object{}) svc, ok := operator.FindObject[*corev1.Service](out, render.CalicoNodeMetricsService) Expect(ok).To(BeTrue()) @@ -70,7 +70,7 @@ var _ = Describe("node metrics service modifier", func() { metrics := 7091 enabled := true ctx := operator.Context{ - Installation: &operatorv1.InstallationSpec{Variant: operatorv1.TigeraSecureEnterprise}, + Installation: &operatorv1.InstallationSpec{Variant: operatorv1.CalicoEnterprise}, FelixConfiguration: &v3.FelixConfiguration{Spec: v3.FelixConfigurationSpec{ PrometheusReporterPort: &reporter, PrometheusMetricsPort: &metrics, @@ -93,4 +93,3 @@ var _ = Describe("node metrics service modifier", func() { Expect(ok).To(BeFalse()) }) }) - diff --git a/pkg/enterprise/typha_test.go b/pkg/enterprise/typha_test.go index 67330b7539..9b41d448fa 100644 --- a/pkg/enterprise/typha_test.go +++ b/pkg/enterprise/typha_test.go @@ -52,7 +52,7 @@ var _ = Describe("typha enterprise modifier", func() { It("adds enterprise RBAC and MULTI_INTERFACE_MODE for the enterprise variant", func() { ctx := operator.Context{Installation: &operatorv1.InstallationSpec{ - Variant: operatorv1.TigeraSecureEnterprise, + Variant: operatorv1.CalicoEnterprise, CalicoNetwork: &operatorv1.CalicoNetworkSpec{MultiInterfaceMode: &multiMode}, }} out := operator.ApplyPatches(render.ComponentNameTypha, ctx, newObjs()) diff --git a/pkg/operator/image_test.go b/pkg/operator/image_test.go index 93e9beb67d..b972665e17 100644 --- a/pkg/operator/image_test.go +++ b/pkg/operator/image_test.go @@ -36,7 +36,7 @@ var _ = Describe("image overrides", func() { return components.ComponentTigeraNode, true }) - ent := &operatorv1.InstallationSpec{Variant: operatorv1.TigeraSecureEnterprise} + ent := &operatorv1.InstallationSpec{Variant: operatorv1.CalicoEnterprise} Expect(operator.ResolveImage("node", components.ComponentCalicoNode, ent)).To(Equal(components.ComponentTigeraNode)) }) diff --git a/pkg/render/node.go b/pkg/render/node.go index 2128d55021..b93775d197 100644 --- a/pkg/render/node.go +++ b/pkg/render/node.go @@ -1772,7 +1772,6 @@ func (c *nodeComponent) nodeLivenessReadinessProbes() (*corev1.Probe, *corev1.Pr return lp, rp } - // getAutodetectionMethod returns the IP auto detection method in a form understandable by the calico/node // startup processing. It returns an empty string if IP auto detection should not be enabled. func getAutodetectionMethod(ad *operatorv1.NodeAddressAutodetection) string { diff --git a/pkg/render/node_test.go b/pkg/render/node_test.go index 30c772a91d..ed02631b6c 100644 --- a/pkg/render/node_test.go +++ b/pkg/render/node_test.go @@ -734,7 +734,6 @@ var _ = Describe("Node rendering tests", func() { verifyProbesAndLifecycle(ds, false, true) }) - It("should render all resources when using Calico CNI on EKS", func() { expectedResources := []struct { name string From fd4e8355b358f25f685b5dfcc82b0c40c5842f23 Mon Sep 17 00:00:00 2001 From: Casey Davenport Date: Wed, 3 Jun 2026 15:31:46 -0700 Subject: [PATCH 15/54] Generalize the variant extension mechanism The registry package is renamed to extensions. The installation controller builds the render context through a registered factory, and the componentHandler applies registered modifiers to component output. The node and typha variant branches now live in enterprise modifiers, and the calico log directory is mounted for both variants. --- cmd/main.go | 6 +- .../installation/core_controller.go | 110 ++--- pkg/controller/utils/component.go | 14 +- .../utils/component_enterprise_test.go | 87 ++++ pkg/controller/utils/component_test.go | 10 +- pkg/enterprise/installation.go | 61 +-- pkg/enterprise/installation_test.go | 51 ++- pkg/enterprise/node.go | 185 ++++++++- pkg/enterprise/node_test.go | 166 ++++++-- pkg/enterprise/typha.go | 10 +- pkg/enterprise/typha_test.go | 15 +- .../extensions_suite_test.go} | 6 +- pkg/extensions/factory.go | 127 ++++++ pkg/extensions/factory_test.go | 73 ++++ pkg/{operator => extensions}/image.go | 2 +- pkg/{operator => extensions}/image_test.go | 14 +- .../patch.go => extensions/modifier.go} | 34 +- .../modifier_test.go} | 34 +- .../rendercontext.go} | 16 +- pkg/operator/extension.go | 59 --- pkg/operator/extension_test.go | 42 -- pkg/render/component.go | 21 +- pkg/render/enterprise_setup_test.go | 13 +- pkg/render/node.go | 123 +----- pkg/render/node_enterprise_test.go | 172 ++++++++ pkg/render/node_test.go | 375 ++---------------- pkg/render/render_test.go | 21 +- 27 files changed, 1037 insertions(+), 810 deletions(-) create mode 100644 pkg/controller/utils/component_enterprise_test.go rename pkg/{operator/operator_suite_test.go => extensions/extensions_suite_test.go} (88%) create mode 100644 pkg/extensions/factory.go create mode 100644 pkg/extensions/factory_test.go rename pkg/{operator => extensions}/image.go (98%) rename pkg/{operator => extensions}/image_test.go (69%) rename pkg/{operator/patch.go => extensions/modifier.go} (50%) rename pkg/{operator/patch_test.go => extensions/modifier_test.go} (53%) rename pkg/{operator/context.go => extensions/rendercontext.go} (70%) delete mode 100644 pkg/operator/extension.go delete mode 100644 pkg/operator/extension_test.go create mode 100644 pkg/render/node_enterprise_test.go diff --git a/cmd/main.go b/cmd/main.go index b086ca605a..0613e68612 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -529,9 +529,9 @@ If a value other than 'all' is specified, the first CRD with a prefix of the spe os.Exit(1) } - // Wire in-repo enterprise modifiers and extensions into the operator - // registries. After the monorepo split this call moves to calico-private's - // main. + // Wire the in-repo Calico Enterprise extensions (the render context factory, + // modifiers, and image overrides) into the operator registries. After the + // monorepo split this call moves to calico-private's main. enterprise.Register() err = controller.AddToManager(mgr, options) diff --git a/pkg/controller/installation/core_controller.go b/pkg/controller/installation/core_controller.go index d1dfceeecb..bbeb8582af 100644 --- a/pkg/controller/installation/core_controller.go +++ b/pkg/controller/installation/core_controller.go @@ -75,9 +75,9 @@ import ( "github.com/tigera/operator/pkg/controller/utils/imageset" "github.com/tigera/operator/pkg/ctrlruntime" "github.com/tigera/operator/pkg/dns" + "github.com/tigera/operator/pkg/extensions" "github.com/tigera/operator/pkg/imports/admission" "github.com/tigera/operator/pkg/imports/crds" - "github.com/tigera/operator/pkg/operator" "github.com/tigera/operator/pkg/render" rcertificatemanagement "github.com/tigera/operator/pkg/render/certificatemanagement" "github.com/tigera/operator/pkg/render/common/networkpolicy" @@ -91,11 +91,11 @@ import ( const ( techPreviewFeatureSeccompApparmor = "tech-preview.operator.tigera.io/node-apparmor-profile" - // The default port used by calico/node to report Calico Enterprise internal metrics. - // This is separate from the calico/node prometheus metrics port, which is user configurable. + // defaultNodeReporterPort is the default port calico/node uses to report Calico + // Enterprise internal metrics. The Linux node path derives this in the + // enterprise node modifier; this copy serves the Windows controller, which + // still carries its enterprise logic inline. defaultNodeReporterPort = 9081 - - defaultFelixMetricsDefaultPort = 9091 ) const InstallationName string = "calico" @@ -1208,47 +1208,28 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile return reconcile.Result{}, err } - // nodeReporterMetricsPort is a port used in Enterprise to host internal metrics. - // Operator is responsible for creating a service which maps to that port. - nodeReporterMetricsPort := defaultNodeReporterPort - felixPrometheusMetricsPort := defaultFelixMetricsDefaultPort calicoVersion := components.CalicoRelease if instance.Spec.Variant.IsEnterprise() { calicoVersion = components.EnterpriseRelease - if felixConfiguration.Spec.PrometheusReporterPort != nil { - nodeReporterMetricsPort = *felixConfiguration.Spec.PrometheusReporterPort - } - if felixConfiguration.Spec.PrometheusMetricsPort != nil { - felixPrometheusMetricsPort = *felixConfiguration.Spec.PrometheusMetricsPort - } - } - - // Run the registered installation extension. For the enterprise variant this - // validates config and creates the node-prometheus certificate, adding it (and - // the prometheus/esgw certs) to the trusted bundle. Returns the render Context - // consumed by registered modifiers. - var modCtx operator.Context - if ext := operator.GetInstallationExtension(); ext != nil { - modCtx, err = ext.Prepare(operator.InstallationPrep{ - Ctx: ctx, - Client: r.client, - Installation: &instance.Spec, - FelixConfiguration: felixConfiguration, - CertificateManager: certificateManager, - TrustedBundle: typhaNodeTLS.TrustedBundle, - ClusterDomain: r.clusterDomain, - }) - if err != nil { - r.status.SetDegraded(operatorv1.ResourceCreateError, "Error preparing enterprise installation", err, reqLogger) - return reconcile.Result{}, err - } - } else { - modCtx = operator.Context{ - Installation: &instance.Spec, - FelixConfiguration: felixConfiguration, - ClusterDomain: r.clusterDomain, - TrustedBundle: typhaNodeTLS.TrustedBundle, - } + } + + // Build the render context handed to registered modifiers. The core operator + // factory returns just the base context; an extension's factory additionally + // does controller-side work for its variant - validating config and creating + // the node-prometheus certificate, adding it (and the prometheus/esgw certs) + // to the trusted bundle - and may abort the reconcile by returning an error. + modCtx, err := extensions.GetRenderContextFactory().New( + extensions.WithContext(ctx), + extensions.WithClient(r.client), + extensions.WithInstallation(&instance.Spec), + extensions.WithFelixConfiguration(felixConfiguration), + extensions.WithCertificateManager(certificateManager), + extensions.WithTrustedBundle(typhaNodeTLS.TrustedBundle), + extensions.WithClusterDomain(r.clusterDomain), + ) + if err != nil { + r.status.SetDegraded(operatorv1.ResourceCreateError, "Error preparing installation extension", err, reqLogger) + return reconcile.Result{}, err } nodePrometheusTLS := modCtx.NodePrometheusTLS @@ -1289,7 +1270,7 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile } // Create a component handler to create or update the rendered components. - handler := r.newComponentHandler(log, r.client, r.scheme, instance, utils.WithContext(modCtx)) + handler := r.newComponentHandler(log, r.client, r.scheme, instance, utils.WithRenderContext(modCtx)) // Render namespaces first - this ensures that any other controllers blocked on namespace existence can proceed. namespaceCfg := &render.NamespaceConfiguration{ @@ -1525,28 +1506,25 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile // Build a configuration for rendering calico/node. nodeCfg := render.NodeConfiguration{ - GoldmaneRunning: goldmaneRunning, - K8sServiceEp: k8sapi.Endpoint, - Installation: &instance.Spec, - IPPools: crdPoolsToOperator(currentPools.Items), - LogCollector: logCollector, - BirdTemplates: birdTemplates, - TLS: typhaNodeTLS, - ClusterDomain: r.clusterDomain, - DefaultDNSPolicy: defaultDNSPolicy, - DefaultDNSConfig: defaultDNSConfig, - GoldmaneIP: goldmaneIP, - NodeReporterMetricsPort: nodeReporterMetricsPort, - BGPLayouts: bgpLayout, - NodeAppArmorProfile: nodeAppArmorProfile, - MigrateNamespaces: needsNamespaceMigration, - CanRemoveCNIFinalizer: canRemoveCNI, - PrometheusServerTLS: nodePrometheusTLS, - FelixHealthPort: *felixConfiguration.Spec.HealthPort, - NodeCgroupV2Path: felixConfiguration.Spec.CgroupV2Path, - FelixPrometheusMetricsEnabled: utils.IsFelixPrometheusMetricsEnabled(felixConfiguration), - FelixPrometheusMetricsPort: felixPrometheusMetricsPort, - V3CRDs: r.v3CRDs, + GoldmaneRunning: goldmaneRunning, + K8sServiceEp: k8sapi.Endpoint, + Installation: &instance.Spec, + IPPools: crdPoolsToOperator(currentPools.Items), + LogCollector: logCollector, + BirdTemplates: birdTemplates, + TLS: typhaNodeTLS, + ClusterDomain: r.clusterDomain, + DefaultDNSPolicy: defaultDNSPolicy, + DefaultDNSConfig: defaultDNSConfig, + GoldmaneIP: goldmaneIP, + BGPLayouts: bgpLayout, + NodeAppArmorProfile: nodeAppArmorProfile, + MigrateNamespaces: needsNamespaceMigration, + CanRemoveCNIFinalizer: canRemoveCNI, + PrometheusServerTLS: nodePrometheusTLS, + FelixHealthPort: *felixConfiguration.Spec.HealthPort, + NodeCgroupV2Path: felixConfiguration.Spec.CgroupV2Path, + V3CRDs: r.v3CRDs, } if bgpConfiguration.Spec.BindMode != nil { diff --git a/pkg/controller/utils/component.go b/pkg/controller/utils/component.go index 7d63736fcd..3e343c4540 100644 --- a/pkg/controller/utils/component.go +++ b/pkg/controller/utils/component.go @@ -47,7 +47,7 @@ import ( "github.com/tigera/operator/pkg/apigroup" "github.com/tigera/operator/pkg/common" "github.com/tigera/operator/pkg/controller/status" - "github.com/tigera/operator/pkg/operator" + "github.com/tigera/operator/pkg/extensions" "github.com/tigera/operator/pkg/render" rmeta "github.com/tigera/operator/pkg/render/common/meta" ) @@ -78,8 +78,9 @@ type ComponentHandler interface { // ComponentHandlerOption configures a componentHandler. type ComponentHandlerOption func(*componentHandler) -// WithContext supplies the operator.Context passed to registered render patches. -func WithContext(ctx operator.Context) ComponentHandlerOption { +// WithRenderContext supplies the extensions.RenderContext passed to registered +// render modifiers. +func WithRenderContext(ctx extensions.RenderContext) ComponentHandlerOption { return func(c *componentHandler) { c.modCtx = ctx } } @@ -106,7 +107,7 @@ type componentHandler struct { log logr.Logger createOnly bool apiGroupEnvs []v1.EnvVar - modCtx operator.Context + modCtx extensions.RenderContext } func (c *componentHandler) SetCreateOnly() { @@ -467,8 +468,8 @@ func (c *componentHandler) CreateOrUpdateOrDelete(ctx context.Context, component var cronJobs []types.NamespacedName objsToCreate, objsToDelete := component.Objects() - if named, ok := component.(render.Named); ok { - objsToCreate = operator.ApplyPatches(named.Name(), c.modCtx, objsToCreate) + if named, ok := component.(render.Extensible); ok { + objsToCreate = extensions.ApplyModifiers(named.Name(), c.modCtx, objsToCreate) } // Load the InstallationSpec once and reuse it for every object: createOrUpdateObject needs it @@ -1150,7 +1151,6 @@ func addComponentLabel(obj metav1.Object, cr metav1.Object) { owner, ok := cr.(runtime.Object) if ok && owner.GetObjectKind() != nil && owner.GetObjectKind() != nil { obj.GetLabels()["app.kubernetes.io/component"] = sanitizeLabel(owner.GetObjectKind().GroupVersionKind().GroupKind().String()) - } } } diff --git a/pkg/controller/utils/component_enterprise_test.go b/pkg/controller/utils/component_enterprise_test.go new file mode 100644 index 0000000000..a59e1394b6 --- /dev/null +++ b/pkg/controller/utils/component_enterprise_test.go @@ -0,0 +1,87 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package utils_test + +import ( + "context" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + rbacv1 "k8s.io/api/rbac/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + logf "sigs.k8s.io/controller-runtime/pkg/log" + + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/apis" + "github.com/tigera/operator/pkg/common" + "github.com/tigera/operator/pkg/controller/certificatemanager" + "github.com/tigera/operator/pkg/controller/k8sapi" + "github.com/tigera/operator/pkg/controller/utils" + ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" + "github.com/tigera/operator/pkg/dns" + "github.com/tigera/operator/pkg/enterprise" + "github.com/tigera/operator/pkg/extensions" + "github.com/tigera/operator/pkg/render" +) + +// This exercises the full path comment-by-comment: a real render component goes +// through CreateOrUpdateOrDelete with an enterprise RenderContext, and the +// registered modifier must match the real render output by name. If render ever +// renames the typha ClusterRole, the modifier silently no-ops and this fails. +var _ = Describe("componentHandler enterprise modifier integration", func() { + BeforeEach(func() { enterprise.Register() }) + AfterEach(func() { extensions.ResetForTest() }) + + It("applies the enterprise typha modifier to real render output", func() { + scheme := runtime.NewScheme() + Expect(apis.AddToScheme(scheme, false)).NotTo(HaveOccurred()) + cli := ctrlrfake.DefaultFakeClientBuilder(scheme).Build() + + certManager, err := certificatemanager.Create(cli, nil, "", common.OperatorNamespace(), certificatemanager.AllowCACreation()) + Expect(err).NotTo(HaveOccurred()) + nodeKeyPair, err := certManager.GetOrCreateKeyPair(cli, render.NodeTLSSecretName, common.OperatorNamespace(), []string{render.FelixCommonName}) + Expect(err).NotTo(HaveOccurred()) + typhaKeyPair, err := certManager.GetOrCreateKeyPair(cli, render.TyphaTLSSecretName, common.OperatorNamespace(), []string{render.TyphaCommonName}) + Expect(err).NotTo(HaveOccurred()) + + instance := &operatorv1.InstallationSpec{ + Variant: operatorv1.CalicoEnterprise, + CNI: &operatorv1.CNISpec{Type: operatorv1.PluginCalico}, + } + comp := render.Typha(&render.TyphaConfiguration{ + K8sServiceEp: k8sapi.ServiceEndpoint{}, + Installation: instance, + ClusterDomain: dns.DefaultClusterDomain, + FelixHealthPort: 9099, + TLS: &render.TyphaNodeTLS{ + TrustedBundle: certManager.CreateTrustedBundle(), + TyphaSecret: typhaKeyPair, + TyphaCommonName: render.TyphaCommonName, + NodeSecret: nodeKeyPair, + NodeCommonName: render.FelixCommonName, + }, + }) + + renderCtx := extensions.RenderContext{Installation: instance} + handler := utils.NewComponentHandler(logf.Log, cli, scheme, nil, utils.WithRenderContext(renderCtx)) + Expect(handler.CreateOrUpdateOrDelete(context.Background(), comp, nil)).NotTo(HaveOccurred()) + + role := &rbacv1.ClusterRole{} + Expect(cli.Get(context.Background(), client.ObjectKey{Name: "calico-typha"}, role)).NotTo(HaveOccurred()) + Expect(role.Rules).To(ContainElement(HaveField("Resources", ContainElement("licensekeys")))) + }) +}) diff --git a/pkg/controller/utils/component_test.go b/pkg/controller/utils/component_test.go index 1aa5d4be63..3081adfe19 100644 --- a/pkg/controller/utils/component_test.go +++ b/pkg/controller/utils/component_test.go @@ -46,7 +46,7 @@ import ( "github.com/tigera/operator/pkg/common" "github.com/tigera/operator/pkg/controller/status" ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" - "github.com/tigera/operator/pkg/operator" + "github.com/tigera/operator/pkg/extensions" "github.com/tigera/operator/pkg/render" rmeta "github.com/tigera/operator/pkg/render/common/meta" ) @@ -2487,13 +2487,13 @@ func (mc *mockClient) SubResource(subResource string) client.SubResourceClient { panic("SubResource not implemented in mockClient") } -var _ = Describe("componentHandler patch application", func() { +var _ = Describe("componentHandler modifier application", func() { AfterEach(func() { - operator.ResetForTest() + extensions.ResetForTest() }) - It("applies registered patches to a named component before create", func() { - operator.Patch("fake", func(ctx operator.Context, objs []client.Object) []client.Object { + It("applies registered modifiers to a named component before create", func() { + extensions.Modify("fake", func(ctx extensions.RenderContext, objs []client.Object) []client.Object { cm := objs[0].(*corev1.ConfigMap) cm.Data = map[string]string{"patched": "yes"} return objs diff --git a/pkg/enterprise/installation.go b/pkg/enterprise/installation.go index dc99e70eaf..596229a635 100644 --- a/pkg/enterprise/installation.go +++ b/pkg/enterprise/installation.go @@ -20,61 +20,64 @@ import ( "github.com/tigera/operator/pkg/common" "github.com/tigera/operator/pkg/dns" - "github.com/tigera/operator/pkg/operator" + "github.com/tigera/operator/pkg/extensions" "github.com/tigera/operator/pkg/render" relasticsearch "github.com/tigera/operator/pkg/render/common/elasticsearch" "github.com/tigera/operator/pkg/render/monitor" ) -type installationExtension struct{} +// installationFactory is the Calico Enterprise RenderContextFactory. It builds +// the base render context and then does the controller-side work the modifiers +// can't: validating config and creating/fetching the certificates that feed the +// trusted bundle. +type installationFactory struct{} func registerInstallation() { - operator.RegisterInstallationExtension(&installationExtension{}) + extensions.RegisterRenderContextFactory(&installationFactory{}) } -func (e *installationExtension) Prepare(p operator.InstallationPrep) (operator.Context, error) { - ctx := operator.Context{ - Installation: p.Installation, - FelixConfiguration: p.FelixConfiguration, - ClusterDomain: p.ClusterDomain, - TrustedBundle: p.TrustedBundle, - } - if !p.Installation.Variant.IsEnterprise() { - return ctx, nil +func (f *installationFactory) New(opts ...extensions.RenderContextOption) (extensions.RenderContext, error) { + in := extensions.ApplyInputs(opts...) + rc := extensions.BaseRenderContext(in) + if in.Installation == nil || !in.Installation.Variant.IsEnterprise() { + return rc, nil } - // Reject the unsupported zero reporter port. (Port value derivation stays in - // the OSS controller; only validation moves here.) - if p.FelixConfiguration.Spec.PrometheusReporterPort != nil && *p.FelixConfiguration.Spec.PrometheusReporterPort == 0 { - return ctx, errors.New("felixConfiguration prometheusReporterPort=0 not supported") + // Reject the unsupported zero reporter port. The port value itself is derived + // in the node modifier; only this validation lives here. + if in.FelixConfiguration.Spec.PrometheusReporterPort != nil && *in.FelixConfiguration.Spec.PrometheusReporterPort == 0 { + return rc, errors.New("felixConfiguration prometheusReporterPort=0 not supported") } - nodePrometheusTLS, err := p.CertificateManager.GetOrCreateKeyPair( - p.Client, render.NodePrometheusTLSServerSecret, common.OperatorNamespace(), - dns.GetServiceDNSNames(render.CalicoNodeMetricsService, common.CalicoNamespace, p.ClusterDomain)) + nodePrometheusTLS, err := in.CertificateManager.GetOrCreateKeyPair( + in.Client, + render.NodePrometheusTLSServerSecret, + common.OperatorNamespace(), + dns.GetServiceDNSNames(render.CalicoNodeMetricsService, common.CalicoNamespace, in.ClusterDomain), + ) if err != nil { - return ctx, fmt.Errorf("error creating node prometheus TLS certificate: %w", err) + return rc, fmt.Errorf("error creating node prometheus TLS certificate: %w", err) } if nodePrometheusTLS != nil { - p.TrustedBundle.AddCertificates(nodePrometheusTLS) + in.TrustedBundle.AddCertificates(nodePrometheusTLS) } - ctx.NodePrometheusTLS = nodePrometheusTLS + rc.NodePrometheusTLS = nodePrometheusTLS - prometheusClientCert, err := p.CertificateManager.GetCertificate(p.Client, monitor.PrometheusClientTLSSecretName, common.OperatorNamespace()) + prometheusClientCert, err := in.CertificateManager.GetCertificate(in.Client, monitor.PrometheusClientTLSSecretName, common.OperatorNamespace()) if err != nil { - return ctx, fmt.Errorf("unable to fetch prometheus certificate: %w", err) + return rc, fmt.Errorf("unable to fetch prometheus certificate: %w", err) } if prometheusClientCert != nil { - p.TrustedBundle.AddCertificates(prometheusClientCert) + in.TrustedBundle.AddCertificates(prometheusClientCert) } - esgwCertificate, err := p.CertificateManager.GetCertificate(p.Client, relasticsearch.PublicCertSecret, common.OperatorNamespace()) + esgwCertificate, err := in.CertificateManager.GetCertificate(in.Client, relasticsearch.PublicCertSecret, common.OperatorNamespace()) if err != nil { - return ctx, fmt.Errorf("failed to retrieve / validate %s: %w", relasticsearch.PublicCertSecret, err) + return rc, fmt.Errorf("failed to retrieve / validate %s: %w", relasticsearch.PublicCertSecret, err) } if esgwCertificate != nil { - p.TrustedBundle.AddCertificates(esgwCertificate) + in.TrustedBundle.AddCertificates(esgwCertificate) } - return ctx, nil + return rc, nil } diff --git a/pkg/enterprise/installation_test.go b/pkg/enterprise/installation_test.go index bf64a2ad7d..18c38c312a 100644 --- a/pkg/enterprise/installation_test.go +++ b/pkg/enterprise/installation_test.go @@ -21,48 +21,45 @@ import ( . "github.com/onsi/gomega" v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" + "k8s.io/apimachinery/pkg/runtime" + operatorv1 "github.com/tigera/operator/api/v1" "github.com/tigera/operator/pkg/apis" "github.com/tigera/operator/pkg/common" "github.com/tigera/operator/pkg/controller/certificatemanager" ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" "github.com/tigera/operator/pkg/enterprise" - "github.com/tigera/operator/pkg/operator" - "k8s.io/apimachinery/pkg/runtime" + "github.com/tigera/operator/pkg/extensions" ) -var _ = Describe("installation enterprise extension", func() { +var _ = Describe("installation render context factory", func() { BeforeEach(func() { enterprise.Register() }) - AfterEach(func() { - operator.ResetForTest() - operator.ResetExtensionsForTest() - }) + AfterEach(func() { extensions.ResetForTest() }) It("rejects a zero prometheus reporter port", func() { port := 0 - p := newPrep(operatorv1.CalicoEnterprise) - p.FelixConfiguration = &v3.FelixConfiguration{Spec: v3.FelixConfigurationSpec{PrometheusReporterPort: &port}} - _, err := operator.GetInstallationExtension().Prepare(p) + opts := newOpts(operatorv1.CalicoEnterprise) + opts = append(opts, extensions.WithFelixConfiguration(&v3.FelixConfiguration{ + Spec: v3.FelixConfigurationSpec{PrometheusReporterPort: &port}, + })) + _, err := extensions.GetRenderContextFactory().New(opts...) Expect(err).To(HaveOccurred()) }) It("creates the node prometheus keypair for the enterprise variant", func() { - p := newPrep(operatorv1.CalicoEnterprise) - p.FelixConfiguration = &v3.FelixConfiguration{} - ctx, err := operator.GetInstallationExtension().Prepare(p) + rc, err := extensions.GetRenderContextFactory().New(newOpts(operatorv1.CalicoEnterprise)...) Expect(err).NotTo(HaveOccurred()) - Expect(ctx.NodePrometheusTLS).NotTo(BeNil()) + Expect(rc.NodePrometheusTLS).NotTo(BeNil()) }) It("is a no-op for the Calico variant", func() { - p := newPrep(operatorv1.Calico) - ctx, err := operator.GetInstallationExtension().Prepare(p) + rc, err := extensions.GetRenderContextFactory().New(newOpts(operatorv1.Calico)...) Expect(err).NotTo(HaveOccurred()) - Expect(ctx.NodePrometheusTLS).To(BeNil()) + Expect(rc.NodePrometheusTLS).To(BeNil()) }) }) -func newPrep(variant operatorv1.ProductVariant) operator.InstallationPrep { +func newOpts(variant operatorv1.ProductVariant) []extensions.RenderContextOption { scheme := runtime.NewScheme() Expect(apis.AddToScheme(scheme, false)).NotTo(HaveOccurred()) c := ctrlrfake.DefaultFakeClientBuilder(scheme).Build() @@ -71,15 +68,13 @@ func newPrep(variant operatorv1.ProductVariant) operator.InstallationPrep { Expect(err).NotTo(HaveOccurred()) trustedBundle := certManager.CreateTrustedBundle() - return operator.InstallationPrep{ - Ctx: context.Background(), - Client: c, - Installation: &operatorv1.InstallationSpec{ - Variant: variant, - }, - FelixConfiguration: &v3.FelixConfiguration{}, - CertificateManager: certManager, - TrustedBundle: trustedBundle, - ClusterDomain: "cluster.local", + return []extensions.RenderContextOption{ + extensions.WithContext(context.Background()), + extensions.WithClient(c), + extensions.WithInstallation(&operatorv1.InstallationSpec{Variant: variant}), + extensions.WithFelixConfiguration(&v3.FelixConfiguration{}), + extensions.WithCertificateManager(certManager), + extensions.WithTrustedBundle(trustedBundle), + extensions.WithClusterDomain("cluster.local"), } } diff --git a/pkg/enterprise/node.go b/pkg/enterprise/node.go index f1ea2bcce2..9bf15ece57 100644 --- a/pkg/enterprise/node.go +++ b/pkg/enterprise/node.go @@ -15,7 +15,12 @@ package enterprise import ( + "fmt" + + v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" client "sigs.k8s.io/controller-runtime/pkg/client" @@ -24,47 +29,167 @@ import ( "github.com/tigera/operator/pkg/common" "github.com/tigera/operator/pkg/components" "github.com/tigera/operator/pkg/controller/utils" - "github.com/tigera/operator/pkg/operator" + "github.com/tigera/operator/pkg/extensions" "github.com/tigera/operator/pkg/render" ) const ( + // defaultNodeReporterPort is the port calico/node reports Enterprise internal + // metrics on when FelixConfiguration does not override prometheusReporterPort. defaultNodeReporterPort = 9081 + + // defaultFelixMetricsPort is the Felix prometheus metrics port used when + // FelixConfiguration does not override prometheusMetricsPort. defaultFelixMetricsPort = 9091 + + installCNIContainerName = "install-cni" ) func registerNode() { - operator.OverrideImage(render.ComponentNameNode, func(in *operatorv1.InstallationSpec) (components.Component, bool) { + extensions.OverrideImage(render.ComponentNameNode, func(in *operatorv1.InstallationSpec) (components.Component, bool) { if !in.Variant.IsEnterprise() { return components.Component{}, false } return components.ComponentTigeraNode, true }) - operator.Patch(render.ComponentNameNode, patchNode) + extensions.Modify(render.ComponentNameNode, modifyNode) } -func patchNode(ctx operator.Context, objs []client.Object) []client.Object { +// modifyNode layers Calico Enterprise behavior onto the rendered calico/node +// objects: the extra RBAC rules, the node-metrics Service, and the Enterprise +// daemonset configuration (flow/DNS log env, prometheus reporter, BGP metrics +// readiness check, multi-interface mode, and the calico log volume). +func modifyNode(ctx extensions.RenderContext, objs []client.Object) []client.Object { if ctx.Installation == nil || !ctx.Installation.Variant.IsEnterprise() { return objs } + + if role, ok := extensions.FindObject[*rbacv1.ClusterRole](objs, render.CalicoNodeObjectName); ok { + role.Rules = append(role.Rules, nodeEnterpriseRules()...) + } + + // The Network resource is only available in Enterprise / Cloud at this time. + if role, ok := extensions.FindObject[*rbacv1.ClusterRole](objs, render.CalicoCNIPluginObjectName); ok { + role.Rules = append(role.Rules, rbacv1.PolicyRule{ + APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, + Resources: []string{"networks"}, + Verbs: []string{"get"}, + }) + } + + if ds, ok := extensions.FindObject[*appsv1.DaemonSet](objs, common.NodeDaemonSetName); ok { + modifyNodeDaemonSet(ctx, ds) + } + return append(objs, nodeMetricsService(ctx)) } -// nodeMetricsService builds the enterprise-only calico-node-metrics Service. -// Ports are derived from FelixConfiguration exactly as the installation controller does. -func nodeMetricsService(ctx operator.Context) *corev1.Service { - reporterPort := defaultNodeReporterPort - felixPort := defaultFelixMetricsPort - felixEnabled := false - if fc := ctx.FelixConfiguration; fc != nil { - if fc.Spec.PrometheusReporterPort != nil { - reporterPort = *fc.Spec.PrometheusReporterPort +// nodeEnterpriseRules are the additional cluster role rules calico/node needs in +// Calico Enterprise. +func nodeEnterpriseRules() []rbacv1.PolicyRule { + return []rbacv1.PolicyRule{ + { + // Calico Enterprise needs to be able to read additional resources. + APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, + Resources: []string{ + "bfdconfigurations", + "egressgatewaypolicies", + "externalnetworks", + "licensekeys", + "networks", + "packetcaptures", + "remoteclusterconfigurations", + }, + Verbs: []string{"get", "list", "watch"}, + }, + { + // Tigera Secure updates status for packet captures. + APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, + Resources: []string{ + "packetcaptures", + "packetcaptures/status", + }, + Verbs: []string{"update"}, + }, + } +} + +// modifyNodeDaemonSet applies the Enterprise-specific daemonset changes that the +// base render leaves out: the Enterprise felix env, multi-interface mode, and the +// BGP metrics readiness check. The calico log volume is mounted by the base +// render for both variants, so it is not handled here. +func modifyNodeDaemonSet(ctx extensions.RenderContext, ds *appsv1.DaemonSet) { + spec := &ds.Spec.Template.Spec + + multiInterfaceMode := multiInterfaceModeEnv(ctx.Installation) + + for i := range spec.InitContainers { + if spec.InitContainers[i].Name == installCNIContainerName && multiInterfaceMode != nil { + spec.InitContainers[i].Env = append(spec.InitContainers[i].Env, *multiInterfaceMode) + } + } + + for i := range spec.Containers { + c := &spec.Containers[i] + if c.Name != render.CalicoNodeObjectName { + continue } - if fc.Spec.PrometheusMetricsPort != nil { - felixPort = *fc.Spec.PrometheusMetricsPort + + c.Env = append(c.Env, nodeEnterpriseEnv(ctx)...) + + // Add the BGP metrics readiness check, but only when the base render kept + // the bird readiness check (i.e. BGP is in use and we're not on VPP). + if c.ReadinessProbe != nil && c.ReadinessProbe.Exec != nil && containsString(c.ReadinessProbe.Exec.Command, "--bird-ready") { + c.ReadinessProbe.Exec.Command = append(c.ReadinessProbe.Exec.Command, "--bgp-metrics-ready") } - felixEnabled = utils.IsFelixPrometheusMetricsEnabled(fc) } +} + +// nodeEnterpriseEnv is the Enterprise felix configuration added to the +// calico/node container. +func nodeEnterpriseEnv(ctx extensions.RenderContext) []corev1.EnvVar { + env := []corev1.EnvVar{ + {Name: "FELIX_PROMETHEUSREPORTERENABLED", Value: "true"}, + {Name: "FELIX_PROMETHEUSREPORTERPORT", Value: fmt.Sprintf("%d", nodeReporterPort(ctx.FelixConfiguration))}, + {Name: "FELIX_FLOWLOGSFILEENABLED", Value: "true"}, + {Name: "FELIX_FLOWLOGSFILEINCLUDELABELS", Value: "true"}, + {Name: "FELIX_FLOWLOGSFILEINCLUDEPOLICIES", Value: "true"}, + {Name: "FELIX_FLOWLOGSFILEINCLUDESERVICE", Value: "true"}, + {Name: "FELIX_FLOWLOGSENABLENETWORKSETS", Value: "true"}, + {Name: "FELIX_FLOWLOGSCOLLECTPROCESSINFO", Value: "true"}, + {Name: "FELIX_DNSLOGSFILEENABLED", Value: "true"}, + {Name: "FELIX_DNSLOGSFILEPERNODELIMIT", Value: "1000"}, + } + + if mode := multiInterfaceModeEnv(ctx.Installation); mode != nil { + env = append(env, *mode) + } + + if ctx.NodePrometheusTLS != nil && ctx.TrustedBundle != nil { + env = append(env, + corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERCERTFILE", Value: ctx.NodePrometheusTLS.VolumeMountCertificateFilePath()}, + corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERKEYFILE", Value: ctx.NodePrometheusTLS.VolumeMountKeyFilePath()}, + corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERCAFILE", Value: ctx.TrustedBundle.MountPath()}, + ) + } + + return env +} + +// multiInterfaceModeEnv returns the MULTI_INTERFACE_MODE env var when the +// installation configures it, or nil otherwise. +func multiInterfaceModeEnv(install *operatorv1.InstallationSpec) *corev1.EnvVar { + if install.CalicoNetwork != nil && install.CalicoNetwork.MultiInterfaceMode != nil { + return &corev1.EnvVar{Name: "MULTI_INTERFACE_MODE", Value: install.CalicoNetwork.MultiInterfaceMode.Value()} + } + return nil +} + +// nodeMetricsService builds the enterprise-only calico-node-metrics Service. +func nodeMetricsService(ctx extensions.RenderContext) *corev1.Service { + reporterPort := nodeReporterPort(ctx.FelixConfiguration) + felixPort := felixMetricsPort(ctx.FelixConfiguration) + felixEnabled := ctx.FelixConfiguration != nil && utils.IsFelixPrometheusMetricsEnabled(ctx.FelixConfiguration) ports := []corev1.ServicePort{ { @@ -103,3 +228,31 @@ func nodeMetricsService(ctx operator.Context) *corev1.Service { }, } } + +// nodeReporterPort returns the reporter metrics port from the FelixConfiguration, +// falling back to the default. The node-metrics Service and the +// FELIX_PROMETHEUSREPORTERPORT env var both derive from here so they can't drift. +func nodeReporterPort(fc *v3.FelixConfiguration) int { + if fc != nil && fc.Spec.PrometheusReporterPort != nil { + return *fc.Spec.PrometheusReporterPort + } + return defaultNodeReporterPort +} + +// felixMetricsPort returns the Felix prometheus metrics port from the +// FelixConfiguration, falling back to the default. +func felixMetricsPort(fc *v3.FelixConfiguration) int { + if fc != nil && fc.Spec.PrometheusMetricsPort != nil { + return *fc.Spec.PrometheusMetricsPort + } + return defaultFelixMetricsPort +} + +func containsString(s []string, v string) bool { + for _, x := range s { + if x == v { + return true + } + } + return false +} diff --git a/pkg/enterprise/node_test.go b/pkg/enterprise/node_test.go index d2370fda6e..2ec7e53b38 100644 --- a/pkg/enterprise/node_test.go +++ b/pkg/enterprise/node_test.go @@ -18,78 +18,178 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" client "sigs.k8s.io/controller-runtime/pkg/client" v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/common" "github.com/tigera/operator/pkg/components" "github.com/tigera/operator/pkg/enterprise" - "github.com/tigera/operator/pkg/operator" + "github.com/tigera/operator/pkg/extensions" "github.com/tigera/operator/pkg/render" ) var _ = Describe("node enterprise image override", func() { BeforeEach(func() { enterprise.Register() }) - AfterEach(func() { - operator.ResetForTest() - operator.ResetExtensionsForTest() - }) + AfterEach(func() { extensions.ResetForTest() }) It("selects the enterprise node image for the enterprise variant", func() { ent := &operatorv1.InstallationSpec{Variant: operatorv1.CalicoEnterprise} - Expect(operator.ResolveImage("node", components.ComponentCalicoNode, ent)).To(Equal(components.ComponentTigeraNode)) + Expect(extensions.ResolveImage("node", components.ComponentCalicoNode, ent)).To(Equal(components.ComponentTigeraNode)) }) It("leaves the default in place for the Calico variant", func() { - oss := &operatorv1.InstallationSpec{Variant: operatorv1.Calico} - Expect(operator.ResolveImage("node", components.ComponentCalicoNode, oss)).To(Equal(components.ComponentCalicoNode)) + calico := &operatorv1.InstallationSpec{Variant: operatorv1.Calico} + Expect(extensions.ResolveImage("node", components.ComponentCalicoNode, calico)).To(Equal(components.ComponentCalicoNode)) }) }) -var _ = Describe("node metrics service modifier", func() { +var _ = Describe("node enterprise modifier", func() { BeforeEach(func() { enterprise.Register() }) - AfterEach(func() { - operator.ResetForTest() - operator.ResetExtensionsForTest() + AfterEach(func() { extensions.ResetForTest() }) + + // newObjs returns the subset of rendered node objects the modifier touches. + newObjs := func() []client.Object { + return []client.Object{ + &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: render.CalicoNodeObjectName}}, + &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: render.CalicoCNIPluginObjectName}}, + &appsv1.DaemonSet{ + ObjectMeta: metav1.ObjectMeta{Name: common.NodeDaemonSetName}, + Spec: appsv1.DaemonSetSpec{Template: corev1.PodTemplateSpec{Spec: corev1.PodSpec{ + InitContainers: []corev1.Container{{Name: "install-cni"}}, + Containers: []corev1.Container{{ + Name: render.CalicoNodeObjectName, + ReadinessProbe: &corev1.Probe{ProbeHandler: corev1.ProbeHandler{Exec: &corev1.ExecAction{ + Command: []string{"/bin/calico-node", "-bird-ready", "--bird-ready", "--felix-ready"}, + }}}, + }}, + }}}, + }, + } + } + + nodeContainer := func(ds *appsv1.DaemonSet) *corev1.Container { + for i := range ds.Spec.Template.Spec.Containers { + if ds.Spec.Template.Spec.Containers[i].Name == render.CalicoNodeObjectName { + return &ds.Spec.Template.Spec.Containers[i] + } + } + return nil + } + + entCtx := func() extensions.RenderContext { + return extensions.RenderContext{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.CalicoEnterprise}} + } + + It("adds the enterprise cluster role rules", func() { + out := extensions.ApplyModifiers(render.ComponentNameNode, entCtx(), newObjs()) + + nodeRole, ok := extensions.FindObject[*rbacv1.ClusterRole](out, render.CalicoNodeObjectName) + Expect(ok).To(BeTrue()) + Expect(nodeRole.Rules).To(ContainElement(HaveField("Resources", ContainElement("licensekeys")))) + + cniRole, ok := extensions.FindObject[*rbacv1.ClusterRole](out, render.CalicoCNIPluginObjectName) + Expect(ok).To(BeTrue()) + Expect(cniRole.Rules).To(ContainElement(HaveField("Resources", ConsistOf("networks")))) + }) + + It("adds the enterprise felix env to the node container", func() { + out := extensions.ApplyModifiers(render.ComponentNameNode, entCtx(), newObjs()) + ds, _ := extensions.FindObject[*appsv1.DaemonSet](out, common.NodeDaemonSetName) + c := nodeContainer(ds) + + Expect(c.Env).To(ContainElements( + corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERENABLED", Value: "true"}, + corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERPORT", Value: "9081"}, + corev1.EnvVar{Name: "FELIX_FLOWLOGSFILEENABLED", Value: "true"}, + corev1.EnvVar{Name: "FELIX_DNSLOGSFILEENABLED", Value: "true"}, + )) + }) + + It("derives the reporter port from FelixConfiguration", func() { + reporter := 7081 + ctx := entCtx() + ctx.FelixConfiguration = &v3.FelixConfiguration{Spec: v3.FelixConfigurationSpec{PrometheusReporterPort: &reporter}} + + out := extensions.ApplyModifiers(render.ComponentNameNode, ctx, newObjs()) + ds, _ := extensions.FindObject[*appsv1.DaemonSet](out, common.NodeDaemonSetName) + Expect(nodeContainer(ds).Env).To(ContainElement(corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERPORT", Value: "7081"})) + }) + + It("appends the BGP metrics readiness check when the bird check is present", func() { + out := extensions.ApplyModifiers(render.ComponentNameNode, entCtx(), newObjs()) + ds, _ := extensions.FindObject[*appsv1.DaemonSet](out, common.NodeDaemonSetName) + Expect(nodeContainer(ds).ReadinessProbe.Exec.Command).To(ContainElement("--bgp-metrics-ready")) }) - It("appends the node metrics service for the enterprise variant", func() { - ctx := operator.Context{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.CalicoEnterprise}} - out := operator.ApplyPatches(render.ComponentNameNode, ctx, []client.Object{}) - svc, ok := operator.FindObject[*corev1.Service](out, render.CalicoNodeMetricsService) + It("does not add the BGP metrics readiness check when the bird check is absent", func() { + objs := newObjs() + ds := objs[2].(*appsv1.DaemonSet) + ds.Spec.Template.Spec.Containers[0].ReadinessProbe.Exec.Command = []string{"/bin/calico-node", "--felix-ready"} + + out := extensions.ApplyModifiers(render.ComponentNameNode, entCtx(), objs) + got, _ := extensions.FindObject[*appsv1.DaemonSet](out, common.NodeDaemonSetName) + Expect(nodeContainer(got).ReadinessProbe.Exec.Command).NotTo(ContainElement("--bgp-metrics-ready")) + }) + + It("adds MULTI_INTERFACE_MODE to the node and install-cni containers when configured", func() { + mode := operatorv1.MultiInterfaceModeMultus + ctx := entCtx() + ctx.Installation.CalicoNetwork = &operatorv1.CalicoNetworkSpec{MultiInterfaceMode: &mode} + + out := extensions.ApplyModifiers(render.ComponentNameNode, ctx, newObjs()) + ds, _ := extensions.FindObject[*appsv1.DaemonSet](out, common.NodeDaemonSetName) + + want := corev1.EnvVar{Name: "MULTI_INTERFACE_MODE", Value: mode.Value()} + Expect(nodeContainer(ds).Env).To(ContainElement(want)) + Expect(ds.Spec.Template.Spec.InitContainers[0].Env).To(ContainElement(want)) + }) + + It("appends the node metrics service", func() { + out := extensions.ApplyModifiers(render.ComponentNameNode, entCtx(), newObjs()) + svc, ok := extensions.FindObject[*corev1.Service](out, render.CalicoNodeMetricsService) Expect(ok).To(BeTrue()) - // default ports when FelixConfiguration is nil: 9081 + 9900, felix-metrics-port absent Expect(svc.Spec.Ports).To(HaveLen(2)) Expect(svc.Spec.Ports[0].Port).To(Equal(int32(9081))) Expect(svc.Spec.Ports[1].Port).To(Equal(int32(9900))) }) - It("derives ports and felix-metrics-port from FelixConfiguration", func() { + It("derives metrics service ports and felix-metrics-port from FelixConfiguration", func() { reporter := 7081 metrics := 7091 enabled := true - ctx := operator.Context{ - Installation: &operatorv1.InstallationSpec{Variant: operatorv1.CalicoEnterprise}, - FelixConfiguration: &v3.FelixConfiguration{Spec: v3.FelixConfigurationSpec{ - PrometheusReporterPort: &reporter, - PrometheusMetricsPort: &metrics, - PrometheusMetricsEnabled: &enabled, - }}, - } - out := operator.ApplyPatches(render.ComponentNameNode, ctx, []client.Object{}) - svc, ok := operator.FindObject[*corev1.Service](out, render.CalicoNodeMetricsService) - Expect(ok).To(BeTrue()) + ctx := entCtx() + ctx.FelixConfiguration = &v3.FelixConfiguration{Spec: v3.FelixConfigurationSpec{ + PrometheusReporterPort: &reporter, + PrometheusMetricsPort: &metrics, + PrometheusMetricsEnabled: &enabled, + }} + + out := extensions.ApplyModifiers(render.ComponentNameNode, ctx, newObjs()) + svc, _ := extensions.FindObject[*corev1.Service](out, render.CalicoNodeMetricsService) Expect(svc.Spec.Ports).To(HaveLen(3)) Expect(svc.Spec.Ports[0].Port).To(Equal(int32(7081))) Expect(svc.Spec.Ports[2].Name).To(Equal("felix-metrics-port")) Expect(svc.Spec.Ports[2].Port).To(Equal(int32(7091))) }) - It("does not append it for the Calico variant", func() { - ctx := operator.Context{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.Calico}} - out := operator.ApplyPatches(render.ComponentNameNode, ctx, []client.Object{}) - _, ok := operator.FindObject[*corev1.Service](out, render.CalicoNodeMetricsService) + It("is a no-op for the Calico variant", func() { + ctx := extensions.RenderContext{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.Calico}} + out := extensions.ApplyModifiers(render.ComponentNameNode, ctx, newObjs()) + + _, ok := extensions.FindObject[*corev1.Service](out, render.CalicoNodeMetricsService) + Expect(ok).To(BeFalse()) + nodeRole, _ := extensions.FindObject[*rbacv1.ClusterRole](out, render.CalicoNodeObjectName) + Expect(nodeRole.Rules).To(BeEmpty()) + }) + + It("does not panic on a zero RenderContext", func() { + out := extensions.ApplyModifiers(render.ComponentNameNode, extensions.RenderContext{}, newObjs()) + _, ok := extensions.FindObject[*corev1.Service](out, render.CalicoNodeMetricsService) Expect(ok).To(BeFalse()) }) }) diff --git a/pkg/enterprise/typha.go b/pkg/enterprise/typha.go index 8deef34b28..5a2471a5fd 100644 --- a/pkg/enterprise/typha.go +++ b/pkg/enterprise/typha.go @@ -20,20 +20,20 @@ import ( rbacv1 "k8s.io/api/rbac/v1" "sigs.k8s.io/controller-runtime/pkg/client" - "github.com/tigera/operator/pkg/operator" + "github.com/tigera/operator/pkg/extensions" "github.com/tigera/operator/pkg/render" ) func registerTypha() { - operator.Patch(render.ComponentNameTypha, patchTypha) + extensions.Modify(render.ComponentNameTypha, modifyTypha) } -func patchTypha(ctx operator.Context, objs []client.Object) []client.Object { +func modifyTypha(ctx extensions.RenderContext, objs []client.Object) []client.Object { if ctx.Installation == nil || !ctx.Installation.Variant.IsEnterprise() { return objs } - if role, ok := operator.FindObject[*rbacv1.ClusterRole](objs, "calico-typha"); ok { + if role, ok := extensions.FindObject[*rbacv1.ClusterRole](objs, "calico-typha"); ok { role.Rules = append(role.Rules, rbacv1.PolicyRule{ APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, Resources: []string{ @@ -50,7 +50,7 @@ func patchTypha(ctx operator.Context, objs []client.Object) []client.Object { }) } - if dep, ok := operator.FindObject[*appsv1.Deployment](objs, "calico-typha"); ok { + if dep, ok := extensions.FindObject[*appsv1.Deployment](objs, "calico-typha"); ok { net := ctx.Installation.CalicoNetwork if net != nil && net.MultiInterfaceMode != nil { for i := range dep.Spec.Template.Spec.Containers { diff --git a/pkg/enterprise/typha_test.go b/pkg/enterprise/typha_test.go index 9b41d448fa..13fcccc2f9 100644 --- a/pkg/enterprise/typha_test.go +++ b/pkg/enterprise/typha_test.go @@ -25,15 +25,14 @@ import ( operatorv1 "github.com/tigera/operator/api/v1" "github.com/tigera/operator/pkg/enterprise" - "github.com/tigera/operator/pkg/operator" + "github.com/tigera/operator/pkg/extensions" "github.com/tigera/operator/pkg/render" ) var _ = Describe("typha enterprise modifier", func() { BeforeEach(func() { enterprise.Register() }) AfterEach(func() { - operator.ResetForTest() - operator.ResetExtensionsForTest() + extensions.ResetForTest() }) multiMode := operatorv1.MultiInterfaceModeMultus @@ -51,11 +50,11 @@ var _ = Describe("typha enterprise modifier", func() { } It("adds enterprise RBAC and MULTI_INTERFACE_MODE for the enterprise variant", func() { - ctx := operator.Context{Installation: &operatorv1.InstallationSpec{ + ctx := extensions.RenderContext{Installation: &operatorv1.InstallationSpec{ Variant: operatorv1.CalicoEnterprise, CalicoNetwork: &operatorv1.CalicoNetworkSpec{MultiInterfaceMode: &multiMode}, }} - out := operator.ApplyPatches(render.ComponentNameTypha, ctx, newObjs()) + out := extensions.ApplyModifiers(render.ComponentNameTypha, ctx, newObjs()) role := out[0].(*rbacv1.ClusterRole) Expect(role.Rules).To(ContainElement(HaveField("Resources", ContainElement("licensekeys")))) @@ -71,18 +70,18 @@ var _ = Describe("typha enterprise modifier", func() { }) It("is a no-op for the Calico variant", func() { - ctx := operator.Context{Installation: &operatorv1.InstallationSpec{ + ctx := extensions.RenderContext{Installation: &operatorv1.InstallationSpec{ Variant: operatorv1.Calico, CalicoNetwork: &operatorv1.CalicoNetworkSpec{MultiInterfaceMode: &multiMode}, }} - out := operator.ApplyPatches(render.ComponentNameTypha, ctx, newObjs()) + out := extensions.ApplyModifiers(render.ComponentNameTypha, ctx, newObjs()) Expect(out[0].(*rbacv1.ClusterRole).Rules).To(BeEmpty()) dep := out[1].(*appsv1.Deployment) Expect(dep.Spec.Template.Spec.Containers[0].Env).To(BeEmpty()) }) It("does not panic on a zero Context (nil Installation)", func() { - out := operator.ApplyPatches(render.ComponentNameTypha, operator.Context{}, newObjs()) + out := extensions.ApplyModifiers(render.ComponentNameTypha, extensions.RenderContext{}, newObjs()) Expect(out[0].(*rbacv1.ClusterRole).Rules).To(BeEmpty()) }) }) diff --git a/pkg/operator/operator_suite_test.go b/pkg/extensions/extensions_suite_test.go similarity index 88% rename from pkg/operator/operator_suite_test.go rename to pkg/extensions/extensions_suite_test.go index 42d3e32f63..791eedb737 100644 --- a/pkg/operator/operator_suite_test.go +++ b/pkg/extensions/extensions_suite_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package operator_test +package extensions_test import ( "testing" @@ -21,7 +21,7 @@ import ( . "github.com/onsi/gomega" ) -func TestOperator(t *testing.T) { +func TestExtensions(t *testing.T) { RegisterFailHandler(Fail) - RunSpecs(t, "pkg/operator Suite") + RunSpecs(t, "pkg/extensions Suite") } diff --git a/pkg/extensions/factory.go b/pkg/extensions/factory.go new file mode 100644 index 0000000000..f6081635ca --- /dev/null +++ b/pkg/extensions/factory.go @@ -0,0 +1,127 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package extensions + +import ( + "context" + + v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" + "sigs.k8s.io/controller-runtime/pkg/client" + + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/controller/certificatemanager" + "github.com/tigera/operator/pkg/tls/certificatemanagement" +) + +// Inputs is the reconcile state a RenderContextFactory builds a RenderContext +// from. The installation controller populates it through the With* options. It +// carries both the values that flow straight into the RenderContext and the +// side-effecting dependencies (Client, CertificateManager) a factory needs to +// produce controller-side artifacts. +type Inputs struct { + Ctx context.Context + Client client.Client + Installation *operatorv1.InstallationSpec + FelixConfiguration *v3.FelixConfiguration + CertificateManager certificatemanager.CertificateManager + TrustedBundle certificatemanagement.TrustedBundle + ClusterDomain string +} + +// RenderContextOption sets a field on the Inputs a factory builds from. +type RenderContextOption func(*Inputs) + +func WithContext(ctx context.Context) RenderContextOption { + return func(in *Inputs) { in.Ctx = ctx } +} + +func WithClient(c client.Client) RenderContextOption { + return func(in *Inputs) { in.Client = c } +} + +func WithInstallation(i *operatorv1.InstallationSpec) RenderContextOption { + return func(in *Inputs) { in.Installation = i } +} + +func WithFelixConfiguration(fc *v3.FelixConfiguration) RenderContextOption { + return func(in *Inputs) { in.FelixConfiguration = fc } +} + +func WithCertificateManager(cm certificatemanager.CertificateManager) RenderContextOption { + return func(in *Inputs) { in.CertificateManager = cm } +} + +func WithTrustedBundle(tb certificatemanagement.TrustedBundle) RenderContextOption { + return func(in *Inputs) { in.TrustedBundle = tb } +} + +func WithClusterDomain(d string) RenderContextOption { + return func(in *Inputs) { in.ClusterDomain = d } +} + +// RenderContextFactory builds the RenderContext handed to render modifiers. New +// applies the options, performs any controller-side work (creating +// certificates, extending the trusted bundle), and returns the assembled +// RenderContext - or an error that aborts the reconcile. +// +// This is the generic seam controllers use to extend base operator behavior; +// its first consumer is Calico Enterprise, but nothing here is enterprise +// specific. The core operator default does no side-effecting work. +type RenderContextFactory interface { + New(opts ...RenderContextOption) (RenderContext, error) +} + +// ApplyInputs returns the Inputs produced by applying opts. Factories use it to +// collect the option-supplied state before doing their work. +func ApplyInputs(opts ...RenderContextOption) Inputs { + var in Inputs + for _, o := range opts { + o(&in) + } + return in +} + +// BaseRenderContext maps the generically-gathered inputs onto a RenderContext. +// The default factory and every registered factory build on it, so the base +// fields are assembled in exactly one place. A factory layers its side-effect +// artifacts (e.g. NodePrometheusTLS) on top of the returned value. +func BaseRenderContext(in Inputs) RenderContext { + return RenderContext{ + Installation: in.Installation, + FelixConfiguration: in.FelixConfiguration, + ClusterDomain: in.ClusterDomain, + TrustedBundle: in.TrustedBundle, + } +} + +// defaultFactory is the core operator's RenderContextFactory. It does no +// side-effecting work and returns just the base RenderContext. +type defaultFactory struct{} + +func (defaultFactory) New(opts ...RenderContextOption) (RenderContext, error) { + return BaseRenderContext(ApplyInputs(opts...)), nil +} + +var renderContextFactory RenderContextFactory = defaultFactory{} + +// RegisterRenderContextFactory installs f as the factory the installation +// controller uses. Registration replaces any prior factory, so it is safe to +// call more than once. Without a registered factory the core operator default +// applies. +func RegisterRenderContextFactory(f RenderContextFactory) { renderContextFactory = f } + +// GetRenderContextFactory returns the registered factory, or the core operator +// default when none is registered. +func GetRenderContextFactory() RenderContextFactory { return renderContextFactory } diff --git a/pkg/extensions/factory_test.go b/pkg/extensions/factory_test.go new file mode 100644 index 0000000000..408704b7ad --- /dev/null +++ b/pkg/extensions/factory_test.go @@ -0,0 +1,73 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package extensions_test + +import ( + "errors" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/extensions" +) + +var _ = Describe("render context factory", func() { + AfterEach(func() { extensions.ResetForTest() }) + + It("returns the base render context from the default factory", func() { + install := &operatorv1.InstallationSpec{Variant: operatorv1.Calico} + rc, err := extensions.GetRenderContextFactory().New( + extensions.WithInstallation(install), + extensions.WithClusterDomain("cluster.local"), + ) + Expect(err).NotTo(HaveOccurred()) + Expect(rc.Installation).To(BeIdenticalTo(install)) + Expect(rc.ClusterDomain).To(Equal("cluster.local")) + Expect(rc.NodePrometheusTLS).To(BeNil()) + }) + + It("uses a registered factory in place of the default", func() { + extensions.RegisterRenderContextFactory(&fakeFactory{}) + rc, err := extensions.GetRenderContextFactory().New() + Expect(err).NotTo(HaveOccurred()) + Expect(rc.ClusterDomain).To(Equal("from-fake")) + }) + + It("surfaces the factory error", func() { + extensions.RegisterRenderContextFactory(&fakeFactory{err: errors.New("boom")}) + _, err := extensions.GetRenderContextFactory().New() + Expect(err).To(MatchError("boom")) + }) + + It("restores the default factory on reset", func() { + extensions.RegisterRenderContextFactory(&fakeFactory{}) + extensions.ResetForTest() + rc, err := extensions.GetRenderContextFactory().New(extensions.WithClusterDomain("real")) + Expect(err).NotTo(HaveOccurred()) + Expect(rc.ClusterDomain).To(Equal("real")) + }) +}) + +type fakeFactory struct { + err error +} + +func (f *fakeFactory) New(_ ...extensions.RenderContextOption) (extensions.RenderContext, error) { + if f.err != nil { + return extensions.RenderContext{}, f.err + } + return extensions.RenderContext{ClusterDomain: "from-fake"}, nil +} diff --git a/pkg/operator/image.go b/pkg/extensions/image.go similarity index 98% rename from pkg/operator/image.go rename to pkg/extensions/image.go index c6de3b917a..776a02bffd 100644 --- a/pkg/operator/image.go +++ b/pkg/extensions/image.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package operator +package extensions import ( operatorv1 "github.com/tigera/operator/api/v1" diff --git a/pkg/operator/image_test.go b/pkg/extensions/image_test.go similarity index 69% rename from pkg/operator/image_test.go rename to pkg/extensions/image_test.go index b972665e17..5a382d6c36 100644 --- a/pkg/operator/image_test.go +++ b/pkg/extensions/image_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package operator_test +package extensions_test import ( . "github.com/onsi/ginkgo/v2" @@ -20,16 +20,16 @@ import ( operatorv1 "github.com/tigera/operator/api/v1" "github.com/tigera/operator/pkg/components" - "github.com/tigera/operator/pkg/operator" + "github.com/tigera/operator/pkg/extensions" ) var _ = Describe("image overrides", func() { AfterEach(func() { - operator.ResetForTest() + extensions.ResetForTest() }) It("uses the override when one matches", func() { - operator.OverrideImage("node", func(in *operatorv1.InstallationSpec) (components.Component, bool) { + extensions.OverrideImage("node", func(in *operatorv1.InstallationSpec) (components.Component, bool) { if !in.Variant.IsEnterprise() { return components.Component{}, false } @@ -37,11 +37,11 @@ var _ = Describe("image overrides", func() { }) ent := &operatorv1.InstallationSpec{Variant: operatorv1.CalicoEnterprise} - Expect(operator.ResolveImage("node", components.ComponentCalicoNode, ent)).To(Equal(components.ComponentTigeraNode)) + Expect(extensions.ResolveImage("node", components.ComponentCalicoNode, ent)).To(Equal(components.ComponentTigeraNode)) }) It("falls back to the default when no override matches", func() { - oss := &operatorv1.InstallationSpec{Variant: operatorv1.Calico} - Expect(operator.ResolveImage("node", components.ComponentCalicoNode, oss)).To(Equal(components.ComponentCalicoNode)) + calico := &operatorv1.InstallationSpec{Variant: operatorv1.Calico} + Expect(extensions.ResolveImage("node", components.ComponentCalicoNode, calico)).To(Equal(components.ComponentCalicoNode)) }) }) diff --git a/pkg/operator/patch.go b/pkg/extensions/modifier.go similarity index 50% rename from pkg/operator/patch.go rename to pkg/extensions/modifier.go index bce2ff783c..5b911edc72 100644 --- a/pkg/operator/patch.go +++ b/pkg/extensions/modifier.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package operator +package extensions import ( "sigs.k8s.io/controller-runtime/pkg/client" @@ -20,22 +20,26 @@ import ( "github.com/tigera/operator/pkg/imageoverride" ) -// PatchFunc post-processes the objects a render component produced. It may -// mutate matched objects and/or append additional objects, and must return the +// Modifier post-processes the objects a render component produced. It may mutate +// matched objects and/or append additional objects, and must return the // (possibly extended) slice. Implementations self-gate on ctx.Installation.Variant. -type PatchFunc func(ctx Context, objs []client.Object) []client.Object +type Modifier func(ctx RenderContext, objs []client.Object) []client.Object -var patches = map[string][]PatchFunc{} +var modifiers = map[string]Modifier{} -// Patch registers fn to run against the named component's objects. Multiple -// patches may be registered for the same component; they run in registration order. -func Patch(component string, fn PatchFunc) { - patches[component] = append(patches[component], fn) +// Modify registers fn as the modifier for the named component. A component has +// at most one modifier; the modifier handles all of that component's +// extension-specific mutations. Registration replaces any prior modifier, so it +// is idempotent and safe to call more than once - matching the image-override +// registry rather than stacking duplicate work. +func Modify(component string, fn Modifier) { + modifiers[component] = fn } -// ApplyPatches runs every patch registered for the named component over objs. -func ApplyPatches(component string, ctx Context, objs []client.Object) []client.Object { - for _, fn := range patches[component] { +// ApplyModifiers runs the modifier registered for the named component over objs, +// returning objs unchanged when none is registered. +func ApplyModifiers(component string, ctx RenderContext, objs []client.Object) []client.Object { + if fn, ok := modifiers[component]; ok { objs = fn(ctx, objs) } return objs @@ -52,8 +56,10 @@ func FindObject[T client.Object](objs []client.Object, name string) (T, bool) { return zero, false } -// ResetForTest clears all registries. Test-only. +// ResetForTest clears every registry: modifiers, image overrides, and the +// render context factory. Test-only. func ResetForTest() { - patches = map[string][]PatchFunc{} + modifiers = map[string]Modifier{} + renderContextFactory = defaultFactory{} imageoverride.ResetForTest() } diff --git a/pkg/operator/patch_test.go b/pkg/extensions/modifier_test.go similarity index 53% rename from pkg/operator/patch_test.go rename to pkg/extensions/modifier_test.go index d9cd4de108..7b40beb740 100644 --- a/pkg/operator/patch_test.go +++ b/pkg/extensions/modifier_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package operator_test +package extensions_test import ( . "github.com/onsi/ginkgo/v2" @@ -21,24 +21,24 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" - "github.com/tigera/operator/pkg/operator" + "github.com/tigera/operator/pkg/extensions" ) -var _ = Describe("patch registry", func() { +var _ = Describe("modifier registry", func() { AfterEach(func() { - operator.ResetForTest() + extensions.ResetForTest() }) - It("applies a registered patch to the matching component", func() { - operator.Patch("test", func(ctx operator.Context, objs []client.Object) []client.Object { - cm, ok := operator.FindObject[*corev1.ConfigMap](objs, "cm") + It("applies a registered modifier to the matching component", func() { + extensions.Modify("test", func(ctx extensions.RenderContext, objs []client.Object) []client.Object { + cm, ok := extensions.FindObject[*corev1.ConfigMap](objs, "cm") Expect(ok).To(BeTrue()) cm.Data = map[string]string{"k": "v"} return append(objs, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "extra"}}) }) in := []client.Object{&corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "cm"}}} - out := operator.ApplyPatches("test", operator.Context{}, in) + out := extensions.ApplyModifiers("test", extensions.RenderContext{}, in) Expect(out).To(HaveLen(2)) cm := out[0].(*corev1.ConfigMap) @@ -46,9 +46,23 @@ var _ = Describe("patch registry", func() { Expect(out[1].GetName()).To(Equal("extra")) }) - It("returns objects unchanged when no patch is registered", func() { + It("returns objects unchanged when no modifier is registered", func() { in := []client.Object{&corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "cm"}}} - out := operator.ApplyPatches("unregistered", operator.Context{}, in) + out := extensions.ApplyModifiers("unregistered", extensions.RenderContext{}, in) Expect(out).To(Equal(in)) }) + + It("replaces rather than stacks when a component is registered twice", func() { + add := func(name string) extensions.Modifier { + return func(_ extensions.RenderContext, objs []client.Object) []client.Object { + return append(objs, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: name}}) + } + } + extensions.Modify("test", add("first")) + extensions.Modify("test", add("second")) + + out := extensions.ApplyModifiers("test", extensions.RenderContext{}, nil) + Expect(out).To(HaveLen(1)) + Expect(out[0].GetName()).To(Equal("second")) + }) }) diff --git a/pkg/operator/context.go b/pkg/extensions/rendercontext.go similarity index 70% rename from pkg/operator/context.go rename to pkg/extensions/rendercontext.go index cb84bbd052..52d0d753c5 100644 --- a/pkg/operator/context.go +++ b/pkg/extensions/rendercontext.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package operator +package extensions import ( v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" @@ -20,14 +20,15 @@ import ( "github.com/tigera/operator/pkg/tls/certificatemanagement" ) -// Context carries reconcile-derived inputs from controllers into render -// modifiers. OSS code never reads these fields - only registered modifiers do. +// RenderContext carries reconcile-derived inputs from controllers into render +// modifiers. Core operator code never reads these fields - only registered +// modifiers do. // Two kinds of value live here: // - raw cluster state gathered generically (Installation, FelixConfiguration, // ClusterDomain) that modifiers derive their own values from, and // - controller-produced artifacts (TrustedBundle, NodePrometheusTLS) that can // only be created controller-side because they have cluster side effects. -type Context struct { +type RenderContext struct { Installation *operatorv1.InstallationSpec FelixConfiguration *v3.FelixConfiguration ClusterDomain string @@ -35,7 +36,10 @@ type Context struct { // TrustedBundle is the shared CA bundle for the calico-system namespace. TrustedBundle certificatemanagement.TrustedBundle - // NodePrometheusTLS is produced by the installation controller's enterprise - // extension and mounted by the node modifier. + // NodePrometheusTLS is created by the enterprise RenderContextFactory (it has + // cluster side effects, so it can't be built in a modifier). Two consumers + // read it: the installation controller passes it to the node configuration + // (PrometheusServerTLS) so the keypair is mounted, and the node modifier uses + // it to set the FELIX_PROMETHEUSREPORTER* certificate env vars. NodePrometheusTLS certificatemanagement.KeyPairInterface } diff --git a/pkg/operator/extension.go b/pkg/operator/extension.go deleted file mode 100644 index de5080c45d..0000000000 --- a/pkg/operator/extension.go +++ /dev/null @@ -1,59 +0,0 @@ -// Copyright (c) 2026 Tigera, Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package operator - -import ( - "context" - - v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" - operatorv1 "github.com/tigera/operator/api/v1" - "github.com/tigera/operator/pkg/controller/certificatemanager" - "github.com/tigera/operator/pkg/tls/certificatemanagement" - "sigs.k8s.io/controller-runtime/pkg/client" -) - -// InstallationPrep is the input to an InstallationExtension's Prepare. It holds -// the generically-gathered reconcile state the extension needs to do its -// side-effecting work (create certs, assemble the trusted bundle). -type InstallationPrep struct { - Ctx context.Context - Client client.Client - Installation *operatorv1.InstallationSpec - FelixConfiguration *v3.FelixConfiguration - CertificateManager certificatemanager.CertificateManager - TrustedBundle certificatemanagement.TrustedBundle - ClusterDomain string -} - -// InstallationExtension is the enterprise hook for the installation controller. -// Prepare runs controller-side before rendering. It performs work modifiers -// can't (cluster side effects, fetching/creating certificates) and may abort -// the reconcile by returning an error. It returns the Context handed to the -// render patches; on the Calico variant it should return an empty Context and -// nil error. -type InstallationExtension interface { - Prepare(p InstallationPrep) (Context, error) -} - -var installationExtension InstallationExtension - -// RegisterInstallationExtension registers the installation controller extension. -func RegisterInstallationExtension(e InstallationExtension) { installationExtension = e } - -// GetInstallationExtension returns the registered extension, or nil. -func GetInstallationExtension() InstallationExtension { return installationExtension } - -// ResetExtensionsForTest clears registered extensions. Test-only. -func ResetExtensionsForTest() { installationExtension = nil } diff --git a/pkg/operator/extension_test.go b/pkg/operator/extension_test.go deleted file mode 100644 index e3ecd95592..0000000000 --- a/pkg/operator/extension_test.go +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright (c) 2026 Tigera, Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package operator_test - -import ( - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - - "github.com/tigera/operator/pkg/operator" -) - -var _ = Describe("controller extensions", func() { - AfterEach(func() { operator.ResetExtensionsForTest() }) - - It("returns the registered installation extension", func() { - ext := &fakeInstallationExtension{} - operator.RegisterInstallationExtension(ext) - Expect(operator.GetInstallationExtension()).To(BeIdenticalTo(ext)) - }) - - It("returns nil when none is registered", func() { - Expect(operator.GetInstallationExtension()).To(BeNil()) - }) -}) - -type fakeInstallationExtension struct{} - -func (f *fakeInstallationExtension) Prepare(_ operator.InstallationPrep) (operator.Context, error) { - return operator.Context{}, nil -} diff --git a/pkg/render/component.go b/pkg/render/component.go index 585355438d..0fa4f5da8b 100644 --- a/pkg/render/component.go +++ b/pkg/render/component.go @@ -40,17 +40,20 @@ type Component interface { SupportedOSType() rmeta.OSType } -// Named is implemented by components that expose enterprise extension points. -// The componentHandler uses Name() to look up registered patches. Components -// without enterprise extensions need not implement it. -type Named interface { +// Extensible is implemented by components that expose extension points. The +// componentHandler uses Name() to look up registered modifiers. Components +// without extensions need not implement it. +// +// Note this interface is structural: any component that grows a Name() string +// method for an unrelated reason becomes modifier-eligible. There are no name +// collisions today, but keep it in mind when adding a Name() method. +type Extensible interface { Name() string } -// Component names used as keys into the operator patch registry. Keep these in -// sync with the Name() methods that return them. +// Component names used as keys into the extension modifier registry. Keep these +// in sync with the Name() methods that return them. const ( - ComponentNameTypha = "typha" - ComponentNameNode = "node" - ComponentNameAPIServer = "apiserver" + ComponentNameTypha = "typha" + ComponentNameNode = "node" ) diff --git a/pkg/render/enterprise_setup_test.go b/pkg/render/enterprise_setup_test.go index 5f464c950b..13766f7035 100644 --- a/pkg/render/enterprise_setup_test.go +++ b/pkg/render/enterprise_setup_test.go @@ -20,10 +20,15 @@ import ( "github.com/tigera/operator/pkg/enterprise" ) -// The render suite asserts enterprise-variant output for components whose -// variant-specific behavior now lives in registered modifiers/overrides -// (e.g. the node image). Register them once so the suite exercises the same -// integrated behavior the operator binary produces. +// Register the enterprise extensions once for the whole render suite. This wires +// two things the suite relies on: +// - the image override, which the Objects()-level render tests pick up through +// ResolveImages (e.g. the enterprise node image), and +// - the modifiers, which node_enterprise_test.go applies explicitly to real +// render output to check they still match it. +// +// The plain Objects()-level tests do not run modifiers - those only run at the +// componentHandler - so registering here does not change their output. var _ = BeforeSuite(func() { enterprise.Register() }) diff --git a/pkg/render/node.go b/pkg/render/node.go index 6bf69d2f23..0e42fe3091 100644 --- a/pkg/render/node.go +++ b/pkg/render/node.go @@ -116,11 +116,10 @@ type NodeConfiguration struct { GoldmaneIP string // Optional fields. - LogCollector *operatorv1.LogCollector - MigrateNamespaces bool - NodeAppArmorProfile string - BirdTemplates map[string]string - NodeReporterMetricsPort int + LogCollector *operatorv1.LogCollector + MigrateNamespaces bool + NodeAppArmorProfile string + BirdTemplates map[string]string // CanRemoveCNIFinalizer specifies whether CNI plugin is still needed during uninstall since the CNI plugin and // associated RBAC resources are required for pod teardown to succeed. Setting this to true removes @@ -147,10 +146,6 @@ type NodeConfiguration struct { // should this value change. BindMode string - FelixPrometheusMetricsEnabled bool - - FelixPrometheusMetricsPort int - V3CRDs bool } @@ -554,34 +549,6 @@ func (c *nodeComponent) nodeRole() *rbacv1.ClusterRole { }, }, } - if c.cfg.Installation.Variant.IsEnterprise() { - extraRules := []rbacv1.PolicyRule{ - { - // Calico Enterprise needs to be able to read additional resources. - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{ - "bfdconfigurations", - "egressgatewaypolicies", - "externalnetworks", - "licensekeys", - "networks", - "packetcaptures", - "remoteclusterconfigurations", - }, - Verbs: []string{"get", "list", "watch"}, - }, - { - // Tigera Secure updates status for packet captures. - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{ - "packetcaptures", - "packetcaptures/status", - }, - Verbs: []string{"update"}, - }, - } - role.Rules = append(role.Rules, extraRules...) - } if c.cfg.Installation.KubernetesProvider.IsOpenShift() { role.Rules = append(role.Rules, rbacv1.PolicyRule{ APIGroups: []string{"security.openshift.io"}, @@ -643,14 +610,6 @@ func (c *nodeComponent) cniPluginRole() *rbacv1.ClusterRole { }, }, } - if c.cfg.Installation.Variant.IsEnterprise() { - // The Network resource is only available in Enterprise / Cloud at this time. - role.Rules = append(role.Rules, rbacv1.PolicyRule{ - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{"networks"}, - Verbs: []string{"get"}, - }) - } return role } @@ -1094,13 +1053,16 @@ func (c *nodeComponent) nodeVolumes() []corev1.Volume { c.cfg.TLS.TrustedBundle.Volume(), c.cfg.TLS.NodeSecret.Volume(), c.varRunCalicoVolume(), - corev1.Volume{Name: "var-lib-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/lib/calico", Type: &dirOrCreate}}}, + {Name: "var-lib-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/lib/calico", Type: &dirOrCreate}}}, + // The Calico log directory. The CNI plugin logs to the cni/ subdirectory of + // this, and Felix writes its flow/DNS logs here on the enterprise variant. + {Name: "var-log-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico", Type: &dirOrCreate}}}, // Volume for the containing directory so that the init container can mount the child bpf directory if needed. - corev1.Volume{Name: "sys-fs", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/sys/fs", Type: &dirOrCreate}}}, + {Name: "sys-fs", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/sys/fs", Type: &dirOrCreate}}}, // Volume for the bpffs itself, used by the main node container. - corev1.Volume{Name: "bpffs", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/sys/fs/bpf", Type: &dirMustExist}}}, + {Name: "bpffs", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/sys/fs/bpf", Type: &dirMustExist}}}, // Volume used by mount-cgroupv2 init container to access root cgroup name space of node. - corev1.Volume{Name: "nodeproc", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/proc"}}}, + {Name: "nodeproc", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/proc"}}}, } if c.vppDataplaneEnabled() { @@ -1114,17 +1076,6 @@ func (c *nodeComponent) nodeVolumes() []corev1.Volume { if c.cfg.Installation.CNI.Type == operatorv1.PluginCalico { volumes = append(volumes, corev1.Volume{Name: "cni-bin-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: *c.cfg.Installation.CNI.BinDir, Type: &dirOrCreate}}}) volumes = append(volumes, corev1.Volume{Name: "cni-net-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: *c.cfg.Installation.CNI.ConfDir}}}) - volumes = append(volumes, corev1.Volume{Name: "cni-log-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico/cni"}}}) - } - - // Override with Tigera-specific config. - if c.cfg.Installation.Variant.IsEnterprise() { - // Add volume for calico logs. - calicoLogVol := corev1.Volume{ - Name: "var-log-calico", - VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico", Type: &dirOrCreate}}, - } - volumes = append(volumes, calicoLogVol) } // Create and append flexvolume @@ -1326,12 +1277,6 @@ func (c *nodeComponent) cniEnvvars() []corev1.EnvVar { envVars = append(envVars, c.cfg.K8sServiceEp.EnvVars()...) - if c.cfg.Installation.Variant.IsEnterprise() { - if c.cfg.Installation.CalicoNetwork != nil && c.cfg.Installation.CalicoNetwork.MultiInterfaceMode != nil { - envVars = append(envVars, corev1.EnvVar{Name: "MULTI_INTERFACE_MODE", Value: c.cfg.Installation.CalicoNetwork.MultiInterfaceMode.Value()}) - } - } - return envVars } @@ -1375,15 +1320,9 @@ func (c *nodeComponent) nodeVolumeMounts() []corev1.VolumeMount { if c.vppDataplaneEnabled() { nodeVolumeMounts = append(nodeVolumeMounts, corev1.VolumeMount{MountPath: "/usr/local/bin/felix-plugins", Name: "felix-plugins", ReadOnly: true}) } - if c.cfg.Installation.Variant.IsEnterprise() { - extraNodeMounts := []corev1.VolumeMount{ - {MountPath: "/var/log/calico", Name: "var-log-calico"}, - } - nodeVolumeMounts = append(nodeVolumeMounts, extraNodeMounts...) - } else if c.cfg.Installation.CNI.Type == operatorv1.PluginCalico { - cniLogMount := corev1.VolumeMount{MountPath: "/var/log/calico/cni", Name: "cni-log-dir", ReadOnly: false} - nodeVolumeMounts = append(nodeVolumeMounts, cniLogMount) - } + // Mount the Calico log directory. The CNI plugin writes to the cni/ subdirectory + // and, on the enterprise variant, Felix writes its flow/DNS logs here too. + nodeVolumeMounts = append(nodeVolumeMounts, corev1.VolumeMount{MountPath: "/var/log/calico", Name: "var-log-calico"}) if c.cfg.Installation.CNI.Type == operatorv1.PluginCalico { nodeVolumeMounts = append(nodeVolumeMounts, corev1.VolumeMount{MountPath: "/host/etc/cni/net.d", Name: "cni-net-dir"}) @@ -1629,35 +1568,6 @@ func (c *nodeComponent) nodeEnvVars() []corev1.EnvVar { nodeEnv = append(nodeEnv, corev1.EnvVar{Name: "FELIX_IPV6SUPPORT", Value: "false"}) } - if c.cfg.Installation.Variant.IsEnterprise() { - // Add in Calico Enterprise specific configuration. - extraNodeEnv := []corev1.EnvVar{ - {Name: "FELIX_PROMETHEUSREPORTERENABLED", Value: "true"}, - {Name: "FELIX_PROMETHEUSREPORTERPORT", Value: fmt.Sprintf("%d", c.cfg.NodeReporterMetricsPort)}, - {Name: "FELIX_FLOWLOGSFILEENABLED", Value: "true"}, - {Name: "FELIX_FLOWLOGSFILEINCLUDELABELS", Value: "true"}, - {Name: "FELIX_FLOWLOGSFILEINCLUDEPOLICIES", Value: "true"}, - {Name: "FELIX_FLOWLOGSFILEINCLUDESERVICE", Value: "true"}, - {Name: "FELIX_FLOWLOGSENABLENETWORKSETS", Value: "true"}, - {Name: "FELIX_FLOWLOGSCOLLECTPROCESSINFO", Value: "true"}, - {Name: "FELIX_DNSLOGSFILEENABLED", Value: "true"}, - {Name: "FELIX_DNSLOGSFILEPERNODELIMIT", Value: "1000"}, - } - - if c.cfg.Installation.CalicoNetwork != nil && c.cfg.Installation.CalicoNetwork.MultiInterfaceMode != nil { - extraNodeEnv = append(extraNodeEnv, corev1.EnvVar{Name: "MULTI_INTERFACE_MODE", Value: c.cfg.Installation.CalicoNetwork.MultiInterfaceMode.Value()}) - } - - if c.cfg.PrometheusServerTLS != nil { - extraNodeEnv = append(extraNodeEnv, - corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERCERTFILE", Value: c.cfg.PrometheusServerTLS.VolumeMountCertificateFilePath()}, - corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERKEYFILE", Value: c.cfg.PrometheusServerTLS.VolumeMountKeyFilePath()}, - corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERCAFILE", Value: c.cfg.TLS.TrustedBundle.MountPath()}, - ) - } - nodeEnv = append(nodeEnv, extraNodeEnv...) - } - if c.cfg.Installation.NodeMetricsPort != nil { // If a node metrics port was given, then enable felix prometheus metrics and set the port. // Note that this takes precedence over any FelixConfiguration resources in the cluster. @@ -1736,10 +1646,7 @@ func (c *nodeComponent) nodeLivenessReadinessProbes() (*corev1.Probe, *corev1.Pr var readinessCmd []string readinessCmd = []string{components.CalicoBinaryPath, "component", "node", "health", "--bird-ready", "--felix-ready"} - if c.cfg.Installation.Variant.IsEnterprise() { - readinessCmd = append(readinessCmd, "--bgp-metrics-ready") - } - // If not using BGP or using VPP, don't check bird status (or bgp metrics server for enterprise). + // If not using BGP or using VPP, don't check bird status. if !bgpEnabled(c.cfg.Installation) || c.vppDataplaneEnabled() { readinessCmd = []string{components.CalicoBinaryPath, "component", "node", "health", "--felix-ready"} } diff --git a/pkg/render/node_enterprise_test.go b/pkg/render/node_enterprise_test.go new file mode 100644 index 0000000000..4cc73d2a70 --- /dev/null +++ b/pkg/render/node_enterprise_test.go @@ -0,0 +1,172 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package render_test + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/apis" + "github.com/tigera/operator/pkg/common" + "github.com/tigera/operator/pkg/controller/certificatemanager" + "github.com/tigera/operator/pkg/controller/k8sapi" + ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" + "github.com/tigera/operator/pkg/dns" + "github.com/tigera/operator/pkg/extensions" + "github.com/tigera/operator/pkg/render" +) + +// These tests run the real node/typha render output through the registered +// enterprise modifiers. The render suite registers the enterprise extensions in +// its BeforeSuite, so this exercises the same integrated behavior the operator +// binary produces - and, importantly, catches a modifier whose FindObject stops +// matching because render renamed an object or container. +var _ = Describe("node enterprise modifier integration", func() { + var ( + cli client.Client + certManager certificatemanager.CertificateManager + typhaNodeTLS *render.TyphaNodeTLS + instance *operatorv1.InstallationSpec + renderCtx extensions.RenderContext + ) + + nodeContainer := func(ds *appsv1.DaemonSet) *corev1.Container { + for i := range ds.Spec.Template.Spec.Containers { + if ds.Spec.Template.Spec.Containers[i].Name == render.CalicoNodeObjectName { + return &ds.Spec.Template.Spec.Containers[i] + } + } + return nil + } + + BeforeEach(func() { + scheme := runtime.NewScheme() + Expect(apis.AddToScheme(scheme, false)).NotTo(HaveOccurred()) + cli = ctrlrfake.DefaultFakeClientBuilder(scheme).Build() + + var err error + certManager, err = certificatemanager.Create(cli, nil, "", common.OperatorNamespace(), certificatemanager.AllowCACreation()) + Expect(err).NotTo(HaveOccurred()) + typhaNodeTLS = getTyphaNodeTLS(cli, certManager) + + nodePrometheusTLS, err := certManager.GetOrCreateKeyPair(cli, render.NodePrometheusTLSServerSecret, common.OperatorNamespace(), []string{"calico-node-metrics"}) + Expect(err).NotTo(HaveOccurred()) + typhaNodeTLS.TrustedBundle.AddCertificates(nodePrometheusTLS) + + confDir, binDir := render.DefaultCNIDirectories(operatorv1.ProviderNone) + bgp := operatorv1.BGPEnabled + instance = &operatorv1.InstallationSpec{ + Variant: operatorv1.CalicoEnterprise, + CNI: &operatorv1.CNISpec{ + Type: operatorv1.PluginCalico, + IPAM: &operatorv1.IPAMSpec{Type: operatorv1.IPAMPluginCalico}, + BinDir: &binDir, + ConfDir: &confDir, + }, + CalicoNetwork: &operatorv1.CalicoNetworkSpec{ + BGP: &bgp, + IPPools: []operatorv1.IPPool{{CIDR: "192.168.1.0/16"}}, + }, + } + + renderCtx = extensions.RenderContext{ + Installation: instance, + TrustedBundle: typhaNodeTLS.TrustedBundle, + NodePrometheusTLS: nodePrometheusTLS, + } + }) + + // renderNodeObjects renders the real node component and applies the registered + // modifier, exactly as the componentHandler does. + renderNodeObjects := func() []client.Object { + cfg := &render.NodeConfiguration{ + K8sServiceEp: k8sapi.ServiceEndpoint{}, + Installation: instance, + TLS: typhaNodeTLS, + ClusterDomain: dns.DefaultClusterDomain, + FelixHealthPort: 9099, + IPPools: instance.CalicoNetwork.IPPools, + PrometheusServerTLS: renderCtx.NodePrometheusTLS, + } + comp := render.Node(cfg) + Expect(comp.ResolveImages(nil)).NotTo(HaveOccurred()) + objs, _ := comp.Objects() + return extensions.ApplyModifiers(render.ComponentNameNode, renderCtx, objs) + } + + It("appends the node metrics service to the real render output", func() { + objs := renderNodeObjects() + svc, ok := extensions.FindObject[*corev1.Service](objs, render.CalicoNodeMetricsService) + Expect(ok).To(BeTrue(), "expected the modifier to append %s", render.CalicoNodeMetricsService) + Expect(svc.Namespace).To(Equal(common.CalicoNamespace)) + }) + + It("adds the enterprise rules to the real cluster roles", func() { + objs := renderNodeObjects() + + nodeRole, ok := extensions.FindObject[*rbacv1.ClusterRole](objs, render.CalicoNodeObjectName) + Expect(ok).To(BeTrue()) + Expect(nodeRole.Rules).To(ContainElement(HaveField("Resources", ContainElement("licensekeys")))) + + cniRole, ok := extensions.FindObject[*rbacv1.ClusterRole](objs, render.CalicoCNIPluginObjectName) + Expect(ok).To(BeTrue()) + Expect(cniRole.Rules).To(ContainElement(HaveField("Resources", ContainElement("networks")))) + }) + + It("rewrites the real node daemonset for enterprise", func() { + objs := renderNodeObjects() + ds, ok := extensions.FindObject[*appsv1.DaemonSet](objs, common.NodeDaemonSetName) + Expect(ok).To(BeTrue()) + + c := nodeContainer(ds) + Expect(c).NotTo(BeNil()) + + Expect(c.Env).To(ContainElements( + corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERENABLED", Value: "true"}, + corev1.EnvVar{Name: "FELIX_FLOWLOGSFILEENABLED", Value: "true"}, + )) + // The reporter cert env is wired from the NodePrometheusTLS keypair the + // factory creates. + Expect(c.Env).To(ContainElement(HaveField("Name", "FELIX_PROMETHEUSREPORTERCERTFILE"))) + + // BGP is enabled, so the bird readiness check is present and the modifier + // adds the BGP metrics check. + Expect(c.ReadinessProbe.Exec.Command).To(ContainElement("--bgp-metrics-ready")) + }) + + It("adds the enterprise rules to the real typha cluster role", func() { + comp := render.Typha(&render.TyphaConfiguration{ + K8sServiceEp: k8sapi.ServiceEndpoint{}, + Installation: instance, + TLS: typhaNodeTLS, + ClusterDomain: dns.DefaultClusterDomain, + FelixHealthPort: 9099, + }) + Expect(comp.ResolveImages(nil)).NotTo(HaveOccurred()) + objs, _ := comp.Objects() + objs = extensions.ApplyModifiers(render.ComponentNameTypha, renderCtx, objs) + + role, ok := extensions.FindObject[*rbacv1.ClusterRole](objs, "calico-typha") + Expect(ok).To(BeTrue()) + Expect(role.Rules).To(ContainElement(HaveField("Resources", ContainElement("licensekeys")))) + }) +}) diff --git a/pkg/render/node_test.go b/pkg/render/node_test.go index 697feda36d..c534759543 100644 --- a/pkg/render/node_test.go +++ b/pkg/render/node_test.go @@ -134,14 +134,12 @@ var _ = Describe("Node rendering tests", func() { // Create a default configuration. cfg = render.NodeConfiguration{ - K8sServiceEp: k8sServiceEp, - Installation: defaultInstance, - TLS: typhaNodeTLS, - ClusterDomain: defaultClusterDomain, - FelixHealthPort: 9099, - IPPools: defaultInstance.CalicoNetwork.IPPools, - FelixPrometheusMetricsEnabled: false, - FelixPrometheusMetricsPort: 9098, + K8sServiceEp: k8sServiceEp, + Installation: defaultInstance, + TLS: typhaNodeTLS, + ClusterDomain: defaultClusterDomain, + FelixHealthPort: 9099, + IPPools: defaultInstance.CalicoNetwork.IPPools, } }) @@ -321,7 +319,7 @@ var _ = Describe("Node rendering tests", func() { {Name: "xtables-lock", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/run/xtables.lock", Type: &fileOrCreate}}}, {Name: "cni-bin-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/opt/cni/bin", Type: &dirOrCreate}}}, {Name: "cni-net-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/etc/cni/net.d"}}}, - {Name: "cni-log-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico/cni"}}}, + {Name: "var-log-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico", Type: &dirOrCreate}}}, {Name: "policysync", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/run/nodeagent", Type: &dirOrCreate}}}, {Name: "sys-fs", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/sys/fs", Type: &dirOrCreate}}}, {Name: "bpffs", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/sys/fs/bpf", Type: &dirMustExist}}}, @@ -359,7 +357,7 @@ var _ = Describe("Node rendering tests", func() { {MountPath: "/var/run/nodeagent", Name: "policysync"}, {MountPath: "/etc/pki/tls/certs", Name: "tigera-ca-bundle", ReadOnly: true}, {MountPath: "/node-certs", Name: render.NodeTLSSecretName, ReadOnly: true}, - {MountPath: "/var/log/calico/cni", Name: "cni-log-dir", ReadOnly: false}, + {MountPath: "/var/log/calico", Name: "var-log-calico"}, {MountPath: "/sys/fs/bpf", Name: "bpffs"}, } Expect(ds.Spec.Template.Spec.Containers[0].VolumeMounts).To(ConsistOf(expectedNodeVolumeMounts)) @@ -367,7 +365,7 @@ var _ = Describe("Node rendering tests", func() { // Verify tolerations. Expect(ds.Spec.Template.Spec.Tolerations).To(ConsistOf(rmeta.TolerateAll)) - verifyProbesAndLifecycle(ds, false, false) + verifyProbesAndLifecycle(ds, false) }) It("should render node correctly for BPF dataplane", func() { @@ -512,7 +510,7 @@ var _ = Describe("Node rendering tests", func() { {Name: "xtables-lock", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/run/xtables.lock", Type: &fileOrCreate}}}, {Name: "cni-bin-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/opt/cni/bin", Type: &dirOrCreate}}}, {Name: "cni-net-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/etc/cni/net.d"}}}, - {Name: "cni-log-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico/cni"}}}, + {Name: "var-log-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico", Type: &dirOrCreate}}}, {Name: "sys-fs", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/sys/fs", Type: &dirOrCreate}}}, {Name: "bpffs", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/sys/fs/bpf", Type: &dirMustExist}}}, {Name: "nodeproc", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/proc"}}}, @@ -550,7 +548,7 @@ var _ = Describe("Node rendering tests", func() { {MountPath: "/var/run/nodeagent", Name: "policysync"}, {MountPath: "/etc/pki/tls/certs", Name: "tigera-ca-bundle", ReadOnly: true}, {MountPath: "/node-certs", Name: render.NodeTLSSecretName, ReadOnly: true}, - {MountPath: "/var/log/calico/cni", Name: "cni-log-dir", ReadOnly: false}, + {MountPath: "/var/log/calico", Name: "var-log-calico"}, {MountPath: "/sys/fs/bpf", Name: "bpffs"}, } Expect(ds.Spec.Template.Spec.Containers[0].VolumeMounts).To(ConsistOf(expectedNodeVolumeMounts)) @@ -558,7 +556,7 @@ var _ = Describe("Node rendering tests", func() { // Verify tolerations. Expect(ds.Spec.Template.Spec.Tolerations).To(ConsistOf(rmeta.TolerateAll)) - verifyProbesAndLifecycle(ds, false, false) + verifyProbesAndLifecycle(ds, false) }) It("should properly render an explicitly configured MTU", func() { @@ -637,103 +635,6 @@ var _ = Describe("Node rendering tests", func() { } }) - It("should render all resources for a default configuration using CalicoEnterprise", func() { - expectedResources := []struct { - name string - ns string - group string - version string - kind string - }{ - {name: "calico-node", ns: common.CalicoNamespace, group: "", version: "v1", kind: "ServiceAccount"}, - {name: "calico-node", ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRole"}, - {name: "calico-node", ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRoleBinding"}, - {name: "calico-cni-plugin", ns: common.CalicoNamespace, group: "", version: "v1", kind: "ServiceAccount"}, - {name: "calico-cni-plugin", ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRole"}, - {name: "calico-cni-plugin", ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRoleBinding"}, - {name: "cni-config", ns: common.CalicoNamespace, group: "", version: "v1", kind: "ConfigMap"}, - {name: common.NodeDaemonSetName, ns: common.CalicoNamespace, group: "apps", version: "v1", kind: "DaemonSet"}, - } - defaultInstance.Variant = operatorv1.CalicoEnterprise - cfg.NodeReporterMetricsPort = 9081 - - component := render.Node(&cfg) - Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() - Expect(len(resources)).To(Equal(len(expectedResources))) - - // Should render the correct resources. - i := 0 - for _, expectedRes := range expectedResources { - rtest.ExpectResourceTypeAndObjectMetadata(resources[i], expectedRes.name, expectedRes.ns, expectedRes.group, expectedRes.version, expectedRes.kind) - i++ - } - - // The DaemonSet should have the correct configuration. - ds := rtest.GetResource(resources, "calico-node", "calico-system", "apps", "v1", "DaemonSet").(*appsv1.DaemonSet) - - // The pod template should have node critical priority - Expect(ds.Spec.Template.Spec.PriorityClassName).To(Equal(render.NodePriorityClassName)) - Expect(ds.Spec.Template.Spec.Containers[0].Image).To(Equal(components.TigeraRegistry + "tigera/node:" + components.ComponentTigeraNode.Version)) - verifyInitContainers(ds, defaultInstance) - - expectedNodeEnv := []corev1.EnvVar{ - // Default envvars. - {Name: "DATASTORE_TYPE", Value: "kubernetes"}, - {Name: "WAIT_FOR_DATASTORE", Value: "true"}, - {Name: "CALICO_MANAGE_CNI", Value: "true"}, - {Name: "CALICO_NETWORKING_BACKEND", Value: "bird"}, - {Name: "CLUSTER_TYPE", Value: "k8s,operator,bgp"}, - {Name: "CALICO_DISABLE_FILE_LOGGING", Value: "false"}, - {Name: "FELIX_DEFAULTENDPOINTTOHOSTACTION", Value: "ACCEPT"}, - {Name: "FELIX_HEALTHENABLED", Value: "true"}, - {Name: "FELIX_HEALTHPORT", Value: "9099"}, - { - Name: "NODENAME", - ValueFrom: &corev1.EnvVarSource{ - FieldRef: &corev1.ObjectFieldSelector{FieldPath: "spec.nodeName"}, - }, - }, - { - Name: "NAMESPACE", - ValueFrom: &corev1.EnvVarSource{ - FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.namespace"}, - }, - }, - {Name: "FELIX_TYPHAK8SNAMESPACE", Value: "calico-system"}, - {Name: "FELIX_TYPHAK8SSERVICENAME", Value: "calico-typha"}, - {Name: "FELIX_TYPHACAFILE", Value: certificatemanagement.TrustedCertBundleMountPath}, - {Name: "FELIX_TYPHACERTFILE", Value: "/node-certs/tls.crt"}, - {Name: "FELIX_TYPHACN", Value: "typha-server"}, - {Name: "FELIX_TYPHAKEYFILE", Value: "/node-certs/tls.key"}, - // Tigera-specific envvars - {Name: "FELIX_PROMETHEUSREPORTERENABLED", Value: "true"}, - {Name: "FELIX_PROMETHEUSREPORTERPORT", Value: "9081"}, - {Name: "FELIX_FLOWLOGSFILEENABLED", Value: "true"}, - {Name: "FELIX_FLOWLOGSFILEINCLUDELABELS", Value: "true"}, - {Name: "FELIX_FLOWLOGSFILEINCLUDEPOLICIES", Value: "true"}, - {Name: "FELIX_FLOWLOGSFILEINCLUDESERVICE", Value: "true"}, - {Name: "FELIX_FLOWLOGSENABLENETWORKSETS", Value: "true"}, - {Name: "FELIX_FLOWLOGSCOLLECTPROCESSINFO", Value: "true"}, - {Name: "FELIX_DNSLOGSFILEENABLED", Value: "true"}, - {Name: "FELIX_DNSLOGSFILEPERNODELIMIT", Value: "1000"}, - {Name: "MULTI_INTERFACE_MODE", Value: operatorv1.MultiInterfaceModeNone.Value()}, - {Name: "NO_DEFAULT_POOLS", Value: "true"}, - } - expectedNodeEnv = configureExpectedNodeEnvIPVersions(expectedNodeEnv, defaultInstance, enableIPv4, enableIPv6) - Expect(ds.Spec.Template.Spec.Containers[0].Env).To(ConsistOf(expectedNodeEnv)) - Expect(len(ds.Spec.Template.Spec.Containers[0].Env)).To(Equal(len(expectedNodeEnv))) - - dirMustExist := corev1.HostPathDirectory - bpfVol := corev1.Volume{Name: "bpffs", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/sys/fs/bpf", Type: &dirMustExist}}} - Expect(ds.Spec.Template.Spec.Volumes).To(ContainElement(bpfVol)) - - bpfVolMount := corev1.VolumeMount{MountPath: "/sys/fs/bpf", Name: "bpffs"} - Expect(ds.Spec.Template.Spec.Containers[0].VolumeMounts).To(ContainElement(bpfVolMount)) - - verifyProbesAndLifecycle(ds, false, true) - }) - It("should render all resources when using Calico CNI on EKS", func() { expectedResources := []struct { name string @@ -871,7 +772,7 @@ var _ = Describe("Node rendering tests", func() { {Name: "xtables-lock", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/run/xtables.lock", Type: &fileOrCreate}}}, {Name: "cni-bin-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/opt/cni/bin", Type: &dirOrCreate}}}, {Name: "cni-net-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/etc/cni/net.d"}}}, - {Name: "cni-log-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico/cni"}}}, + {Name: "var-log-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico", Type: &dirOrCreate}}}, {Name: "policysync", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/run/nodeagent", Type: &dirOrCreate}}}, {Name: "sys-fs", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/sys/fs", Type: &dirOrCreate}}}, {Name: "bpffs", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/sys/fs/bpf", Type: &dirMustExist}}}, @@ -909,7 +810,7 @@ var _ = Describe("Node rendering tests", func() { {MountPath: "/var/run/nodeagent", Name: "policysync"}, {MountPath: "/etc/pki/tls/certs", Name: "tigera-ca-bundle", ReadOnly: true}, {MountPath: "/node-certs", Name: render.NodeTLSSecretName, ReadOnly: true}, - {MountPath: "/var/log/calico/cni", Name: "cni-log-dir", ReadOnly: false}, + {MountPath: "/var/log/calico", Name: "var-log-calico"}, {MountPath: "/sys/fs/bpf", Name: "bpffs"}, } Expect(ds.Spec.Template.Spec.Containers[0].VolumeMounts).To(ConsistOf(expectedNodeVolumeMounts)) @@ -919,7 +820,7 @@ var _ = Describe("Node rendering tests", func() { // Verify readiness and liveness probes. - verifyProbesAndLifecycle(ds, false, false) + verifyProbesAndLifecycle(ds, false) }) It("should properly render a configuration using the AmazonVPC CNI plugin", func() { @@ -1008,6 +909,7 @@ var _ = Describe("Node rendering tests", func() { {Name: "lib-modules", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/lib/modules"}}}, {Name: "var-run-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/run/calico", Type: &dirOrCreate}}}, {Name: "var-lib-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/lib/calico", Type: &dirOrCreate}}}, + {Name: "var-log-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico", Type: &dirOrCreate}}}, {Name: "xtables-lock", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/run/xtables.lock", Type: &fileOrCreate}}}, {Name: "policysync", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/run/nodeagent", Type: &dirOrCreate}}}, {Name: "sys-fs", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/sys/fs", Type: &dirOrCreate}}}, @@ -1042,6 +944,7 @@ var _ = Describe("Node rendering tests", func() { {MountPath: "/run/xtables.lock", Name: "xtables-lock"}, {MountPath: "/var/run/calico", Name: "var-run-calico"}, {MountPath: "/var/lib/calico", Name: "var-lib-calico"}, + {MountPath: "/var/log/calico", Name: "var-log-calico"}, {MountPath: "/var/run/nodeagent", Name: "policysync"}, {MountPath: "/etc/pki/tls/certs", Name: "tigera-ca-bundle", ReadOnly: true}, {MountPath: "/node-certs", Name: render.NodeTLSSecretName, ReadOnly: true}, @@ -1053,7 +956,7 @@ var _ = Describe("Node rendering tests", func() { Expect(ds.Spec.Template.Spec.Tolerations).To(ConsistOf(rmeta.TolerateAll)) // Verify readiness and liveness probes. - verifyProbesAndLifecycle(ds, false, false) + verifyProbesAndLifecycle(ds, false) }) It("should return customized CNI directories when specified", func() { @@ -1073,7 +976,7 @@ var _ = Describe("Node rendering tests", func() { expectedVols := []corev1.Volume{ {Name: "cni-bin-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/custom/cni/bin", Type: &dirOrCreate}}}, {Name: "cni-net-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/custom/cni/net.d"}}}, - {Name: "cni-log-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico/cni"}}}, + {Name: "var-log-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico", Type: &dirOrCreate}}}, } Expect(ds.Spec.Template.Spec.Volumes).To(ContainElements(expectedVols)) verifyInitContainers(ds, cfg.Installation) @@ -1121,7 +1024,7 @@ var _ = Describe("Node rendering tests", func() { } // Verify readiness and liveness probes. - verifyProbesAndLifecycle(ds, false, false) + verifyProbesAndLifecycle(ds, false) }, Entry("GKE", operatorv1.PluginGKE, operatorv1.IPAMPluginHostLocal, []corev1.EnvVar{ {Name: "FELIX_INTERFACEPREFIX", Value: "gke"}, @@ -1273,7 +1176,7 @@ var _ = Describe("Node rendering tests", func() { {Name: "xtables-lock", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/run/xtables.lock", Type: &fileOrCreate}}}, {Name: "cni-bin-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/opt/cni/bin", Type: &dirOrCreate}}}, {Name: "cni-net-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/etc/cni/net.d"}}}, - {Name: "cni-log-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico/cni"}}}, + {Name: "var-log-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico", Type: &dirOrCreate}}}, {Name: "policysync", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/run/nodeagent", Type: &dirOrCreate}}}, {Name: "sys-fs", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/sys/fs", Type: &dirOrCreate}}}, {Name: "bpffs", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/sys/fs/bpf", Type: &dirMustExist}}}, @@ -1311,7 +1214,7 @@ var _ = Describe("Node rendering tests", func() { {MountPath: "/var/run/nodeagent", Name: "policysync"}, {MountPath: "/etc/pki/tls/certs", Name: "tigera-ca-bundle", ReadOnly: true}, {MountPath: "/node-certs", Name: render.NodeTLSSecretName, ReadOnly: true}, - {MountPath: "/var/log/calico/cni", Name: "cni-log-dir", ReadOnly: false}, + {MountPath: "/var/log/calico", Name: "var-log-calico"}, {MountPath: "/sys/fs/bpf", Name: "bpffs"}, } Expect(ds.Spec.Template.Spec.Containers[0].VolumeMounts).To(ConsistOf(expectedNodeVolumeMounts)) @@ -1320,7 +1223,7 @@ var _ = Describe("Node rendering tests", func() { Expect(ds.Spec.Template.Spec.Tolerations).To(ConsistOf(rmeta.TolerateAll)) // Verify readiness and liveness probes. - verifyProbesAndLifecycle(ds, false, false) + verifyProbesAndLifecycle(ds, false) }) It("should properly render a configuration using the AmazonVPC CNI plugin", func() { @@ -1407,6 +1310,7 @@ var _ = Describe("Node rendering tests", func() { {Name: "lib-modules", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/lib/modules"}}}, {Name: "var-run-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/run/calico", Type: &dirOrCreate}}}, {Name: "var-lib-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/lib/calico", Type: &dirOrCreate}}}, + {Name: "var-log-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico", Type: &dirOrCreate}}}, {Name: "xtables-lock", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/run/xtables.lock", Type: &fileOrCreate}}}, {Name: "policysync", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/run/nodeagent", Type: &dirOrCreate}}}, {Name: "sys-fs", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/sys/fs", Type: &dirOrCreate}}}, @@ -1441,6 +1345,7 @@ var _ = Describe("Node rendering tests", func() { {MountPath: "/run/xtables.lock", Name: "xtables-lock"}, {MountPath: "/var/run/calico", Name: "var-run-calico"}, {MountPath: "/var/lib/calico", Name: "var-lib-calico"}, + {MountPath: "/var/log/calico", Name: "var-log-calico"}, {MountPath: "/var/run/nodeagent", Name: "policysync"}, {MountPath: "/etc/pki/tls/certs", Name: "tigera-ca-bundle", ReadOnly: true}, {MountPath: "/node-certs", Name: render.NodeTLSSecretName, ReadOnly: true}, @@ -1452,7 +1357,7 @@ var _ = Describe("Node rendering tests", func() { Expect(ds.Spec.Template.Spec.Tolerations).To(ConsistOf(rmeta.TolerateAll)) // Verify readiness and liveness probes. - verifyProbesAndLifecycle(ds, false, false) + verifyProbesAndLifecycle(ds, false) }) It("should render all resources when running on openshift", func() { @@ -1519,7 +1424,7 @@ var _ = Describe("Node rendering tests", func() { {Name: "xtables-lock", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/run/xtables.lock", Type: &fileOrCreate}}}, {Name: "cni-bin-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/lib/cni/bin", Type: &dirOrCreate}}}, {Name: "cni-net-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/run/multus/cni/net.d"}}}, - {Name: "cni-log-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico/cni"}}}, + {Name: "var-log-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico", Type: &dirOrCreate}}}, {Name: "policysync", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/run/nodeagent", Type: &dirOrCreate}}}, {Name: "flexvol-driver-host", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/etc/kubernetes/kubelet-plugins/volume/exec/nodeagent~uds", Type: &dirOrCreate}}}, {Name: "sys-fs", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/sys/fs", Type: &dirOrCreate}}}, @@ -1582,209 +1487,7 @@ var _ = Describe("Node rendering tests", func() { Expect(ds.Spec.Template.Spec.Containers[0].Env).To(ConsistOf(expectedNodeEnv)) Expect(len(ds.Spec.Template.Spec.Containers[0].Env)).To(Equal(len(expectedNodeEnv))) - verifyProbesAndLifecycle(ds, true, false) - }) - - It("should render all resources when variant is CalicoEnterprise and running on openshift", func() { - expectedResources := []struct { - name string - ns string - group string - version string - kind string - }{ - {name: "calico-node", ns: common.CalicoNamespace, group: "", version: "v1", kind: "ServiceAccount"}, - {name: "calico-node", ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRole"}, - {name: "calico-node", ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRoleBinding"}, - {name: "calico-cni-plugin", ns: common.CalicoNamespace, group: "", version: "v1", kind: "ServiceAccount"}, - {name: "calico-cni-plugin", ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRole"}, - {name: "calico-cni-plugin", ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRoleBinding"}, - {name: "cni-config", ns: common.CalicoNamespace, group: "", version: "v1", kind: "ConfigMap"}, - {name: common.NodeDaemonSetName, ns: common.CalicoNamespace, group: "apps", version: "v1", kind: "DaemonSet"}, - } - - defaultInstance.Variant = operatorv1.CalicoEnterprise - defaultInstance.KubernetesProvider = operatorv1.ProviderOpenShift - defaultCNIConfDir, defaultCNIBinDir := render.DefaultCNIDirectories(defaultInstance.KubernetesProvider) - defaultInstance.CNI.ConfDir, defaultInstance.CNI.BinDir = &defaultCNIConfDir, &defaultCNIBinDir - cfg.NodeReporterMetricsPort = 9081 - cfg.FelixHealthPort = 9199 - - component := render.Node(&cfg) - Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() - Expect(len(resources)).To(Equal(len(expectedResources))) - - // Should render the correct resources. - i := 0 - for _, expectedRes := range expectedResources { - rtest.ExpectResourceTypeAndObjectMetadata(resources[i], expectedRes.name, expectedRes.ns, expectedRes.group, expectedRes.version, expectedRes.kind) - i++ - } - - // calico-node clusterRole should have openshift securitycontextconstraints PolicyRule - nodeRole := rtest.GetResource(resources, "calico-node", "", "rbac.authorization.k8s.io", "v1", "ClusterRole").(*rbacv1.ClusterRole) - Expect(nodeRole.Rules).To(ContainElement(rbacv1.PolicyRule{ - APIGroups: []string{"security.openshift.io"}, - Resources: []string{"securitycontextconstraints"}, - Verbs: []string{"use"}, - ResourceNames: []string{"privileged"}, - })) - - // The DaemonSet should have the correct configuration. - ds := rtest.GetResource(resources, "calico-node", "calico-system", "apps", "v1", "DaemonSet").(*appsv1.DaemonSet) - Expect(ds.Spec.Template.Spec.Containers[0].Image).To(Equal(components.TigeraRegistry + "tigera/node:" + components.ComponentTigeraNode.Version)) - - // The pod template should have node critical priority - Expect(ds.Spec.Template.Spec.PriorityClassName).To(Equal(render.NodePriorityClassName)) - - verifyInitContainers(ds, defaultInstance) - expectedNodeEnv := []corev1.EnvVar{ - // Default envvars. - {Name: "DATASTORE_TYPE", Value: "kubernetes"}, - {Name: "WAIT_FOR_DATASTORE", Value: "true"}, - {Name: "CALICO_MANAGE_CNI", Value: "true"}, - {Name: "CALICO_NETWORKING_BACKEND", Value: "bird"}, - {Name: "CLUSTER_TYPE", Value: "k8s,operator,openshift,bgp"}, - {Name: "CALICO_DISABLE_FILE_LOGGING", Value: "false"}, - {Name: "FELIX_DEFAULTENDPOINTTOHOSTACTION", Value: "ACCEPT"}, - {Name: "FELIX_HEALTHENABLED", Value: "true"}, - {Name: "FELIX_HEALTHPORT", Value: "9199"}, - { - Name: "NODENAME", - ValueFrom: &corev1.EnvVarSource{ - FieldRef: &corev1.ObjectFieldSelector{FieldPath: "spec.nodeName"}, - }, - }, - { - Name: "NAMESPACE", - ValueFrom: &corev1.EnvVarSource{ - FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.namespace"}, - }, - }, - {Name: "FELIX_TYPHAK8SNAMESPACE", Value: "calico-system"}, - {Name: "FELIX_TYPHAK8SSERVICENAME", Value: "calico-typha"}, - {Name: "FELIX_TYPHACAFILE", Value: certificatemanagement.TrustedCertBundleMountPath}, - {Name: "FELIX_TYPHACERTFILE", Value: "/node-certs/tls.crt"}, - {Name: "FELIX_TYPHACN", Value: "typha-server"}, - {Name: "FELIX_TYPHAKEYFILE", Value: "/node-certs/tls.key"}, - // Tigera-specific envvars - {Name: "FELIX_PROMETHEUSREPORTERENABLED", Value: "true"}, - {Name: "FELIX_PROMETHEUSREPORTERPORT", Value: "9081"}, - {Name: "FELIX_FLOWLOGSFILEENABLED", Value: "true"}, - {Name: "FELIX_FLOWLOGSFILEINCLUDELABELS", Value: "true"}, - {Name: "FELIX_FLOWLOGSFILEINCLUDEPOLICIES", Value: "true"}, - {Name: "FELIX_FLOWLOGSFILEINCLUDESERVICE", Value: "true"}, - {Name: "FELIX_FLOWLOGSENABLENETWORKSETS", Value: "true"}, - {Name: "FELIX_FLOWLOGSCOLLECTPROCESSINFO", Value: "true"}, - {Name: "FELIX_DNSLOGSFILEENABLED", Value: "true"}, - {Name: "FELIX_DNSLOGSFILEPERNODELIMIT", Value: "1000"}, - {Name: "MULTI_INTERFACE_MODE", Value: operatorv1.MultiInterfaceModeNone.Value()}, - {Name: "NO_DEFAULT_POOLS", Value: "true"}, - } - expectedNodeEnv = configureExpectedNodeEnvIPVersions(expectedNodeEnv, defaultInstance, enableIPv4, enableIPv6) - Expect(ds.Spec.Template.Spec.Containers[0].Env).To(ConsistOf(expectedNodeEnv)) - Expect(len(ds.Spec.Template.Spec.Containers[0].Env)).To(Equal(len(expectedNodeEnv))) - - verifyProbesAndLifecycle(ds, true, true) - }) - - It("should render all resources when variant is CalicoEnterprise and running on RKE2", func() { - expectedResources := []struct { - name string - ns string - group string - version string - kind string - }{ - {name: "calico-node", ns: common.CalicoNamespace, group: "", version: "v1", kind: "ServiceAccount"}, - {name: "calico-node", ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRole"}, - {name: "calico-node", ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRoleBinding"}, - {name: "calico-cni-plugin", ns: common.CalicoNamespace, group: "", version: "v1", kind: "ServiceAccount"}, - {name: "calico-cni-plugin", ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRole"}, - {name: "calico-cni-plugin", ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRoleBinding"}, - {name: "cni-config", ns: common.CalicoNamespace, group: "", version: "v1", kind: "ConfigMap"}, - {name: common.NodeDaemonSetName, ns: common.CalicoNamespace, group: "apps", version: "v1", kind: "DaemonSet"}, - } - - defaultInstance.Variant = operatorv1.CalicoEnterprise - defaultInstance.KubernetesProvider = operatorv1.ProviderRKE2 - defaultCNIConfDir, defaultCNIBinDir := render.DefaultCNIDirectories(defaultInstance.KubernetesProvider) - defaultInstance.CNI.ConfDir, defaultInstance.CNI.BinDir = &defaultCNIConfDir, &defaultCNIBinDir - cfg.NodeReporterMetricsPort = 9081 - cfg.FelixHealthPort = 9199 - - component := render.Node(&cfg) - Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() - Expect(len(resources)).To(Equal(len(expectedResources)), fmt.Sprintf("Actual resources: %#v", resources)) - - // Should render the correct resources. - i := 0 - for _, expectedRes := range expectedResources { - rtest.ExpectResourceTypeAndObjectMetadata(resources[i], expectedRes.name, expectedRes.ns, expectedRes.group, expectedRes.version, expectedRes.kind) - i++ - } - - // The DaemonSet should have the correct configuration. - ds := rtest.GetResource(resources, "calico-node", "calico-system", "apps", "v1", "DaemonSet").(*appsv1.DaemonSet) - Expect(ds.Spec.Template.Spec.Containers[0].Image).To(Equal(components.TigeraRegistry + "tigera/node:" + components.ComponentTigeraNode.Version)) - - // The pod template should have node critical priority - Expect(ds.Spec.Template.Spec.PriorityClassName).To(Equal(render.NodePriorityClassName)) - - verifyInitContainers(ds, defaultInstance) - - expectedNodeEnv := []corev1.EnvVar{ - // Default envvars. - {Name: "DATASTORE_TYPE", Value: "kubernetes"}, - {Name: "WAIT_FOR_DATASTORE", Value: "true"}, - {Name: "CALICO_MANAGE_CNI", Value: "true"}, - {Name: "CALICO_NETWORKING_BACKEND", Value: "bird"}, - {Name: "CLUSTER_TYPE", Value: "k8s,operator,bgp"}, - {Name: "CALICO_DISABLE_FILE_LOGGING", Value: "false"}, - {Name: "FELIX_DEFAULTENDPOINTTOHOSTACTION", Value: "ACCEPT"}, - {Name: "FELIX_HEALTHENABLED", Value: "true"}, - {Name: "FELIX_HEALTHPORT", Value: "9199"}, - { - Name: "NODENAME", - ValueFrom: &corev1.EnvVarSource{ - FieldRef: &corev1.ObjectFieldSelector{FieldPath: "spec.nodeName"}, - }, - }, - { - Name: "NAMESPACE", - ValueFrom: &corev1.EnvVarSource{ - FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.namespace"}, - }, - }, - {Name: "FELIX_TYPHAK8SNAMESPACE", Value: "calico-system"}, - {Name: "FELIX_TYPHAK8SSERVICENAME", Value: "calico-typha"}, - {Name: "FELIX_TYPHACAFILE", Value: certificatemanagement.TrustedCertBundleMountPath}, - {Name: "FELIX_TYPHACERTFILE", Value: "/node-certs/tls.crt"}, - {Name: "FELIX_TYPHACN", Value: "typha-server"}, - {Name: "FELIX_TYPHAKEYFILE", Value: "/node-certs/tls.key"}, - {Name: "NO_DEFAULT_POOLS", Value: "true"}, - // Tigera-specific envvars - {Name: "FELIX_PROMETHEUSREPORTERENABLED", Value: "true"}, - {Name: "FELIX_PROMETHEUSREPORTERPORT", Value: "9081"}, - {Name: "FELIX_FLOWLOGSFILEENABLED", Value: "true"}, - {Name: "FELIX_FLOWLOGSFILEINCLUDELABELS", Value: "true"}, - {Name: "FELIX_FLOWLOGSFILEINCLUDEPOLICIES", Value: "true"}, - {Name: "FELIX_FLOWLOGSFILEINCLUDESERVICE", Value: "true"}, - {Name: "FELIX_FLOWLOGSENABLENETWORKSETS", Value: "true"}, - {Name: "FELIX_FLOWLOGSCOLLECTPROCESSINFO", Value: "true"}, - {Name: "FELIX_DNSLOGSFILEENABLED", Value: "true"}, - {Name: "FELIX_DNSLOGSFILEPERNODELIMIT", Value: "1000"}, - - // The RKE2 envvar overrides. - {Name: "MULTI_INTERFACE_MODE", Value: operatorv1.MultiInterfaceModeNone.Value()}, - } - expectedNodeEnv = configureExpectedNodeEnvIPVersions(expectedNodeEnv, defaultInstance, enableIPv4, enableIPv6) - Expect(ds.Spec.Template.Spec.Containers[0].Env).To(ConsistOf(expectedNodeEnv)) - Expect(len(ds.Spec.Template.Spec.Containers[0].Env)).To(Equal(len(expectedNodeEnv))) - - verifyProbesAndLifecycle(ds, true, true) + verifyProbesAndLifecycle(ds, true) }) It("should render volumes and node volumemounts when bird templates are provided", func() { @@ -2043,7 +1746,6 @@ var _ = Describe("Node rendering tests", func() { It("should not enable prometheus metrics if NodeMetricsPort is nil", func() { defaultInstance.Variant = operatorv1.CalicoEnterprise defaultInstance.NodeMetricsPort = nil - cfg.NodeReporterMetricsPort = 9081 component := render.Node(&cfg) Expect(component.ResolveImages(nil)).To(BeNil()) @@ -2057,7 +1759,8 @@ var _ = Describe("Node rendering tests", func() { ds := dsResource.(*appsv1.DaemonSet) Expect(ds.Spec.Template.Spec.Containers[0].Env).ToNot(ContainElement(notExpectedEnvVar)) - // It should have the reporter port, though. + // The reporter port env is added by the enterprise node modifier, not the + // base render, so it should be absent here. expected := corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERPORT"} Expect(ds.Spec.Template.Spec.Containers[0].Env).ToNot(ContainElement(expected)) }) @@ -2869,7 +2572,7 @@ var _ = Describe("Node rendering tests", func() { Expect(ds.Spec.Template.Spec.Containers[0].Env).To(ConsistOf(expectedNodeEnv)) // Verify readiness and liveness probes. - verifyProbesAndLifecycle(ds, false, false) + verifyProbesAndLifecycle(ds, false) }) DescribeTable("test node probes", @@ -2894,7 +2597,7 @@ var _ = Describe("Node rendering tests", func() { Expect(dsResource).ToNot(BeNil()) ds := dsResource.(*appsv1.DaemonSet) - verifyProbesAndLifecycle(ds, isOpenshift, isEnterprise) + verifyProbesAndLifecycle(ds, isOpenshift) }, Entry("k8s Calico OS no BGP", false, false, operatorv1.BGPDisabled), @@ -3212,7 +2915,7 @@ var _ = Describe("Node rendering tests", func() { }) // verifyProbesAndLifecycle asserts the expected node liveness and readiness probe plus pod lifecycle settings. -func verifyProbesAndLifecycle(ds *appsv1.DaemonSet, isOpenshift, isEnterprise bool) { +func verifyProbesAndLifecycle(ds *appsv1.DaemonSet, isOpenshift bool) { // Verify readiness and liveness probes. expectedReadiness := &corev1.Probe{ PeriodSeconds: 10, @@ -3246,14 +2949,14 @@ func verifyProbesAndLifecycle(ds *appsv1.DaemonSet, isOpenshift, isEnterprise bo } ExpectWithOffset(1, found).To(BeTrue()) + // The base render produces the same readiness command for all variants; the + // enterprise --bgp-metrics-ready check is added by the node modifier and is + // covered in the enterprise package tests. var expectedReadinessCmd []string - switch { - case !bgp: - expectedReadinessCmd = []string{"/usr/bin/calico", "component", "node", "health", "--felix-ready"} - case bgp && isEnterprise: - expectedReadinessCmd = []string{"/usr/bin/calico", "component", "node", "health", "--bird-ready", "--felix-ready", "--bgp-metrics-ready"} - case bgp: + if bgp { expectedReadinessCmd = []string{"/usr/bin/calico", "component", "node", "health", "--bird-ready", "--felix-ready"} + } else { + expectedReadinessCmd = []string{"/usr/bin/calico", "component", "node", "health", "--felix-ready"} } expectedReadiness.ProbeHandler = corev1.ProbeHandler{Exec: &corev1.ExecAction{Command: expectedReadinessCmd}} diff --git a/pkg/render/render_test.go b/pkg/render/render_test.go index 42afa04570..a172ce4f00 100644 --- a/pkg/render/render_test.go +++ b/pkg/render/render_test.go @@ -79,17 +79,16 @@ func allCalicoComponents( secretsAndConfigMaps := render.NewCreationPassthrough(objs...) nodeCfg := &render.NodeConfiguration{ - K8sServiceEp: k8sServiceEp, - Installation: cr, - TLS: typhaNodeTLS, - NodeAppArmorProfile: nodeAppArmorProfile, - ClusterDomain: clusterDomain, - NodeReporterMetricsPort: nodeReporterMetricsPort, - BGPLayouts: bgpLayout, - LogCollector: logCollector, - BirdTemplates: bt, - MigrateNamespaces: up, - FelixHealthPort: 9099, + K8sServiceEp: k8sServiceEp, + Installation: cr, + TLS: typhaNodeTLS, + NodeAppArmorProfile: nodeAppArmorProfile, + ClusterDomain: clusterDomain, + BGPLayouts: bgpLayout, + LogCollector: logCollector, + BirdTemplates: bt, + MigrateNamespaces: up, + FelixHealthPort: 9099, } typhaCfg := &render.TyphaConfiguration{ K8sServiceEp: k8sServiceEp, From 311b79ef731b80b59c88d118eb939593c41ad456 Mon Sep 17 00:00:00 2001 From: Casey Davenport Date: Tue, 9 Jun 2026 09:55:01 -0700 Subject: [PATCH 16/54] Simplify render context extension entrypoints Drop the functional-options builder for Inputs (one call site, all fields always set) in favor of a plain struct literal, and replace the single-method RenderContextFactory interface with a registered builder func. All three extension seams now register a func. --- .../installation/core_controller.go | 20 ++-- pkg/enterprise/installation.go | 15 ++- pkg/enterprise/installation_test.go | 32 +++---- pkg/extensions/factory.go | 91 +++++-------------- pkg/extensions/factory_test.go | 44 +++++---- pkg/extensions/modifier.go | 4 +- 6 files changed, 78 insertions(+), 128 deletions(-) diff --git a/pkg/controller/installation/core_controller.go b/pkg/controller/installation/core_controller.go index bbeb8582af..584c27e087 100644 --- a/pkg/controller/installation/core_controller.go +++ b/pkg/controller/installation/core_controller.go @@ -1214,19 +1214,19 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile } // Build the render context handed to registered modifiers. The core operator - // factory returns just the base context; an extension's factory additionally + // builder returns just the base context; an extension's builder additionally // does controller-side work for its variant - validating config and creating // the node-prometheus certificate, adding it (and the prometheus/esgw certs) // to the trusted bundle - and may abort the reconcile by returning an error. - modCtx, err := extensions.GetRenderContextFactory().New( - extensions.WithContext(ctx), - extensions.WithClient(r.client), - extensions.WithInstallation(&instance.Spec), - extensions.WithFelixConfiguration(felixConfiguration), - extensions.WithCertificateManager(certificateManager), - extensions.WithTrustedBundle(typhaNodeTLS.TrustedBundle), - extensions.WithClusterDomain(r.clusterDomain), - ) + modCtx, err := extensions.BuildRenderContext(extensions.Inputs{ + Ctx: ctx, + Client: r.client, + Installation: &instance.Spec, + FelixConfiguration: felixConfiguration, + CertificateManager: certificateManager, + TrustedBundle: typhaNodeTLS.TrustedBundle, + ClusterDomain: r.clusterDomain, + }) if err != nil { r.status.SetDegraded(operatorv1.ResourceCreateError, "Error preparing installation extension", err, reqLogger) return reconcile.Result{}, err diff --git a/pkg/enterprise/installation.go b/pkg/enterprise/installation.go index 596229a635..98fba501ae 100644 --- a/pkg/enterprise/installation.go +++ b/pkg/enterprise/installation.go @@ -26,18 +26,15 @@ import ( "github.com/tigera/operator/pkg/render/monitor" ) -// installationFactory is the Calico Enterprise RenderContextFactory. It builds -// the base render context and then does the controller-side work the modifiers -// can't: validating config and creating/fetching the certificates that feed the -// trusted bundle. -type installationFactory struct{} - func registerInstallation() { - extensions.RegisterRenderContextFactory(&installationFactory{}) + extensions.RegisterRenderContextBuilder(buildRenderContext) } -func (f *installationFactory) New(opts ...extensions.RenderContextOption) (extensions.RenderContext, error) { - in := extensions.ApplyInputs(opts...) +// buildRenderContext is the Calico Enterprise RenderContextBuilder. It builds +// the base render context and then does the controller-side work the modifiers +// can't: validating config and creating/fetching the certificates that feed the +// trusted bundle. +func buildRenderContext(in extensions.Inputs) (extensions.RenderContext, error) { rc := extensions.BaseRenderContext(in) if in.Installation == nil || !in.Installation.Variant.IsEnterprise() { return rc, nil diff --git a/pkg/enterprise/installation_test.go b/pkg/enterprise/installation_test.go index 18c38c312a..f3aa09f887 100644 --- a/pkg/enterprise/installation_test.go +++ b/pkg/enterprise/installation_test.go @@ -32,34 +32,34 @@ import ( "github.com/tigera/operator/pkg/extensions" ) -var _ = Describe("installation render context factory", func() { +var _ = Describe("installation render context builder", func() { BeforeEach(func() { enterprise.Register() }) AfterEach(func() { extensions.ResetForTest() }) It("rejects a zero prometheus reporter port", func() { port := 0 - opts := newOpts(operatorv1.CalicoEnterprise) - opts = append(opts, extensions.WithFelixConfiguration(&v3.FelixConfiguration{ + in := newInputs(operatorv1.CalicoEnterprise) + in.FelixConfiguration = &v3.FelixConfiguration{ Spec: v3.FelixConfigurationSpec{PrometheusReporterPort: &port}, - })) - _, err := extensions.GetRenderContextFactory().New(opts...) + } + _, err := extensions.BuildRenderContext(in) Expect(err).To(HaveOccurred()) }) It("creates the node prometheus keypair for the enterprise variant", func() { - rc, err := extensions.GetRenderContextFactory().New(newOpts(operatorv1.CalicoEnterprise)...) + rc, err := extensions.BuildRenderContext(newInputs(operatorv1.CalicoEnterprise)) Expect(err).NotTo(HaveOccurred()) Expect(rc.NodePrometheusTLS).NotTo(BeNil()) }) It("is a no-op for the Calico variant", func() { - rc, err := extensions.GetRenderContextFactory().New(newOpts(operatorv1.Calico)...) + rc, err := extensions.BuildRenderContext(newInputs(operatorv1.Calico)) Expect(err).NotTo(HaveOccurred()) Expect(rc.NodePrometheusTLS).To(BeNil()) }) }) -func newOpts(variant operatorv1.ProductVariant) []extensions.RenderContextOption { +func newInputs(variant operatorv1.ProductVariant) extensions.Inputs { scheme := runtime.NewScheme() Expect(apis.AddToScheme(scheme, false)).NotTo(HaveOccurred()) c := ctrlrfake.DefaultFakeClientBuilder(scheme).Build() @@ -68,13 +68,13 @@ func newOpts(variant operatorv1.ProductVariant) []extensions.RenderContextOption Expect(err).NotTo(HaveOccurred()) trustedBundle := certManager.CreateTrustedBundle() - return []extensions.RenderContextOption{ - extensions.WithContext(context.Background()), - extensions.WithClient(c), - extensions.WithInstallation(&operatorv1.InstallationSpec{Variant: variant}), - extensions.WithFelixConfiguration(&v3.FelixConfiguration{}), - extensions.WithCertificateManager(certManager), - extensions.WithTrustedBundle(trustedBundle), - extensions.WithClusterDomain("cluster.local"), + return extensions.Inputs{ + Ctx: context.Background(), + Client: c, + Installation: &operatorv1.InstallationSpec{Variant: variant}, + FelixConfiguration: &v3.FelixConfiguration{}, + CertificateManager: certManager, + TrustedBundle: trustedBundle, + ClusterDomain: "cluster.local", } } diff --git a/pkg/extensions/factory.go b/pkg/extensions/factory.go index f6081635ca..bf2d46f246 100644 --- a/pkg/extensions/factory.go +++ b/pkg/extensions/factory.go @@ -25,11 +25,11 @@ import ( "github.com/tigera/operator/pkg/tls/certificatemanagement" ) -// Inputs is the reconcile state a RenderContextFactory builds a RenderContext -// from. The installation controller populates it through the With* options. It -// carries both the values that flow straight into the RenderContext and the -// side-effecting dependencies (Client, CertificateManager) a factory needs to -// produce controller-side artifacts. +// Inputs is the reconcile state a RenderContextBuilder builds a RenderContext +// from. The installation controller populates it directly. It carries both the +// values that flow straight into the RenderContext and the side-effecting +// dependencies (Client, CertificateManager) a builder needs to produce +// controller-side artifacts. type Inputs struct { Ctx context.Context Client client.Client @@ -40,62 +40,19 @@ type Inputs struct { ClusterDomain string } -// RenderContextOption sets a field on the Inputs a factory builds from. -type RenderContextOption func(*Inputs) - -func WithContext(ctx context.Context) RenderContextOption { - return func(in *Inputs) { in.Ctx = ctx } -} - -func WithClient(c client.Client) RenderContextOption { - return func(in *Inputs) { in.Client = c } -} - -func WithInstallation(i *operatorv1.InstallationSpec) RenderContextOption { - return func(in *Inputs) { in.Installation = i } -} - -func WithFelixConfiguration(fc *v3.FelixConfiguration) RenderContextOption { - return func(in *Inputs) { in.FelixConfiguration = fc } -} - -func WithCertificateManager(cm certificatemanager.CertificateManager) RenderContextOption { - return func(in *Inputs) { in.CertificateManager = cm } -} - -func WithTrustedBundle(tb certificatemanagement.TrustedBundle) RenderContextOption { - return func(in *Inputs) { in.TrustedBundle = tb } -} - -func WithClusterDomain(d string) RenderContextOption { - return func(in *Inputs) { in.ClusterDomain = d } -} - -// RenderContextFactory builds the RenderContext handed to render modifiers. New -// applies the options, performs any controller-side work (creating -// certificates, extending the trusted bundle), and returns the assembled -// RenderContext - or an error that aborts the reconcile. +// RenderContextBuilder builds the RenderContext handed to render modifiers. It +// performs any controller-side work (creating certificates, extending the +// trusted bundle) and returns the assembled RenderContext - or an error that +// aborts the reconcile. // // This is the generic seam controllers use to extend base operator behavior; // its first consumer is Calico Enterprise, but nothing here is enterprise // specific. The core operator default does no side-effecting work. -type RenderContextFactory interface { - New(opts ...RenderContextOption) (RenderContext, error) -} - -// ApplyInputs returns the Inputs produced by applying opts. Factories use it to -// collect the option-supplied state before doing their work. -func ApplyInputs(opts ...RenderContextOption) Inputs { - var in Inputs - for _, o := range opts { - o(&in) - } - return in -} +type RenderContextBuilder func(in Inputs) (RenderContext, error) // BaseRenderContext maps the generically-gathered inputs onto a RenderContext. -// The default factory and every registered factory build on it, so the base -// fields are assembled in exactly one place. A factory layers its side-effect +// The default builder and every registered builder build on it, so the base +// fields are assembled in exactly one place. A builder layers its side-effect // artifacts (e.g. NodePrometheusTLS) on top of the returned value. func BaseRenderContext(in Inputs) RenderContext { return RenderContext{ @@ -106,22 +63,20 @@ func BaseRenderContext(in Inputs) RenderContext { } } -// defaultFactory is the core operator's RenderContextFactory. It does no +// defaultRenderContextBuilder is the core operator's builder. It does no // side-effecting work and returns just the base RenderContext. -type defaultFactory struct{} - -func (defaultFactory) New(opts ...RenderContextOption) (RenderContext, error) { - return BaseRenderContext(ApplyInputs(opts...)), nil +func defaultRenderContextBuilder(in Inputs) (RenderContext, error) { + return BaseRenderContext(in), nil } -var renderContextFactory RenderContextFactory = defaultFactory{} +var renderContextBuilder RenderContextBuilder = defaultRenderContextBuilder -// RegisterRenderContextFactory installs f as the factory the installation -// controller uses. Registration replaces any prior factory, so it is safe to -// call more than once. Without a registered factory the core operator default +// RegisterRenderContextBuilder installs f as the builder the installation +// controller uses. Registration replaces any prior builder, so it is safe to +// call more than once. Without a registered builder the core operator default // applies. -func RegisterRenderContextFactory(f RenderContextFactory) { renderContextFactory = f } +func RegisterRenderContextBuilder(f RenderContextBuilder) { renderContextBuilder = f } -// GetRenderContextFactory returns the registered factory, or the core operator -// default when none is registered. -func GetRenderContextFactory() RenderContextFactory { return renderContextFactory } +// BuildRenderContext builds the RenderContext from in using the registered +// builder, or the core operator default when none is registered. +func BuildRenderContext(in Inputs) (RenderContext, error) { return renderContextBuilder(in) } diff --git a/pkg/extensions/factory_test.go b/pkg/extensions/factory_test.go index 408704b7ad..7954f402bd 100644 --- a/pkg/extensions/factory_test.go +++ b/pkg/extensions/factory_test.go @@ -24,50 +24,48 @@ import ( "github.com/tigera/operator/pkg/extensions" ) -var _ = Describe("render context factory", func() { +var _ = Describe("render context builder", func() { AfterEach(func() { extensions.ResetForTest() }) - It("returns the base render context from the default factory", func() { + It("returns the base render context from the default builder", func() { install := &operatorv1.InstallationSpec{Variant: operatorv1.Calico} - rc, err := extensions.GetRenderContextFactory().New( - extensions.WithInstallation(install), - extensions.WithClusterDomain("cluster.local"), - ) + rc, err := extensions.BuildRenderContext(extensions.Inputs{ + Installation: install, + ClusterDomain: "cluster.local", + }) Expect(err).NotTo(HaveOccurred()) Expect(rc.Installation).To(BeIdenticalTo(install)) Expect(rc.ClusterDomain).To(Equal("cluster.local")) Expect(rc.NodePrometheusTLS).To(BeNil()) }) - It("uses a registered factory in place of the default", func() { - extensions.RegisterRenderContextFactory(&fakeFactory{}) - rc, err := extensions.GetRenderContextFactory().New() + It("uses a registered builder in place of the default", func() { + extensions.RegisterRenderContextBuilder(fakeBuilder(nil)) + rc, err := extensions.BuildRenderContext(extensions.Inputs{}) Expect(err).NotTo(HaveOccurred()) Expect(rc.ClusterDomain).To(Equal("from-fake")) }) - It("surfaces the factory error", func() { - extensions.RegisterRenderContextFactory(&fakeFactory{err: errors.New("boom")}) - _, err := extensions.GetRenderContextFactory().New() + It("surfaces the builder error", func() { + extensions.RegisterRenderContextBuilder(fakeBuilder(errors.New("boom"))) + _, err := extensions.BuildRenderContext(extensions.Inputs{}) Expect(err).To(MatchError("boom")) }) - It("restores the default factory on reset", func() { - extensions.RegisterRenderContextFactory(&fakeFactory{}) + It("restores the default builder on reset", func() { + extensions.RegisterRenderContextBuilder(fakeBuilder(nil)) extensions.ResetForTest() - rc, err := extensions.GetRenderContextFactory().New(extensions.WithClusterDomain("real")) + rc, err := extensions.BuildRenderContext(extensions.Inputs{ClusterDomain: "real"}) Expect(err).NotTo(HaveOccurred()) Expect(rc.ClusterDomain).To(Equal("real")) }) }) -type fakeFactory struct { - err error -} - -func (f *fakeFactory) New(_ ...extensions.RenderContextOption) (extensions.RenderContext, error) { - if f.err != nil { - return extensions.RenderContext{}, f.err +func fakeBuilder(err error) extensions.RenderContextBuilder { + return func(_ extensions.Inputs) (extensions.RenderContext, error) { + if err != nil { + return extensions.RenderContext{}, err + } + return extensions.RenderContext{ClusterDomain: "from-fake"}, nil } - return extensions.RenderContext{ClusterDomain: "from-fake"}, nil } diff --git a/pkg/extensions/modifier.go b/pkg/extensions/modifier.go index 5b911edc72..b872638531 100644 --- a/pkg/extensions/modifier.go +++ b/pkg/extensions/modifier.go @@ -57,9 +57,9 @@ func FindObject[T client.Object](objs []client.Object, name string) (T, bool) { } // ResetForTest clears every registry: modifiers, image overrides, and the -// render context factory. Test-only. +// render context builder. Test-only. func ResetForTest() { modifiers = map[string]Modifier{} - renderContextFactory = defaultFactory{} + renderContextBuilder = defaultRenderContextBuilder imageoverride.ResetForTest() } From 5a7cad95955cb9a0f2016e4dcac8b6a1c554351d Mon Sep 17 00:00:00 2001 From: Casey Davenport Date: Tue, 9 Jun 2026 10:27:37 -0700 Subject: [PATCH 17/54] Variant-scope extension registration and tighten the seams Register modifiers, image overrides, and the render context builder per variant. The registries now gate on the installation variant, so the enterprise funcs drop their self-gate guards (the IsEnterprise checks the PR set out to remove) and the image override drops its decline bool - it only runs for its own variant. Move the node prometheus reporter keypair mounting (volume, mount, cert-management init container, pod hash annotation) into the node modifier, and remove NodeConfiguration.PrometheusServerTLS along with the round-trip through the installation controller. Core node render no longer carries a prometheus mount; in calico the keypair is never created. Rename Extensible.Name() to ModifierKey() so an unrelated Name() method can't make a component modifier-eligible by accident. --- .../installation/core_controller.go | 1 - pkg/controller/utils/component.go | 4 +- pkg/controller/utils/component_test.go | 7 ++- pkg/enterprise/installation.go | 6 +- pkg/enterprise/node.go | 56 ++++++++++++++----- pkg/enterprise/typha.go | 7 +-- pkg/extensions/factory.go | 42 +++++++------- pkg/extensions/factory_test.go | 34 ++++++++--- pkg/extensions/image.go | 14 ++--- pkg/extensions/image_test.go | 15 ++--- pkg/extensions/modifier.go | 39 ++++++++----- pkg/extensions/modifier_test.go | 34 ++++++++--- pkg/extensions/rendercontext.go | 9 ++- pkg/imageoverride/imageoverride.go | 35 +++++++----- pkg/render/component.go | 14 ++--- pkg/render/node.go | 18 +----- pkg/render/node_enterprise_test.go | 19 ++++--- pkg/render/typha.go | 2 +- 18 files changed, 210 insertions(+), 146 deletions(-) diff --git a/pkg/controller/installation/core_controller.go b/pkg/controller/installation/core_controller.go index 584c27e087..568a59f954 100644 --- a/pkg/controller/installation/core_controller.go +++ b/pkg/controller/installation/core_controller.go @@ -1521,7 +1521,6 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile NodeAppArmorProfile: nodeAppArmorProfile, MigrateNamespaces: needsNamespaceMigration, CanRemoveCNIFinalizer: canRemoveCNI, - PrometheusServerTLS: nodePrometheusTLS, FelixHealthPort: *felixConfiguration.Spec.HealthPort, NodeCgroupV2Path: felixConfiguration.Spec.CgroupV2Path, V3CRDs: r.v3CRDs, diff --git a/pkg/controller/utils/component.go b/pkg/controller/utils/component.go index 3e343c4540..57238556da 100644 --- a/pkg/controller/utils/component.go +++ b/pkg/controller/utils/component.go @@ -468,8 +468,8 @@ func (c *componentHandler) CreateOrUpdateOrDelete(ctx context.Context, component var cronJobs []types.NamespacedName objsToCreate, objsToDelete := component.Objects() - if named, ok := component.(render.Extensible); ok { - objsToCreate = extensions.ApplyModifiers(named.Name(), c.modCtx, objsToCreate) + if ext, ok := component.(render.Extensible); ok { + objsToCreate = extensions.ApplyModifiers(ext.ModifierKey(), c.modCtx, objsToCreate) } // Load the InstallationSpec once and reuse it for every object: createOrUpdateObject needs it diff --git a/pkg/controller/utils/component_test.go b/pkg/controller/utils/component_test.go index 3081adfe19..eeb989e501 100644 --- a/pkg/controller/utils/component_test.go +++ b/pkg/controller/utils/component_test.go @@ -2493,7 +2493,7 @@ var _ = Describe("componentHandler modifier application", func() { }) It("applies registered modifiers to a named component before create", func() { - extensions.Modify("fake", func(ctx extensions.RenderContext, objs []client.Object) []client.Object { + extensions.Modify(operatorv1.CalicoEnterprise, "fake", func(ctx extensions.RenderContext, objs []client.Object) []client.Object { cm := objs[0].(*corev1.ConfigMap) cm.Data = map[string]string{"patched": "yes"} return objs @@ -2504,7 +2504,8 @@ var _ = Describe("componentHandler modifier application", func() { Expect(corev1.SchemeBuilder.AddToScheme(s)).NotTo(HaveOccurred()) c := ctrlrfake.DefaultFakeClientBuilder(s).Build() - handler := NewComponentHandler(logf.Log, c, s, nil) + modCtx := extensions.RenderContext{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.CalicoEnterprise}} + handler := NewComponentHandler(logf.Log, c, s, nil, WithRenderContext(modCtx)) comp := &namedFakeComponent{name: "fake", obj: &corev1.ConfigMap{ TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"}, ObjectMeta: metav1.ObjectMeta{Name: "cm", Namespace: "default"}, @@ -2523,7 +2524,7 @@ type namedFakeComponent struct { obj client.Object } -func (f *namedFakeComponent) Name() string { return f.name } +func (f *namedFakeComponent) ModifierKey() string { return f.name } func (f *namedFakeComponent) ResolveImages(*operatorv1.ImageSet) error { return nil } func (f *namedFakeComponent) Objects() ([]client.Object, []client.Object) { return []client.Object{f.obj}, nil diff --git a/pkg/enterprise/installation.go b/pkg/enterprise/installation.go index 98fba501ae..cd49480c68 100644 --- a/pkg/enterprise/installation.go +++ b/pkg/enterprise/installation.go @@ -18,6 +18,7 @@ import ( "errors" "fmt" + operatorv1 "github.com/tigera/operator/api/v1" "github.com/tigera/operator/pkg/common" "github.com/tigera/operator/pkg/dns" "github.com/tigera/operator/pkg/extensions" @@ -27,7 +28,7 @@ import ( ) func registerInstallation() { - extensions.RegisterRenderContextBuilder(buildRenderContext) + extensions.RegisterRenderContextBuilder(operatorv1.CalicoEnterprise, buildRenderContext) } // buildRenderContext is the Calico Enterprise RenderContextBuilder. It builds @@ -36,9 +37,6 @@ func registerInstallation() { // trusted bundle. func buildRenderContext(in extensions.Inputs) (extensions.RenderContext, error) { rc := extensions.BaseRenderContext(in) - if in.Installation == nil || !in.Installation.Variant.IsEnterprise() { - return rc, nil - } // Reject the unsupported zero reporter port. The port value itself is derived // in the node modifier; only this validation lives here. diff --git a/pkg/enterprise/node.go b/pkg/enterprise/node.go index 9bf15ece57..455c7b403f 100644 --- a/pkg/enterprise/node.go +++ b/pkg/enterprise/node.go @@ -31,6 +31,7 @@ import ( "github.com/tigera/operator/pkg/controller/utils" "github.com/tigera/operator/pkg/extensions" "github.com/tigera/operator/pkg/render" + rmeta "github.com/tigera/operator/pkg/render/common/meta" ) const ( @@ -46,13 +47,10 @@ const ( ) func registerNode() { - extensions.OverrideImage(render.ComponentNameNode, func(in *operatorv1.InstallationSpec) (components.Component, bool) { - if !in.Variant.IsEnterprise() { - return components.Component{}, false - } - return components.ComponentTigeraNode, true + extensions.OverrideImage(operatorv1.CalicoEnterprise, render.ComponentNameNode, func(in *operatorv1.InstallationSpec) components.Component { + return components.ComponentTigeraNode }) - extensions.Modify(render.ComponentNameNode, modifyNode) + extensions.Modify(operatorv1.CalicoEnterprise, render.ComponentNameNode, modifyNode) } // modifyNode layers Calico Enterprise behavior onto the rendered calico/node @@ -60,10 +58,6 @@ func registerNode() { // daemonset configuration (flow/DNS log env, prometheus reporter, BGP metrics // readiness check, multi-interface mode, and the calico log volume). func modifyNode(ctx extensions.RenderContext, objs []client.Object) []client.Object { - if ctx.Installation == nil || !ctx.Installation.Variant.IsEnterprise() { - return objs - } - if role, ok := extensions.FindObject[*rbacv1.ClusterRole](objs, render.CalicoNodeObjectName); ok { role.Rules = append(role.Rules, nodeEnterpriseRules()...) } @@ -115,9 +109,10 @@ func nodeEnterpriseRules() []rbacv1.PolicyRule { } // modifyNodeDaemonSet applies the Enterprise-specific daemonset changes that the -// base render leaves out: the Enterprise felix env, multi-interface mode, and the -// BGP metrics readiness check. The calico log volume is mounted by the base -// render for both variants, so it is not handled here. +// base render leaves out: the Enterprise felix env, multi-interface mode, the +// BGP metrics readiness check, and the prometheus reporter keypair mount. The +// calico log volume is mounted by the base render for both variants, so it is +// not handled here. func modifyNodeDaemonSet(ctx extensions.RenderContext, ds *appsv1.DaemonSet) { spec := &ds.Spec.Template.Spec @@ -143,6 +138,41 @@ func modifyNodeDaemonSet(ctx extensions.RenderContext, ds *appsv1.DaemonSet) { c.ReadinessProbe.Exec.Command = append(c.ReadinessProbe.Exec.Command, "--bgp-metrics-ready") } } + + mountNodePrometheusTLS(ctx, ds) +} + +// mountNodePrometheusTLS mounts the node prometheus reporter keypair onto the +// daemonset: the volume, the calico-node volume mount, the cert-management init +// container (when in use), and the pod hash annotation that rolls the pods on +// cert rotation. The keypair has cluster side effects, so the enterprise render +// context builder creates it and hands it in via ctx rather than the modifier +// building it. In core (calico) the keypair is never created, so the base node +// render carries no prometheus mount at all. +func mountNodePrometheusTLS(ctx extensions.RenderContext, ds *appsv1.DaemonSet) { + if ctx.NodePrometheusTLS == nil { + return + } + tls := ctx.NodePrometheusTLS + spec := &ds.Spec.Template.Spec + + spec.Volumes = append(spec.Volumes, tls.Volume()) + + for i := range spec.Containers { + c := &spec.Containers[i] + if c.Name != render.CalicoNodeObjectName { + continue + } + c.VolumeMounts = append(c.VolumeMounts, tls.VolumeMount(rmeta.OSTypeLinux)) + if tls.UseCertificateManagement() { + spec.InitContainers = append(spec.InitContainers, tls.InitContainer(common.CalicoNamespace, c.SecurityContext)) + } + } + + if ds.Spec.Template.Annotations == nil { + ds.Spec.Template.Annotations = map[string]string{} + } + ds.Spec.Template.Annotations[tls.HashAnnotationKey()] = tls.HashAnnotationValue() } // nodeEnterpriseEnv is the Enterprise felix configuration added to the diff --git a/pkg/enterprise/typha.go b/pkg/enterprise/typha.go index 5a2471a5fd..368e71e835 100644 --- a/pkg/enterprise/typha.go +++ b/pkg/enterprise/typha.go @@ -20,19 +20,16 @@ import ( rbacv1 "k8s.io/api/rbac/v1" "sigs.k8s.io/controller-runtime/pkg/client" + operatorv1 "github.com/tigera/operator/api/v1" "github.com/tigera/operator/pkg/extensions" "github.com/tigera/operator/pkg/render" ) func registerTypha() { - extensions.Modify(render.ComponentNameTypha, modifyTypha) + extensions.Modify(operatorv1.CalicoEnterprise, render.ComponentNameTypha, modifyTypha) } func modifyTypha(ctx extensions.RenderContext, objs []client.Object) []client.Object { - if ctx.Installation == nil || !ctx.Installation.Variant.IsEnterprise() { - return objs - } - if role, ok := extensions.FindObject[*rbacv1.ClusterRole](objs, "calico-typha"); ok { role.Rules = append(role.Rules, rbacv1.PolicyRule{ APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, diff --git a/pkg/extensions/factory.go b/pkg/extensions/factory.go index bf2d46f246..48d615fd6d 100644 --- a/pkg/extensions/factory.go +++ b/pkg/extensions/factory.go @@ -47,13 +47,14 @@ type Inputs struct { // // This is the generic seam controllers use to extend base operator behavior; // its first consumer is Calico Enterprise, but nothing here is enterprise -// specific. The core operator default does no side-effecting work. +// specific. A builder is registered per variant, so it only runs for its own +// variant and need not re-check it. type RenderContextBuilder func(in Inputs) (RenderContext, error) // BaseRenderContext maps the generically-gathered inputs onto a RenderContext. -// The default builder and every registered builder build on it, so the base -// fields are assembled in exactly one place. A builder layers its side-effect -// artifacts (e.g. NodePrometheusTLS) on top of the returned value. +// Every registered builder builds on it, so the base fields are assembled in +// exactly one place. A builder layers its side-effect artifacts (e.g. +// NodePrometheusTLS) on top of the returned value. func BaseRenderContext(in Inputs) RenderContext { return RenderContext{ Installation: in.Installation, @@ -63,20 +64,23 @@ func BaseRenderContext(in Inputs) RenderContext { } } -// defaultRenderContextBuilder is the core operator's builder. It does no -// side-effecting work and returns just the base RenderContext. -func defaultRenderContextBuilder(in Inputs) (RenderContext, error) { - return BaseRenderContext(in), nil -} +var renderContextBuilders = map[operatorv1.ProductVariant]RenderContextBuilder{} -var renderContextBuilder RenderContextBuilder = defaultRenderContextBuilder - -// RegisterRenderContextBuilder installs f as the builder the installation -// controller uses. Registration replaces any prior builder, so it is safe to -// call more than once. Without a registered builder the core operator default -// applies. -func RegisterRenderContextBuilder(f RenderContextBuilder) { renderContextBuilder = f } +// RegisterRenderContextBuilder installs f as the builder for the given variant. +// Registration replaces any prior builder, so it is safe to call more than +// once. Variants without a registered builder get the base render context. +func RegisterRenderContextBuilder(variant operatorv1.ProductVariant, f RenderContextBuilder) { + renderContextBuilders[variant] = f +} -// BuildRenderContext builds the RenderContext from in using the registered -// builder, or the core operator default when none is registered. -func BuildRenderContext(in Inputs) (RenderContext, error) { return renderContextBuilder(in) } +// BuildRenderContext builds the RenderContext from in using the builder +// registered for the installation variant, or the base render context when the +// variant has no registered builder. +func BuildRenderContext(in Inputs) (RenderContext, error) { + if in.Installation != nil { + if f, ok := renderContextBuilders[in.Installation.Variant]; ok { + return f(in) + } + } + return BaseRenderContext(in), nil +} diff --git a/pkg/extensions/factory_test.go b/pkg/extensions/factory_test.go index 7954f402bd..8b8ad825ce 100644 --- a/pkg/extensions/factory_test.go +++ b/pkg/extensions/factory_test.go @@ -27,7 +27,7 @@ import ( var _ = Describe("render context builder", func() { AfterEach(func() { extensions.ResetForTest() }) - It("returns the base render context from the default builder", func() { + It("returns the base render context when the variant has no builder", func() { install := &operatorv1.InstallationSpec{Variant: operatorv1.Calico} rc, err := extensions.BuildRenderContext(extensions.Inputs{ Installation: install, @@ -39,28 +39,44 @@ var _ = Describe("render context builder", func() { Expect(rc.NodePrometheusTLS).To(BeNil()) }) - It("uses a registered builder in place of the default", func() { - extensions.RegisterRenderContextBuilder(fakeBuilder(nil)) - rc, err := extensions.BuildRenderContext(extensions.Inputs{}) + It("uses the builder registered for the installation variant", func() { + extensions.RegisterRenderContextBuilder(operatorv1.CalicoEnterprise, fakeBuilder(nil)) + rc, err := extensions.BuildRenderContext(enterpriseInputs()) Expect(err).NotTo(HaveOccurred()) Expect(rc.ClusterDomain).To(Equal("from-fake")) }) + It("ignores a builder registered for a different variant", func() { + extensions.RegisterRenderContextBuilder(operatorv1.CalicoEnterprise, fakeBuilder(nil)) + rc, err := extensions.BuildRenderContext(extensions.Inputs{ + Installation: &operatorv1.InstallationSpec{Variant: operatorv1.Calico}, + ClusterDomain: "real", + }) + Expect(err).NotTo(HaveOccurred()) + Expect(rc.ClusterDomain).To(Equal("real")) + }) + It("surfaces the builder error", func() { - extensions.RegisterRenderContextBuilder(fakeBuilder(errors.New("boom"))) - _, err := extensions.BuildRenderContext(extensions.Inputs{}) + extensions.RegisterRenderContextBuilder(operatorv1.CalicoEnterprise, fakeBuilder(errors.New("boom"))) + _, err := extensions.BuildRenderContext(enterpriseInputs()) Expect(err).To(MatchError("boom")) }) - It("restores the default builder on reset", func() { - extensions.RegisterRenderContextBuilder(fakeBuilder(nil)) + It("restores the base builder on reset", func() { + extensions.RegisterRenderContextBuilder(operatorv1.CalicoEnterprise, fakeBuilder(nil)) extensions.ResetForTest() - rc, err := extensions.BuildRenderContext(extensions.Inputs{ClusterDomain: "real"}) + in := enterpriseInputs() + in.ClusterDomain = "real" + rc, err := extensions.BuildRenderContext(in) Expect(err).NotTo(HaveOccurred()) Expect(rc.ClusterDomain).To(Equal("real")) }) }) +func enterpriseInputs() extensions.Inputs { + return extensions.Inputs{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.CalicoEnterprise}} +} + func fakeBuilder(err error) extensions.RenderContextBuilder { return func(_ extensions.Inputs) (extensions.RenderContext, error) { if err != nil { diff --git a/pkg/extensions/image.go b/pkg/extensions/image.go index 776a02bffd..1063261993 100644 --- a/pkg/extensions/image.go +++ b/pkg/extensions/image.go @@ -20,15 +20,15 @@ import ( "github.com/tigera/operator/pkg/imageoverride" ) -// ImageOverride returns the component image to use for an installation, and -// false to decline (leaving the default in place). Implementations self-gate -// on in.Variant. +// ImageOverride returns the component image to use for an installation. An +// override is registered per variant, so it only runs for its own variant and +// need not re-check it. type ImageOverride = imageoverride.Override -// OverrideImage registers an image override under key. The key is the render -// component's image identifier (e.g. "node"). -func OverrideImage(key string, fn ImageOverride) { - imageoverride.Register(key, fn) +// OverrideImage registers an image override under key for the given variant. +// The key is the render component's image identifier (e.g. "node"). +func OverrideImage(variant operatorv1.ProductVariant, key string, fn ImageOverride) { + imageoverride.Register(variant, key, fn) } // ResolveImage returns the override registered for key if it applies to in, diff --git a/pkg/extensions/image_test.go b/pkg/extensions/image_test.go index 5a382d6c36..7f05c6cac1 100644 --- a/pkg/extensions/image_test.go +++ b/pkg/extensions/image_test.go @@ -28,19 +28,20 @@ var _ = Describe("image overrides", func() { extensions.ResetForTest() }) - It("uses the override when one matches", func() { - extensions.OverrideImage("node", func(in *operatorv1.InstallationSpec) (components.Component, bool) { - if !in.Variant.IsEnterprise() { - return components.Component{}, false - } - return components.ComponentTigeraNode, true + It("uses the override registered for the installation variant", func() { + extensions.OverrideImage(operatorv1.CalicoEnterprise, "node", func(in *operatorv1.InstallationSpec) components.Component { + return components.ComponentTigeraNode }) ent := &operatorv1.InstallationSpec{Variant: operatorv1.CalicoEnterprise} Expect(extensions.ResolveImage("node", components.ComponentCalicoNode, ent)).To(Equal(components.ComponentTigeraNode)) }) - It("falls back to the default when no override matches", func() { + It("falls back to the default for a variant with no override", func() { + extensions.OverrideImage(operatorv1.CalicoEnterprise, "node", func(in *operatorv1.InstallationSpec) components.Component { + return components.ComponentTigeraNode + }) + calico := &operatorv1.InstallationSpec{Variant: operatorv1.Calico} Expect(extensions.ResolveImage("node", components.ComponentCalicoNode, calico)).To(Equal(components.ComponentCalicoNode)) }) diff --git a/pkg/extensions/modifier.go b/pkg/extensions/modifier.go index b872638531..7a6fbf8828 100644 --- a/pkg/extensions/modifier.go +++ b/pkg/extensions/modifier.go @@ -17,29 +17,40 @@ package extensions import ( "sigs.k8s.io/controller-runtime/pkg/client" + operatorv1 "github.com/tigera/operator/api/v1" "github.com/tigera/operator/pkg/imageoverride" ) // Modifier post-processes the objects a render component produced. It may mutate // matched objects and/or append additional objects, and must return the -// (possibly extended) slice. Implementations self-gate on ctx.Installation.Variant. +// (possibly extended) slice. A modifier is registered per variant, so it only +// runs for its own variant and need not re-check it. type Modifier func(ctx RenderContext, objs []client.Object) []client.Object -var modifiers = map[string]Modifier{} +type modifierKey struct { + variant operatorv1.ProductVariant + component string +} + +var modifiers = map[modifierKey]Modifier{} -// Modify registers fn as the modifier for the named component. A component has -// at most one modifier; the modifier handles all of that component's -// extension-specific mutations. Registration replaces any prior modifier, so it -// is idempotent and safe to call more than once - matching the image-override -// registry rather than stacking duplicate work. -func Modify(component string, fn Modifier) { - modifiers[component] = fn +// Modify registers fn as the modifier for the named component under the given +// variant. A (variant, component) pair has at most one modifier; it handles all +// of that component's extension-specific mutations for that variant. +// Registration replaces any prior modifier, so it is idempotent and safe to +// call more than once. +func Modify(variant operatorv1.ProductVariant, component string, fn Modifier) { + modifiers[modifierKey{variant, component}] = fn } -// ApplyModifiers runs the modifier registered for the named component over objs, -// returning objs unchanged when none is registered. +// ApplyModifiers runs the modifier registered for the named component and the +// installation's variant over objs, returning objs unchanged when none is +// registered (or when no installation is set). func ApplyModifiers(component string, ctx RenderContext, objs []client.Object) []client.Object { - if fn, ok := modifiers[component]; ok { + if ctx.Installation == nil { + return objs + } + if fn, ok := modifiers[modifierKey{ctx.Installation.Variant, component}]; ok { objs = fn(ctx, objs) } return objs @@ -59,7 +70,7 @@ func FindObject[T client.Object](objs []client.Object, name string) (T, bool) { // ResetForTest clears every registry: modifiers, image overrides, and the // render context builder. Test-only. func ResetForTest() { - modifiers = map[string]Modifier{} - renderContextBuilder = defaultRenderContextBuilder + modifiers = map[modifierKey]Modifier{} + renderContextBuilders = map[operatorv1.ProductVariant]RenderContextBuilder{} imageoverride.ResetForTest() } diff --git a/pkg/extensions/modifier_test.go b/pkg/extensions/modifier_test.go index 7b40beb740..82decda600 100644 --- a/pkg/extensions/modifier_test.go +++ b/pkg/extensions/modifier_test.go @@ -21,6 +21,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" + operatorv1 "github.com/tigera/operator/api/v1" "github.com/tigera/operator/pkg/extensions" ) @@ -29,8 +30,10 @@ var _ = Describe("modifier registry", func() { extensions.ResetForTest() }) - It("applies a registered modifier to the matching component", func() { - extensions.Modify("test", func(ctx extensions.RenderContext, objs []client.Object) []client.Object { + entCtx := extensions.RenderContext{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.CalicoEnterprise}} + + It("applies a registered modifier to the matching component and variant", func() { + extensions.Modify(operatorv1.CalicoEnterprise, "test", func(ctx extensions.RenderContext, objs []client.Object) []client.Object { cm, ok := extensions.FindObject[*corev1.ConfigMap](objs, "cm") Expect(ok).To(BeTrue()) cm.Data = map[string]string{"k": "v"} @@ -38,7 +41,7 @@ var _ = Describe("modifier registry", func() { }) in := []client.Object{&corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "cm"}}} - out := extensions.ApplyModifiers("test", extensions.RenderContext{}, in) + out := extensions.ApplyModifiers("test", entCtx, in) Expect(out).To(HaveLen(2)) cm := out[0].(*corev1.ConfigMap) @@ -48,20 +51,35 @@ var _ = Describe("modifier registry", func() { It("returns objects unchanged when no modifier is registered", func() { in := []client.Object{&corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "cm"}}} - out := extensions.ApplyModifiers("unregistered", extensions.RenderContext{}, in) + out := extensions.ApplyModifiers("unregistered", entCtx, in) Expect(out).To(Equal(in)) }) - It("replaces rather than stacks when a component is registered twice", func() { + It("does not apply a modifier registered for a different variant", func() { + extensions.Modify(operatorv1.CalicoEnterprise, "test", func(_ extensions.RenderContext, objs []client.Object) []client.Object { + return append(objs, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "extra"}}) + }) + + calicoCtx := extensions.RenderContext{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.Calico}} + in := []client.Object{&corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "cm"}}} + Expect(extensions.ApplyModifiers("test", calicoCtx, in)).To(Equal(in)) + }) + + It("returns objects unchanged when no installation is set", func() { + in := []client.Object{&corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "cm"}}} + Expect(extensions.ApplyModifiers("test", extensions.RenderContext{}, in)).To(Equal(in)) + }) + + It("replaces rather than stacks when a (variant, component) is registered twice", func() { add := func(name string) extensions.Modifier { return func(_ extensions.RenderContext, objs []client.Object) []client.Object { return append(objs, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: name}}) } } - extensions.Modify("test", add("first")) - extensions.Modify("test", add("second")) + extensions.Modify(operatorv1.CalicoEnterprise, "test", add("first")) + extensions.Modify(operatorv1.CalicoEnterprise, "test", add("second")) - out := extensions.ApplyModifiers("test", extensions.RenderContext{}, nil) + out := extensions.ApplyModifiers("test", entCtx, nil) Expect(out).To(HaveLen(1)) Expect(out[0].GetName()).To(Equal("second")) }) diff --git a/pkg/extensions/rendercontext.go b/pkg/extensions/rendercontext.go index 52d0d753c5..9ea25e1b6b 100644 --- a/pkg/extensions/rendercontext.go +++ b/pkg/extensions/rendercontext.go @@ -36,10 +36,9 @@ type RenderContext struct { // TrustedBundle is the shared CA bundle for the calico-system namespace. TrustedBundle certificatemanagement.TrustedBundle - // NodePrometheusTLS is created by the enterprise RenderContextFactory (it has - // cluster side effects, so it can't be built in a modifier). Two consumers - // read it: the installation controller passes it to the node configuration - // (PrometheusServerTLS) so the keypair is mounted, and the node modifier uses - // it to set the FELIX_PROMETHEUSREPORTER* certificate env vars. + // NodePrometheusTLS is created by the enterprise render context builder (it + // has cluster side effects, so it can't be built in a modifier). The node + // modifier is its only consumer: it mounts the keypair onto the daemonset and + // sets the FELIX_PROMETHEUSREPORTER* certificate env vars. NodePrometheusTLS certificatemanagement.KeyPairInterface } diff --git a/pkg/imageoverride/imageoverride.go b/pkg/imageoverride/imageoverride.go index bf8950cf84..c3bc471bc2 100644 --- a/pkg/imageoverride/imageoverride.go +++ b/pkg/imageoverride/imageoverride.go @@ -22,30 +22,35 @@ import ( "github.com/tigera/operator/pkg/components" ) -// Override selects the component image to use for an installation, returning -// false to decline (leaving the default in place). -type Override func(in *operatorv1.InstallationSpec) (components.Component, bool) +// Override selects the component image to use for an installation. +type Override func(in *operatorv1.InstallationSpec) components.Component -var registry = map[string]Override{} +type overrideKey struct { + variant operatorv1.ProductVariant + key string +} + +var registry = map[overrideKey]Override{} -// Register stores fn under key. The key is the render component's image -// identifier (e.g. "node"). -func Register(key string, fn Override) { - registry[key] = fn +// Register stores fn under key for the given variant. The key is the render +// component's image identifier (e.g. "node"). +func Register(variant operatorv1.ProductVariant, key string, fn Override) { + registry[overrideKey{variant, key}] = fn } -// Resolve returns the override registered for key if it applies to in, -// otherwise def. +// Resolve returns the override registered for key under the installation's +// variant, otherwise def. func Resolve(key string, def components.Component, in *operatorv1.InstallationSpec) components.Component { - if fn, ok := registry[key]; ok { - if c, override := fn(in); override { - return c - } + if in == nil { + return def + } + if fn, ok := registry[overrideKey{in.Variant, key}]; ok { + return fn(in) } return def } // ResetForTest clears the registry. Test-only. func ResetForTest() { - registry = map[string]Override{} + registry = map[overrideKey]Override{} } diff --git a/pkg/render/component.go b/pkg/render/component.go index 0fa4f5da8b..7785834dcf 100644 --- a/pkg/render/component.go +++ b/pkg/render/component.go @@ -41,18 +41,16 @@ type Component interface { } // Extensible is implemented by components that expose extension points. The -// componentHandler uses Name() to look up registered modifiers. Components -// without extensions need not implement it. -// -// Note this interface is structural: any component that grows a Name() string -// method for an unrelated reason becomes modifier-eligible. There are no name -// collisions today, but keep it in mind when adding a Name() method. +// componentHandler uses ModifierKey() to look up registered modifiers. +// Components without extensions need not implement it. The method name is +// deliberately specific (not a generic Name()) so an unrelated method can't +// make a component modifier-eligible by accident. type Extensible interface { - Name() string + ModifierKey() string } // Component names used as keys into the extension modifier registry. Keep these -// in sync with the Name() methods that return them. +// in sync with the ModifierKey() methods that return them. const ( ComponentNameTypha = "typha" ComponentNameNode = "node" diff --git a/pkg/render/node.go b/pkg/render/node.go index 0e42fe3091..a8d397e1a1 100644 --- a/pkg/render/node.go +++ b/pkg/render/node.go @@ -127,8 +127,6 @@ type NodeConfiguration struct { // For details on why this is needed see 'Node and Installation finalizer' in the core_controller. CanRemoveCNIFinalizer bool - PrometheusServerTLS certificatemanagement.KeyPairInterface - // BGPLayouts is returned by the rendering code after modifying its namespace // so that it can be deployed into the cluster. // TODO: The controller should pass the contents, the renderer should build its own @@ -193,7 +191,7 @@ func (c *nodeComponent) SupportedOSType() rmeta.OSType { return rmeta.OSTypeLinux } -func (c *nodeComponent) Name() string { +func (c *nodeComponent) ModifierKey() string { return ComponentNameNode } @@ -897,18 +895,11 @@ func (c *nodeComponent) nodeDaemonset(cniCfgMap *corev1.ConfigMap) *appsv1.Daemo if len(c.cfg.BirdTemplates) != 0 { annotations[birdTemplateHashAnnotation] = rmeta.AnnotationHash(c.cfg.BirdTemplates) } - if c.cfg.PrometheusServerTLS != nil { - annotations[c.cfg.PrometheusServerTLS.HashAnnotationKey()] = c.cfg.PrometheusServerTLS.HashAnnotationValue() - } if c.cfg.TLS.NodeSecret.UseCertificateManagement() { initContainers = append(initContainers, c.cfg.TLS.NodeSecret.InitContainer(common.CalicoNamespace, nodeContainer.SecurityContext)) } - if c.cfg.PrometheusServerTLS != nil && c.cfg.PrometheusServerTLS.UseCertificateManagement() { - initContainers = append(initContainers, c.cfg.PrometheusServerTLS.InitContainer(common.CalicoNamespace, nodeContainer.SecurityContext)) - } - if cniCfgMap != nil { annotations[nodeCniConfigAnnotation] = rmeta.AnnotationHash(cniCfgMap.Data) } @@ -1114,10 +1105,6 @@ func (c *nodeComponent) nodeVolumes() []corev1.Volume { }, }) } - if c.cfg.PrometheusServerTLS != nil { - volumes = append(volumes, c.cfg.PrometheusServerTLS.Volume()) - } - return volumes } @@ -1358,9 +1345,6 @@ func (c *nodeComponent) nodeVolumeMounts() []corev1.VolumeMount { SubPath: BGPLayoutConfigMapKey, }) } - if c.cfg.PrometheusServerTLS != nil { - nodeVolumeMounts = append(nodeVolumeMounts, c.cfg.PrometheusServerTLS.VolumeMount(c.SupportedOSType())) - } return nodeVolumeMounts } diff --git a/pkg/render/node_enterprise_test.go b/pkg/render/node_enterprise_test.go index 4cc73d2a70..9e0d9c493b 100644 --- a/pkg/render/node_enterprise_test.go +++ b/pkg/render/node_enterprise_test.go @@ -33,6 +33,7 @@ import ( "github.com/tigera/operator/pkg/dns" "github.com/tigera/operator/pkg/extensions" "github.com/tigera/operator/pkg/render" + rmeta "github.com/tigera/operator/pkg/render/common/meta" ) // These tests run the real node/typha render output through the registered @@ -99,13 +100,12 @@ var _ = Describe("node enterprise modifier integration", func() { // modifier, exactly as the componentHandler does. renderNodeObjects := func() []client.Object { cfg := &render.NodeConfiguration{ - K8sServiceEp: k8sapi.ServiceEndpoint{}, - Installation: instance, - TLS: typhaNodeTLS, - ClusterDomain: dns.DefaultClusterDomain, - FelixHealthPort: 9099, - IPPools: instance.CalicoNetwork.IPPools, - PrometheusServerTLS: renderCtx.NodePrometheusTLS, + K8sServiceEp: k8sapi.ServiceEndpoint{}, + Installation: instance, + TLS: typhaNodeTLS, + ClusterDomain: dns.DefaultClusterDomain, + FelixHealthPort: 9099, + IPPools: instance.CalicoNetwork.IPPools, } comp := render.Node(cfg) Expect(comp.ResolveImages(nil)).NotTo(HaveOccurred()) @@ -145,8 +145,11 @@ var _ = Describe("node enterprise modifier integration", func() { corev1.EnvVar{Name: "FELIX_FLOWLOGSFILEENABLED", Value: "true"}, )) // The reporter cert env is wired from the NodePrometheusTLS keypair the - // factory creates. + // builder creates, and the modifier mounts that keypair onto the daemonset. Expect(c.Env).To(ContainElement(HaveField("Name", "FELIX_PROMETHEUSREPORTERCERTFILE"))) + Expect(ds.Spec.Template.Spec.Volumes).To(ContainElement(renderCtx.NodePrometheusTLS.Volume())) + Expect(c.VolumeMounts).To(ContainElement(renderCtx.NodePrometheusTLS.VolumeMount(rmeta.OSTypeLinux))) + Expect(ds.Spec.Template.Annotations).To(HaveKey(renderCtx.NodePrometheusTLS.HashAnnotationKey())) // BGP is enabled, so the bird readiness check is present and the modifier // adds the BGP metrics check. diff --git a/pkg/render/typha.go b/pkg/render/typha.go index 9b1209a3e7..3632978f81 100644 --- a/pkg/render/typha.go +++ b/pkg/render/typha.go @@ -113,7 +113,7 @@ func (c *typhaComponent) SupportedOSType() rmeta.OSType { return rmeta.OSTypeLinux } -func (c *typhaComponent) Name() string { return ComponentNameTypha } +func (c *typhaComponent) ModifierKey() string { return ComponentNameTypha } func (c *typhaComponent) Objects() ([]client.Object, []client.Object) { pdb := c.typhaPodDisruptionBudget() From 6a33636c85b4aee3a0e91d51f963dc0c66afec74 Mon Sep 17 00:00:00 2001 From: Casey Davenport Date: Tue, 9 Jun 2026 11:14:16 -0700 Subject: [PATCH 18/54] Collapse extension seams into Extension and Setup Merge the per-component image override and modifier into a single Extension{Image, Modify} registered once per (variant, component) via extensions.Register, so all of a component's variance lives in one place. The image half still lands in the imageoverride leaf so render resolves it without an import cycle; the fan-out is internal to Register. Rename the variant-level render context builder to Setup (RegisterSetup/RunSetup). That names the two phases a reader has to hold: Setup is the controller-side work that builds the RenderContext baton, and Extension hooks are the pure render-time funcs. Three registries with three key schemes become two concepts split by when they run. --- .../installation/core_controller.go | 13 ++--- pkg/controller/utils/component_test.go | 10 ++-- pkg/enterprise/installation.go | 11 ++-- pkg/enterprise/installation_test.go | 8 +-- pkg/enterprise/node.go | 8 +-- pkg/enterprise/typha.go | 4 +- pkg/extensions/{modifier.go => extension.go} | 43 ++++++++++----- .../{modifier_test.go => extension_test.go} | 32 ++++++----- pkg/extensions/image.go | 15 ++---- pkg/extensions/image_test.go | 12 +++-- pkg/extensions/{factory.go => setup.go} | 53 +++++++++---------- .../{factory_test.go => setup_test.go} | 32 +++++------ 12 files changed, 135 insertions(+), 106 deletions(-) rename pkg/extensions/{modifier.go => extension.go} (58%) rename pkg/extensions/{modifier_test.go => extension_test.go} (71%) rename pkg/extensions/{factory.go => setup.go} (50%) rename pkg/extensions/{factory_test.go => setup_test.go} (65%) diff --git a/pkg/controller/installation/core_controller.go b/pkg/controller/installation/core_controller.go index 568a59f954..c173827d23 100644 --- a/pkg/controller/installation/core_controller.go +++ b/pkg/controller/installation/core_controller.go @@ -1213,12 +1213,13 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile calicoVersion = components.EnterpriseRelease } - // Build the render context handed to registered modifiers. The core operator - // builder returns just the base context; an extension's builder additionally - // does controller-side work for its variant - validating config and creating - // the node-prometheus certificate, adding it (and the prometheus/esgw certs) - // to the trusted bundle - and may abort the reconcile by returning an error. - modCtx, err := extensions.BuildRenderContext(extensions.Inputs{ + // Run the variant setup to build the render context handed to registered + // modifiers. The core operator has no setup, so it gets just the base + // context; an extension's setup additionally does controller-side work for + // its variant - validating config and creating the node-prometheus + // certificate, adding it (and the prometheus/esgw certs) to the trusted + // bundle - and may abort the reconcile by returning an error. + modCtx, err := extensions.RunSetup(extensions.Inputs{ Ctx: ctx, Client: r.client, Installation: &instance.Spec, diff --git a/pkg/controller/utils/component_test.go b/pkg/controller/utils/component_test.go index eeb989e501..93c888812d 100644 --- a/pkg/controller/utils/component_test.go +++ b/pkg/controller/utils/component_test.go @@ -2493,10 +2493,12 @@ var _ = Describe("componentHandler modifier application", func() { }) It("applies registered modifiers to a named component before create", func() { - extensions.Modify(operatorv1.CalicoEnterprise, "fake", func(ctx extensions.RenderContext, objs []client.Object) []client.Object { - cm := objs[0].(*corev1.ConfigMap) - cm.Data = map[string]string{"patched": "yes"} - return objs + extensions.Register(operatorv1.CalicoEnterprise, "fake", extensions.Extension{ + Modify: func(ctx extensions.RenderContext, objs []client.Object) []client.Object { + cm := objs[0].(*corev1.ConfigMap) + cm.Data = map[string]string{"patched": "yes"} + return objs + }, }) s := runtime.NewScheme() diff --git a/pkg/enterprise/installation.go b/pkg/enterprise/installation.go index cd49480c68..ffb0dd7c98 100644 --- a/pkg/enterprise/installation.go +++ b/pkg/enterprise/installation.go @@ -28,14 +28,13 @@ import ( ) func registerInstallation() { - extensions.RegisterRenderContextBuilder(operatorv1.CalicoEnterprise, buildRenderContext) + extensions.RegisterSetup(operatorv1.CalicoEnterprise, setup) } -// buildRenderContext is the Calico Enterprise RenderContextBuilder. It builds -// the base render context and then does the controller-side work the modifiers -// can't: validating config and creating/fetching the certificates that feed the -// trusted bundle. -func buildRenderContext(in extensions.Inputs) (extensions.RenderContext, error) { +// setup is the Calico Enterprise setup phase. It builds the base render context +// and then does the controller-side work the modifiers can't: validating config +// and creating/fetching the certificates that feed the trusted bundle. +func setup(in extensions.Inputs) (extensions.RenderContext, error) { rc := extensions.BaseRenderContext(in) // Reject the unsupported zero reporter port. The port value itself is derived diff --git a/pkg/enterprise/installation_test.go b/pkg/enterprise/installation_test.go index f3aa09f887..1ebf7bf52b 100644 --- a/pkg/enterprise/installation_test.go +++ b/pkg/enterprise/installation_test.go @@ -32,7 +32,7 @@ import ( "github.com/tigera/operator/pkg/extensions" ) -var _ = Describe("installation render context builder", func() { +var _ = Describe("installation setup", func() { BeforeEach(func() { enterprise.Register() }) AfterEach(func() { extensions.ResetForTest() }) @@ -42,18 +42,18 @@ var _ = Describe("installation render context builder", func() { in.FelixConfiguration = &v3.FelixConfiguration{ Spec: v3.FelixConfigurationSpec{PrometheusReporterPort: &port}, } - _, err := extensions.BuildRenderContext(in) + _, err := extensions.RunSetup(in) Expect(err).To(HaveOccurred()) }) It("creates the node prometheus keypair for the enterprise variant", func() { - rc, err := extensions.BuildRenderContext(newInputs(operatorv1.CalicoEnterprise)) + rc, err := extensions.RunSetup(newInputs(operatorv1.CalicoEnterprise)) Expect(err).NotTo(HaveOccurred()) Expect(rc.NodePrometheusTLS).NotTo(BeNil()) }) It("is a no-op for the Calico variant", func() { - rc, err := extensions.BuildRenderContext(newInputs(operatorv1.Calico)) + rc, err := extensions.RunSetup(newInputs(operatorv1.Calico)) Expect(err).NotTo(HaveOccurred()) Expect(rc.NodePrometheusTLS).To(BeNil()) }) diff --git a/pkg/enterprise/node.go b/pkg/enterprise/node.go index 455c7b403f..335bac63c1 100644 --- a/pkg/enterprise/node.go +++ b/pkg/enterprise/node.go @@ -47,10 +47,12 @@ const ( ) func registerNode() { - extensions.OverrideImage(operatorv1.CalicoEnterprise, render.ComponentNameNode, func(in *operatorv1.InstallationSpec) components.Component { - return components.ComponentTigeraNode + extensions.Register(operatorv1.CalicoEnterprise, render.ComponentNameNode, extensions.Extension{ + Image: func(in *operatorv1.InstallationSpec) components.Component { + return components.ComponentTigeraNode + }, + Modify: modifyNode, }) - extensions.Modify(operatorv1.CalicoEnterprise, render.ComponentNameNode, modifyNode) } // modifyNode layers Calico Enterprise behavior onto the rendered calico/node diff --git a/pkg/enterprise/typha.go b/pkg/enterprise/typha.go index 368e71e835..457243842a 100644 --- a/pkg/enterprise/typha.go +++ b/pkg/enterprise/typha.go @@ -26,7 +26,9 @@ import ( ) func registerTypha() { - extensions.Modify(operatorv1.CalicoEnterprise, render.ComponentNameTypha, modifyTypha) + extensions.Register(operatorv1.CalicoEnterprise, render.ComponentNameTypha, extensions.Extension{ + Modify: modifyTypha, + }) } func modifyTypha(ctx extensions.RenderContext, objs []client.Object) []client.Object { diff --git a/pkg/extensions/modifier.go b/pkg/extensions/extension.go similarity index 58% rename from pkg/extensions/modifier.go rename to pkg/extensions/extension.go index 7a6fbf8828..4f57d64dd2 100644 --- a/pkg/extensions/modifier.go +++ b/pkg/extensions/extension.go @@ -21,10 +21,24 @@ import ( "github.com/tigera/operator/pkg/imageoverride" ) +// Extension is everything a variant layers onto one render component. Every +// field is optional: a component that only needs a different image sets Image +// and leaves Modify nil, and vice versa. This is the single registration a +// variant makes per component, so all of that component's variance lives in one +// place. +type Extension struct { + // Image overrides the component's image. Resolved during ResolveImages, in + // the render package, via the imageoverride leaf. + Image ImageOverride + + // Modify post-processes the component's rendered objects, after Objects(). + Modify Modifier +} + // Modifier post-processes the objects a render component produced. It may mutate // matched objects and/or append additional objects, and must return the -// (possibly extended) slice. A modifier is registered per variant, so it only -// runs for its own variant and need not re-check it. +// (possibly extended) slice. A modifier runs only for the variant it was +// registered under, so it need not re-check the variant. type Modifier func(ctx RenderContext, objs []client.Object) []client.Object type modifierKey struct { @@ -34,13 +48,18 @@ type modifierKey struct { var modifiers = map[modifierKey]Modifier{} -// Modify registers fn as the modifier for the named component under the given -// variant. A (variant, component) pair has at most one modifier; it handles all -// of that component's extension-specific mutations for that variant. -// Registration replaces any prior modifier, so it is idempotent and safe to -// call more than once. -func Modify(variant operatorv1.ProductVariant, component string, fn Modifier) { - modifiers[modifierKey{variant, component}] = fn +// Register installs e as the extension for the named component under the given +// variant. A (variant, component) pair has at most one extension; registration +// replaces any prior one, so it is idempotent and safe to call more than once. +// The image override lives in the imageoverride leaf (so the render package can +// resolve it without an import cycle); the modifier lives here. +func Register(variant operatorv1.ProductVariant, component string, e Extension) { + if e.Image != nil { + imageoverride.Register(variant, component, e.Image) + } + if e.Modify != nil { + modifiers[modifierKey{variant, component}] = e.Modify + } } // ApplyModifiers runs the modifier registered for the named component and the @@ -67,10 +86,10 @@ func FindObject[T client.Object](objs []client.Object, name string) (T, bool) { return zero, false } -// ResetForTest clears every registry: modifiers, image overrides, and the -// render context builder. Test-only. +// ResetForTest clears every registry: modifiers, image overrides, and variant +// setups. Test-only. func ResetForTest() { modifiers = map[modifierKey]Modifier{} - renderContextBuilders = map[operatorv1.ProductVariant]RenderContextBuilder{} + setups = map[operatorv1.ProductVariant]Setup{} imageoverride.ResetForTest() } diff --git a/pkg/extensions/modifier_test.go b/pkg/extensions/extension_test.go similarity index 71% rename from pkg/extensions/modifier_test.go rename to pkg/extensions/extension_test.go index 82decda600..c935dac9f0 100644 --- a/pkg/extensions/modifier_test.go +++ b/pkg/extensions/extension_test.go @@ -25,7 +25,7 @@ import ( "github.com/tigera/operator/pkg/extensions" ) -var _ = Describe("modifier registry", func() { +var _ = Describe("extension registry", func() { AfterEach(func() { extensions.ResetForTest() }) @@ -33,11 +33,13 @@ var _ = Describe("modifier registry", func() { entCtx := extensions.RenderContext{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.CalicoEnterprise}} It("applies a registered modifier to the matching component and variant", func() { - extensions.Modify(operatorv1.CalicoEnterprise, "test", func(ctx extensions.RenderContext, objs []client.Object) []client.Object { - cm, ok := extensions.FindObject[*corev1.ConfigMap](objs, "cm") - Expect(ok).To(BeTrue()) - cm.Data = map[string]string{"k": "v"} - return append(objs, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "extra"}}) + extensions.Register(operatorv1.CalicoEnterprise, "test", extensions.Extension{ + Modify: func(ctx extensions.RenderContext, objs []client.Object) []client.Object { + cm, ok := extensions.FindObject[*corev1.ConfigMap](objs, "cm") + Expect(ok).To(BeTrue()) + cm.Data = map[string]string{"k": "v"} + return append(objs, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "extra"}}) + }, }) in := []client.Object{&corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "cm"}}} @@ -56,8 +58,10 @@ var _ = Describe("modifier registry", func() { }) It("does not apply a modifier registered for a different variant", func() { - extensions.Modify(operatorv1.CalicoEnterprise, "test", func(_ extensions.RenderContext, objs []client.Object) []client.Object { - return append(objs, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "extra"}}) + extensions.Register(operatorv1.CalicoEnterprise, "test", extensions.Extension{ + Modify: func(_ extensions.RenderContext, objs []client.Object) []client.Object { + return append(objs, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "extra"}}) + }, }) calicoCtx := extensions.RenderContext{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.Calico}} @@ -71,13 +75,15 @@ var _ = Describe("modifier registry", func() { }) It("replaces rather than stacks when a (variant, component) is registered twice", func() { - add := func(name string) extensions.Modifier { - return func(_ extensions.RenderContext, objs []client.Object) []client.Object { - return append(objs, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: name}}) + add := func(name string) extensions.Extension { + return extensions.Extension{ + Modify: func(_ extensions.RenderContext, objs []client.Object) []client.Object { + return append(objs, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: name}}) + }, } } - extensions.Modify(operatorv1.CalicoEnterprise, "test", add("first")) - extensions.Modify(operatorv1.CalicoEnterprise, "test", add("second")) + extensions.Register(operatorv1.CalicoEnterprise, "test", add("first")) + extensions.Register(operatorv1.CalicoEnterprise, "test", add("second")) out := extensions.ApplyModifiers("test", entCtx, nil) Expect(out).To(HaveLen(1)) diff --git a/pkg/extensions/image.go b/pkg/extensions/image.go index 1063261993..477c89deef 100644 --- a/pkg/extensions/image.go +++ b/pkg/extensions/image.go @@ -20,19 +20,14 @@ import ( "github.com/tigera/operator/pkg/imageoverride" ) -// ImageOverride returns the component image to use for an installation. An -// override is registered per variant, so it only runs for its own variant and -// need not re-check it. +// ImageOverride returns the component image to use for an installation. It is +// the Image field of an Extension. An override runs only for the variant it was +// registered under, so it need not re-check the variant. type ImageOverride = imageoverride.Override -// OverrideImage registers an image override under key for the given variant. -// The key is the render component's image identifier (e.g. "node"). -func OverrideImage(variant operatorv1.ProductVariant, key string, fn ImageOverride) { - imageoverride.Register(variant, key, fn) -} - // ResolveImage returns the override registered for key if it applies to in, -// otherwise def. Render components call this inside ResolveImages. +// otherwise def. The render package resolves images through the imageoverride +// leaf directly; this is the same lookup for callers already inside extensions. func ResolveImage(key string, def components.Component, in *operatorv1.InstallationSpec) components.Component { return imageoverride.Resolve(key, def, in) } diff --git a/pkg/extensions/image_test.go b/pkg/extensions/image_test.go index 7f05c6cac1..25926c1a4f 100644 --- a/pkg/extensions/image_test.go +++ b/pkg/extensions/image_test.go @@ -29,8 +29,10 @@ var _ = Describe("image overrides", func() { }) It("uses the override registered for the installation variant", func() { - extensions.OverrideImage(operatorv1.CalicoEnterprise, "node", func(in *operatorv1.InstallationSpec) components.Component { - return components.ComponentTigeraNode + extensions.Register(operatorv1.CalicoEnterprise, "node", extensions.Extension{ + Image: func(in *operatorv1.InstallationSpec) components.Component { + return components.ComponentTigeraNode + }, }) ent := &operatorv1.InstallationSpec{Variant: operatorv1.CalicoEnterprise} @@ -38,8 +40,10 @@ var _ = Describe("image overrides", func() { }) It("falls back to the default for a variant with no override", func() { - extensions.OverrideImage(operatorv1.CalicoEnterprise, "node", func(in *operatorv1.InstallationSpec) components.Component { - return components.ComponentTigeraNode + extensions.Register(operatorv1.CalicoEnterprise, "node", extensions.Extension{ + Image: func(in *operatorv1.InstallationSpec) components.Component { + return components.ComponentTigeraNode + }, }) calico := &operatorv1.InstallationSpec{Variant: operatorv1.Calico} diff --git a/pkg/extensions/factory.go b/pkg/extensions/setup.go similarity index 50% rename from pkg/extensions/factory.go rename to pkg/extensions/setup.go index 48d615fd6d..136863b0ff 100644 --- a/pkg/extensions/factory.go +++ b/pkg/extensions/setup.go @@ -25,11 +25,11 @@ import ( "github.com/tigera/operator/pkg/tls/certificatemanagement" ) -// Inputs is the reconcile state a RenderContextBuilder builds a RenderContext -// from. The installation controller populates it directly. It carries both the -// values that flow straight into the RenderContext and the side-effecting -// dependencies (Client, CertificateManager) a builder needs to produce -// controller-side artifacts. +// Inputs is the reconcile state a Setup builds a RenderContext from. The +// installation controller populates it directly. It carries both the values +// that flow straight into the RenderContext and the side-effecting dependencies +// (Client, CertificateManager) a setup needs to produce controller-side +// artifacts. type Inputs struct { Ctx context.Context Client client.Client @@ -40,21 +40,21 @@ type Inputs struct { ClusterDomain string } -// RenderContextBuilder builds the RenderContext handed to render modifiers. It -// performs any controller-side work (creating certificates, extending the -// trusted bundle) and returns the assembled RenderContext - or an error that -// aborts the reconcile. +// Setup is a variant's controller-side reconcile phase. It performs the work +// modifiers can't (creating certificates, extending the trusted bundle, +// validating config) and returns the RenderContext that is then handed to that +// variant's modifiers - or an error that aborts the reconcile. // // This is the generic seam controllers use to extend base operator behavior; // its first consumer is Calico Enterprise, but nothing here is enterprise -// specific. A builder is registered per variant, so it only runs for its own -// variant and need not re-check it. -type RenderContextBuilder func(in Inputs) (RenderContext, error) +// specific. A setup runs only for the variant it was registered under, so it +// need not re-check the variant. +type Setup func(in Inputs) (RenderContext, error) // BaseRenderContext maps the generically-gathered inputs onto a RenderContext. -// Every registered builder builds on it, so the base fields are assembled in -// exactly one place. A builder layers its side-effect artifacts (e.g. -// NodePrometheusTLS) on top of the returned value. +// Every setup builds on it, so the base fields are assembled in exactly one +// place. A setup layers its side-effect artifacts (e.g. NodePrometheusTLS) on +// top of the returned value. func BaseRenderContext(in Inputs) RenderContext { return RenderContext{ Installation: in.Installation, @@ -64,22 +64,21 @@ func BaseRenderContext(in Inputs) RenderContext { } } -var renderContextBuilders = map[operatorv1.ProductVariant]RenderContextBuilder{} +var setups = map[operatorv1.ProductVariant]Setup{} -// RegisterRenderContextBuilder installs f as the builder for the given variant. -// Registration replaces any prior builder, so it is safe to call more than -// once. Variants without a registered builder get the base render context. -func RegisterRenderContextBuilder(variant operatorv1.ProductVariant, f RenderContextBuilder) { - renderContextBuilders[variant] = f +// RegisterSetup installs s as the setup for the given variant. Registration +// replaces any prior setup, so it is safe to call more than once. Variants +// without a registered setup get the base render context. +func RegisterSetup(variant operatorv1.ProductVariant, s Setup) { + setups[variant] = s } -// BuildRenderContext builds the RenderContext from in using the builder -// registered for the installation variant, or the base render context when the -// variant has no registered builder. -func BuildRenderContext(in Inputs) (RenderContext, error) { +// RunSetup runs the setup registered for the installation variant and returns +// its RenderContext, or the base render context when the variant has no setup. +func RunSetup(in Inputs) (RenderContext, error) { if in.Installation != nil { - if f, ok := renderContextBuilders[in.Installation.Variant]; ok { - return f(in) + if s, ok := setups[in.Installation.Variant]; ok { + return s(in) } } return BaseRenderContext(in), nil diff --git a/pkg/extensions/factory_test.go b/pkg/extensions/setup_test.go similarity index 65% rename from pkg/extensions/factory_test.go rename to pkg/extensions/setup_test.go index 8b8ad825ce..9967bfe209 100644 --- a/pkg/extensions/factory_test.go +++ b/pkg/extensions/setup_test.go @@ -24,12 +24,12 @@ import ( "github.com/tigera/operator/pkg/extensions" ) -var _ = Describe("render context builder", func() { +var _ = Describe("variant setup", func() { AfterEach(func() { extensions.ResetForTest() }) - It("returns the base render context when the variant has no builder", func() { + It("returns the base render context when the variant has no setup", func() { install := &operatorv1.InstallationSpec{Variant: operatorv1.Calico} - rc, err := extensions.BuildRenderContext(extensions.Inputs{ + rc, err := extensions.RunSetup(extensions.Inputs{ Installation: install, ClusterDomain: "cluster.local", }) @@ -39,16 +39,16 @@ var _ = Describe("render context builder", func() { Expect(rc.NodePrometheusTLS).To(BeNil()) }) - It("uses the builder registered for the installation variant", func() { - extensions.RegisterRenderContextBuilder(operatorv1.CalicoEnterprise, fakeBuilder(nil)) - rc, err := extensions.BuildRenderContext(enterpriseInputs()) + It("uses the setup registered for the installation variant", func() { + extensions.RegisterSetup(operatorv1.CalicoEnterprise, fakeSetup(nil)) + rc, err := extensions.RunSetup(enterpriseInputs()) Expect(err).NotTo(HaveOccurred()) Expect(rc.ClusterDomain).To(Equal("from-fake")) }) - It("ignores a builder registered for a different variant", func() { - extensions.RegisterRenderContextBuilder(operatorv1.CalicoEnterprise, fakeBuilder(nil)) - rc, err := extensions.BuildRenderContext(extensions.Inputs{ + It("ignores a setup registered for a different variant", func() { + extensions.RegisterSetup(operatorv1.CalicoEnterprise, fakeSetup(nil)) + rc, err := extensions.RunSetup(extensions.Inputs{ Installation: &operatorv1.InstallationSpec{Variant: operatorv1.Calico}, ClusterDomain: "real", }) @@ -56,18 +56,18 @@ var _ = Describe("render context builder", func() { Expect(rc.ClusterDomain).To(Equal("real")) }) - It("surfaces the builder error", func() { - extensions.RegisterRenderContextBuilder(operatorv1.CalicoEnterprise, fakeBuilder(errors.New("boom"))) - _, err := extensions.BuildRenderContext(enterpriseInputs()) + It("surfaces the setup error", func() { + extensions.RegisterSetup(operatorv1.CalicoEnterprise, fakeSetup(errors.New("boom"))) + _, err := extensions.RunSetup(enterpriseInputs()) Expect(err).To(MatchError("boom")) }) - It("restores the base builder on reset", func() { - extensions.RegisterRenderContextBuilder(operatorv1.CalicoEnterprise, fakeBuilder(nil)) + It("restores the base context on reset", func() { + extensions.RegisterSetup(operatorv1.CalicoEnterprise, fakeSetup(nil)) extensions.ResetForTest() in := enterpriseInputs() in.ClusterDomain = "real" - rc, err := extensions.BuildRenderContext(in) + rc, err := extensions.RunSetup(in) Expect(err).NotTo(HaveOccurred()) Expect(rc.ClusterDomain).To(Equal("real")) }) @@ -77,7 +77,7 @@ func enterpriseInputs() extensions.Inputs { return extensions.Inputs{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.CalicoEnterprise}} } -func fakeBuilder(err error) extensions.RenderContextBuilder { +func fakeSetup(err error) extensions.Setup { return func(_ extensions.Inputs) (extensions.RenderContext, error) { if err != nil { return extensions.RenderContext{}, err From 9197ec92bb9bc4410df323c4d8b8df8cf4b67643 Mon Sep 17 00:00:00 2001 From: Casey Davenport Date: Tue, 9 Jun 2026 11:19:14 -0700 Subject: [PATCH 19/54] Rename modCtx to renderCtx modCtx read like "modifier context"; the value is an extensions.RenderContext, so name it for what it is. --- pkg/controller/installation/core_controller.go | 6 +++--- pkg/controller/utils/component.go | 6 +++--- pkg/controller/utils/component_test.go | 4 ++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/pkg/controller/installation/core_controller.go b/pkg/controller/installation/core_controller.go index c173827d23..e4f32f7934 100644 --- a/pkg/controller/installation/core_controller.go +++ b/pkg/controller/installation/core_controller.go @@ -1219,7 +1219,7 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile // its variant - validating config and creating the node-prometheus // certificate, adding it (and the prometheus/esgw certs) to the trusted // bundle - and may abort the reconcile by returning an error. - modCtx, err := extensions.RunSetup(extensions.Inputs{ + renderCtx, err := extensions.RunSetup(extensions.Inputs{ Ctx: ctx, Client: r.client, Installation: &instance.Spec, @@ -1232,7 +1232,7 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile r.status.SetDegraded(operatorv1.ResourceCreateError, "Error preparing installation extension", err, reqLogger) return reconcile.Result{}, err } - nodePrometheusTLS := modCtx.NodePrometheusTLS + nodePrometheusTLS := renderCtx.NodePrometheusTLS kubeControllersMetricsPort, err := utils.GetKubeControllerMetricsPort(ctx, r.client) if err != nil { @@ -1271,7 +1271,7 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile } // Create a component handler to create or update the rendered components. - handler := r.newComponentHandler(log, r.client, r.scheme, instance, utils.WithRenderContext(modCtx)) + handler := r.newComponentHandler(log, r.client, r.scheme, instance, utils.WithRenderContext(renderCtx)) // Render namespaces first - this ensures that any other controllers blocked on namespace existence can proceed. namespaceCfg := &render.NamespaceConfiguration{ diff --git a/pkg/controller/utils/component.go b/pkg/controller/utils/component.go index 57238556da..550063296d 100644 --- a/pkg/controller/utils/component.go +++ b/pkg/controller/utils/component.go @@ -81,7 +81,7 @@ type ComponentHandlerOption func(*componentHandler) // WithRenderContext supplies the extensions.RenderContext passed to registered // render modifiers. func WithRenderContext(ctx extensions.RenderContext) ComponentHandlerOption { - return func(c *componentHandler) { c.modCtx = ctx } + return func(c *componentHandler) { c.renderCtx = ctx } } // cr is allowed to be nil in the case we don't want to put ownership on a resource, @@ -107,7 +107,7 @@ type componentHandler struct { log logr.Logger createOnly bool apiGroupEnvs []v1.EnvVar - modCtx extensions.RenderContext + renderCtx extensions.RenderContext } func (c *componentHandler) SetCreateOnly() { @@ -469,7 +469,7 @@ func (c *componentHandler) CreateOrUpdateOrDelete(ctx context.Context, component objsToCreate, objsToDelete := component.Objects() if ext, ok := component.(render.Extensible); ok { - objsToCreate = extensions.ApplyModifiers(ext.ModifierKey(), c.modCtx, objsToCreate) + objsToCreate = extensions.ApplyModifiers(ext.ModifierKey(), c.renderCtx, objsToCreate) } // Load the InstallationSpec once and reuse it for every object: createOrUpdateObject needs it diff --git a/pkg/controller/utils/component_test.go b/pkg/controller/utils/component_test.go index 93c888812d..ecbe90afe4 100644 --- a/pkg/controller/utils/component_test.go +++ b/pkg/controller/utils/component_test.go @@ -2506,8 +2506,8 @@ var _ = Describe("componentHandler modifier application", func() { Expect(corev1.SchemeBuilder.AddToScheme(s)).NotTo(HaveOccurred()) c := ctrlrfake.DefaultFakeClientBuilder(s).Build() - modCtx := extensions.RenderContext{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.CalicoEnterprise}} - handler := NewComponentHandler(logf.Log, c, s, nil, WithRenderContext(modCtx)) + renderCtx := extensions.RenderContext{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.CalicoEnterprise}} + handler := NewComponentHandler(logf.Log, c, s, nil, WithRenderContext(renderCtx)) comp := &namedFakeComponent{name: "fake", obj: &corev1.ConfigMap{ TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"}, ObjectMeta: metav1.ObjectMeta{Name: "cm", Namespace: "default"}, From 6375a348d5ea318d97682bf6f486655c5b0eb850 Mon Sep 17 00:00:00 2001 From: Casey Davenport Date: Tue, 9 Jun 2026 11:40:44 -0700 Subject: [PATCH 20/54] Document the extensions package and drop stale builder wording Add a package doc that lays out the two-phase model (Setup vs Extension) so the whole seam is legible from `go doc`. Fix two comments that still called the setup a render context builder. --- pkg/enterprise/node.go | 8 +++---- pkg/extensions/doc.go | 38 +++++++++++++++++++++++++++++++++ pkg/extensions/rendercontext.go | 8 +++---- 3 files changed, 46 insertions(+), 8 deletions(-) create mode 100644 pkg/extensions/doc.go diff --git a/pkg/enterprise/node.go b/pkg/enterprise/node.go index 335bac63c1..8e1de4832b 100644 --- a/pkg/enterprise/node.go +++ b/pkg/enterprise/node.go @@ -147,10 +147,10 @@ func modifyNodeDaemonSet(ctx extensions.RenderContext, ds *appsv1.DaemonSet) { // mountNodePrometheusTLS mounts the node prometheus reporter keypair onto the // daemonset: the volume, the calico-node volume mount, the cert-management init // container (when in use), and the pod hash annotation that rolls the pods on -// cert rotation. The keypair has cluster side effects, so the enterprise render -// context builder creates it and hands it in via ctx rather than the modifier -// building it. In core (calico) the keypair is never created, so the base node -// render carries no prometheus mount at all. +// cert rotation. The keypair has cluster side effects, so the enterprise setup +// creates it and hands it in via ctx rather than the modifier building it. In +// core (calico) the keypair is never created, so the base node render carries +// no prometheus mount at all. func mountNodePrometheusTLS(ctx extensions.RenderContext, ds *appsv1.DaemonSet) { if ctx.NodePrometheusTLS == nil { return diff --git a/pkg/extensions/doc.go b/pkg/extensions/doc.go new file mode 100644 index 0000000000..73adc07fa0 --- /dev/null +++ b/pkg/extensions/doc.go @@ -0,0 +1,38 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package extensions is the seam other product variants (today just Calico +// Enterprise) use to layer variant-specific behavior onto the core operator's +// render output, so core code never branches on variant. +// +// Everything keys off the installation Variant, and registration is per +// variant, so a registered hook only ever runs for its own variant and never +// re-checks it. There are two phases: +// +// Setup is the controller-side phase. It runs once per reconcile in the +// installation controller, has cluster access (Client, CertificateManager), and +// does the side-effecting work a pure render hook can't: creating certificates, +// extending the trusted bundle, validating config. It returns the RenderContext +// - the read-only baton passed to the render phase. Register one per variant +// with RegisterSetup; the controller runs it with RunSetup. +// +// Extension is the render phase: pure, per-component hooks that run after a +// component builds its objects. Its Image field overrides the component's image +// (resolved during ResolveImages), and its Modify field post-processes the +// rendered objects (run at the componentHandler). Register one per component +// with Register. +// +// A variant wires up its setup and extensions in one place at startup - see +// pkg/enterprise. +package extensions diff --git a/pkg/extensions/rendercontext.go b/pkg/extensions/rendercontext.go index 9ea25e1b6b..9653825a59 100644 --- a/pkg/extensions/rendercontext.go +++ b/pkg/extensions/rendercontext.go @@ -36,9 +36,9 @@ type RenderContext struct { // TrustedBundle is the shared CA bundle for the calico-system namespace. TrustedBundle certificatemanagement.TrustedBundle - // NodePrometheusTLS is created by the enterprise render context builder (it - // has cluster side effects, so it can't be built in a modifier). The node - // modifier is its only consumer: it mounts the keypair onto the daemonset and - // sets the FELIX_PROMETHEUSREPORTER* certificate env vars. + // NodePrometheusTLS is created by the enterprise setup (it has cluster side + // effects, so it can't be built in a modifier). The node modifier is its only + // consumer: it mounts the keypair onto the daemonset and sets the + // FELIX_PROMETHEUSREPORTER* certificate env vars. NodePrometheusTLS certificatemanagement.KeyPairInterface } From 147ec15feebbc538d5a2f72ad4d33612f974ff3a Mon Sep 17 00:00:00 2001 From: Casey Davenport Date: Tue, 9 Jun 2026 14:23:53 -0700 Subject: [PATCH 21/54] De-variant windows; add per-component context to extensions Add a per-component context channel: a component implements render.ExtensionContextProvider to hand its modifier config a modifier can't derive from the shared RenderContext (config only the component's controller has). The componentHandler reads it into RenderContext.Component before applying the modifier. node's setup-produced keypair keeps its own field; this is for component-config-derived inputs. Move windows's enterprise branches into a pkg/enterprise extension: the two windows image overrides, the node-metrics Service, the calico log volume (swapped in for the OSS cni-log mount), the enterprise felix env, the trusted DNS servers for openshift/rke2, and the prometheus reporter keypair mount. The windows component exposes its reporter port, keypair, and trusted bundle via ExtensionContext; the windows controller wires the render context into its handler. Core windows render is now OSS-only. --- .../installation/windows_controller.go | 6 +- pkg/controller/utils/component.go | 6 +- pkg/enterprise/register.go | 1 + pkg/enterprise/windows.go | 189 ++++++++++++++++++ pkg/enterprise/windows_test.go | 160 +++++++++++++++ pkg/extensions/rendercontext.go | 16 +- pkg/render/component.go | 17 ++ pkg/render/render_test.go | 14 +- pkg/render/windows.go | 147 +++----------- pkg/render/windows_test.go | 42 ++-- 10 files changed, 450 insertions(+), 148 deletions(-) create mode 100644 pkg/enterprise/windows.go create mode 100644 pkg/enterprise/windows_test.go diff --git a/pkg/controller/installation/windows_controller.go b/pkg/controller/installation/windows_controller.go index e23bf55db7..e3cb1389b2 100644 --- a/pkg/controller/installation/windows_controller.go +++ b/pkg/controller/installation/windows_controller.go @@ -51,6 +51,7 @@ import ( "github.com/tigera/operator/pkg/controller/utils/imageset" "github.com/tigera/operator/pkg/ctrlruntime" "github.com/tigera/operator/pkg/dns" + "github.com/tigera/operator/pkg/extensions" "github.com/tigera/operator/pkg/render" "github.com/tigera/operator/pkg/render/monitor" "github.com/tigera/operator/pkg/tls/certificatemanagement" @@ -406,7 +407,10 @@ func (r *ReconcileWindows) Reconcile(ctx context.Context, request reconcile.Requ } // Create a component handler to create or update the rendered components. - handler := utils.NewComponentHandler(logw, r.client, r.scheme, instance) + // The render context carries the installation so registered modifiers gate on + // variant; the windows component supplies its own per-component context (the + // reporter port and prometheus keypair) via ExtensionContext. + handler := utils.NewComponentHandler(logw, r.client, r.scheme, instance, utils.WithRenderContext(extensions.RenderContext{Installation: &instance.Spec})) if err := handler.CreateOrUpdateOrDelete(ctx, component, nil); err != nil { r.status.SetDegraded(operatorv1.ResourceUpdateError, "Error creating / updating resource", err, reqLogger) return reconcile.Result{}, err diff --git a/pkg/controller/utils/component.go b/pkg/controller/utils/component.go index 550063296d..d2424c99c7 100644 --- a/pkg/controller/utils/component.go +++ b/pkg/controller/utils/component.go @@ -469,7 +469,11 @@ func (c *componentHandler) CreateOrUpdateOrDelete(ctx context.Context, component objsToCreate, objsToDelete := component.Objects() if ext, ok := component.(render.Extensible); ok { - objsToCreate = extensions.ApplyModifiers(ext.ModifierKey(), c.renderCtx, objsToCreate) + rc := c.renderCtx + if p, ok := component.(render.ExtensionContextProvider); ok { + rc.Component = p.ExtensionContext() + } + objsToCreate = extensions.ApplyModifiers(ext.ModifierKey(), rc, objsToCreate) } // Load the InstallationSpec once and reuse it for every object: createOrUpdateObject needs it diff --git a/pkg/enterprise/register.go b/pkg/enterprise/register.go index a01c9b7cfa..761f1736b7 100644 --- a/pkg/enterprise/register.go +++ b/pkg/enterprise/register.go @@ -20,5 +20,6 @@ package enterprise func Register() { registerTypha() registerNode() + registerWindows() registerInstallation() } diff --git a/pkg/enterprise/windows.go b/pkg/enterprise/windows.go new file mode 100644 index 0000000000..12f62cd6c6 --- /dev/null +++ b/pkg/enterprise/windows.go @@ -0,0 +1,189 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package enterprise + +import ( + "fmt" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + "sigs.k8s.io/controller-runtime/pkg/client" + + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/common" + "github.com/tigera/operator/pkg/components" + "github.com/tigera/operator/pkg/extensions" + "github.com/tigera/operator/pkg/render" + rmeta "github.com/tigera/operator/pkg/render/common/meta" +) + +// windowsNodeContainers are the calico-node-windows containers that share the +// felix env and node volume mounts, so they receive the same enterprise layering. +var windowsNodeContainers = map[string]bool{"felix": true, "node": true, "confd": true} + +func registerWindows() { + extensions.Register(operatorv1.CalicoEnterprise, render.ComponentNameWindowsNodeImg, extensions.Extension{ + Image: func(*operatorv1.InstallationSpec) components.Component { return components.ComponentTigeraNodeWindows }, + }) + extensions.Register(operatorv1.CalicoEnterprise, render.ComponentNameWindowsCNIImg, extensions.Extension{ + Image: func(*operatorv1.InstallationSpec) components.Component { return components.ComponentTigeraCNIWindows }, + }) + extensions.Register(operatorv1.CalicoEnterprise, render.ComponentNameWindows, extensions.Extension{ + Modify: modifyWindows, + }) +} + +// modifyWindows layers Calico Enterprise behavior onto the rendered +// calico-node-windows objects: the node-metrics Service and the Enterprise +// daemonset configuration (flow/DNS log env, prometheus reporter, trusted DNS +// servers, the calico log volume, and the prometheus reporter keypair mount). +func modifyWindows(ctx extensions.RenderContext, objs []client.Object) []client.Object { + wc, _ := ctx.Component.(render.WindowsExtensionContext) + + if ds, ok := extensions.FindObject[*appsv1.DaemonSet](objs, common.WindowsDaemonSetName); ok { + modifyWindowsDaemonSet(ctx, wc, ds) + } + + return append(objs, windowsNodeMetricsService(wc)) +} + +func modifyWindowsDaemonSet(ctx extensions.RenderContext, wc render.WindowsExtensionContext, ds *appsv1.DaemonSet) { + dirOrCreate := corev1.HostPathDirectoryOrCreate + spec := &ds.Spec.Template.Spec + + spec.Volumes = append(spec.Volumes, corev1.Volume{ + Name: "var-log-calico", + VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico", Type: &dirOrCreate}}, + }) + + for i := range spec.Containers { + c := &spec.Containers[i] + if !windowsNodeContainers[c.Name] { + continue + } + + c.Env = append(c.Env, windowsEnterpriseEnv(ctx, wc)...) + + // Enterprise mounts the calico log directory in place of the OSS CNI log + // directory, so drop the OSS mount before adding the enterprise one. + c.VolumeMounts = removeVolumeMount(c.VolumeMounts, "cni-log-dir") + c.VolumeMounts = append(c.VolumeMounts, corev1.VolumeMount{MountPath: "/var/log/calico", Name: "var-log-calico"}) + } + + mountWindowsPrometheusTLS(wc, ds) +} + +// windowsEnterpriseEnv is the Enterprise felix configuration added to the +// calico-node-windows containers. +func windowsEnterpriseEnv(ctx extensions.RenderContext, wc render.WindowsExtensionContext) []corev1.EnvVar { + env := []corev1.EnvVar{ + {Name: "FELIX_PROMETHEUSREPORTERENABLED", Value: "true"}, + {Name: "FELIX_PROMETHEUSREPORTERPORT", Value: fmt.Sprintf("%d", wc.NodeReporterMetricsPort)}, + {Name: "FELIX_FLOWLOGSFILEENABLED", Value: "true"}, + {Name: "FELIX_FLOWLOGSFILEINCLUDELABELS", Value: "true"}, + {Name: "FELIX_FLOWLOGSFILEINCLUDEPOLICIES", Value: "true"}, + {Name: "FELIX_FLOWLOGSFILEINCLUDESERVICE", Value: "true"}, + {Name: "FELIX_FLOWLOGSENABLENETWORKSETS", Value: "true"}, + {Name: "FELIX_FLOWLOGSCOLLECTPROCESSINFO", Value: "true"}, + {Name: "FELIX_DNSLOGSFILEENABLED", Value: "true"}, + {Name: "FELIX_DNSLOGSFILEPERNODELIMIT", Value: "1000"}, + } + + if wc.PrometheusServerTLS != nil && wc.TrustedBundle != nil { + env = append(env, + corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERCERTFILE", Value: wc.PrometheusServerTLS.VolumeMountCertificateFilePath()}, + corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERKEYFILE", Value: wc.PrometheusServerTLS.VolumeMountKeyFilePath()}, + corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERCAFILE", Value: wc.TrustedBundle.MountPath()}, + ) + } + + // Providers without a kube-dns service need a non-default trusted DNS server. + switch ctx.Installation.KubernetesProvider { + case operatorv1.ProviderOpenShift: + env = append(env, corev1.EnvVar{Name: "FELIX_DNSTRUSTEDSERVERS", Value: "k8s-service:openshift-dns/dns-default"}) + case operatorv1.ProviderRKE2: + env = append(env, corev1.EnvVar{Name: "FELIX_DNSTRUSTEDSERVERS", Value: "k8s-service:kube-system/rke2-coredns-rke2-coredns"}) + } + + return env +} + +// mountWindowsPrometheusTLS mounts the node prometheus reporter keypair onto the +// windows daemonset: the volume, the volume mount on each node container, and +// the pod hash annotation that rolls the pods on cert rotation. +func mountWindowsPrometheusTLS(wc render.WindowsExtensionContext, ds *appsv1.DaemonSet) { + if wc.PrometheusServerTLS == nil { + return + } + tls := wc.PrometheusServerTLS + spec := &ds.Spec.Template.Spec + + spec.Volumes = append(spec.Volumes, tls.Volume()) + + for i := range spec.Containers { + c := &spec.Containers[i] + if windowsNodeContainers[c.Name] { + c.VolumeMounts = append(c.VolumeMounts, tls.VolumeMount(rmeta.OSTypeWindows)) + } + } + + if ds.Spec.Template.Annotations == nil { + ds.Spec.Template.Annotations = map[string]string{} + } + ds.Spec.Template.Annotations[tls.HashAnnotationKey()] = tls.HashAnnotationValue() +} + +// windowsNodeMetricsService builds the enterprise-only calico-node-metrics-windows +// Service. +func windowsNodeMetricsService(wc render.WindowsExtensionContext) *corev1.Service { + return &corev1.Service{ + TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: render.WindowsNodeMetricsService, + Namespace: common.CalicoNamespace, + Labels: map[string]string{"k8s-app": render.WindowsNodeObjectName}, + }, + Spec: corev1.ServiceSpec{ + Selector: map[string]string{"k8s-app": render.WindowsNodeObjectName}, + ClusterIP: "None", + Ports: []corev1.ServicePort{ + { + Name: "calico-metrics-port", + Port: int32(wc.NodeReporterMetricsPort), + TargetPort: intstr.FromInt(wc.NodeReporterMetricsPort), + Protocol: corev1.ProtocolTCP, + }, + { + Name: "calico-bgp-metrics-port", + Port: render.NodeBGPReporterPort, + TargetPort: intstr.FromInt(int(render.NodeBGPReporterPort)), + Protocol: corev1.ProtocolTCP, + }, + }, + }, + } +} + +func removeVolumeMount(mounts []corev1.VolumeMount, name string) []corev1.VolumeMount { + out := mounts[:0] + for _, m := range mounts { + if m.Name != name { + out = append(out, m) + } + } + return out +} diff --git a/pkg/enterprise/windows_test.go b/pkg/enterprise/windows_test.go new file mode 100644 index 0000000000..b289941bdb --- /dev/null +++ b/pkg/enterprise/windows_test.go @@ -0,0 +1,160 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package enterprise_test + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + client "sigs.k8s.io/controller-runtime/pkg/client" + + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/apis" + "github.com/tigera/operator/pkg/common" + "github.com/tigera/operator/pkg/components" + "github.com/tigera/operator/pkg/controller/certificatemanager" + ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" + "github.com/tigera/operator/pkg/enterprise" + "github.com/tigera/operator/pkg/extensions" + "github.com/tigera/operator/pkg/render" + "github.com/tigera/operator/pkg/tls/certificatemanagement" +) + +var _ = Describe("windows enterprise image override", func() { + BeforeEach(func() { enterprise.Register() }) + AfterEach(func() { extensions.ResetForTest() }) + + ent := &operatorv1.InstallationSpec{Variant: operatorv1.CalicoEnterprise} + calico := &operatorv1.InstallationSpec{Variant: operatorv1.Calico} + + It("selects the enterprise windows images for the enterprise variant", func() { + Expect(extensions.ResolveImage(render.ComponentNameWindowsNodeImg, components.ComponentCalicoNodeWindows, ent)).To(Equal(components.ComponentTigeraNodeWindows)) + Expect(extensions.ResolveImage(render.ComponentNameWindowsCNIImg, components.ComponentCalicoCNIWindows, ent)).To(Equal(components.ComponentTigeraCNIWindows)) + }) + + It("leaves the defaults in place for the Calico variant", func() { + Expect(extensions.ResolveImage(render.ComponentNameWindowsNodeImg, components.ComponentCalicoNodeWindows, calico)).To(Equal(components.ComponentCalicoNodeWindows)) + Expect(extensions.ResolveImage(render.ComponentNameWindowsCNIImg, components.ComponentCalicoCNIWindows, calico)).To(Equal(components.ComponentCalicoCNIWindows)) + }) +}) + +var _ = Describe("windows enterprise modifier", func() { + BeforeEach(func() { enterprise.Register() }) + AfterEach(func() { extensions.ResetForTest() }) + + // newObjs returns a windows daemonset with the node containers and the OSS + // cni-log-dir mount the modifier swaps out. + newObjs := func() []client.Object { + nodeContainer := func(name string) corev1.Container { + return corev1.Container{ + Name: name, + VolumeMounts: []corev1.VolumeMount{{MountPath: "/var/log/calico/cni", Name: "cni-log-dir"}}, + } + } + return []client.Object{ + &appsv1.DaemonSet{ + ObjectMeta: metav1.ObjectMeta{Name: common.WindowsDaemonSetName}, + Spec: appsv1.DaemonSetSpec{Template: corev1.PodTemplateSpec{Spec: corev1.PodSpec{ + Containers: []corev1.Container{nodeContainer("felix"), nodeContainer("node"), nodeContainer("confd")}, + }}}, + }, + } + } + + ds := func(objs []client.Object) *appsv1.DaemonSet { + d, _ := extensions.FindObject[*appsv1.DaemonSet](objs, common.WindowsDaemonSetName) + return d + } + container := func(d *appsv1.DaemonSet, name string) *corev1.Container { + for i := range d.Spec.Template.Spec.Containers { + if d.Spec.Template.Spec.Containers[i].Name == name { + return &d.Spec.Template.Spec.Containers[i] + } + } + return nil + } + + ctxFor := func(provider operatorv1.Provider, tls certificatemanagement.KeyPairInterface, bundle certificatemanagement.TrustedBundleRO) extensions.RenderContext { + return extensions.RenderContext{ + Installation: &operatorv1.InstallationSpec{Variant: operatorv1.CalicoEnterprise, KubernetesProvider: provider}, + Component: render.WindowsExtensionContext{ + NodeReporterMetricsPort: 9081, + PrometheusServerTLS: tls, + TrustedBundle: bundle, + }, + } + } + + It("appends the node-metrics service", func() { + out := extensions.ApplyModifiers(render.ComponentNameWindows, ctxFor(operatorv1.ProviderNone, nil, nil), newObjs()) + svc, ok := extensions.FindObject[*corev1.Service](out, render.WindowsNodeMetricsService) + Expect(ok).To(BeTrue()) + Expect(svc.Namespace).To(Equal(common.CalicoNamespace)) + Expect(svc.Spec.Ports[0].Port).To(Equal(int32(9081))) + }) + + It("swaps the cni log mount for the calico log volume and adds enterprise env", func() { + out := extensions.ApplyModifiers(render.ComponentNameWindows, ctxFor(operatorv1.ProviderNone, nil, nil), newObjs()) + d := ds(out) + + Expect(d.Spec.Template.Spec.Volumes).To(ContainElement(HaveField("Name", "var-log-calico"))) + for _, name := range []string{"felix", "node", "confd"} { + c := container(d, name) + Expect(c.VolumeMounts).To(ContainElement(HaveField("Name", "var-log-calico"))) + Expect(c.VolumeMounts).NotTo(ContainElement(HaveField("Name", "cni-log-dir"))) + Expect(c.Env).To(ContainElements( + corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERENABLED", Value: "true"}, + corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERPORT", Value: "9081"}, + corev1.EnvVar{Name: "FELIX_DNSLOGSFILEENABLED", Value: "true"}, + )) + } + }) + + It("sets the trusted DNS server on openshift", func() { + out := extensions.ApplyModifiers(render.ComponentNameWindows, ctxFor(operatorv1.ProviderOpenShift, nil, nil), newObjs()) + Expect(container(ds(out), "node").Env).To(ContainElement(corev1.EnvVar{Name: "FELIX_DNSTRUSTEDSERVERS", Value: "k8s-service:openshift-dns/dns-default"})) + }) + + It("mounts the prometheus reporter keypair when present", func() { + scheme := runtime.NewScheme() + Expect(apis.AddToScheme(scheme, false)).NotTo(HaveOccurred()) + cli := ctrlrfake.DefaultFakeClientBuilder(scheme).Build() + cm, err := certificatemanager.Create(cli, nil, "", common.OperatorNamespace(), certificatemanager.AllowCACreation()) + Expect(err).NotTo(HaveOccurred()) + tls, err := cm.GetOrCreateKeyPair(cli, render.NodePrometheusTLSServerSecret, common.OperatorNamespace(), []string{"calico-node-metrics-windows"}) + Expect(err).NotTo(HaveOccurred()) + bundle := cm.CreateTrustedBundle() + + out := extensions.ApplyModifiers(render.ComponentNameWindows, ctxFor(operatorv1.ProviderNone, tls, bundle), newObjs()) + d := ds(out) + + Expect(d.Spec.Template.Spec.Volumes).To(ContainElement(tls.Volume())) + Expect(d.Spec.Template.Annotations).To(HaveKey(tls.HashAnnotationKey())) + Expect(container(d, "node").Env).To(ContainElement(HaveField("Name", "FELIX_PROMETHEUSREPORTERCERTFILE"))) + Expect(container(d, "node").VolumeMounts).To(ContainElement(tls.VolumeMount(render.Windows(&render.WindowsConfiguration{}).SupportedOSType()))) + }) + + It("does nothing for the Calico variant", func() { + ctx := extensions.RenderContext{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.Calico}} + out := extensions.ApplyModifiers(render.ComponentNameWindows, ctx, newObjs()) + _, ok := extensions.FindObject[*corev1.Service](out, render.WindowsNodeMetricsService) + Expect(ok).To(BeFalse()) + Expect(ds(out).Spec.Template.Spec.Volumes).To(BeEmpty()) + }) +}) diff --git a/pkg/extensions/rendercontext.go b/pkg/extensions/rendercontext.go index 9653825a59..c03389c153 100644 --- a/pkg/extensions/rendercontext.go +++ b/pkg/extensions/rendercontext.go @@ -23,11 +23,13 @@ import ( // RenderContext carries reconcile-derived inputs from controllers into render // modifiers. Core operator code never reads these fields - only registered // modifiers do. -// Two kinds of value live here: +// Three kinds of value live here: // - raw cluster state gathered generically (Installation, FelixConfiguration, -// ClusterDomain) that modifiers derive their own values from, and +// ClusterDomain) that modifiers derive their own values from, // - controller-produced artifacts (TrustedBundle, NodePrometheusTLS) that can -// only be created controller-side because they have cluster side effects. +// only be created controller-side because they have cluster side effects, and +// - Component, the per-component context the component being modified supplies +// for config a modifier can't derive from the fields above. type RenderContext struct { Installation *operatorv1.InstallationSpec FelixConfiguration *v3.FelixConfiguration @@ -41,4 +43,12 @@ type RenderContext struct { // consumer: it mounts the keypair onto the daemonset and sets the // FELIX_PROMETHEUSREPORTER* certificate env vars. NodePrometheusTLS certificatemanagement.KeyPairInterface + + // Component is per-component context that the component being modified supplies + // via render.ExtensionContextProvider - config a modifier needs but can't + // derive from the fields above (e.g. a keypair the component's own controller + // created, or a CR field only that controller reads). The componentHandler + // sets it per component before applying the modifier; a modifier type-asserts + // it to the component's own context type. Nil when the component supplies none. + Component any } diff --git a/pkg/render/component.go b/pkg/render/component.go index 7785834dcf..3063e7f842 100644 --- a/pkg/render/component.go +++ b/pkg/render/component.go @@ -49,9 +49,26 @@ type Extensible interface { ModifierKey() string } +// ExtensionContextProvider is an optional companion to Extensible. A component +// implements it to hand its modifier component-specific context that can't be +// derived from the shared extensions.RenderContext - config only the component's +// controller has, such as a keypair the controller created. The componentHandler +// reads the returned value into RenderContext.Component before applying the +// modifier, and the modifier type-asserts it to the component's own context type. +type ExtensionContextProvider interface { + ExtensionContext() any +} + // Component names used as keys into the extension modifier registry. Keep these // in sync with the ModifierKey() methods that return them. const ( ComponentNameTypha = "typha" ComponentNameNode = "node" + + // ComponentNameWindows keys the windows daemonset modifier. The two windows + // images resolve through their own override keys, since one component renders + // both. + ComponentNameWindows = "windows" + ComponentNameWindowsNodeImg = "windows-node-image" + ComponentNameWindowsCNIImg = "windows-cni-image" ) diff --git a/pkg/render/render_test.go b/pkg/render/render_test.go index a172ce4f00..242c821bb2 100644 --- a/pkg/render/render_test.go +++ b/pkg/render/render_test.go @@ -224,16 +224,16 @@ var _ = Describe("Rendering tests", func() { }) It("should render all resources when variant is Tigera Secure", func() { - // For this scenario, we expect the basic resources plus the following for Tigera Secure: - // - X Same as default config - // - 1 Service to expose calico/node metrics. - // - 1 Service to expose Windows calico/node metrics. + // For this scenario, we expect the basic resources plus the following for Tigera Secure. + // The calico/node and Windows calico/node metrics Services are added by the + // enterprise modifiers at the componentHandler, not by Objects(), so they do + // not appear in this render-only aggregation. var nodeMetricsPort int32 = 9081 instance.Variant = operatorv1.CalicoEnterprise instance.NodeMetricsPort = &nodeMetricsPort c, err := allCalicoComponents(k8sServiceEp, instance, nil, nil, nil, typhaNodeTLS, nil, nil, false, "", dns.DefaultClusterDomain, 9094, 0, nil, nil) Expect(err).To(BeNil(), "Expected Calico to create successfully %s", err) - Expect(componentCount(c)).To(Equal((5 + 3 + 4 + 1 + 6 + 6 + 1 + 2) + 1)) + Expect(componentCount(c)).To(Equal(5 + 3 + 4 + 1 + 6 + 6 + 1 + 2)) }) It("should render all resources when variant is Tigera Secure and Management Cluster", func() { @@ -277,8 +277,8 @@ var _ = Describe("Rendering tests", func() { &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: common.KubeControllersDeploymentName, Namespace: common.CalicoNamespace}, TypeMeta: metav1.TypeMeta{Kind: "Deployment", APIVersion: "apps/v1"}}, &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: "calico-kube-controllers-metrics", Namespace: common.CalicoNamespace}, TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}}, - // Windows node objects. - &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: render.WindowsNodeMetricsService, Namespace: common.CalicoNamespace}, TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}}, + // Windows node objects. The Windows node-metrics Service is added by the + // enterprise modifier at the componentHandler, so it is not in this output. &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "cni-config-windows", Namespace: common.CalicoNamespace}, TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"}}, &appsv1.DaemonSet{ObjectMeta: metav1.ObjectMeta{Name: common.WindowsDaemonSetName, Namespace: common.CalicoNamespace}, TypeMeta: metav1.TypeMeta{Kind: "DaemonSet", APIVersion: "apps/v1"}}, diff --git a/pkg/render/windows.go b/pkg/render/windows.go index c8af5dc762..15f388ecc8 100644 --- a/pkg/render/windows.go +++ b/pkg/render/windows.go @@ -24,13 +24,13 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/intstr" "sigs.k8s.io/controller-runtime/pkg/client" operatorv1 "github.com/tigera/operator/api/v1" "github.com/tigera/operator/pkg/common" "github.com/tigera/operator/pkg/components" "github.com/tigera/operator/pkg/controller/k8sapi" + "github.com/tigera/operator/pkg/imageoverride" rcomp "github.com/tigera/operator/pkg/render/common/components" rmeta "github.com/tigera/operator/pkg/render/common/meta" "github.com/tigera/operator/pkg/render/common/securitycontext" @@ -77,13 +77,10 @@ func (c *windowsComponent) ResolveImages(is *operatorv1.ImageSet) error { return imageName } - if c.cfg.Installation.Variant.IsEnterprise() { - c.cniImage = appendIfErr(components.GetReference(components.ComponentTigeraCNIWindows, reg, path, prefix, is)) - c.nodeImage = appendIfErr(components.GetReference(components.ComponentTigeraNodeWindows, reg, path, prefix, is)) - } else { - c.cniImage = appendIfErr(components.GetReference(components.ComponentCalicoCNIWindows, reg, path, prefix, is)) - c.nodeImage = appendIfErr(components.GetReference(components.ComponentCalicoNodeWindows, reg, path, prefix, is)) - } + cniImage := imageoverride.Resolve(ComponentNameWindowsCNIImg, components.ComponentCalicoCNIWindows, c.cfg.Installation) + nodeImage := imageoverride.Resolve(ComponentNameWindowsNodeImg, components.ComponentCalicoNodeWindows, c.cfg.Installation) + c.cniImage = appendIfErr(components.GetReference(cniImage, reg, path, prefix, is)) + c.nodeImage = appendIfErr(components.GetReference(nodeImage, reg, path, prefix, is)) if len(errMsgs) != 0 { return fmt.Errorf("%s", strings.Join(errMsgs, ",")) @@ -95,6 +92,27 @@ func (c *windowsComponent) SupportedOSType() rmeta.OSType { return rmeta.OSTypeWindows } +func (c *windowsComponent) ModifierKey() string { return ComponentNameWindows } + +// WindowsExtensionContext is the per-component context the windows modifier +// reads (via RenderContext.Component). It carries the enterprise inputs the +// windows controller has but a modifier can't derive from the installation: the +// reporter metrics port, the node prometheus keypair, and the trusted bundle +// the cert env vars reference. +type WindowsExtensionContext struct { + NodeReporterMetricsPort int + PrometheusServerTLS certificatemanagement.KeyPairInterface + TrustedBundle certificatemanagement.TrustedBundleRO +} + +func (c *windowsComponent) ExtensionContext() any { + return WindowsExtensionContext{ + NodeReporterMetricsPort: c.cfg.NodeReporterMetricsPort, + PrometheusServerTLS: c.cfg.PrometheusServerTLS, + TrustedBundle: c.cfg.TLS.TrustedBundle, + } +} + func (c *windowsComponent) Objects() ([]client.Object, []client.Object) { // Clean up old windows upgrader daemonset if present objsToDelete := []client.Object{ @@ -116,11 +134,6 @@ func (c *windowsComponent) Objects() ([]client.Object, []client.Object) { objs := []client.Object{} - if c.cfg.Installation.Variant.IsEnterprise() { - // Include Service for exposing node metrics. - objs = append(objs, c.nodeMetricsService()) - } - cniConfig := c.windowsCNIConfigMap() if cniConfig != nil { objs = append(objs, cniConfig) @@ -135,43 +148,6 @@ func (c *windowsComponent) Ready() bool { return true } -// nodeMetricsService creates a Service which exposes two endpoints on calico/node for -// reporting Prometheus metrics (for policy enforcement activity and BGP stats). -// This service is used internally by Calico Enterprise and is separate from general -// Prometheus metrics which are user-configurable. -func (c *windowsComponent) nodeMetricsService() *corev1.Service { - return &corev1.Service{ - TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}, - ObjectMeta: metav1.ObjectMeta{ - Name: WindowsNodeMetricsService, - Namespace: common.CalicoNamespace, - Labels: map[string]string{"k8s-app": WindowsNodeObjectName}, - }, - Spec: corev1.ServiceSpec{ - Selector: map[string]string{"k8s-app": WindowsNodeObjectName}, - // Important: "None" tells Kubernetes that we want a headless service with - // no kube-proxy load balancer. If we omit this then kube-proxy will render - // a huge set of iptables rules for this service since there's an instance - // on every node. - ClusterIP: "None", - Ports: []corev1.ServicePort{ - { - Name: "calico-metrics-port", - Port: int32(c.cfg.NodeReporterMetricsPort), - TargetPort: intstr.FromInt(c.cfg.NodeReporterMetricsPort), - Protocol: corev1.ProtocolTCP, - }, - { - Name: "calico-bgp-metrics-port", - Port: NodeBGPReporterPort, - TargetPort: intstr.FromInt(int(NodeBGPReporterPort)), - Protocol: corev1.ProtocolTCP, - }, - }, - }, - } -} - // windowsCNIConfigMap returns a config map containing the CNI network config to be installed on each node. // Returns nil if no configmap is needed. func (c *windowsComponent) windowsCNIConfigMap() *corev1.ConfigMap { @@ -380,8 +356,8 @@ func (c *windowsComponent) windowsVolumes() []corev1.Volume { {Name: "policysync", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/run/nodeagent", Type: &dirOrCreate}}}, c.cfg.TLS.TrustedBundle.Volume(), c.cfg.TLS.NodeSecret.Volume(), - corev1.Volume{Name: "var-run-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/run/calico", Type: &dirOrCreate}}}, - corev1.Volume{Name: "var-lib-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/lib/calico", Type: &dirOrCreate}}}, + {Name: "var-run-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/run/calico", Type: &dirOrCreate}}}, + {Name: "var-lib-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/lib/calico", Type: &dirOrCreate}}}, } // If needed for this configuration, then include the CNI volumes. @@ -392,20 +368,6 @@ func (c *windowsComponent) windowsVolumes() []corev1.Volume { volumes = append(volumes, corev1.Volume{Name: "cni-log-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: c.cfg.Installation.WindowsNodes.CNILogDir, Type: &dirOrCreate}}}) } - // Override with Tigera-specific config. - if c.cfg.Installation.Variant.IsEnterprise() { - // Add volume for calico logs. - calicoLogVol := corev1.Volume{ - Name: "var-log-calico", - VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico", Type: &dirOrCreate}}, - } - volumes = append(volumes, calicoLogVol) - } - - if c.cfg.PrometheusServerTLS != nil { - volumes = append(volumes, c.cfg.PrometheusServerTLS.Volume()) - } - return volumes } @@ -479,7 +441,6 @@ func (c *windowsComponent) nodeContainer() corev1.Container { // felixContainer creates the windows felix container. func (c *windowsComponent) felixContainer() corev1.Container { - lp, rp := c.windowsLivenessReadinessProbes() return corev1.Container{ @@ -659,31 +620,6 @@ func (c *windowsComponent) windowsEnvVars() []corev1.EnvVar { windowsEnv = append(windowsEnv, corev1.EnvVar{Name: "FELIX_IPV6SUPPORT", Value: "false"}) } - if c.cfg.Installation.Variant.IsEnterprise() { - // Add in Calico Enterprise specific configuration. - extraNodeEnv := []corev1.EnvVar{ - {Name: "FELIX_PROMETHEUSREPORTERENABLED", Value: "true"}, - {Name: "FELIX_PROMETHEUSREPORTERPORT", Value: fmt.Sprintf("%d", c.cfg.NodeReporterMetricsPort)}, - {Name: "FELIX_FLOWLOGSFILEENABLED", Value: "true"}, - {Name: "FELIX_FLOWLOGSFILEINCLUDELABELS", Value: "true"}, - {Name: "FELIX_FLOWLOGSFILEINCLUDEPOLICIES", Value: "true"}, - {Name: "FELIX_FLOWLOGSFILEINCLUDESERVICE", Value: "true"}, - {Name: "FELIX_FLOWLOGSENABLENETWORKSETS", Value: "true"}, - {Name: "FELIX_FLOWLOGSCOLLECTPROCESSINFO", Value: "true"}, - {Name: "FELIX_DNSLOGSFILEENABLED", Value: "true"}, - {Name: "FELIX_DNSLOGSFILEPERNODELIMIT", Value: "1000"}, - } - - if c.cfg.PrometheusServerTLS != nil { - extraNodeEnv = append(extraNodeEnv, - corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERCERTFILE", Value: c.cfg.PrometheusServerTLS.VolumeMountCertificateFilePath()}, - corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERKEYFILE", Value: c.cfg.PrometheusServerTLS.VolumeMountKeyFilePath()}, - corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERCAFILE", Value: c.cfg.TLS.TrustedBundle.MountPath()}, - ) - } - windowsEnv = append(windowsEnv, extraNodeEnv...) - } - if c.cfg.Installation.NodeMetricsPort != nil { // If a node metrics port was given, then enable felix prometheus metrics and set the port. // Note that this takes precedence over any FelixConfiguration resources in the cluster. @@ -694,20 +630,6 @@ func (c *windowsComponent) windowsEnvVars() []corev1.EnvVar { windowsEnv = append(windowsEnv, extraNodeEnv...) } - // Configure provider specific environment variables here. - switch c.cfg.Installation.KubernetesProvider { - case operatorv1.ProviderOpenShift: - if c.cfg.Installation.Variant.IsEnterprise() { - // We need to configure a non-default trusted DNS server, since there's no kube-dns. - windowsEnv = append(windowsEnv, corev1.EnvVar{Name: "FELIX_DNSTRUSTEDSERVERS", Value: "k8s-service:openshift-dns/dns-default"}) - } - case operatorv1.ProviderRKE2: - // For RKE2, configure a non-default trusted DNS server, as the DNS service is not named "kube-dns". - if c.cfg.Installation.Variant.IsEnterprise() { - windowsEnv = append(windowsEnv, corev1.EnvVar{Name: "FELIX_DNSTRUSTEDSERVERS", Value: "k8s-service:kube-system/rke2-coredns-rke2-coredns"}) - } - } - if c.cfg.Installation.CNI.Type != operatorv1.PluginCalico { windowsEnv = append(windowsEnv, corev1.EnvVar{Name: "FELIX_ROUTESOURCE", Value: "WorkloadIPs"}) } @@ -726,12 +648,7 @@ func (c *windowsComponent) windowsVolumeMounts() []corev1.VolumeMount { corev1.VolumeMount{MountPath: "/var/run/calico", Name: "var-run-calico"}, corev1.VolumeMount{MountPath: "/var/lib/calico", Name: "var-lib-calico"}) - if c.cfg.Installation.Variant.IsEnterprise() { - extraNodeMounts := []corev1.VolumeMount{ - {MountPath: "/var/log/calico", Name: "var-log-calico"}, - } - windowsVolumeMounts = append(windowsVolumeMounts, extraNodeMounts...) - } else if c.cfg.Installation.CNI.Type == operatorv1.PluginCalico { + if c.cfg.Installation.CNI.Type == operatorv1.PluginCalico { windowsVolumeMounts = append(windowsVolumeMounts, corev1.VolumeMount{MountPath: "/var/log/calico/cni", Name: "cni-log-dir", ReadOnly: false}) } @@ -739,9 +656,6 @@ func (c *windowsComponent) windowsVolumeMounts() []corev1.VolumeMount { windowsVolumeMounts = append(windowsVolumeMounts, corev1.VolumeMount{MountPath: "/host/etc/cni/net.d", Name: "cni-net-dir"}) } - if c.cfg.PrometheusServerTLS != nil { - windowsVolumeMounts = append(windowsVolumeMounts, c.cfg.PrometheusServerTLS.VolumeMount(c.SupportedOSType())) - } return windowsVolumeMounts } @@ -797,9 +711,6 @@ func (c *windowsComponent) windowsDaemonset(cniCfgMap *corev1.ConfigMap) *appsv1 initContainers := []corev1.Container{c.uninstallContainer()} annotations := c.cfg.TLS.TrustedBundle.HashAnnotations() - if c.cfg.PrometheusServerTLS != nil { - annotations[c.cfg.PrometheusServerTLS.HashAnnotationKey()] = c.cfg.PrometheusServerTLS.HashAnnotationValue() - } if cniCfgMap != nil { annotations[nodeCniConfigAnnotation] = rmeta.AnnotationHash(cniCfgMap.Data) diff --git a/pkg/render/windows_test.go b/pkg/render/windows_test.go index ebb1c4d9b1..1eb52c0849 100644 --- a/pkg/render/windows_test.go +++ b/pkg/render/windows_test.go @@ -35,12 +35,28 @@ import ( "github.com/tigera/operator/pkg/controller/certificatemanager" "github.com/tigera/operator/pkg/controller/k8sapi" ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" + "github.com/tigera/operator/pkg/extensions" "github.com/tigera/operator/pkg/render" rmeta "github.com/tigera/operator/pkg/render/common/meta" rtest "github.com/tigera/operator/pkg/render/common/test" "github.com/tigera/operator/pkg/tls/certificatemanagement" ) +// renderWindows renders the windows component and applies the registered +// enterprise modifier the way the componentHandler does, so enterprise tests +// exercise the integrated output (image overrides come from ResolveImages; the +// metrics service, env, volumes and mounts come from the modifier). +func renderWindows(cfg *render.WindowsConfiguration) []client.Object { + comp := render.Windows(cfg) + ExpectWithOffset(1, comp.ResolveImages(nil)).To(BeNil()) + objs, _ := comp.Objects() + rc := extensions.RenderContext{Installation: cfg.Installation} + if p, ok := comp.(render.ExtensionContextProvider); ok { + rc.Component = p.ExtensionContext() + } + return extensions.ApplyModifiers(render.ComponentNameWindows, rc, objs) +} + var _ = Describe("Windows rendering tests", func() { var defaultInstance *operatorv1.InstallationSpec var typhaNodeTLS *render.TyphaNodeTLS @@ -694,16 +710,14 @@ var _ = Describe("Windows rendering tests", func() { version string kind string }{ - {name: "calico-node-metrics-windows", ns: "calico-system", group: "", version: "v1", kind: "Service"}, {name: "cni-config-windows", ns: common.CalicoNamespace, group: "", version: "v1", kind: "ConfigMap"}, {name: common.WindowsDaemonSetName, ns: common.CalicoNamespace, group: "apps", version: "v1", kind: "DaemonSet"}, + {name: "calico-node-metrics-windows", ns: "calico-system", group: "", version: "v1", kind: "Service"}, } defaultInstance.Variant = operatorv1.CalicoEnterprise cfg.NodeReporterMetricsPort = 9081 - component := render.Windows(&cfg) - Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() + resources := renderWindows(&cfg) Expect(len(resources)).To(Equal(len(expectedResources))) // Should render the correct resources. @@ -1686,18 +1700,16 @@ var _ = Describe("Windows rendering tests", func() { version string kind string }{ - {name: "calico-node-metrics-windows", ns: "calico-system", group: "", version: "v1", kind: "Service"}, {name: "cni-config-windows", ns: common.CalicoNamespace, group: "", version: "v1", kind: "ConfigMap"}, {name: common.WindowsDaemonSetName, ns: common.CalicoNamespace, group: "apps", version: "v1", kind: "DaemonSet"}, + {name: "calico-node-metrics-windows", ns: "calico-system", group: "", version: "v1", kind: "Service"}, } defaultInstance.Variant = operatorv1.CalicoEnterprise defaultInstance.KubernetesProvider = operatorv1.ProviderOpenShift cfg.NodeReporterMetricsPort = 9081 - component := render.Windows(&cfg) - Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() + resources := renderWindows(&cfg) Expect(len(resources)).To(Equal(len(expectedResources))) // Should render the correct resources. @@ -1842,18 +1854,16 @@ var _ = Describe("Windows rendering tests", func() { version string kind string }{ - {name: "calico-node-metrics-windows", ns: "calico-system", group: "", version: "v1", kind: "Service"}, {name: "cni-config-windows", ns: common.CalicoNamespace, group: "", version: "v1", kind: "ConfigMap"}, {name: common.WindowsDaemonSetName, ns: common.CalicoNamespace, group: "apps", version: "v1", kind: "DaemonSet"}, + {name: "calico-node-metrics-windows", ns: "calico-system", group: "", version: "v1", kind: "Service"}, } defaultInstance.Variant = operatorv1.CalicoEnterprise defaultInstance.KubernetesProvider = operatorv1.ProviderRKE2 cfg.NodeReporterMetricsPort = 9081 - component := render.Windows(&cfg) - Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() + resources := renderWindows(&cfg) Expect(len(resources)).To(Equal(len(expectedResources)), fmt.Sprintf("Actual resources: %#v", resources)) // Should render the correct resources. @@ -2138,9 +2148,7 @@ var _ = Describe("Windows rendering tests", func() { defaultInstance.NodeMetricsPort = nil cfg.NodeReporterMetricsPort = 9081 - component := render.Windows(&cfg) - Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() + resources := renderWindows(&cfg) Expect(len(resources)).To(Equal(defaultNumExpectedResources + 1)) dsResource := rtest.GetResource(resources, "calico-node-windows", "calico-system", "apps", "v1", "DaemonSet") @@ -2159,9 +2167,7 @@ var _ = Describe("Windows rendering tests", func() { var nodeMetricsPort int32 = 1234 defaultInstance.Variant = operatorv1.CalicoEnterprise defaultInstance.NodeMetricsPort = &nodeMetricsPort - component := render.Windows(&cfg) - Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() + resources := renderWindows(&cfg) Expect(len(resources)).To(Equal(defaultNumExpectedResources + 1)) dsResource := rtest.GetResource(resources, "calico-node-windows", "calico-system", "apps", "v1", "DaemonSet") From ffe993e59db246b1aa8e97151ca0590161ed83a5 Mon Sep 17 00:00:00 2001 From: Casey Davenport Date: Tue, 9 Jun 2026 15:21:14 -0700 Subject: [PATCH 22/54] De-variant the guardian component into an enterprise extension Move guardian's enterprise branches out of core render into a pkg/enterprise extension: the secrets Role/RoleBinding and default UI settings, the elasticsearch/kibana service ports, the management-cluster-request cluster role rules (which wholly replace the OSS rules, plus impersonation and the OpenShift SCC), and the CA bundle env vars. rulesForManagementClusterRequests moves to enterprise; the UI-settings helpers are exported from the manager render so the modifier can build them. The guardian component exposes its OpenShift flag, impersonation config, and trusted bundle path via ExtensionContext; the clusterconnection controller wires the render context into its handler. Core guardian render is now OSS-only except the network policy, which is the next commit. --- .../clusterconnection_controller.go | 6 +- .../clusterconnection_suite_test.go | 7 + pkg/enterprise/guardian.go | 485 ++++++++++++++++ pkg/enterprise/guardian_test.go | 109 ++++ pkg/enterprise/register.go | 1 + pkg/render/guardian.go | 529 ++---------------- pkg/render/guardian_test.go | 46 +- pkg/render/manager.go | 24 +- 8 files changed, 709 insertions(+), 498 deletions(-) create mode 100644 pkg/enterprise/guardian.go create mode 100644 pkg/enterprise/guardian_test.go diff --git a/pkg/controller/clusterconnection/clusterconnection_controller.go b/pkg/controller/clusterconnection/clusterconnection_controller.go index 4b1b7dc8f6..b46ad0c205 100644 --- a/pkg/controller/clusterconnection/clusterconnection_controller.go +++ b/pkg/controller/clusterconnection/clusterconnection_controller.go @@ -49,6 +49,7 @@ import ( "github.com/tigera/operator/pkg/controller/utils/imageset" "github.com/tigera/operator/pkg/ctrlruntime" "github.com/tigera/operator/pkg/dns" + "github.com/tigera/operator/pkg/extensions" "github.com/tigera/operator/pkg/render" "github.com/tigera/operator/pkg/render/common/networkpolicy" "github.com/tigera/operator/pkg/render/goldmane" @@ -443,7 +444,10 @@ func (r *ReconcileConnection) Reconcile(ctx context.Context, request reconcile.R return reconcile.Result{}, err } - ch := utils.NewComponentHandler(log, r.cli, r.scheme, managementClusterConnection) + // The render context carries the installation so registered modifiers gate on + // variant; the guardian component supplies its own per-component context (the + // impersonation config, OpenShift, and CA bundle path) via ExtensionContext. + ch := utils.NewComponentHandler(log, r.cli, r.scheme, managementClusterConnection, utils.WithRenderContext(extensions.RenderContext{Installation: installationSpec})) guardianCfg := &render.GuardianConfiguration{ URL: managementClusterConnection.Spec.ManagementClusterAddr, PodProxies: r.resolvedPodProxies, diff --git a/pkg/controller/clusterconnection/clusterconnection_suite_test.go b/pkg/controller/clusterconnection/clusterconnection_suite_test.go index 8967498282..09a8254690 100644 --- a/pkg/controller/clusterconnection/clusterconnection_suite_test.go +++ b/pkg/controller/clusterconnection/clusterconnection_suite_test.go @@ -22,11 +22,18 @@ import ( "sigs.k8s.io/controller-runtime/pkg/log/zap" logf "sigs.k8s.io/controller-runtime/pkg/log" + + "github.com/tigera/operator/pkg/enterprise" ) func TestStatus(t *testing.T) { logf.SetLogger(zap.New(zap.WriteTo(ginkgo.GinkgoWriter))) gomega.RegisterFailHandler(ginkgo.Fail) + + // Wire the enterprise extensions so the guardian modifier runs the way it + // does in the operator binary, which is what these controller tests exercise. + enterprise.Register() + suiteConfig, reporterConfig := ginkgo.GinkgoConfiguration() reporterConfig.JUnitReport = "../../../report/ut/clusterconnection_controller_suite.xml" ginkgo.RunSpecs(t, "pkg/controller/Management Cluster Connection Suite", suiteConfig, reporterConfig) diff --git a/pkg/enterprise/guardian.go b/pkg/enterprise/guardian.go new file mode 100644 index 0000000000..bcac8a48e1 --- /dev/null +++ b/pkg/enterprise/guardian.go @@ -0,0 +1,485 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package enterprise + +import ( + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + "sigs.k8s.io/controller-runtime/pkg/client" + + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/common" + "github.com/tigera/operator/pkg/extensions" + "github.com/tigera/operator/pkg/render" + "github.com/tigera/operator/pkg/render/common/securitycontextconstraints" +) + +func registerGuardian() { + extensions.Register(operatorv1.CalicoEnterprise, render.GuardianName, extensions.Extension{ + Modify: modifyGuardian, + }) +} + +// modifyGuardian layers Calico Enterprise behavior onto the rendered guardian +// objects: the secrets Role/RoleBinding and default UI settings, the +// elasticsearch/kibana service ports, the management-cluster-request cluster +// role rules (which replace the OSS rules), and the CA bundle env vars. +func modifyGuardian(ctx extensions.RenderContext, objs []client.Object) []client.Object { + gc, _ := ctx.Component.(render.GuardianExtensionContext) + + if role, ok := extensions.FindObject[*rbacv1.ClusterRole](objs, render.GuardianClusterRoleName); ok { + role.Rules = guardianEnterpriseRules(gc) + } + + if svc, ok := extensions.FindObject[*corev1.Service](objs, render.GuardianServiceName); ok { + svc.Spec.Ports = append(svc.Spec.Ports, guardianEnterpriseServicePorts()...) + } + + if dep, ok := extensions.FindObject[*appsv1.Deployment](objs, render.GuardianDeploymentName); ok { + addGuardianEnterpriseEnv(gc, dep) + } + + return append(objs, + guardianSecretsRole(), + guardianSecretRoleBinding(), + // Default UI settings for this managed cluster. + render.ManagerClusterWideSettingsGroup(), + render.ManagerUserSpecificSettingsGroup(), + render.ManagerClusterWideTigeraLayer(), + render.ManagerClusterWideDefaultView(), + ) +} + +// guardianEnterpriseRules are the cluster role rules guardian needs in Calico +// Enterprise. They wholly replace the OSS rules: the management cluster drives +// guardian over the tunnel, so it needs the union of the rules its components +// require, plus any configured impersonation and the OpenShift SCC. +func guardianEnterpriseRules(gc render.GuardianExtensionContext) []rbacv1.PolicyRule { + var rules []rbacv1.PolicyRule + + if imp := gc.Impersonation; imp != nil { + if imp.Users != nil { + rules = append(rules, rbacv1.PolicyRule{ + APIGroups: []string{""}, + Resources: []string{"users"}, + ResourceNames: imp.Users, + Verbs: []string{"impersonate"}, + }) + } + if imp.Groups != nil { + rules = append(rules, rbacv1.PolicyRule{ + APIGroups: []string{""}, + Resources: []string{"groups"}, + ResourceNames: imp.Groups, + Verbs: []string{"impersonate"}, + }) + } + if imp.ServiceAccounts != nil { + rules = append(rules, rbacv1.PolicyRule{ + APIGroups: []string{""}, + Resources: []string{"serviceaccounts"}, + ResourceNames: imp.ServiceAccounts, + Verbs: []string{"impersonate"}, + }) + } + } + + rules = append(rules, rulesForManagementClusterRequests(gc.OpenShift)...) + + if gc.OpenShift { + rules = append(rules, rbacv1.PolicyRule{ + APIGroups: []string{"security.openshift.io"}, + Resources: []string{"securitycontextconstraints"}, + Verbs: []string{"use"}, + ResourceNames: []string{securitycontextconstraints.NonRootV2}, + }) + } + + return rules +} + +func guardianEnterpriseServicePorts() []corev1.ServicePort { + return []corev1.ServicePort{ + { + Name: "elasticsearch", + Port: 9200, + TargetPort: intstr.IntOrString{Type: intstr.Int, IntVal: 8080}, + Protocol: corev1.ProtocolTCP, + }, + { + Name: "kibana", + Port: 5601, + TargetPort: intstr.IntOrString{Type: intstr.Int, IntVal: 8080}, + Protocol: corev1.ProtocolTCP, + }, + } +} + +func addGuardianEnterpriseEnv(gc render.GuardianExtensionContext, dep *appsv1.Deployment) { + for i := range dep.Spec.Template.Spec.Containers { + c := &dep.Spec.Template.Spec.Containers[i] + if c.Name != render.GuardianContainerName { + continue + } + c.Env = append(c.Env, + corev1.EnvVar{Name: "GUARDIAN_PACKET_CAPTURE_CA_BUNDLE_PATH", Value: gc.TrustedBundleMountPath}, + corev1.EnvVar{Name: "GUARDIAN_PROMETHEUS_CA_BUNDLE_PATH", Value: gc.TrustedBundleMountPath}, + corev1.EnvVar{Name: "GUARDIAN_QUERYSERVER_CA_BUNDLE_PATH", Value: gc.TrustedBundleMountPath}, + ) + } +} + +// guardianSecretsRole creates a Role that allows the management cluster to +// provision secrets to the tigera-operator Namespace, used to push secrets the +// managed cluster needs to access / authenticate with the management cluster. +func guardianSecretsRole() *rbacv1.Role { + return &rbacv1.Role{ + TypeMeta: metav1.TypeMeta{Kind: "Role", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: render.GuardianSecretsRole, + Namespace: common.OperatorNamespace(), + }, + Rules: []rbacv1.PolicyRule{ + { + APIGroups: []string{""}, + Resources: []string{"secrets"}, + Verbs: []string{"create", "delete", "deletecollection", "update"}, + }, + }, + } +} + +func guardianSecretRoleBinding() *rbacv1.RoleBinding { + return &rbacv1.RoleBinding{ + TypeMeta: metav1.TypeMeta{Kind: "RoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: render.GuardianSecretsRoleBindingName, + Namespace: common.OperatorNamespace(), + }, + RoleRef: rbacv1.RoleRef{ + APIGroup: "rbac.authorization.k8s.io", + Kind: "Role", + Name: render.GuardianSecretsRole, + }, + Subjects: []rbacv1.Subject{ + { + Kind: "ServiceAccount", + Name: render.GuardianServiceAccountName, + Namespace: render.GuardianNamespace, + }, + }, + } +} + +// rulesForManagementClusterRequests returns the set of RBAC rules guardian needs +// to satisfy requests from the management cluster over the tunnel. +func rulesForManagementClusterRequests(isOpenShift bool) []rbacv1.PolicyRule { + rules := []rbacv1.PolicyRule{ + // Common rules required to handle requests from multiple components in the management cluster. + { + // ID uses read-only permissions and kube-controllers uses both read and write verbs. + APIGroups: []string{""}, + Resources: []string{"configmaps"}, + Verbs: []string{"create", "delete", "get", "list", "update", "watch"}, + }, + { + // Allows Linseed to watch namespaces before copying its token. + // Also enables PolicyRecommendation to watch namespaces, + // and Manager/kube-controllers to list them. + APIGroups: []string{""}, + Resources: []string{"namespaces"}, + Verbs: []string{"get", "list", "watch"}, + }, + { + // kube-controllers watches Nodes to monitor for deletions. + // Manager performs a list operation on Nodes. + APIGroups: []string{""}, + Resources: []string{"nodes"}, + Verbs: []string{"get", "list", "watch"}, + }, + { + // kube-controllers watches Pods to verify existence for IPAM garbage collection. + // Manager performs get operations on Pods. + APIGroups: []string{""}, + Resources: []string{"pods"}, + Verbs: []string{"get", "list", "watch"}, + }, + { + // The Federated Services Controller needs access to the remote kubeconfig secret + // in order to create a remote syncer. + APIGroups: []string{""}, + Resources: []string{"secrets"}, + Verbs: []string{"get", "list", "watch"}, + }, + { + // Manager uses list; kube-controllers uses 'get', 'list', 'watch', 'update'. + APIGroups: []string{""}, + Resources: []string{"services"}, + Verbs: []string{"get", "list", "update", "watch"}, + }, + { + // Needed by kube-controllers to validate licenses; also used by ID. + APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, + Resources: []string{"licensekeys"}, + Verbs: []string{"get", "watch"}, + }, + { + // Manager uses list; PolicyRecommendation & ID uses all verbs. + APIGroups: []string{"projectcalico.org"}, + Resources: []string{ + "globalnetworksets", + "networkpolicies", + "tier.networkpolicies", + "stagednetworkpolicies", + "tier.stagednetworkpolicies", + }, + Verbs: []string{"create", "delete", "get", "list", "patch", "update", "watch"}, + }, + { + // Manager uses list; PolicyRecommendation uses all verbs. + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"tiers"}, + Verbs: []string{"create", "delete", "get", "list", "patch", "update", "watch"}, + }, + // Rules needed by guardian to handle manager authorization reviews. + { + APIGroups: []string{"rbac.authorization.k8s.io"}, + Resources: []string{"clusterroles", "clusterrolebindings", "roles", "rolebindings"}, + Verbs: []string{"list", "get"}, + }, + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"uisettings", "uisettingsgroups"}, + Verbs: []string{"list", "get"}, + }, + + // Rules needed by guardian to handle other manager requests. + { + APIGroups: []string{""}, + Resources: []string{"events"}, + Verbs: []string{"list"}, + }, + { + APIGroups: []string{""}, + Resources: []string{"serviceaccounts"}, + Verbs: []string{"list"}, + }, + { + // Allow query server talk to Prometheus via the manager user. + APIGroups: []string{""}, + Resources: []string{"services/proxy"}, + ResourceNames: []string{ + "calico-node-prometheus:9090", + "https:calico-api:8080", + }, + Verbs: []string{"create", "get"}, + }, + { + APIGroups: []string{"apps"}, + Resources: []string{"daemonsets", "replicasets", "statefulsets"}, + Verbs: []string{"list"}, + }, + { + APIGroups: []string{"authentication.k8s.io"}, + Resources: []string{"tokenreviews"}, + Verbs: []string{"create"}, + }, + { + APIGroups: []string{"authorization.k8s.io"}, + Resources: []string{"subjectaccessreviews"}, + Verbs: []string{"create"}, + }, + { + APIGroups: []string{"networking.k8s.io"}, + Resources: []string{"networkpolicies"}, + Verbs: []string{"get", "list"}, + }, + { + APIGroups: []string{"policy.networking.k8s.io"}, + Resources: []string{ + "clusternetworkpolicies", + "adminnetworkpolicies", + "baselineadminnetworkpolicies", + }, + Verbs: []string{"list"}, + }, + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"alertexceptions"}, + Verbs: []string{"get", "list", "update"}, + }, + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"felixconfigurations"}, + ResourceNames: []string{"default"}, + Verbs: []string{"get"}, + }, + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{ + "globalnetworkpolicies", + "networksets", + "stagedglobalnetworkpolicies", + "stagedkubernetesnetworkpolicies", + "tier.globalnetworkpolicies", + "tier.stagedglobalnetworkpolicies", + }, + Verbs: []string{"list"}, + }, + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"hostendpoints"}, + Verbs: []string{"list"}, + }, + + // Rules needed by guardian to handle policy recommendation requests. + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{ + "policyrecommendationscopes", + "policyrecommendationscopes/status", + }, + Verbs: []string{"create", "delete", "get", "list", "patch", "update", "watch"}, + }, + + // Rules needed by guardian to handle calico-kube-controller requests. + { + // Nodes are watched to monitor for deletions. + APIGroups: []string{""}, + Resources: []string{"endpoints"}, + Verbs: []string{"create", "delete", "get", "list", "update", "watch"}, + }, + { + APIGroups: []string{""}, + Resources: []string{"services/status"}, + Verbs: []string{"get", "list", "update", "watch"}, + }, + { + // Needs to manage hostendpoints. + APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, + Resources: []string{"hostendpoints"}, + Verbs: []string{"create", "delete", "get", "list", "update", "watch"}, + }, + { + // Needs access to update clusterinformations. + APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, + Resources: []string{"clusterinformations"}, + Verbs: []string{"create", "get", "list", "update", "watch"}, + }, + { + // Needs to manipulate kubecontrollersconfiguration, which contains its config. + // It creates a default if none exists, and updates status as well. + APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, + Resources: []string{"kubecontrollersconfigurations"}, + Verbs: []string{"create", "get", "list", "update", "watch"}, + }, + { + APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, + Resources: []string{"tiers"}, + Verbs: []string{"create"}, + }, + { + APIGroups: []string{"crd.projectcalico.org", "projectcalico.org"}, + Resources: []string{"deeppacketinspections"}, + Verbs: []string{"get", "list", "watch"}, + }, + { + APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, + Resources: []string{"deeppacketinspections/status"}, + Verbs: []string{"update"}, + }, + { + APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, + Resources: []string{"packetcaptures"}, + Verbs: []string{"get", "list", "update"}, + }, + { + APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, + Resources: []string{"remoteclusterconfigurations"}, + Verbs: []string{"get", "list", "watch"}, + }, + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"licensekeys"}, + Verbs: []string{"create", "get", "list", "update", "watch"}, + }, + { + // Grant permissions to access ClusterInformation resources in managed clusters. + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"clusterinformations"}, + Verbs: []string{"get", "list", "watch"}, + }, + { + APIGroups: []string{"usage.tigera.io"}, + Resources: []string{"licenseusagereports"}, + Verbs: []string{"create", "delete", "get", "list", "update", "watch"}, + }, + + // Rules needed by guardian to handle Intrusion detection requests. + { + APIGroups: []string{""}, + Resources: []string{"podtemplates"}, + Verbs: []string{"get"}, + }, + { + APIGroups: []string{"apps"}, + Resources: []string{"deployments"}, + Verbs: []string{"get"}, + }, + { + APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, + Resources: []string{"alertexceptions"}, + Verbs: []string{"get", "list"}, + }, + { + APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, + Resources: []string{"securityeventwebhooks"}, + Verbs: []string{"get", "list", "update", "watch"}, + }, + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{ + "globalalerts", + "globalalerts/status", + "globalthreatfeeds", + "globalthreatfeeds/status", + }, + Verbs: []string{"create", "delete", "get", "list", "patch", "update", "watch"}, + }, + // Rules needed to fetch the compliance reports + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"globalreporttypes", "globalreports"}, + Verbs: []string{"get", "list", "watch"}, + }, + } + + // Rules needed by policy recommendation in openshift. + if isOpenShift { + rules = append(rules, + rbacv1.PolicyRule{ + APIGroups: []string{"security.openshift.io"}, + Resources: []string{"securitycontextconstraints"}, + Verbs: []string{"use"}, + ResourceNames: []string{securitycontextconstraints.HostNetworkV2}, + }, + ) + } + + return rules +} diff --git a/pkg/enterprise/guardian_test.go b/pkg/enterprise/guardian_test.go new file mode 100644 index 0000000000..b68ad61b04 --- /dev/null +++ b/pkg/enterprise/guardian_test.go @@ -0,0 +1,109 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package enterprise_test + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + client "sigs.k8s.io/controller-runtime/pkg/client" + + v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/enterprise" + "github.com/tigera/operator/pkg/extensions" + "github.com/tigera/operator/pkg/render" +) + +var _ = Describe("guardian enterprise modifier", func() { + BeforeEach(func() { enterprise.Register() }) + AfterEach(func() { extensions.ResetForTest() }) + + // newObjs returns the subset of rendered guardian objects the modifier touches. + newObjs := func() []client.Object { + return []client.Object{ + &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: render.GuardianClusterRoleName}, Rules: []rbacv1.PolicyRule{{Verbs: []string{"get"}}}}, + &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{Name: render.GuardianServiceName, Namespace: render.GuardianNamespace}, + Spec: corev1.ServiceSpec{Ports: []corev1.ServicePort{{Name: "https", Port: 443}}}, + }, + &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{Name: render.GuardianDeploymentName, Namespace: render.GuardianNamespace}, + Spec: appsv1.DeploymentSpec{Template: corev1.PodTemplateSpec{Spec: corev1.PodSpec{ + Containers: []corev1.Container{{Name: render.GuardianContainerName}}, + }}}, + }, + } + } + + ctxWith := func(c render.GuardianExtensionContext) extensions.RenderContext { + return extensions.RenderContext{ + Installation: &operatorv1.InstallationSpec{Variant: operatorv1.CalicoEnterprise}, + Component: c, + } + } + + It("appends the secrets RBAC and UI settings", func() { + out := extensions.ApplyModifiers(render.GuardianName, ctxWith(render.GuardianExtensionContext{}), newObjs()) + _, ok := extensions.FindObject[*rbacv1.Role](out, render.GuardianSecretsRole) + Expect(ok).To(BeTrue()) + _, ok = extensions.FindObject[*rbacv1.RoleBinding](out, render.GuardianSecretsRoleBindingName) + Expect(ok).To(BeTrue()) + _, ok = extensions.FindObject[*v3.UISettingsGroup](out, render.ManagerClusterSettings) + Expect(ok).To(BeTrue()) + }) + + It("adds the elasticsearch and kibana service ports", func() { + out := extensions.ApplyModifiers(render.GuardianName, ctxWith(render.GuardianExtensionContext{}), newObjs()) + svc, _ := extensions.FindObject[*corev1.Service](out, render.GuardianServiceName) + names := []string{} + for _, p := range svc.Spec.Ports { + names = append(names, p.Name) + } + Expect(names).To(ContainElements("https", "elasticsearch", "kibana")) + }) + + It("replaces the cluster role rules and adds impersonation", func() { + gc := render.GuardianExtensionContext{ + Impersonation: &operatorv1.Impersonation{Users: []string{"foo"}, Groups: []string{"bar"}}, + } + out := extensions.ApplyModifiers(render.GuardianName, ctxWith(gc), newObjs()) + role, _ := extensions.FindObject[*rbacv1.ClusterRole](out, render.GuardianClusterRoleName) + + // The single OSS placeholder rule is gone, replaced by the enterprise set. + Expect(role.Rules).NotTo(ContainElement(rbacv1.PolicyRule{Verbs: []string{"get"}})) + Expect(role.Rules).To(ContainElement(HaveField("ResourceNames", Equal([]string{"foo"})))) + Expect(role.Rules).To(ContainElement(HaveField("ResourceNames", Equal([]string{"bar"})))) + }) + + It("adds the CA bundle env to the guardian container", func() { + gc := render.GuardianExtensionContext{TrustedBundleMountPath: "/ca/bundle"} + out := extensions.ApplyModifiers(render.GuardianName, ctxWith(gc), newObjs()) + dep, _ := extensions.FindObject[*appsv1.Deployment](out, render.GuardianDeploymentName) + Expect(dep.Spec.Template.Spec.Containers[0].Env).To(ContainElement(corev1.EnvVar{Name: "GUARDIAN_PROMETHEUS_CA_BUNDLE_PATH", Value: "/ca/bundle"})) + }) + + It("does nothing for the Calico variant", func() { + ctx := extensions.RenderContext{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.Calico}} + out := extensions.ApplyModifiers(render.GuardianName, ctx, newObjs()) + Expect(out).To(HaveLen(len(newObjs()))) + role, _ := extensions.FindObject[*rbacv1.ClusterRole](out, render.GuardianClusterRoleName) + Expect(role.Rules).To(Equal([]rbacv1.PolicyRule{{Verbs: []string{"get"}}})) + }) +}) diff --git a/pkg/enterprise/register.go b/pkg/enterprise/register.go index 761f1736b7..20589ad0fa 100644 --- a/pkg/enterprise/register.go +++ b/pkg/enterprise/register.go @@ -21,5 +21,6 @@ func Register() { registerTypha() registerNode() registerWindows() + registerGuardian() registerInstallation() } diff --git a/pkg/render/guardian.go b/pkg/render/guardian.go index 8fd24d35b6..8c1f30cf3c 100644 --- a/pkg/render/guardian.go +++ b/pkg/render/guardian.go @@ -45,7 +45,6 @@ import ( "github.com/tigera/operator/pkg/render/common/networkpolicy" "github.com/tigera/operator/pkg/render/common/secret" "github.com/tigera/operator/pkg/render/common/securitycontext" - "github.com/tigera/operator/pkg/render/common/securitycontextconstraints" "github.com/tigera/operator/pkg/tls/certificatemanagement" ) @@ -150,32 +149,40 @@ func (c *GuardianComponent) SupportedOSType() rmeta.OSType { return rmeta.OSTypeLinux } +func (c *GuardianComponent) ModifierKey() string { return GuardianName } + +// GuardianExtensionContext is the per-component context the guardian modifier +// reads (via RenderContext.Component). It carries the inputs the enterprise +// guardian behavior needs that a modifier can't derive from the installation: +// the management cluster's impersonation config, whether we're on OpenShift, +// and the trusted bundle mount path the CA env vars reference. +type GuardianExtensionContext struct { + OpenShift bool + Impersonation *operatorv1.Impersonation + TrustedBundleMountPath string +} + +func (c *GuardianComponent) ExtensionContext() any { + var impersonation *operatorv1.Impersonation + if c.cfg.ManagementClusterConnection != nil { + impersonation = c.cfg.ManagementClusterConnection.Spec.Impersonation + } + return GuardianExtensionContext{ + OpenShift: c.cfg.OpenShift, + Impersonation: impersonation, + TrustedBundleMountPath: c.cfg.TrustedCertBundle.MountPath(), + } +} + func (c *GuardianComponent) Objects() ([]client.Object, []client.Object) { objs := []client.Object{ - // common RBAC for EE and OSS c.serviceAccount(), c.clusterRole(), c.clusterRoleBinding(), - } - - if c.cfg.Installation.Variant.IsEnterprise() { - // Enterprise-specific RBAC and settings - objs = append(objs, - c.secretsRole(), - c.secretRoleBinding(), - // Install default UI settings for this managed cluster. - managerClusterWideSettingsGroup(), - managerUserSpecificSettingsGroup(), - managerClusterWideTigeraLayer(), - managerClusterWideDefaultView(), - ) - } - - objs = append(objs, c.deployment(), c.service(), secret.CopyToNamespace(GuardianNamespace, c.cfg.TunnelSecret)[0], - ) + } return objs, deprecatedObjects() } @@ -197,28 +204,6 @@ func (c *GuardianComponent) service() *corev1.Service { }, } - if c.cfg.Installation.Variant.IsEnterprise() { - ports = append(ports, - corev1.ServicePort{ - Name: "elasticsearch", - Port: 9200, - TargetPort: intstr.IntOrString{ - Type: intstr.Int, - IntVal: 8080, - }, - Protocol: corev1.ProtocolTCP, - }, - corev1.ServicePort{ - Name: "kibana", - Port: 5601, - TargetPort: intstr.IntOrString{ - Type: intstr.Int, - IntVal: 8080, - }, - Protocol: corev1.ProtocolTCP, - }, - ) - } return &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: GuardianServiceName, @@ -241,87 +226,42 @@ func (c *GuardianComponent) serviceAccount() *corev1.ServiceAccount { } func (c *GuardianComponent) clusterRole() *rbacv1.ClusterRole { - var policyRules []rbacv1.PolicyRule - if c.cfg.Installation.Variant.IsEnterprise() { - impersonation := c.cfg.ManagementClusterConnection.Spec.Impersonation - if impersonation != nil { - if impersonation.Users != nil { - policyRules = append(policyRules, - rbacv1.PolicyRule{ - APIGroups: []string{""}, - Resources: []string{"users"}, - ResourceNames: impersonation.Users, - Verbs: []string{"impersonate"}, - }) - } - if impersonation.Groups != nil { - policyRules = append(policyRules, - rbacv1.PolicyRule{ - APIGroups: []string{""}, - Resources: []string{"groups"}, - ResourceNames: impersonation.Groups, - Verbs: []string{"impersonate"}, - }) - } - if impersonation.ServiceAccounts != nil { - policyRules = append(policyRules, - rbacv1.PolicyRule{ - APIGroups: []string{""}, - Resources: []string{"serviceaccounts"}, - ResourceNames: impersonation.ServiceAccounts, - Verbs: []string{"impersonate"}, - }) - } - } - - policyRules = append(policyRules, rulesForManagementClusterRequests(c.cfg.OpenShift)...) - - if c.cfg.OpenShift { - policyRules = append(policyRules, rbacv1.PolicyRule{ - APIGroups: []string{"security.openshift.io"}, - Resources: []string{"securitycontextconstraints"}, - Verbs: []string{"use"}, - ResourceNames: []string{securitycontextconstraints.NonRootV2}, - }) - } - } else { - policyRules = append(policyRules, - rbacv1.PolicyRule{ - APIGroups: []string{""}, - Resources: []string{"namespaces", "services", "pods"}, - Verbs: []string{"get", "list", "watch"}, - }, - rbacv1.PolicyRule{ - APIGroups: []string{"apps"}, - Resources: []string{"deployments", "replicasets", "statefulsets", "daemonsets"}, - Verbs: []string{"get", "list", "watch"}, - }, - rbacv1.PolicyRule{ - APIGroups: []string{"networking.k8s.io"}, - Resources: []string{"networkpolicies"}, - Verbs: []string{"get", "list", "watch"}, - }, - rbacv1.PolicyRule{ - APIGroups: []string{"projectcalico.org"}, - Resources: []string{ - "clusterinformations", - "tiers", - "stagednetworkpolicies", - "tier.stagednetworkpolicies", - "stagedglobalnetworkpolicies", - "tier.stagedglobalnetworkpolicies", - "stagedkubernetesnetworkpolicies", - "tier.stagedkubernetesnetworkpolicies", - "networkpolicies", - "tier.networkpolicies", - "globalnetworkpolicies", - "tier.globalnetworkpolicies", - "globalnetworksets", - "networksets", - }, - Verbs: []string{"get", "list", "watch"}, + policyRules := []rbacv1.PolicyRule{ + { + APIGroups: []string{""}, + Resources: []string{"namespaces", "services", "pods"}, + Verbs: []string{"get", "list", "watch"}, + }, + { + APIGroups: []string{"apps"}, + Resources: []string{"deployments", "replicasets", "statefulsets", "daemonsets"}, + Verbs: []string{"get", "list", "watch"}, + }, + { + APIGroups: []string{"networking.k8s.io"}, + Resources: []string{"networkpolicies"}, + Verbs: []string{"get", "list", "watch"}, + }, + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{ + "clusterinformations", + "tiers", + "stagednetworkpolicies", + "tier.stagednetworkpolicies", + "stagedglobalnetworkpolicies", + "tier.stagedglobalnetworkpolicies", + "stagedkubernetesnetworkpolicies", + "tier.stagedkubernetesnetworkpolicies", + "networkpolicies", + "tier.networkpolicies", + "globalnetworkpolicies", + "tier.globalnetworkpolicies", + "globalnetworksets", + "networksets", }, - ) + Verbs: []string{"get", "list", "watch"}, + }, } return &rbacv1.ClusterRole{ @@ -354,47 +294,6 @@ func (c *GuardianComponent) clusterRoleBinding() *rbacv1.ClusterRoleBinding { } } -// secretRole creates a Role that allows the management cluster to provision secrets to the tigera-operator Namespace. -// This is used to push secrets used by the managed cluster to access / authenticate with the management cluster. -func (c *GuardianComponent) secretsRole() *rbacv1.Role { - return &rbacv1.Role{ - TypeMeta: metav1.TypeMeta{Kind: "Role", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{ - Name: GuardianSecretsRole, - Namespace: common.OperatorNamespace(), - }, - Rules: []rbacv1.PolicyRule{ - { - APIGroups: []string{""}, - Resources: []string{"secrets"}, - Verbs: []string{"create", "delete", "deletecollection", "update"}, - }, - }, - } -} - -func (c *GuardianComponent) secretRoleBinding() *rbacv1.RoleBinding { - return &rbacv1.RoleBinding{ - TypeMeta: metav1.TypeMeta{Kind: "RoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{ - Name: GuardianSecretsRoleBindingName, - Namespace: common.OperatorNamespace(), - }, - RoleRef: rbacv1.RoleRef{ - APIGroup: "rbac.authorization.k8s.io", - Kind: "Role", - Name: GuardianSecretsRole, - }, - Subjects: []rbacv1.Subject{ - { - Kind: "ServiceAccount", - Name: GuardianServiceAccountName, - Namespace: GuardianNamespace, - }, - }, - } -} - func (c *GuardianComponent) deployment() *appsv1.Deployment { var replicas int32 = 1 @@ -468,14 +367,6 @@ func (c *GuardianComponent) container() []corev1.Container { } envVars = append(envVars, c.cfg.Installation.Proxy.EnvVars()...) - if c.cfg.Installation.Variant.IsEnterprise() { - envVars = append(envVars, - corev1.EnvVar{Name: "GUARDIAN_PACKET_CAPTURE_CA_BUNDLE_PATH", Value: c.cfg.TrustedCertBundle.MountPath()}, - corev1.EnvVar{Name: "GUARDIAN_PROMETHEUS_CA_BUNDLE_PATH", Value: c.cfg.TrustedCertBundle.MountPath()}, - corev1.EnvVar{Name: "GUARDIAN_QUERYSERVER_CA_BUNDLE_PATH", Value: c.cfg.TrustedCertBundle.MountPath()}, - ) - } - if c.cfg.GuardianClientKeyPair != nil { envVars = append(envVars, corev1.EnvVar{ @@ -782,304 +673,6 @@ func GuardianService(clusterDomain string) string { return fmt.Sprintf("https://%s.%s.svc.%s:%d", GuardianServiceName, GuardianNamespace, clusterDomain, 443) } -// rulesForManagementClusterRequests returns the set of RBAC rules needed by Guardian in order to -// satisfy requests from the management cluster over the tunnel. -func rulesForManagementClusterRequests(isOpenShift bool) []rbacv1.PolicyRule { - rules := []rbacv1.PolicyRule{ - // Common rules required to handle requests from multiple components in the management cluster. - { - // ID uses read-only permissions and kube-controllers uses both read and write verbs. - APIGroups: []string{""}, - Resources: []string{"configmaps"}, - Verbs: []string{"create", "delete", "get", "list", "update", "watch"}, - }, - { - // Allows Linseed to watch namespaces before copying its token. - // Also enables PolicyRecommendation to watch namespaces, - // and Manager/kube-controllers to list them. - APIGroups: []string{""}, - Resources: []string{"namespaces"}, - Verbs: []string{"get", "list", "watch"}, - }, - { - // kube-controllers watches Nodes to monitor for deletions. - // Manager performs a list operation on Nodes. - APIGroups: []string{""}, - Resources: []string{"nodes"}, - Verbs: []string{"get", "list", "watch"}, - }, - { - // kube-controllers watches Pods to verify existence for IPAM garbage collection. - // Manager performs get operations on Pods. - APIGroups: []string{""}, - Resources: []string{"pods"}, - Verbs: []string{"get", "list", "watch"}, - }, - { - // The Federated Services Controller needs access to the remote kubeconfig secret - // in order to create a remote syncer. - APIGroups: []string{""}, - Resources: []string{"secrets"}, - Verbs: []string{"get", "list", "watch"}, - }, - { - // Manager uses list; kube-controllers uses 'get', 'list', 'watch', 'update'. - APIGroups: []string{""}, - Resources: []string{"services"}, - Verbs: []string{"get", "list", "update", "watch"}, - }, - { - // Needed by kube-controllers to validate licenses; also used by ID. - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{"licensekeys"}, - Verbs: []string{"get", "watch"}, - }, - { - // Manager uses list; PolicyRecommendation & ID uses all verbs. - APIGroups: []string{"projectcalico.org"}, - Resources: []string{ - "globalnetworksets", - "networkpolicies", - "tier.networkpolicies", - "stagednetworkpolicies", - "tier.stagednetworkpolicies", - }, - Verbs: []string{"create", "delete", "get", "list", "patch", "update", "watch"}, - }, - { - // Manager uses list; PolicyRecommendation uses all verbs. - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"tiers"}, - Verbs: []string{"create", "delete", "get", "list", "patch", "update", "watch"}, - }, - // Rules needed by guardian to handle manager authorization reviews. - { - APIGroups: []string{"rbac.authorization.k8s.io"}, - Resources: []string{"clusterroles", "clusterrolebindings", "roles", "rolebindings"}, - Verbs: []string{"list", "get"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"uisettings", "uisettingsgroups"}, - Verbs: []string{"list", "get"}, - }, - - // Rules needed by guardian to handle other manager requests. - { - APIGroups: []string{""}, - Resources: []string{"events"}, - Verbs: []string{"list"}, - }, - { - APIGroups: []string{""}, - Resources: []string{"serviceaccounts"}, - Verbs: []string{"list"}, - }, - { - // Allow query server talk to Prometheus via the manager user. - APIGroups: []string{""}, - Resources: []string{"services/proxy"}, - ResourceNames: []string{ - "calico-node-prometheus:9090", - "https:calico-api:8080", - }, - Verbs: []string{"create", "get"}, - }, - { - APIGroups: []string{"apps"}, - Resources: []string{"daemonsets", "replicasets", "statefulsets"}, - Verbs: []string{"list"}, - }, - { - APIGroups: []string{"authentication.k8s.io"}, - Resources: []string{"tokenreviews"}, - Verbs: []string{"create"}, - }, - { - APIGroups: []string{"authorization.k8s.io"}, - Resources: []string{"subjectaccessreviews"}, - Verbs: []string{"create"}, - }, - { - APIGroups: []string{"networking.k8s.io"}, - Resources: []string{"networkpolicies"}, - Verbs: []string{"get", "list"}, - }, - { - APIGroups: []string{"policy.networking.k8s.io"}, - Resources: []string{ - "clusternetworkpolicies", - "adminnetworkpolicies", - "baselineadminnetworkpolicies", - }, - Verbs: []string{"list"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"alertexceptions"}, - Verbs: []string{"get", "list", "update"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"felixconfigurations"}, - ResourceNames: []string{"default"}, - Verbs: []string{"get"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{ - "globalnetworkpolicies", - "networksets", - "stagedglobalnetworkpolicies", - "stagedkubernetesnetworkpolicies", - "tier.globalnetworkpolicies", - "tier.stagedglobalnetworkpolicies", - }, - Verbs: []string{"list"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"hostendpoints"}, - Verbs: []string{"list"}, - }, - - // Rules needed by guardian to handle policy recommendation requests. - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{ - "policyrecommendationscopes", - "policyrecommendationscopes/status", - }, - Verbs: []string{"create", "delete", "get", "list", "patch", "update", "watch"}, - }, - - // Rules needed by guardian to handle calico-kube-controller requests. - { - // Nodes are watched to monitor for deletions. - APIGroups: []string{""}, - Resources: []string{"endpoints"}, - Verbs: []string{"create", "delete", "get", "list", "update", "watch"}, - }, - { - APIGroups: []string{""}, - Resources: []string{"services/status"}, - Verbs: []string{"get", "list", "update", "watch"}, - }, - { - // Needs to manage hostendpoints. - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{"hostendpoints"}, - Verbs: []string{"create", "delete", "get", "list", "update", "watch"}, - }, - { - // Needs access to update clusterinformations. - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{"clusterinformations"}, - Verbs: []string{"create", "get", "list", "update", "watch"}, - }, - { - // Needs to manipulate kubecontrollersconfiguration, which contains its config. - // It creates a default if none exists, and updates status as well. - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{"kubecontrollersconfigurations"}, - Verbs: []string{"create", "get", "list", "update", "watch"}, - }, - { - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{"tiers"}, - Verbs: []string{"create"}, - }, - { - APIGroups: []string{"crd.projectcalico.org", "projectcalico.org"}, - Resources: []string{"deeppacketinspections"}, - Verbs: []string{"get", "list", "watch"}, - }, - { - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{"deeppacketinspections/status"}, - Verbs: []string{"update"}, - }, - { - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{"packetcaptures"}, - Verbs: []string{"get", "list", "update"}, - }, - { - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{"remoteclusterconfigurations"}, - Verbs: []string{"get", "list", "watch"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"licensekeys"}, - Verbs: []string{"create", "get", "list", "update", "watch"}, - }, - { - // Grant permissions to access ClusterInformation resources in managed clusters. - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"clusterinformations"}, - Verbs: []string{"get", "list", "watch"}, - }, - { - APIGroups: []string{"usage.tigera.io"}, - Resources: []string{"licenseusagereports"}, - Verbs: []string{"create", "delete", "get", "list", "update", "watch"}, - }, - - // Rules needed by guardian to handle Intrusion detection requests. - { - APIGroups: []string{""}, - Resources: []string{"podtemplates"}, - Verbs: []string{"get"}, - }, - { - APIGroups: []string{"apps"}, - Resources: []string{"deployments"}, - Verbs: []string{"get"}, - }, - { - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{"alertexceptions"}, - Verbs: []string{"get", "list"}, - }, - { - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{"securityeventwebhooks"}, - Verbs: []string{"get", "list", "update", "watch"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{ - "globalalerts", - "globalalerts/status", - "globalthreatfeeds", - "globalthreatfeeds/status", - }, - Verbs: []string{"create", "delete", "get", "list", "patch", "update", "watch"}, - }, - // Rules needed to fetch the compliance reports - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"globalreporttypes", "globalreports"}, - Verbs: []string{"get", "list", "watch"}, - }, - } - - // Rules needed by policy recommendation in openshift. - if isOpenShift { - rules = append(rules, - rbacv1.PolicyRule{ - APIGroups: []string{"security.openshift.io"}, - Resources: []string{"securitycontextconstraints"}, - Verbs: []string{"use"}, - ResourceNames: []string{securitycontextconstraints.HostNetworkV2}, - }, - ) - } - - return rules -} - func deprecatedObjects() []client.Object { return []client.Object{ // All the Guardian objects were moved to "calico-system" circa Calico v3.30, and so the legacy tigera-guardian diff --git a/pkg/render/guardian_test.go b/pkg/render/guardian_test.go index 7e82472a41..7d2b454abb 100644 --- a/pkg/render/guardian_test.go +++ b/pkg/render/guardian_test.go @@ -36,6 +36,7 @@ import ( "github.com/tigera/operator/pkg/components" "github.com/tigera/operator/pkg/controller/certificatemanager" ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" + "github.com/tigera/operator/pkg/extensions" "github.com/tigera/operator/pkg/render" rmeta "github.com/tigera/operator/pkg/render/common/meta" "github.com/tigera/operator/pkg/render/common/networkpolicy" @@ -43,6 +44,19 @@ import ( "github.com/tigera/operator/pkg/render/testutils" ) +// guardianObjects renders the guardian component and applies the registered +// enterprise modifier the way the componentHandler does. +func guardianObjects(cfg *render.GuardianConfiguration) []client.Object { + g := render.Guardian(cfg) + ExpectWithOffset(1, g.ResolveImages(nil)).To(BeNil()) + objs, _ := g.Objects() + rc := extensions.RenderContext{Installation: cfg.Installation} + if p, ok := g.(render.ExtensionContextProvider); ok { + rc.Component = p.ExtensionContext() + } + return extensions.ApplyModifiers(render.GuardianName, rc, objs) +} + var _ = Describe("Rendering tests", func() { var cfg *render.GuardianConfiguration var g render.Component @@ -95,6 +109,13 @@ var _ = Describe("Rendering tests", func() { g = render.Guardian(cfg) Expect(g.ResolveImages(nil)).To(BeNil()) resources, deleteResources = g.Objects() + // Apply the registered enterprise modifier the way the componentHandler + // does, so these enterprise tests exercise the integrated output. + rc := extensions.RenderContext{Installation: cfg.Installation} + if p, ok := g.(render.ExtensionContextProvider); ok { + rc.Component = p.ExtensionContext() + } + resources = extensions.ApplyModifiers(render.GuardianName, rc, resources) } BeforeEach(func() { @@ -189,8 +210,7 @@ var _ = Describe("Rendering tests", func() { }, } - g := render.Guardian(cfg) - resources, _ := g.Objects() + resources := guardianObjects(cfg) Expect(resources).ToNot(BeNil()) clusterRole, ok := rtest.GetResource(resources, render.GuardianClusterRoleName, "", "rbac.authorization.k8s.io", "v1", "ClusterRole").(*rbacv1.ClusterRole) @@ -230,8 +250,7 @@ var _ = Describe("Rendering tests", func() { }, } - g := render.Guardian(cfg) - resources, _ := g.Objects() + resources := guardianObjects(cfg) Expect(resources).ToNot(BeNil()) clusterRole, ok := rtest.GetResource(resources, render.GuardianClusterRoleName, "", "rbac.authorization.k8s.io", "v1", "ClusterRole").(*rbacv1.ClusterRole) @@ -270,8 +289,7 @@ var _ = Describe("Rendering tests", func() { }, } - g := render.Guardian(cfg) - resources, _ := g.Objects() + resources := guardianObjects(cfg) Expect(resources).ToNot(BeNil()) clusterRole, ok := rtest.GetResource(resources, render.GuardianClusterRoleName, "", "rbac.authorization.k8s.io", "v1", "ClusterRole").(*rbacv1.ClusterRole) @@ -301,9 +319,7 @@ var _ = Describe("Rendering tests", func() { It("should render SecurityContextConstrains properly when provider is OpenShift", func() { cfg.Installation.KubernetesProvider = operatorv1.ProviderOpenShift cfg.OpenShift = true - component := render.Guardian(cfg) - Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() + resources := guardianObjects(cfg) role := rtest.GetResource(resources, render.GuardianClusterRoleName, "", "rbac.authorization.k8s.io", "v1", "ClusterRole").(*rbacv1.ClusterRole) Expect(role.Rules).To(ContainElement(rbacv1.PolicyRule{ @@ -441,8 +457,7 @@ var _ = Describe("guardian", func() { } }) It("should render when disabled", func() { - g := render.Guardian(cfg) - resources, _ := g.Objects() + resources := guardianObjects(cfg) Expect(resources).ToNot(BeNil()) deployment := rtest.GetResource(resources, render.GuardianDeploymentName, render.GuardianNamespace, "apps", "v1", "Deployment").(*appsv1.Deployment) @@ -452,8 +467,7 @@ var _ = Describe("guardian", func() { It("should render when set to disabled", func() { cfg.TunnelCAType = operatorv1.CATypeTigera - g := render.Guardian(cfg) - resources, _ := g.Objects() + resources := guardianObjects(cfg) Expect(resources).ToNot(BeNil()) deployment := rtest.GetResource(resources, render.GuardianDeploymentName, render.GuardianNamespace, "apps", "v1", "Deployment").(*appsv1.Deployment) @@ -464,8 +478,7 @@ var _ = Describe("guardian", func() { It("should render when enabled", func() { cfg.TunnelCAType = operatorv1.CATypePublic - g := render.Guardian(cfg) - resources, _ := g.Objects() + resources := guardianObjects(cfg) Expect(resources).ToNot(BeNil()) deployment := rtest.GetResource(resources, render.GuardianDeploymentName, render.GuardianNamespace, "apps", "v1", "Deployment").(*appsv1.Deployment) @@ -503,8 +516,7 @@ var _ = Describe("guardian", func() { }, } - g := render.Guardian(cfg) - resources, _ := g.Objects() + resources := guardianObjects(cfg) Expect(resources).ToNot(BeNil()) deployment, ok := rtest.GetResource(resources, render.GuardianDeploymentName, render.GuardianNamespace, "apps", "v1", "Deployment").(*appsv1.Deployment) diff --git a/pkg/render/manager.go b/pkg/render/manager.go index eaa20519db..d08937a686 100644 --- a/pkg/render/manager.go +++ b/pkg/render/manager.go @@ -262,10 +262,10 @@ func (c *managerComponent) Objects() ([]client.Object, []client.Object) { // For multi-tenant environments, the management cluster itself isn't shown in the UI so we only need to create these // when there is no tenant. objsToCreate = append(objsToCreate, - managerClusterWideSettingsGroup(), - managerUserSpecificSettingsGroup(), - managerClusterWideTigeraLayer(), - managerClusterWideDefaultView(), + ManagerClusterWideSettingsGroup(), + ManagerUserSpecificSettingsGroup(), + ManagerClusterWideTigeraLayer(), + ManagerClusterWideDefaultView(), ) // Continue to create the legacy namespace so that we can create our external name service that points to the new // manager service. This will help ease transition for customers and avoid outages caused by the name and namespace @@ -1337,10 +1337,10 @@ func (c *managerComponent) multiTenantManagedClustersAccess() []client.Object { return objects } -// managerClusterWideSettingsGroup returns a UISettingsGroup with the description "cluster-wide settings" +// ManagerClusterWideSettingsGroup returns a UISettingsGroup with the description "cluster-wide settings" // // Calico Enterprise only -func managerClusterWideSettingsGroup() *v3.UISettingsGroup { +func ManagerClusterWideSettingsGroup() *v3.UISettingsGroup { return &v3.UISettingsGroup{ TypeMeta: metav1.TypeMeta{Kind: "UISettingsGroup", APIVersion: "projectcalico.org/v3"}, ObjectMeta: metav1.ObjectMeta{ @@ -1352,10 +1352,10 @@ func managerClusterWideSettingsGroup() *v3.UISettingsGroup { } } -// managerUserSpecificSettingsGroup returns a UISettingsGroup with the description "user settings" +// ManagerUserSpecificSettingsGroup returns a UISettingsGroup with the description "user settings" // // Calico Enterprise only -func managerUserSpecificSettingsGroup() *v3.UISettingsGroup { +func ManagerUserSpecificSettingsGroup() *v3.UISettingsGroup { return &v3.UISettingsGroup{ TypeMeta: metav1.TypeMeta{Kind: "UISettingsGroup", APIVersion: "projectcalico.org/v3"}, ObjectMeta: metav1.ObjectMeta{ @@ -1368,11 +1368,11 @@ func managerUserSpecificSettingsGroup() *v3.UISettingsGroup { } } -// managerClusterWideTigeraLayer returns a UISettings layer belonging to the cluster-wide settings group that contains +// ManagerClusterWideTigeraLayer returns a UISettings layer belonging to the cluster-wide settings group that contains // all of the tigera namespaces. // // Calico Enterprise only -func managerClusterWideTigeraLayer() *v3.UISettings { +func ManagerClusterWideTigeraLayer() *v3.UISettings { namespaces := []string{ "tigera-compliance", "tigera-dex", @@ -1418,11 +1418,11 @@ func managerClusterWideTigeraLayer() *v3.UISettings { } } -// managerClusterWideDefaultView returns a UISettings view belonging to the cluster-wide settings group that shows +// ManagerClusterWideDefaultView returns a UISettings view belonging to the cluster-wide settings group that shows // everything and uses the tigera-infrastructure layer. // // Calico Enterprise only -func managerClusterWideDefaultView() *v3.UISettings { +func ManagerClusterWideDefaultView() *v3.UISettings { return &v3.UISettings{ TypeMeta: metav1.TypeMeta{Kind: "UISettings", APIVersion: "projectcalico.org/v3"}, ObjectMeta: metav1.ObjectMeta{ From 0351e221f5511b00537320cdc7e8662d4b0bbd21 Mon Sep 17 00:00:00 2001 From: Casey Davenport Date: Tue, 9 Jun 2026 15:35:14 -0700 Subject: [PATCH 23/54] De-variant the guardian network policy Move the enterprise guardian network policy out of core render. GuardianPolicy is now an extensible component that renders the OSS policy; the enterprise modifier replaces it with the management-cluster policy, built entirely from the component's ExtensionContext (URL, pod proxies, OpenShift, egress-policy flag). Building the egress rules can fail on proxy URL parsing - the modifier drops the policy on failure, matching the old behavior of omitting it rather than installing a partial one. guardian.go now carries no IsEnterprise branches. --- pkg/enterprise/guardian.go | 172 ++++++++++++++++++++++++ pkg/render/guardian.go | 256 ++++++------------------------------ pkg/render/guardian_test.go | 10 +- 3 files changed, 222 insertions(+), 216 deletions(-) diff --git a/pkg/enterprise/guardian.go b/pkg/enterprise/guardian.go index bcac8a48e1..c85bad2680 100644 --- a/pkg/enterprise/guardian.go +++ b/pkg/enterprise/guardian.go @@ -15,6 +15,10 @@ package enterprise import ( + "net" + "net/url" + + "github.com/sirupsen/logrus" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" @@ -22,17 +26,185 @@ import ( "k8s.io/apimachinery/pkg/util/intstr" "sigs.k8s.io/controller-runtime/pkg/client" + v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" + "github.com/tigera/api/pkg/lib/numorstring" + operatorv1 "github.com/tigera/operator/api/v1" "github.com/tigera/operator/pkg/common" "github.com/tigera/operator/pkg/extensions" "github.com/tigera/operator/pkg/render" + "github.com/tigera/operator/pkg/render/common/networkpolicy" "github.com/tigera/operator/pkg/render/common/securitycontextconstraints" + operatorurl "github.com/tigera/operator/pkg/url" ) func registerGuardian() { extensions.Register(operatorv1.CalicoEnterprise, render.GuardianName, extensions.Extension{ Modify: modifyGuardian, }) + extensions.Register(operatorv1.CalicoEnterprise, render.ComponentNameGuardianPolicy, extensions.Extension{ + Modify: modifyGuardianPolicy, + }) +} + +// modifyGuardianPolicy replaces the core OSS guardian network policy with the +// enterprise management-cluster policy. Building the enterprise egress rules can +// fail (proxy URL parsing); on failure we drop the policy entirely, matching the +// core behavior of omitting it rather than installing a partial policy. +func modifyGuardianPolicy(ctx extensions.RenderContext, objs []client.Object) []client.Object { + gpc, _ := ctx.Component.(render.GuardianPolicyExtensionContext) + + policy, ok := extensions.FindObject[*v3.NetworkPolicy](objs, render.GuardianPolicyName) + if !ok { + return objs + } + + spec, err := enterpriseGuardianPolicySpec(gpc) + if err != nil { + logrus.WithError(err).Error("Failed to build guardian network policy, policy will be omitted") + return removeObject(objs, policy) + } + policy.Spec = spec + return objs +} + +func removeObject(objs []client.Object, drop client.Object) []client.Object { + out := objs[:0] + for _, o := range objs { + if o != drop { + out = append(out, o) + } + } + return out +} + +// enterpriseGuardianPolicySpec builds the network policy spec for guardian in a +// managed cluster: egress to the management cluster components and the tunnel +// destination(s), and ingress from the management-cluster components that reach +// back over the tunnel. +func enterpriseGuardianPolicySpec(gpc render.GuardianPolicyExtensionContext) (v3.NetworkPolicySpec, error) { + egressRules := []v3.Rule{ + { + Action: v3.Allow, + Protocol: &networkpolicy.TCPProtocol, + Destination: render.PacketCaptureEntityRule, + }, + } + egressRules = networkpolicy.AppendDNSEgressRules(egressRules, gpc.OpenShift) + egressRules = append(egressRules, []v3.Rule{ + { + Action: v3.Allow, + Protocol: &networkpolicy.TCPProtocol, + Destination: networkpolicy.KubeAPIServerEntityRule, + }, + { + Action: v3.Allow, + Protocol: &networkpolicy.TCPProtocol, + Destination: networkpolicy.PrometheusEntityRule, + }, + { + Action: v3.Allow, + Protocol: &networkpolicy.TCPProtocol, + Destination: render.TigeraAPIServerEntityRule, + }, + }...) + + // Create an egress rule for each unique destination the guardian pods connect + // to. With multiple pods whose proxy settings differ, there are multiple + // destinations that must be allowed. + allowedDestinations := map[string]bool{} + for _, podProxyConfig := range render.ProcessPodProxies(gpc.PodProxies) { + var proxyURL *url.URL + var err error + if podProxyConfig != nil && podProxyConfig.HTTPSProxy != "" { + // The scheme is HTTPS, as we establish an mTLS session with the target. + // We expect the URL to be of the form host:port. + targetURL := &url.URL{Scheme: "https", Host: gpc.URL} + proxyURL, err = podProxyConfig.ProxyFunc()(targetURL) + if err != nil { + return v3.NetworkPolicySpec{}, err + } + } + + var tunnelDestinationHostPort string + if proxyURL != nil { + proxyHostPort, err := operatorurl.ParseHostPortFromHTTPProxyURL(proxyURL) + if err != nil { + return v3.NetworkPolicySpec{}, err + } + tunnelDestinationHostPort = proxyHostPort + } else { + // gpc.URL has host:port form. + tunnelDestinationHostPort = gpc.URL + } + + if allowedDestinations[tunnelDestinationHostPort] { + continue + } + + host, port, err := net.SplitHostPort(tunnelDestinationHostPort) + if err != nil { + return v3.NetworkPolicySpec{}, err + } + parsedPort, err := numorstring.PortFromString(port) + if err != nil { + return v3.NetworkPolicySpec{}, err + } + parsedIP := net.ParseIP(host) + if parsedIP == nil { + // Domain-based egress rules require the EgressAccessControl license feature. + if !gpc.IncludeEgressNetworkPolicy { + continue + } + egressRules = append(egressRules, v3.Rule{ + Action: v3.Allow, + Protocol: &networkpolicy.TCPProtocol, + Destination: v3.EntityRule{ + Domains: []string{host}, + Ports: []numorstring.Port{parsedPort}, + }, + }) + allowedDestinations[tunnelDestinationHostPort] = true + } else { + netSuffix := "/32" + if parsedIP.To4() == nil { + netSuffix = "/128" + } + egressRules = append(egressRules, v3.Rule{ + Action: v3.Allow, + Protocol: &networkpolicy.TCPProtocol, + Destination: v3.EntityRule{ + Nets: []string{parsedIP.String() + netSuffix}, + Ports: []numorstring.Port{parsedPort}, + }, + }) + allowedDestinations[tunnelDestinationHostPort] = true + } + } + + egressRules = append(egressRules, v3.Rule{Action: v3.Pass}) + + dest := v3.EntityRule{Ports: networkpolicy.Ports(render.GuardianTargetPort)} + helper := networkpolicy.DefaultHelper() + ingressRules := []v3.Rule{ + {Action: v3.Allow, Protocol: &networkpolicy.TCPProtocol, Source: render.FluentdSourceEntityRule, Destination: dest}, + {Action: v3.Allow, Protocol: &networkpolicy.TCPProtocol, Source: helper.ComplianceBenchmarkerSourceEntityRule(), Destination: dest}, + {Action: v3.Allow, Protocol: &networkpolicy.TCPProtocol, Source: helper.ComplianceReporterSourceEntityRule(), Destination: dest}, + {Action: v3.Allow, Protocol: &networkpolicy.TCPProtocol, Source: helper.ComplianceSnapshotterSourceEntityRule(), Destination: dest}, + {Action: v3.Allow, Protocol: &networkpolicy.TCPProtocol, Source: helper.ComplianceControllerSourceEntityRule(), Destination: dest}, + {Action: v3.Allow, Protocol: &networkpolicy.TCPProtocol, Source: render.IntrusionDetectionSourceEntityRule, Destination: dest}, + {Action: v3.Allow, Protocol: &networkpolicy.TCPProtocol, Source: render.IntrusionDetectionInstallerSourceEntityRule, Destination: dest}, + {Action: v3.Allow, Protocol: &networkpolicy.TCPProtocol, Destination: dest}, + } + + return v3.NetworkPolicySpec{ + Order: &networkpolicy.HighPrecedenceOrder, + Tier: networkpolicy.CalicoTierName, + Selector: networkpolicy.KubernetesAppSelector(render.GuardianName), + Types: []v3.PolicyType{v3.PolicyTypeIngress, v3.PolicyTypeEgress}, + Ingress: ingressRules, + Egress: egressRules, + }, nil } // modifyGuardian layers Calico Enterprise behavior onto the rendered guardian diff --git a/pkg/render/guardian.go b/pkg/render/guardian.go index 8c1f30cf3c..f723469447 100644 --- a/pkg/render/guardian.go +++ b/pkg/render/guardian.go @@ -18,13 +18,9 @@ package render import ( "fmt" - "net" - "net/url" "golang.org/x/net/http/httpproxy" - operatorurl "github.com/tigera/operator/pkg/url" - appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" netv1 "k8s.io/api/networking/v1" @@ -35,8 +31,6 @@ import ( v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" - "github.com/tigera/api/pkg/lib/numorstring" - operatorv1 "github.com/tigera/operator/api/v1" "github.com/tigera/operator/pkg/common" "github.com/tigera/operator/pkg/components" @@ -84,27 +78,51 @@ func Guardian(cfg *GuardianConfiguration) Component { } } +// GuardianPolicy renders the guardian network policy. The core operator renders +// the OSS policy; the enterprise modifier (keyed ComponentNameGuardianPolicy) +// replaces it with the management-cluster policy. The error return is retained +// for callers but is always nil now that the fallible enterprise computation +// lives in the modifier. func GuardianPolicy(cfg *GuardianConfiguration) (Component, error) { - var policies []client.Object + return &guardianPolicyComponent{cfg: cfg}, nil +} - guardianAccessPolicy, err := guardianCalicoSystemPolicy(cfg) - if err != nil { - return nil, err - } - if guardianAccessPolicy != nil { - policies = []client.Object{ - guardianAccessPolicy, - } +const ComponentNameGuardianPolicy = "guardian-policy" + +type guardianPolicyComponent struct { + cfg *GuardianConfiguration +} + +func (c *guardianPolicyComponent) ResolveImages(*operatorv1.ImageSet) error { return nil } +func (c *guardianPolicyComponent) SupportedOSType() rmeta.OSType { return rmeta.OSTypeAny } +func (c *guardianPolicyComponent) Ready() bool { return true } +func (c *guardianPolicyComponent) ModifierKey() string { return ComponentNameGuardianPolicy } + +// GuardianPolicyExtensionContext is the per-component context the guardian +// policy modifier reads (via RenderContext.Component). The enterprise guardian +// network policy is built entirely from these inputs. +type GuardianPolicyExtensionContext struct { + URL string + PodProxies []*httpproxy.Config + OpenShift bool + IncludeEgressNetworkPolicy bool +} + +func (c *guardianPolicyComponent) ExtensionContext() any { + return GuardianPolicyExtensionContext{ + URL: c.cfg.URL, + PodProxies: c.cfg.PodProxies, + OpenShift: c.cfg.OpenShift, + IncludeEgressNetworkPolicy: c.cfg.IncludeEgressNetworkPolicy, } +} - return NewPassthrough( - policies, - []client.Object{ - // allow-tigera Tier was renamed to calico-system - networkpolicy.DeprecatedAllowTigeraNetworkPolicyObject("guardian-access", GuardianNamespace), - networkpolicy.DeprecatedAllowTigeraNetworkPolicyObject("default-deny", GuardianNamespace), - }, - ), nil +func (c *guardianPolicyComponent) Objects() ([]client.Object, []client.Object) { + return []client.Object{ossNetworkPolicy()}, []client.Object{ + // allow-tigera Tier was renamed to calico-system + networkpolicy.DeprecatedAllowTigeraNetworkPolicyObject("guardian-access", GuardianNamespace), + networkpolicy.DeprecatedAllowTigeraNetworkPolicyObject("default-deny", GuardianNamespace), + } } // GuardianConfiguration contains all the config information needed to render the component. @@ -467,198 +485,6 @@ func ossNetworkPolicy() *v3.NetworkPolicy { } } -func guardianCalicoSystemPolicy(cfg *GuardianConfiguration) (*v3.NetworkPolicy, error) { - if !cfg.Installation.Variant.IsEnterprise() { - return ossNetworkPolicy(), nil - } - - egressRules := []v3.Rule{ - { - Action: v3.Allow, - Protocol: &networkpolicy.TCPProtocol, - Destination: PacketCaptureEntityRule, - }, - } - egressRules = networkpolicy.AppendDNSEgressRules(egressRules, cfg.OpenShift) - egressRules = append(egressRules, []v3.Rule{ - { - Action: v3.Allow, - Protocol: &networkpolicy.TCPProtocol, - Destination: networkpolicy.KubeAPIServerEntityRule, - }, - { - Action: v3.Allow, - Protocol: &networkpolicy.TCPProtocol, - Destination: networkpolicy.PrometheusEntityRule, - }, - { - Action: v3.Allow, - Protocol: &networkpolicy.TCPProtocol, - Destination: TigeraAPIServerEntityRule, - }, - }...) - - // The loop below creates an egress rule for each unique destination that the Guardian pods connect to. If there are - // multiple guardian pods and their proxy settings differ, then there are multiple destinations that must have egress allowed. - allowedDestinations := map[string]bool{} - processedPodProxies := ProcessPodProxies(cfg.PodProxies) - for _, podProxyConfig := range processedPodProxies { - var proxyURL *url.URL - var err error - if podProxyConfig != nil && podProxyConfig.HTTPSProxy != "" { - targetURL := &url.URL{ - // The scheme should be HTTPS, as we are establishing an mTLS session with the target. - Scheme: "https", - - // We expect `target` to be of the form host:port. - Host: cfg.URL, - } - - proxyURL, err = podProxyConfig.ProxyFunc()(targetURL) - if err != nil { - return nil, err - } - } - - var tunnelDestinationHostPort string - if proxyURL != nil { - proxyHostPort, err := operatorurl.ParseHostPortFromHTTPProxyURL(proxyURL) - if err != nil { - return nil, err - } - - tunnelDestinationHostPort = proxyHostPort - } else { - // cfg.URL has host:port form - tunnelDestinationHostPort = cfg.URL - } - - // Check if we've already created an egress rule for this destination. - if allowedDestinations[tunnelDestinationHostPort] { - continue - } - - host, port, err := net.SplitHostPort(tunnelDestinationHostPort) - if err != nil { - return nil, err - } - parsedPort, err := numorstring.PortFromString(port) - if err != nil { - return nil, err - } - parsedIp := net.ParseIP(host) - if parsedIp == nil { - // Domain-based egress rules require the EgressAccessControl license feature. - if !cfg.IncludeEgressNetworkPolicy { - continue - } - // Assume host is a valid hostname. - egressRules = append(egressRules, v3.Rule{ - Action: v3.Allow, - Protocol: &networkpolicy.TCPProtocol, - Destination: v3.EntityRule{ - Domains: []string{host}, - Ports: []numorstring.Port{parsedPort}, - }, - }) - allowedDestinations[tunnelDestinationHostPort] = true - - } else { - var netSuffix string - if parsedIp.To4() != nil { - netSuffix = "/32" - } else { - netSuffix = "/128" - } - - egressRules = append(egressRules, v3.Rule{ - Action: v3.Allow, - Protocol: &networkpolicy.TCPProtocol, - Destination: v3.EntityRule{ - Nets: []string{parsedIp.String() + netSuffix}, - Ports: []numorstring.Port{parsedPort}, - }, - }) - allowedDestinations[tunnelDestinationHostPort] = true - } - } - - egressRules = append(egressRules, v3.Rule{Action: v3.Pass}) - - guardianIngressDestinationEntityRule := v3.EntityRule{Ports: networkpolicy.Ports(GuardianTargetPort)} - networkpolicyHelper := networkpolicy.DefaultHelper() - var ingressRules []v3.Rule - if cfg.Installation.Variant.IsEnterprise() { - ingressRules = append(ingressRules, []v3.Rule{ - { - Action: v3.Allow, - Protocol: &networkpolicy.TCPProtocol, - Source: FluentdSourceEntityRule, - Destination: guardianIngressDestinationEntityRule, - }, - { - Action: v3.Allow, - Protocol: &networkpolicy.TCPProtocol, - Source: networkpolicyHelper.ComplianceBenchmarkerSourceEntityRule(), - Destination: guardianIngressDestinationEntityRule, - }, - { - Action: v3.Allow, - Protocol: &networkpolicy.TCPProtocol, - Source: networkpolicyHelper.ComplianceReporterSourceEntityRule(), - Destination: guardianIngressDestinationEntityRule, - }, - { - Action: v3.Allow, - Protocol: &networkpolicy.TCPProtocol, - Source: networkpolicyHelper.ComplianceSnapshotterSourceEntityRule(), - Destination: guardianIngressDestinationEntityRule, - }, - { - Action: v3.Allow, - Protocol: &networkpolicy.TCPProtocol, - Source: networkpolicyHelper.ComplianceControllerSourceEntityRule(), - Destination: guardianIngressDestinationEntityRule, - }, - { - Action: v3.Allow, - Protocol: &networkpolicy.TCPProtocol, - Source: IntrusionDetectionSourceEntityRule, - Destination: guardianIngressDestinationEntityRule, - }, - { - Action: v3.Allow, - Protocol: &networkpolicy.TCPProtocol, - Source: IntrusionDetectionInstallerSourceEntityRule, - Destination: guardianIngressDestinationEntityRule, - }, - { - Action: v3.Allow, - Protocol: &networkpolicy.TCPProtocol, - Destination: guardianIngressDestinationEntityRule, - }, - }...) - } - - policy := &v3.NetworkPolicy{ - TypeMeta: metav1.TypeMeta{Kind: "NetworkPolicy", APIVersion: "projectcalico.org/v3"}, - ObjectMeta: metav1.ObjectMeta{ - Name: GuardianPolicyName, - Namespace: GuardianNamespace, - }, - Spec: v3.NetworkPolicySpec{ - Order: &networkpolicy.HighPrecedenceOrder, - Tier: networkpolicy.CalicoTierName, - Selector: networkpolicy.KubernetesAppSelector(GuardianName), - Types: []v3.PolicyType{v3.PolicyTypeIngress, v3.PolicyTypeEgress}, - Ingress: ingressRules, - Egress: egressRules, - }, - } - - return policy, nil -} - func ProcessPodProxies(podProxies []*httpproxy.Config) []*httpproxy.Config { // If pod proxies are empty, then pod proxy resolution has not yet occurred. // Assume that a single Guardian pod is running without a proxy. diff --git a/pkg/render/guardian_test.go b/pkg/render/guardian_test.go index 7d2b454abb..244e1c6fa5 100644 --- a/pkg/render/guardian_test.go +++ b/pkg/render/guardian_test.go @@ -343,7 +343,15 @@ var _ = Describe("Rendering tests", func() { cfg.IncludeEgressNetworkPolicy = includeEgressNetworkPolicy g, err := render.GuardianPolicy(cfg) Expect(err).NotTo(HaveOccurred()) - resources, _ = g.Objects() + objs, _ := g.Objects() + // Apply the registered enterprise modifier the way the componentHandler + // does, so the enterprise policy is exercised. For the Calico variant the + // modifier is a no-op and the OSS policy is returned. + rc := extensions.RenderContext{Installation: cfg.Installation} + if p, ok := g.(render.ExtensionContextProvider); ok { + rc.Component = p.ExtensionContext() + } + resources = extensions.ApplyModifiers(render.ComponentNameGuardianPolicy, rc, objs) } Context("policy rendering based on variant and IncludeEgressNetworkPolicy", func() { From 3de36391a2024da761a21b67bb1f1d820cec5057 Mon Sep 17 00:00:00 2001 From: Casey Davenport Date: Tue, 9 Jun 2026 18:50:08 -0700 Subject: [PATCH 24/54] De-variant the apiserver into an enterprise extension The apiserver was the last render component with IsEnterprise branches. Its enterprise behavior (query server container, audit logging, the tigera-* RBAC, UI settings roles, managed cluster access) moves into pkg/enterprise as the apiserver modifier, leaving render with only the shared objects. The deployment-existence check no longer keys off the variant. The controller sets RequiresQueryServer (true for Enterprise) and render branches on RequiresAggregationServer || RequiresQueryServer, so the variant decision lives in the controller rather than render. The apiserver also needs to delete enterprise objects when switching to Calico, and to drop the management-cluster RBAC when not a management cluster, so Modifier now takes and returns both the create and delete lists. The other modifiers pass the delete list through unchanged. --- .../apiserver/apiserver_controller.go | 9 +- .../apiserver/apiserver_suite_test.go | 5 + pkg/controller/utils/component.go | 2 +- pkg/controller/utils/component_test.go | 4 +- pkg/enterprise/apiserver.go | 1146 +++++++++++++++ pkg/enterprise/guardian.go | 12 +- pkg/enterprise/guardian_test.go | 10 +- pkg/enterprise/node.go | 4 +- pkg/enterprise/node_test.go | 20 +- pkg/enterprise/register.go | 1 + pkg/enterprise/typha.go | 4 +- pkg/enterprise/typha_test.go | 6 +- pkg/enterprise/windows.go | 4 +- pkg/enterprise/windows_test.go | 10 +- pkg/extensions/extension.go | 24 +- pkg/extensions/extension_test.go | 38 +- pkg/render/apiserver.go | 1269 ++--------------- pkg/render/apiserver_test.go | 117 +- pkg/render/guardian_test.go | 7 +- pkg/render/node_enterprise_test.go | 5 +- pkg/render/windows_test.go | 3 +- 21 files changed, 1413 insertions(+), 1287 deletions(-) create mode 100644 pkg/enterprise/apiserver.go diff --git a/pkg/controller/apiserver/apiserver_controller.go b/pkg/controller/apiserver/apiserver_controller.go index 668756cc10..0e762c1717 100644 --- a/pkg/controller/apiserver/apiserver_controller.go +++ b/pkg/controller/apiserver/apiserver_controller.go @@ -49,6 +49,7 @@ import ( "github.com/tigera/operator/pkg/controller/utils/imageset" "github.com/tigera/operator/pkg/ctrlruntime" "github.com/tigera/operator/pkg/dns" + "github.com/tigera/operator/pkg/extensions" "github.com/tigera/operator/pkg/render" rcertificatemanagement "github.com/tigera/operator/pkg/render/certificatemanagement" "github.com/tigera/operator/pkg/render/common/authentication" @@ -473,8 +474,11 @@ func (r *ReconcileAPIServer) Reconcile(ctx context.Context, request reconcile.Re return reconcile.Result{}, err } - // Create a component handler to manage the rendered component. - handler := utils.NewComponentHandler(log, r.client, r.scheme, instance) + // Create a component handler to manage the rendered component. The render context + // carries the installation so the componentHandler applies the variant's API server + // modifier (query server, audit logging, Enterprise RBAC) to the rendered objects. + handler := utils.NewComponentHandler(log, r.client, r.scheme, instance, + utils.WithRenderContext(extensions.RenderContext{Installation: installationSpec})) // Render the desired objects from the CRD and create or update them. reqLogger.V(3).Info("rendering components") @@ -497,6 +501,7 @@ func (r *ReconcileAPIServer) Reconcile(ctx context.Context, request reconcile.Re KubernetesVersion: r.opts.KubernetesVersion, ClusterDomain: r.opts.ClusterDomain, RequiresAggregationServer: !r.opts.UseV3CRDs, + RequiresQueryServer: installationSpec.Variant.IsEnterprise(), QueryServerTLSKeyPairCertificateManagementOnly: queryServerTLSSecretCertificateManagementOnly, } diff --git a/pkg/controller/apiserver/apiserver_suite_test.go b/pkg/controller/apiserver/apiserver_suite_test.go index ae0a254ba0..17f3e98c8d 100644 --- a/pkg/controller/apiserver/apiserver_suite_test.go +++ b/pkg/controller/apiserver/apiserver_suite_test.go @@ -24,9 +24,14 @@ import ( logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" + + "github.com/tigera/operator/pkg/enterprise" ) func TestStatus(t *testing.T) { + // Wire the enterprise extensions so the componentHandler applies the API server + // modifier (query server, audit logging, Enterprise RBAC) during reconciliation. + enterprise.Register() logf.SetLogger(zap.New(zap.WriteTo(ginkgo.GinkgoWriter), zap.UseDevMode(true), zap.Level(uzap.NewAtomicLevelAt(uzap.DebugLevel)))) gomega.RegisterFailHandler(ginkgo.Fail) suiteConfig, reporterConfig := ginkgo.GinkgoConfiguration() diff --git a/pkg/controller/utils/component.go b/pkg/controller/utils/component.go index d2424c99c7..0eda36688b 100644 --- a/pkg/controller/utils/component.go +++ b/pkg/controller/utils/component.go @@ -473,7 +473,7 @@ func (c *componentHandler) CreateOrUpdateOrDelete(ctx context.Context, component if p, ok := component.(render.ExtensionContextProvider); ok { rc.Component = p.ExtensionContext() } - objsToCreate = extensions.ApplyModifiers(ext.ModifierKey(), rc, objsToCreate) + objsToCreate, objsToDelete = extensions.ApplyModifiers(ext.ModifierKey(), rc, objsToCreate, objsToDelete) } // Load the InstallationSpec once and reuse it for every object: createOrUpdateObject needs it diff --git a/pkg/controller/utils/component_test.go b/pkg/controller/utils/component_test.go index ecbe90afe4..fad9e6fedb 100644 --- a/pkg/controller/utils/component_test.go +++ b/pkg/controller/utils/component_test.go @@ -2494,10 +2494,10 @@ var _ = Describe("componentHandler modifier application", func() { It("applies registered modifiers to a named component before create", func() { extensions.Register(operatorv1.CalicoEnterprise, "fake", extensions.Extension{ - Modify: func(ctx extensions.RenderContext, objs []client.Object) []client.Object { + Modify: func(ctx extensions.RenderContext, objs, del []client.Object) ([]client.Object, []client.Object) { cm := objs[0].(*corev1.ConfigMap) cm.Data = map[string]string{"patched": "yes"} - return objs + return objs, del }, }) diff --git a/pkg/enterprise/apiserver.go b/pkg/enterprise/apiserver.go new file mode 100644 index 0000000000..178be099a3 --- /dev/null +++ b/pkg/enterprise/apiserver.go @@ -0,0 +1,1146 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package enterprise + +import ( + "fmt" + "strings" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" + + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/common" + "github.com/tigera/operator/pkg/components" + "github.com/tigera/operator/pkg/extensions" + "github.com/tigera/operator/pkg/render" + rcomp "github.com/tigera/operator/pkg/render/common/components" + relasticsearch "github.com/tigera/operator/pkg/render/common/elasticsearch" + rmeta "github.com/tigera/operator/pkg/render/common/meta" + "github.com/tigera/operator/pkg/render/common/securitycontext" + "github.com/tigera/operator/pkg/tls/certificatemanagement" +) + +const ( + auditLogsVolumeName = "calico-audit-logs" + auditPolicyVolumeName = "calico-audit-policy" +) + +// apiServer carries the rendered API server configuration and resolved image so the +// enterprise builders (moved verbatim from the render package) can construct the +// Enterprise-only objects and deployment additions. +type apiServer struct { + cfg *render.APIServerConfiguration + calicoImage string +} + +func registerAPIServer() { + extensions.Register(operatorv1.CalicoEnterprise, render.ComponentNameAPIServer, extensions.Extension{ + Modify: modifyAPIServer, + }) + // When running Calico, clean up any Enterprise objects left behind by a prior + // Enterprise installation. + extensions.Register(operatorv1.Calico, render.ComponentNameAPIServer, extensions.Extension{ + Modify: cleanupAPIServer, + }) +} + +// modifyAPIServer layers Calico Enterprise behavior onto the rendered API server objects: +// the query server container and its volumes, audit logging on the aggregation API server +// container, the Enterprise RBAC objects, and the query server port on the Service. +func modifyAPIServer(ctx extensions.RenderContext, create, del []client.Object) ([]client.Object, []client.Object) { + ec, _ := ctx.Component.(render.APIServerExtensionContext) + c := &apiServer{cfg: ec.Config, calicoImage: ec.CalicoImage} + + if dep, ok := extensions.FindObject[*appsv1.Deployment](create, render.APIServerName); ok { + c.layerDeployment(dep) + } + if svc, ok := extensions.FindObject[*corev1.Service](create, render.APIServerServiceName); ok { + c.addQueryServerPort(svc) + } + // Enterprise serves staged policies through the tiered-policy passthrough role. + if role, ok := extensions.FindObject[*rbacv1.ClusterRole](create, "calico-tiered-policy-passthrough"); ok { + for i := range role.Rules { + if contains(role.Rules[i].Resources, "networkpolicies") { + role.Rules[i].Resources = append(role.Rules[i].Resources, "stagednetworkpolicies", "stagedglobalnetworkpolicies") + } + } + } + + // Global Enterprise RBAC. + create = append(create, c.tigeraAPIServerClusterRole(), c.tigeraAPIServerClusterRoleBinding()) + if !c.cfg.MultiTenant { + // These resources are only installed in zero-tenant clusters. + create = append(create, c.tigeraUserClusterRole(), c.tigeraNetworkAdminClusterRole()) + } + if c.cfg.ManagementCluster != nil { + create = append(create, c.managedClusterWatchClusterRole()) + if c.cfg.MultiTenant { + create = append(create, c.multiTenantSecretsRBAC()...) + create = append(create, c.multiTenantManagedClusterAccessClusterRoles()...) + } else { + create = append(create, c.secretsRBAC()...) + } + } else { + // If we're not a management cluster, the API server doesn't need permissions to access secrets. + del = append(del, c.multiTenantSecretsRBAC()...) + del = append(del, c.secretsRBAC()...) + del = append(del, c.multiTenantManagedClusterAccessClusterRoles()...) + del = append(del, c.managedClusterWatchClusterRole()) + } + + // Namespaced Enterprise objects. + if c.cfg.TrustedBundle != nil { + create = append(create, c.cfg.TrustedBundle.ConfigMap(render.QueryserverNamespace)) + } + if c.cfg.ManagementClusterConnection != nil { + create = append(create, c.externalLinseedRoleBinding()) + } + + // Objects that only exist alongside the aggregation API server. + aggregationObjects := []client.Object{ + c.uiSettingsGroupGetterClusterRole(), + c.kubeControllerManagerUISettingsGroupGetterClusterRoleBinding(), + c.uiSettingsPassthruClusterRole(), + c.uiSettingsPassthruClusterRolebinding(), + c.auditPolicyConfigMap(), + } + if c.cfg.RequiresAggregationServer { + create = append(create, aggregationObjects...) + } else { + del = append(del, aggregationObjects...) + } + + // Clean up cluster-scoped resources that were created with the 'tigera' prefix. + del = append(del, c.deprecatedResources()...) + + // Re-apply deployment overrides so the modifier-added query server container picks up + // any per-container overrides. The override appliers use replace/merge semantics, so + // re-running over the render-applied containers is idempotent. + if dep, ok := extensions.FindObject[*appsv1.Deployment](create, render.APIServerName); ok { + if overrides := c.cfg.APIServer.APIServerDeployment; overrides != nil { + rcomp.ApplyDeploymentOverrides(dep, overrides) + } + } + + return create, del +} + +// cleanupAPIServer deletes the Enterprise API server objects when running Calico, so a +// cluster switched from Enterprise to Calico does not leave them behind. +func cleanupAPIServer(ctx extensions.RenderContext, create, del []client.Object) ([]client.Object, []client.Object) { + ec, _ := ctx.Component.(render.APIServerExtensionContext) + c := &apiServer{cfg: ec.Config} + + del = append(del, c.tigeraAPIServerClusterRole(), c.tigeraAPIServerClusterRoleBinding()) + if !c.cfg.MultiTenant { + del = append(del, c.tigeraUserClusterRole(), c.tigeraNetworkAdminClusterRole()) + } + del = append(del, c.multiTenantSecretsRBAC()...) + del = append(del, c.secretsRBAC()...) + del = append(del, c.multiTenantManagedClusterAccessClusterRoles()...) + del = append(del, c.managedClusterWatchClusterRole()) + + return create, del +} + +// layerDeployment adds the Enterprise query server container, audit logging, and the +// query server / trusted bundle volumes to the rendered API server deployment. +func (c *apiServer) layerDeployment(d *appsv1.Deployment) { + // Audit logging is performed through the aggregation API server container, which is + // only present when the aggregation API server is running. + if c.cfg.RequiresAggregationServer { + for i := range d.Spec.Template.Spec.Containers { + ctr := &d.Spec.Template.Spec.Containers[i] + if ctr.Name != string(render.APIServerContainerName) { + continue + } + ctr.VolumeMounts = append(ctr.VolumeMounts, + corev1.VolumeMount{Name: auditLogsVolumeName, MountPath: "/var/log/calico/audit"}, + corev1.VolumeMount{Name: auditPolicyVolumeName, MountPath: "/etc/tigera/audit"}, + ) + ctr.Args = append(ctr.Args, + "--audit-policy-file=/etc/tigera/audit/policy.conf", + "--audit-log-path=/var/log/calico/audit/tsee-audit.log", + ) + // In case of OpenShift, apiserver needs privileged access to write audit logs to the + // host path volume. Audit logs are owned by root on hosts so we need to be root. + ctr.SecurityContext = securitycontext.NewRootContext(c.cfg.OpenShift) + } + + d.Spec.Template.Spec.Volumes = append(d.Spec.Template.Spec.Volumes, c.auditVolumes()...) + } + + d.Spec.Template.Spec.Containers = append(d.Spec.Template.Spec.Containers, c.queryServerContainer()) + + if c.cfg.TrustedBundle != nil { + d.Spec.Template.Spec.Volumes = append(d.Spec.Template.Spec.Volumes, c.cfg.TrustedBundle.Volume()) + for k, v := range c.cfg.TrustedBundle.HashAnnotations() { + d.Spec.Template.Annotations[k] = v + } + } +} + +// auditVolumes are the host-path audit log and audit policy volumes used by the +// aggregation API server container. +func (c *apiServer) auditVolumes() []corev1.Volume { + return []corev1.Volume{ + { + Name: auditLogsVolumeName, + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: "/var/log/calico/audit", + Type: ptr.To(corev1.HostPathDirectoryOrCreate), + }, + }, + }, + { + Name: auditPolicyVolumeName, + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{Name: auditPolicyVolumeName}, + Items: []corev1.KeyToPath{ + { + Key: "config", + Path: "policy.conf", + }, + }, + }, + }, + }, + } +} + +func (c *apiServer) addQueryServerPort(s *corev1.Service) { + queryServerTargetPort := render.GetContainerPort(c.cfg, render.TigeraAPIServerQueryServerContainerName) + s.Spec.Ports = append(s.Spec.Ports, corev1.ServicePort{ + Name: render.QueryServerPortName, + Port: render.QueryServerPort, + Protocol: corev1.ProtocolTCP, + TargetPort: intstr.FromInt32(queryServerTargetPort.ContainerPort), + }) +} + +func contains(s []string, v string) bool { + for _, x := range s { + if x == v { + return true + } + } + return false +} + +func (c *apiServer) multiTenantSecretsRBAC() []client.Object { + return render.TunnelSecretRBAC(render.APIServerSecretsRBACName, render.APIServerServiceAccountName, c.cfg.ManagementCluster, true) +} + +func (c *apiServer) secretsRBAC() []client.Object { + return render.TunnelSecretRBAC(render.APIServerSecretsRBACName, render.APIServerServiceAccountName, c.cfg.ManagementCluster, false) +} + +func (c *apiServer) queryServerContainer() corev1.Container { + queryServerTargetPort := render.GetContainerPort(c.cfg, render.TigeraAPIServerQueryServerContainerName).ContainerPort + + var tlsSecret certificatemanagement.KeyPairInterface + if c.cfg.QueryServerTLSKeyPairCertificateManagementOnly != nil { + tlsSecret = c.cfg.QueryServerTLSKeyPairCertificateManagementOnly + } else { + tlsSecret = c.cfg.TLSKeyPair + } + env := []corev1.EnvVar{ + {Name: "DATASTORE_TYPE", Value: "kubernetes"}, + {Name: "LISTEN_ADDR", Value: fmt.Sprintf(":%d", queryServerTargetPort)}, + {Name: "TLS_CERT", Value: fmt.Sprintf("/%s/tls.crt", tlsSecret.GetName())}, + {Name: "TLS_KEY", Value: fmt.Sprintf("/%s/tls.key", tlsSecret.GetName())}, + } + if c.cfg.TrustedBundle != nil { + env = append(env, corev1.EnvVar{Name: "TRUSTED_BUNDLE_PATH", Value: c.cfg.TrustedBundle.MountPath()}) + } + + if render.HostNetwork(c.cfg) { + env = append(env, c.cfg.K8SServiceEndpoint.EnvVars()...) + } else { + env = append(env, c.cfg.K8SServiceEndpointPodNetwork.EnvVars()...) + } + + if c.cfg.Installation.CalicoNetwork != nil && c.cfg.Installation.CalicoNetwork.MultiInterfaceMode != nil { + env = append(env, corev1.EnvVar{Name: "MULTI_INTERFACE_MODE", Value: c.cfg.Installation.CalicoNetwork.MultiInterfaceMode.Value()}) + } + + if c.cfg.KeyValidatorConfig != nil { + env = append(env, c.cfg.KeyValidatorConfig.RequiredEnv("")...) + } + + linseedURL := relasticsearch.LinseedEndpoint(rmeta.OSTypeLinux, c.cfg.ClusterDomain, render.ElasticsearchNamespace, c.cfg.ManagementClusterConnection != nil, false) + env = append(env, + corev1.EnvVar{Name: "LINSEED_URL", Value: linseedURL}, + corev1.EnvVar{Name: "LINSEED_CLIENT_CERT", Value: fmt.Sprintf("/%s/tls.crt", tlsSecret.GetName())}, + corev1.EnvVar{Name: "LINSEED_CLIENT_KEY", Value: fmt.Sprintf("/%s/tls.key", tlsSecret.GetName())}, + ) + if c.cfg.ManagementClusterConnection != nil { + env = append(env, + corev1.EnvVar{Name: "CLUSTER_ID", Value: ""}, + corev1.EnvVar{Name: "LINSEED_TOKEN", Value: render.GetLinseedTokenPath(true)}, + ) + } + if c.cfg.TrustedBundle != nil { + env = append(env, corev1.EnvVar{Name: "LINSEED_CA", Value: c.cfg.TrustedBundle.MountPath()}) + } + + // set LogLEVEL for queryserver container + if logging := c.cfg.APIServer.Logging; logging != nil && + logging.QueryServerLogging != nil && logging.QueryServerLogging.LogSeverity != nil { + env = append(env, + corev1.EnvVar{Name: "LOGLEVEL", Value: strings.ToLower(string(*logging.QueryServerLogging.LogSeverity))}) + } else { + // set default LOGLEVEL to info when not set by the user + env = append(env, corev1.EnvVar{Name: "LOGLEVEL", Value: "info"}) + } + + volumeMounts := []corev1.VolumeMount{ + tlsSecret.VolumeMount(rmeta.OSTypeLinux), + } + if c.cfg.TrustedBundle != nil { + volumeMounts = append(volumeMounts, c.cfg.TrustedBundle.VolumeMounts(rmeta.OSTypeLinux)...) + } + if c.cfg.ManagementClusterConnection != nil { + volumeMounts = append(volumeMounts, corev1.VolumeMount{ + Name: render.LinseedTokenVolumeName, + MountPath: render.LinseedVolumeMountPath, + }) + } + + container := corev1.Container{ + Name: string(render.TigeraAPIServerQueryServerContainerName), + Image: c.calicoImage, + Command: []string{components.CalicoBinaryPath, "component", "queryserver"}, + Env: env, + LivenessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/version", + Port: intstr.FromInt32(queryServerTargetPort), + Scheme: corev1.URISchemeHTTPS, + }, + }, + InitialDelaySeconds: 90, + }, + SecurityContext: securitycontext.NewNonRootContext(), + VolumeMounts: volumeMounts, + } + return container +} + +func (c *apiServer) externalLinseedRoleBinding() *rbacv1.RoleBinding { + return &rbacv1.RoleBinding{ + TypeMeta: metav1.TypeMeta{Kind: "RoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "tigera-linseed", + Namespace: render.APIServerNamespace, + }, + RoleRef: rbacv1.RoleRef{ + APIGroup: "rbac.authorization.k8s.io", + Kind: "ClusterRole", + Name: render.TigeraLinseedSecretsClusterRole, + }, + Subjects: []rbacv1.Subject{ + { + Kind: "ServiceAccount", + Name: render.GuardianServiceAccountName, + Namespace: render.GuardianNamespace, + }, + }, + } +} + +func (c *apiServer) tigeraAPIServerClusterRole() *rbacv1.ClusterRole { + rules := []rbacv1.PolicyRule{ + { + // Read access to Linseed policy activity data for queryserver enrichment. + APIGroups: []string{"linseed.tigera.io"}, + Resources: []string{"policyactivity"}, + Verbs: []string{"get"}, + }, + { + // Calico Enterprise backing storage. + APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, + Resources: []string{ + "alertexceptions", + "bfdconfigurations", + "deeppacketinspections", + "deeppacketinspections/status", + "egressgatewaypolicies", + "externalnetworks", + "globalalerts", + "globalalerts/status", + "globalalerttemplates", + "globalreports", + "globalreports/status", + "globalreporttypes", + "globalthreatfeeds", + "globalthreatfeeds/status", + "licensekeys", + "managedclusters", + "managedclusters/status", + "networks", + "packetcaptures", + "packetcaptures/status", + "policyrecommendationscopes", + "policyrecommendationscopes/status", + "remoteclusterconfigurations", + "securityeventwebhooks", + "securityeventwebhooks/status", + "uisettings", + "uisettingsgroups", + }, + Verbs: []string{ + "get", + "list", + "watch", + "create", + "update", + "delete", + "patch", + }, + }, + { + // The queryserver's RBAC calculator needs to list tiers, + // uisettingsgroups, and managedclusters via the aggregated + // API to evaluate user permissions for the /policies endpoint. + APIGroups: []string{"projectcalico.org"}, + Resources: []string{ + "tiers", + "uisettingsgroups", + "managedclusters", + }, + Verbs: []string{"get", "list", "watch"}, + }, + { + // Required by the AuthorizationReview calculator in queryserver to evaluate + // RBAC permissions for users. + APIGroups: []string{"rbac.authorization.k8s.io"}, + Resources: []string{ + "clusterroles", + "clusterrolebindings", + "roles", + "rolebindings", + }, + Verbs: []string{"get", "list", "watch"}, + }, + } + + return &rbacv1.ClusterRole{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: render.APIServerName, + }, + Rules: rules, + } +} + +func (c *apiServer) tigeraAPIServerClusterRoleBinding() *rbacv1.ClusterRoleBinding { + return &rbacv1.ClusterRoleBinding{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: render.APIServerName, + }, + Subjects: []rbacv1.Subject{ + { + Kind: "ServiceAccount", + Name: render.APIServerServiceAccountName, + Namespace: render.APIServerNamespace, + }, + }, + RoleRef: rbacv1.RoleRef{ + Kind: "ClusterRole", + Name: render.APIServerName, + APIGroup: "rbac.authorization.k8s.io", + }, + } +} + +func (c *apiServer) uiSettingsGroupGetterClusterRole() *rbacv1.ClusterRole { + return &rbacv1.ClusterRole{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "calico-uisettingsgroup-getter", + }, + Rules: []rbacv1.PolicyRule{ + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{ + "uisettingsgroups", + }, + Verbs: []string{"get"}, + }, + }, + } +} + +func (c *apiServer) kubeControllerManagerUISettingsGroupGetterClusterRoleBinding() *rbacv1.ClusterRoleBinding { + return &rbacv1.ClusterRoleBinding{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "calico-uisettingsgroup-getter", + }, + RoleRef: rbacv1.RoleRef{ + Kind: "ClusterRole", + Name: "calico-uisettingsgroup-getter", + APIGroup: "rbac.authorization.k8s.io", + }, + Subjects: []rbacv1.Subject{ + { + Kind: "User", + Name: "system:kube-controller-manager", + APIGroup: "rbac.authorization.k8s.io", + }, + }, + } +} + +func (c *apiServer) tigeraUserClusterRole() *rbacv1.ClusterRole { + rules := []rbacv1.PolicyRule{ + // List requests that the Tigera manager needs. + { + APIGroups: []string{ + "projectcalico.org", + "networking.k8s.io", + "extensions", + "", + }, + // Use both the networkpolicies and tier.networkpolicies resource types to ensure identical behavior + // irrespective of the Calico RBAC scheme (see the ClusterRole "calico-tiered-policy-passthrough" for + // more details). Similar for all tiered policy resource types. + Resources: []string{ + "tiers", + "networkpolicies", + "tier.networkpolicies", + "globalnetworkpolicies", + "tier.globalnetworkpolicies", + "namespaces", + "globalnetworksets", + "networksets", + "managedclusters", + "stagedglobalnetworkpolicies", + "tier.stagedglobalnetworkpolicies", + "stagednetworkpolicies", + "tier.stagednetworkpolicies", + "stagedkubernetesnetworkpolicies", + "policyrecommendationscopes", + }, + Verbs: []string{"watch", "list"}, + }, + { + APIGroups: []string{"policy.networking.k8s.io"}, + Resources: []string{ + "clusternetworkpolicies", + "adminnetworkpolicies", + "baselineadminnetworkpolicies", + }, + Verbs: []string{"watch", "list"}, + }, + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"packetcaptures/files"}, + Verbs: []string{"get"}, + }, + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"packetcaptures"}, + Verbs: []string{"get", "list", "watch"}, + }, + // Additional "list" requests required to view flows. + { + APIGroups: []string{""}, + Resources: []string{"pods"}, + Verbs: []string{"list"}, + }, + // Additional "list" requests required to view serviceaccount labels. + { + APIGroups: []string{""}, + Resources: []string{"serviceaccounts"}, + Verbs: []string{"list"}, + }, + // Access for WAF API to read in coreruleset configmap + { + APIGroups: []string{""}, + Resources: []string{"configmaps"}, + ResourceNames: []string{"coreruleset-default"}, + Verbs: []string{"get"}, + }, + // Access to statistics. + { + APIGroups: []string{""}, + Resources: []string{"services/proxy"}, + ResourceNames: []string{ + "https:calico-api:8080", "calico-node-prometheus:9090", + }, + Verbs: []string{"get", "create"}, + }, + // Access to policies in all tiers + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"tiers"}, + Verbs: []string{"get"}, + }, + // List and download the reports in the Tigera Secure manager. + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"globalreports"}, + Verbs: []string{"get", "list"}, + }, + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"globalreporttypes"}, + Verbs: []string{"get"}, + }, + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"clusterinformations"}, + Verbs: []string{"get", "list"}, + }, + // Access to hostendpoints from the UI ServiceGraph. + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"hostendpoints"}, + Verbs: []string{"get", "list"}, + }, + // List and view the threat defense configuration + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{ + "alertexceptions", + "globalalerts", + "globalalerts/status", + "globalalerttemplates", + "globalthreatfeeds", + "globalthreatfeeds/status", + "securityeventwebhooks", + }, + Verbs: []string{"get", "watch", "list"}, + }, + // User can: + // - read UISettings in the cluster-settings group + // - read and write UISettings in the user-settings group + // Default settings group and settings are created in manager.go. + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"uisettingsgroups"}, + Verbs: []string{"get"}, + ResourceNames: []string{"cluster-settings", "user-settings"}, + }, + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"uisettingsgroups/data"}, + Verbs: []string{"get", "list", "watch"}, + ResourceNames: []string{"cluster-settings"}, + }, + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"uisettingsgroups/data"}, + Verbs: []string{"*"}, + ResourceNames: []string{"user-settings"}, + }, + // Allow the user to read applicationlayers to detect if WAF is enabled/disabled. + { + APIGroups: []string{"operator.tigera.io"}, + Resources: []string{"applicationlayers", "packetcaptureapis", "compliances", "intrusiondetections"}, + Verbs: []string{"get"}, + }, + { + APIGroups: []string{"apps"}, + Resources: []string{"deployments"}, + Verbs: []string{"get", "list", "watch"}, + }, + // Allow the user to read services to view WAF configuration. + { + APIGroups: []string{""}, + Resources: []string{"services"}, + Verbs: []string{"get", "list", "watch"}, + }, + // Allow the user to read felixconfigurations to detect if wireguard and/or other features are enabled. + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"felixconfigurations"}, + Verbs: []string{"get", "list"}, + }, + // Allow the user to only view securityeventwebhooks. + { + APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, + Resources: []string{"securityeventwebhooks"}, + Verbs: []string{"get", "list"}, + }, + } + + // Privileges for lma.tigera.io have no effect on managed clusters. + if c.cfg.ManagementClusterConnection == nil { + // Access to flow logs, audit logs, and statistics. + // Access to log into Kibana for oidc users. + rules = append(rules, rbacv1.PolicyRule{ + APIGroups: []string{"lma.tigera.io"}, + Resources: []string{"*"}, + ResourceNames: []string{ + "flows", "audit*", "l7", "events", "dns", "waf", "kibana_login", "recommendations", + }, + Verbs: []string{"get"}, + }) + } + + return &rbacv1.ClusterRole{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "tigera-ui-user", + }, + Rules: rules, + } +} + +func (c *apiServer) tigeraNetworkAdminClusterRole() *rbacv1.ClusterRole { + rules := []rbacv1.PolicyRule{ + // Full access to all network policies + { + APIGroups: []string{ + "projectcalico.org", + "networking.k8s.io", + "extensions", + }, + // Use both the networkpolicies and tier.networkpolicies resource types to ensure identical behavior + // irrespective of the Calico RBAC scheme (see the ClusterRole "calico-tiered-policy-passthrough" for + // more details). Similar for all tiered policy resource types. + Resources: []string{ + "tiers", + "networkpolicies", + "tier.networkpolicies", + "globalnetworkpolicies", + "tier.globalnetworkpolicies", + "stagedglobalnetworkpolicies", + "tier.stagedglobalnetworkpolicies", + "stagednetworkpolicies", + "tier.stagednetworkpolicies", + "stagedkubernetesnetworkpolicies", + "globalnetworksets", + "networksets", + "managedclusters", + "packetcaptures", + "policyrecommendationscopes", + }, + Verbs: []string{"create", "update", "delete", "patch", "get", "watch", "list"}, + }, + { + APIGroups: []string{ + "policy.networking.k8s.io", + }, + Resources: []string{ + "clusternetworkpolicies", + "adminnetworkpolicies", + "baselineadminnetworkpolicies", + }, + Verbs: []string{"create", "update", "delete", "patch", "get", "watch", "list"}, + }, + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"packetcaptures/files"}, + Verbs: []string{"get", "delete"}, + }, + // Additional "list" requests that the Tigera Secure manager needs + { + APIGroups: []string{""}, + Resources: []string{"namespaces"}, + Verbs: []string{"watch", "list"}, + }, + // Additional "list" requests required to view flows. + { + APIGroups: []string{""}, + Resources: []string{"pods"}, + Verbs: []string{"list"}, + }, + // Additional "list" requests required to view serviceaccount labels. + { + APIGroups: []string{""}, + Resources: []string{"serviceaccounts"}, + Verbs: []string{"list"}, + }, + // Access for WAF API to read in coreruleset configmap + { + APIGroups: []string{""}, + Resources: []string{"configmaps"}, + ResourceNames: []string{"coreruleset-default"}, + Verbs: []string{"get"}, + }, + // Access to statistics. + { + APIGroups: []string{""}, + Resources: []string{"services/proxy"}, + ResourceNames: []string{ + "https:calico-api:8080", "calico-node-prometheus:9090", + }, + Verbs: []string{"get", "create"}, + }, + // Manage globalreport configuration, view report generation status, and list reports in the Tigera Secure manager. + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"globalreports"}, + Verbs: []string{"*"}, + }, + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"globalreports/status"}, + Verbs: []string{"get", "list", "watch"}, + }, + // List and download the reports in the Tigera Secure manager. + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"globalreporttypes"}, + Verbs: []string{"get"}, + }, + // Access to cluster information containing Calico and EE versions from the UI. + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"clusterinformations"}, + Verbs: []string{"get", "list"}, + }, + // Access to hostendpoints from the UI ServiceGraph. + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"hostendpoints"}, + Verbs: []string{"get", "list"}, + }, + // Manage the threat defense configuration + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{ + "alertexceptions", + "globalalerts", + "globalalerts/status", + "globalalerttemplates", + "globalthreatfeeds", + "globalthreatfeeds/status", + "securityeventwebhooks", + }, + Verbs: []string{"create", "update", "delete", "patch", "get", "watch", "list"}, + }, + // User can: + // - read and write UISettings in the cluster-settings group, and rename the group + // - read and write UISettings in the user-settings group, and rename the group + // Default settings group and settings are created in manager.go. + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"uisettingsgroups"}, + Verbs: []string{"get", "patch", "update"}, + ResourceNames: []string{"cluster-settings", "user-settings"}, + }, + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"uisettingsgroups/data"}, + Verbs: []string{"*"}, + ResourceNames: []string{"cluster-settings", "user-settings"}, + }, + // Allow the user to read and write applicationlayers to enable/disable WAF. + { + APIGroups: []string{"operator.tigera.io"}, + Resources: []string{"applicationlayers", "packetcaptureapis", "compliances", "intrusiondetections"}, + Verbs: []string{"get", "update", "patch", "create", "delete"}, + }, + // Allow the user to read deployments to view WAF configuration. + { + APIGroups: []string{"apps"}, + Resources: []string{"deployments"}, + Verbs: []string{"get", "list", "watch", "patch"}, + }, + { + APIGroups: []string{""}, + Resources: []string{"services"}, + Verbs: []string{"get", "list", "watch", "patch"}, + }, + // Allow the user to read felixconfigurations to detect if wireguard and/or other features are enabled. + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"felixconfigurations"}, + Verbs: []string{"get", "list"}, + }, + // Allow the user to perform CRUD operations on securityeventwebhooks. + { + APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, + Resources: []string{"securityeventwebhooks"}, + Verbs: []string{"get", "list", "update", "patch", "create", "delete"}, + }, + // Allow the user to create secrets. + { + APIGroups: []string{""}, + Resources: []string{ + "secrets", + }, + Verbs: []string{"create"}, + }, + // Allow the user to patch webhooks-secret secret. + { + APIGroups: []string{""}, + Resources: []string{ + "secrets", + }, + ResourceNames: []string{ + "webhooks-secret", + }, + Verbs: []string{"patch"}, + }, + } + + // Privileges for lma.tigera.io have no effect on managed clusters. + if c.cfg.ManagementClusterConnection == nil { + // Access to flow logs, audit logs, and statistics. + // Elasticsearch superuser access once logged into Kibana. + rules = append(rules, rbacv1.PolicyRule{ + APIGroups: []string{"lma.tigera.io"}, + Resources: []string{"*"}, + ResourceNames: []string{ + "flows", "audit*", "l7", "events", "dns", "waf", "kibana_login", "elasticsearch_superuser", "recommendations", + }, + Verbs: []string{"get"}, + }) + } + + // In v3 CRD / webhooks mode there is no aggregated apiserver, and the + // calico-uisettings-passthrough ClusterRole that normally grants the broad + // uisettings permission isn't deployed. Grant write verbs here so the + // calico-webhooks UISettings handler (which narrows access via a SAR on + // uisettingsgroups/data) gets invoked instead of being short-circuited by + // kube-apiserver RBAC. + if !c.cfg.RequiresAggregationServer { + rules = append(rules, rbacv1.PolicyRule{ + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"uisettings"}, + Verbs: []string{"create", "update", "delete", "patch"}, + }) + } + + return &rbacv1.ClusterRole{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "tigera-network-admin", + }, + Rules: rules, + } +} + +func (c *apiServer) uiSettingsPassthruClusterRole() *rbacv1.ClusterRole { + return &rbacv1.ClusterRole{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "calico-uisettings-passthrough", + }, + Rules: []rbacv1.PolicyRule{ + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"uisettings"}, + Verbs: []string{"*"}, + }, + }, + } +} + +func (c *apiServer) uiSettingsPassthruClusterRolebinding() *rbacv1.ClusterRoleBinding { + return &rbacv1.ClusterRoleBinding{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "calico-uisettings-passthrough", + }, + Subjects: []rbacv1.Subject{ + { + Kind: "Group", + Name: "system:authenticated", + APIGroup: "rbac.authorization.k8s.io", + }, + }, + RoleRef: rbacv1.RoleRef{ + Kind: "ClusterRole", + Name: "calico-uisettings-passthrough", + APIGroup: "rbac.authorization.k8s.io", + }, + } +} + +func (c *apiServer) auditPolicyConfigMap() *corev1.ConfigMap { + const defaultAuditPolicy = `apiVersion: audit.k8s.io/v1 +kind: Policy +rules: +- level: RequestResponse + omitStages: + - RequestReceived + verbs: + - create + - patch + - update + - delete + resources: + - group: projectcalico.org + resources: + - globalnetworkpolicies + - networkpolicies + - stagedglobalnetworkpolicies + - stagednetworkpolicies + - stagedkubernetesnetworkpolicies + - globalnetworksets + - networksets + - tiers + - hostendpoints` + + return &corev1.ConfigMap{ + TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"}, + ObjectMeta: metav1.ObjectMeta{ + // This object is for Enterprise only, so pass it explicitly. + Namespace: render.APIServerNamespace, + Name: auditPolicyVolumeName, + }, + Data: map[string]string{ + "config": defaultAuditPolicy, + }, + } +} + +func (c *apiServer) multiTenantManagedClusterAccessClusterRoles() []client.Object { + var objects []client.Object + objects = append(objects, &rbacv1.ClusterRole{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{Name: render.MultiTenantManagedClustersAccessClusterRoleName}, + Rules: []rbacv1.PolicyRule{ + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"managedclusters"}, + Verbs: []string{ + // The Authentication Proxy in Voltron checks if Enterprise Components (using impersonation headers for + // the service in the canonical namespace) can get a managed clusters before sending the request down the tunnel. + // This ClusterRole will be assigned to each component using a RoleBinding in the canonical or tenant namespace. + "get", + }, + }, + }, + }) + + return objects +} + +func (c *apiServer) managedClusterWatchClusterRole() client.Object { + return &rbacv1.ClusterRole{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{Name: render.ManagedClustersWatchClusterRoleName}, + Rules: []rbacv1.PolicyRule{ + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"managedclusters"}, + Verbs: []string{ + "get", "list", "watch", + }, + }, + }, + } +} + +func (c *apiServer) deprecatedResources() []client.Object { + return []client.Object{ + &rbacv1.ClusterRole{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{Name: "tigera-extension-apiserver-secrets-access"}, + }, + &rbacv1.ClusterRoleBinding{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{Name: "tigera-extension-apiserver-secrets-access"}, + }, + + // delegateAuthClusterRoleBinding + &rbacv1.ClusterRoleBinding{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{Name: "tigera-apiserver-delegate-auth"}, + }, + + // authClusterRole + &rbacv1.ClusterRole{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{Name: "tigera-extension-apiserver-auth-access"}, + }, + + // authClusterRoleBinding + &rbacv1.ClusterRoleBinding{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{Name: "tigera-extension-apiserver-auth-access"}, + }, + // authReaderRoleBinding - need clean up in diff namespace kube-system + &rbacv1.RoleBinding{ + TypeMeta: metav1.TypeMeta{Kind: "RoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "tigera-auth-reader", + Namespace: "kube-system", + }, + }, + // webhookReaderClusterRole + &rbacv1.ClusterRole{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{Name: "tigera-webhook-reader"}, + }, + + // webhookReaderClusterRoleBinding + &rbacv1.ClusterRoleBinding{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{Name: "tigera-apiserver-webhook-reader"}, + }, + + // calico-apiserver CR and CRB + &rbacv1.ClusterRole{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{Name: "tigera-apiserver"}, + }, + &rbacv1.ClusterRoleBinding{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{Name: "tigera-apiserver"}, + }, + + &rbacv1.ClusterRole{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{Name: "tigera-uisettingsgroup-getter"}, + }, + &rbacv1.ClusterRoleBinding{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{Name: "tigera-uisettingsgroup-getter"}, + }, + + &rbacv1.ClusterRole{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{Name: "tigera-tiered-policy-passthrough"}, + }, + &rbacv1.ClusterRoleBinding{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{Name: "tigera-tiered-policy-passthrough"}, + }, + + &rbacv1.ClusterRole{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{Name: "tigera-uisettings-passthrough"}, + }, + &rbacv1.ClusterRoleBinding{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{Name: "tigera-uisettings-passthrough"}, + }, + + // Clean up legacy secrets in the tigera-operator namespace + &corev1.Secret{ + TypeMeta: metav1.TypeMeta{Kind: "Secret", APIVersion: "v1"}, + ObjectMeta: metav1.ObjectMeta{Name: "tigera-api-cert", Namespace: common.OperatorNamespace()}, + }, + } +} diff --git a/pkg/enterprise/guardian.go b/pkg/enterprise/guardian.go index c85bad2680..1d16e13b19 100644 --- a/pkg/enterprise/guardian.go +++ b/pkg/enterprise/guardian.go @@ -51,21 +51,21 @@ func registerGuardian() { // enterprise management-cluster policy. Building the enterprise egress rules can // fail (proxy URL parsing); on failure we drop the policy entirely, matching the // core behavior of omitting it rather than installing a partial policy. -func modifyGuardianPolicy(ctx extensions.RenderContext, objs []client.Object) []client.Object { +func modifyGuardianPolicy(ctx extensions.RenderContext, objs, del []client.Object) ([]client.Object, []client.Object) { gpc, _ := ctx.Component.(render.GuardianPolicyExtensionContext) policy, ok := extensions.FindObject[*v3.NetworkPolicy](objs, render.GuardianPolicyName) if !ok { - return objs + return objs, del } spec, err := enterpriseGuardianPolicySpec(gpc) if err != nil { logrus.WithError(err).Error("Failed to build guardian network policy, policy will be omitted") - return removeObject(objs, policy) + return removeObject(objs, policy), del } policy.Spec = spec - return objs + return objs, del } func removeObject(objs []client.Object, drop client.Object) []client.Object { @@ -211,7 +211,7 @@ func enterpriseGuardianPolicySpec(gpc render.GuardianPolicyExtensionContext) (v3 // objects: the secrets Role/RoleBinding and default UI settings, the // elasticsearch/kibana service ports, the management-cluster-request cluster // role rules (which replace the OSS rules), and the CA bundle env vars. -func modifyGuardian(ctx extensions.RenderContext, objs []client.Object) []client.Object { +func modifyGuardian(ctx extensions.RenderContext, objs, del []client.Object) ([]client.Object, []client.Object) { gc, _ := ctx.Component.(render.GuardianExtensionContext) if role, ok := extensions.FindObject[*rbacv1.ClusterRole](objs, render.GuardianClusterRoleName); ok { @@ -234,7 +234,7 @@ func modifyGuardian(ctx extensions.RenderContext, objs []client.Object) []client render.ManagerUserSpecificSettingsGroup(), render.ManagerClusterWideTigeraLayer(), render.ManagerClusterWideDefaultView(), - ) + ), del } // guardianEnterpriseRules are the cluster role rules guardian needs in Calico diff --git a/pkg/enterprise/guardian_test.go b/pkg/enterprise/guardian_test.go index b68ad61b04..9b43423199 100644 --- a/pkg/enterprise/guardian_test.go +++ b/pkg/enterprise/guardian_test.go @@ -60,7 +60,7 @@ var _ = Describe("guardian enterprise modifier", func() { } It("appends the secrets RBAC and UI settings", func() { - out := extensions.ApplyModifiers(render.GuardianName, ctxWith(render.GuardianExtensionContext{}), newObjs()) + out, _ := extensions.ApplyModifiers(render.GuardianName, ctxWith(render.GuardianExtensionContext{}), newObjs(), nil) _, ok := extensions.FindObject[*rbacv1.Role](out, render.GuardianSecretsRole) Expect(ok).To(BeTrue()) _, ok = extensions.FindObject[*rbacv1.RoleBinding](out, render.GuardianSecretsRoleBindingName) @@ -70,7 +70,7 @@ var _ = Describe("guardian enterprise modifier", func() { }) It("adds the elasticsearch and kibana service ports", func() { - out := extensions.ApplyModifiers(render.GuardianName, ctxWith(render.GuardianExtensionContext{}), newObjs()) + out, _ := extensions.ApplyModifiers(render.GuardianName, ctxWith(render.GuardianExtensionContext{}), newObjs(), nil) svc, _ := extensions.FindObject[*corev1.Service](out, render.GuardianServiceName) names := []string{} for _, p := range svc.Spec.Ports { @@ -83,7 +83,7 @@ var _ = Describe("guardian enterprise modifier", func() { gc := render.GuardianExtensionContext{ Impersonation: &operatorv1.Impersonation{Users: []string{"foo"}, Groups: []string{"bar"}}, } - out := extensions.ApplyModifiers(render.GuardianName, ctxWith(gc), newObjs()) + out, _ := extensions.ApplyModifiers(render.GuardianName, ctxWith(gc), newObjs(), nil) role, _ := extensions.FindObject[*rbacv1.ClusterRole](out, render.GuardianClusterRoleName) // The single OSS placeholder rule is gone, replaced by the enterprise set. @@ -94,14 +94,14 @@ var _ = Describe("guardian enterprise modifier", func() { It("adds the CA bundle env to the guardian container", func() { gc := render.GuardianExtensionContext{TrustedBundleMountPath: "/ca/bundle"} - out := extensions.ApplyModifiers(render.GuardianName, ctxWith(gc), newObjs()) + out, _ := extensions.ApplyModifiers(render.GuardianName, ctxWith(gc), newObjs(), nil) dep, _ := extensions.FindObject[*appsv1.Deployment](out, render.GuardianDeploymentName) Expect(dep.Spec.Template.Spec.Containers[0].Env).To(ContainElement(corev1.EnvVar{Name: "GUARDIAN_PROMETHEUS_CA_BUNDLE_PATH", Value: "/ca/bundle"})) }) It("does nothing for the Calico variant", func() { ctx := extensions.RenderContext{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.Calico}} - out := extensions.ApplyModifiers(render.GuardianName, ctx, newObjs()) + out, _ := extensions.ApplyModifiers(render.GuardianName, ctx, newObjs(), nil) Expect(out).To(HaveLen(len(newObjs()))) role, _ := extensions.FindObject[*rbacv1.ClusterRole](out, render.GuardianClusterRoleName) Expect(role.Rules).To(Equal([]rbacv1.PolicyRule{{Verbs: []string{"get"}}})) diff --git a/pkg/enterprise/node.go b/pkg/enterprise/node.go index 8e1de4832b..d1f2023563 100644 --- a/pkg/enterprise/node.go +++ b/pkg/enterprise/node.go @@ -59,7 +59,7 @@ func registerNode() { // objects: the extra RBAC rules, the node-metrics Service, and the Enterprise // daemonset configuration (flow/DNS log env, prometheus reporter, BGP metrics // readiness check, multi-interface mode, and the calico log volume). -func modifyNode(ctx extensions.RenderContext, objs []client.Object) []client.Object { +func modifyNode(ctx extensions.RenderContext, objs, del []client.Object) ([]client.Object, []client.Object) { if role, ok := extensions.FindObject[*rbacv1.ClusterRole](objs, render.CalicoNodeObjectName); ok { role.Rules = append(role.Rules, nodeEnterpriseRules()...) } @@ -77,7 +77,7 @@ func modifyNode(ctx extensions.RenderContext, objs []client.Object) []client.Obj modifyNodeDaemonSet(ctx, ds) } - return append(objs, nodeMetricsService(ctx)) + return append(objs, nodeMetricsService(ctx)), del } // nodeEnterpriseRules are the additional cluster role rules calico/node needs in diff --git a/pkg/enterprise/node_test.go b/pkg/enterprise/node_test.go index 2ec7e53b38..22a5408754 100644 --- a/pkg/enterprise/node_test.go +++ b/pkg/enterprise/node_test.go @@ -86,7 +86,7 @@ var _ = Describe("node enterprise modifier", func() { } It("adds the enterprise cluster role rules", func() { - out := extensions.ApplyModifiers(render.ComponentNameNode, entCtx(), newObjs()) + out, _ := extensions.ApplyModifiers(render.ComponentNameNode, entCtx(), newObjs(), nil) nodeRole, ok := extensions.FindObject[*rbacv1.ClusterRole](out, render.CalicoNodeObjectName) Expect(ok).To(BeTrue()) @@ -98,7 +98,7 @@ var _ = Describe("node enterprise modifier", func() { }) It("adds the enterprise felix env to the node container", func() { - out := extensions.ApplyModifiers(render.ComponentNameNode, entCtx(), newObjs()) + out, _ := extensions.ApplyModifiers(render.ComponentNameNode, entCtx(), newObjs(), nil) ds, _ := extensions.FindObject[*appsv1.DaemonSet](out, common.NodeDaemonSetName) c := nodeContainer(ds) @@ -115,13 +115,13 @@ var _ = Describe("node enterprise modifier", func() { ctx := entCtx() ctx.FelixConfiguration = &v3.FelixConfiguration{Spec: v3.FelixConfigurationSpec{PrometheusReporterPort: &reporter}} - out := extensions.ApplyModifiers(render.ComponentNameNode, ctx, newObjs()) + out, _ := extensions.ApplyModifiers(render.ComponentNameNode, ctx, newObjs(), nil) ds, _ := extensions.FindObject[*appsv1.DaemonSet](out, common.NodeDaemonSetName) Expect(nodeContainer(ds).Env).To(ContainElement(corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERPORT", Value: "7081"})) }) It("appends the BGP metrics readiness check when the bird check is present", func() { - out := extensions.ApplyModifiers(render.ComponentNameNode, entCtx(), newObjs()) + out, _ := extensions.ApplyModifiers(render.ComponentNameNode, entCtx(), newObjs(), nil) ds, _ := extensions.FindObject[*appsv1.DaemonSet](out, common.NodeDaemonSetName) Expect(nodeContainer(ds).ReadinessProbe.Exec.Command).To(ContainElement("--bgp-metrics-ready")) }) @@ -131,7 +131,7 @@ var _ = Describe("node enterprise modifier", func() { ds := objs[2].(*appsv1.DaemonSet) ds.Spec.Template.Spec.Containers[0].ReadinessProbe.Exec.Command = []string{"/bin/calico-node", "--felix-ready"} - out := extensions.ApplyModifiers(render.ComponentNameNode, entCtx(), objs) + out, _ := extensions.ApplyModifiers(render.ComponentNameNode, entCtx(), objs, nil) got, _ := extensions.FindObject[*appsv1.DaemonSet](out, common.NodeDaemonSetName) Expect(nodeContainer(got).ReadinessProbe.Exec.Command).NotTo(ContainElement("--bgp-metrics-ready")) }) @@ -141,7 +141,7 @@ var _ = Describe("node enterprise modifier", func() { ctx := entCtx() ctx.Installation.CalicoNetwork = &operatorv1.CalicoNetworkSpec{MultiInterfaceMode: &mode} - out := extensions.ApplyModifiers(render.ComponentNameNode, ctx, newObjs()) + out, _ := extensions.ApplyModifiers(render.ComponentNameNode, ctx, newObjs(), nil) ds, _ := extensions.FindObject[*appsv1.DaemonSet](out, common.NodeDaemonSetName) want := corev1.EnvVar{Name: "MULTI_INTERFACE_MODE", Value: mode.Value()} @@ -150,7 +150,7 @@ var _ = Describe("node enterprise modifier", func() { }) It("appends the node metrics service", func() { - out := extensions.ApplyModifiers(render.ComponentNameNode, entCtx(), newObjs()) + out, _ := extensions.ApplyModifiers(render.ComponentNameNode, entCtx(), newObjs(), nil) svc, ok := extensions.FindObject[*corev1.Service](out, render.CalicoNodeMetricsService) Expect(ok).To(BeTrue()) Expect(svc.Spec.Ports).To(HaveLen(2)) @@ -169,7 +169,7 @@ var _ = Describe("node enterprise modifier", func() { PrometheusMetricsEnabled: &enabled, }} - out := extensions.ApplyModifiers(render.ComponentNameNode, ctx, newObjs()) + out, _ := extensions.ApplyModifiers(render.ComponentNameNode, ctx, newObjs(), nil) svc, _ := extensions.FindObject[*corev1.Service](out, render.CalicoNodeMetricsService) Expect(svc.Spec.Ports).To(HaveLen(3)) Expect(svc.Spec.Ports[0].Port).To(Equal(int32(7081))) @@ -179,7 +179,7 @@ var _ = Describe("node enterprise modifier", func() { It("is a no-op for the Calico variant", func() { ctx := extensions.RenderContext{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.Calico}} - out := extensions.ApplyModifiers(render.ComponentNameNode, ctx, newObjs()) + out, _ := extensions.ApplyModifiers(render.ComponentNameNode, ctx, newObjs(), nil) _, ok := extensions.FindObject[*corev1.Service](out, render.CalicoNodeMetricsService) Expect(ok).To(BeFalse()) @@ -188,7 +188,7 @@ var _ = Describe("node enterprise modifier", func() { }) It("does not panic on a zero RenderContext", func() { - out := extensions.ApplyModifiers(render.ComponentNameNode, extensions.RenderContext{}, newObjs()) + out, _ := extensions.ApplyModifiers(render.ComponentNameNode, extensions.RenderContext{}, newObjs(), nil) _, ok := extensions.FindObject[*corev1.Service](out, render.CalicoNodeMetricsService) Expect(ok).To(BeFalse()) }) diff --git a/pkg/enterprise/register.go b/pkg/enterprise/register.go index 20589ad0fa..8d6bdc0cd8 100644 --- a/pkg/enterprise/register.go +++ b/pkg/enterprise/register.go @@ -23,4 +23,5 @@ func Register() { registerWindows() registerGuardian() registerInstallation() + registerAPIServer() } diff --git a/pkg/enterprise/typha.go b/pkg/enterprise/typha.go index 457243842a..38733c871c 100644 --- a/pkg/enterprise/typha.go +++ b/pkg/enterprise/typha.go @@ -31,7 +31,7 @@ func registerTypha() { }) } -func modifyTypha(ctx extensions.RenderContext, objs []client.Object) []client.Object { +func modifyTypha(ctx extensions.RenderContext, objs, del []client.Object) ([]client.Object, []client.Object) { if role, ok := extensions.FindObject[*rbacv1.ClusterRole](objs, "calico-typha"); ok { role.Rules = append(role.Rules, rbacv1.PolicyRule{ APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, @@ -61,5 +61,5 @@ func modifyTypha(ctx extensions.RenderContext, objs []client.Object) []client.Ob } } - return objs + return objs, del } diff --git a/pkg/enterprise/typha_test.go b/pkg/enterprise/typha_test.go index 13fcccc2f9..2ebb853f8c 100644 --- a/pkg/enterprise/typha_test.go +++ b/pkg/enterprise/typha_test.go @@ -54,7 +54,7 @@ var _ = Describe("typha enterprise modifier", func() { Variant: operatorv1.CalicoEnterprise, CalicoNetwork: &operatorv1.CalicoNetworkSpec{MultiInterfaceMode: &multiMode}, }} - out := extensions.ApplyModifiers(render.ComponentNameTypha, ctx, newObjs()) + out, _ := extensions.ApplyModifiers(render.ComponentNameTypha, ctx, newObjs(), nil) role := out[0].(*rbacv1.ClusterRole) Expect(role.Rules).To(ContainElement(HaveField("Resources", ContainElement("licensekeys")))) @@ -74,14 +74,14 @@ var _ = Describe("typha enterprise modifier", func() { Variant: operatorv1.Calico, CalicoNetwork: &operatorv1.CalicoNetworkSpec{MultiInterfaceMode: &multiMode}, }} - out := extensions.ApplyModifiers(render.ComponentNameTypha, ctx, newObjs()) + out, _ := extensions.ApplyModifiers(render.ComponentNameTypha, ctx, newObjs(), nil) Expect(out[0].(*rbacv1.ClusterRole).Rules).To(BeEmpty()) dep := out[1].(*appsv1.Deployment) Expect(dep.Spec.Template.Spec.Containers[0].Env).To(BeEmpty()) }) It("does not panic on a zero Context (nil Installation)", func() { - out := extensions.ApplyModifiers(render.ComponentNameTypha, extensions.RenderContext{}, newObjs()) + out, _ := extensions.ApplyModifiers(render.ComponentNameTypha, extensions.RenderContext{}, newObjs(), nil) Expect(out[0].(*rbacv1.ClusterRole).Rules).To(BeEmpty()) }) }) diff --git a/pkg/enterprise/windows.go b/pkg/enterprise/windows.go index 12f62cd6c6..720ea80c20 100644 --- a/pkg/enterprise/windows.go +++ b/pkg/enterprise/windows.go @@ -51,14 +51,14 @@ func registerWindows() { // calico-node-windows objects: the node-metrics Service and the Enterprise // daemonset configuration (flow/DNS log env, prometheus reporter, trusted DNS // servers, the calico log volume, and the prometheus reporter keypair mount). -func modifyWindows(ctx extensions.RenderContext, objs []client.Object) []client.Object { +func modifyWindows(ctx extensions.RenderContext, objs, del []client.Object) ([]client.Object, []client.Object) { wc, _ := ctx.Component.(render.WindowsExtensionContext) if ds, ok := extensions.FindObject[*appsv1.DaemonSet](objs, common.WindowsDaemonSetName); ok { modifyWindowsDaemonSet(ctx, wc, ds) } - return append(objs, windowsNodeMetricsService(wc)) + return append(objs, windowsNodeMetricsService(wc)), del } func modifyWindowsDaemonSet(ctx extensions.RenderContext, wc render.WindowsExtensionContext, ds *appsv1.DaemonSet) { diff --git a/pkg/enterprise/windows_test.go b/pkg/enterprise/windows_test.go index b289941bdb..18668c9079 100644 --- a/pkg/enterprise/windows_test.go +++ b/pkg/enterprise/windows_test.go @@ -102,7 +102,7 @@ var _ = Describe("windows enterprise modifier", func() { } It("appends the node-metrics service", func() { - out := extensions.ApplyModifiers(render.ComponentNameWindows, ctxFor(operatorv1.ProviderNone, nil, nil), newObjs()) + out, _ := extensions.ApplyModifiers(render.ComponentNameWindows, ctxFor(operatorv1.ProviderNone, nil, nil), newObjs(), nil) svc, ok := extensions.FindObject[*corev1.Service](out, render.WindowsNodeMetricsService) Expect(ok).To(BeTrue()) Expect(svc.Namespace).To(Equal(common.CalicoNamespace)) @@ -110,7 +110,7 @@ var _ = Describe("windows enterprise modifier", func() { }) It("swaps the cni log mount for the calico log volume and adds enterprise env", func() { - out := extensions.ApplyModifiers(render.ComponentNameWindows, ctxFor(operatorv1.ProviderNone, nil, nil), newObjs()) + out, _ := extensions.ApplyModifiers(render.ComponentNameWindows, ctxFor(operatorv1.ProviderNone, nil, nil), newObjs(), nil) d := ds(out) Expect(d.Spec.Template.Spec.Volumes).To(ContainElement(HaveField("Name", "var-log-calico"))) @@ -127,7 +127,7 @@ var _ = Describe("windows enterprise modifier", func() { }) It("sets the trusted DNS server on openshift", func() { - out := extensions.ApplyModifiers(render.ComponentNameWindows, ctxFor(operatorv1.ProviderOpenShift, nil, nil), newObjs()) + out, _ := extensions.ApplyModifiers(render.ComponentNameWindows, ctxFor(operatorv1.ProviderOpenShift, nil, nil), newObjs(), nil) Expect(container(ds(out), "node").Env).To(ContainElement(corev1.EnvVar{Name: "FELIX_DNSTRUSTEDSERVERS", Value: "k8s-service:openshift-dns/dns-default"})) }) @@ -141,7 +141,7 @@ var _ = Describe("windows enterprise modifier", func() { Expect(err).NotTo(HaveOccurred()) bundle := cm.CreateTrustedBundle() - out := extensions.ApplyModifiers(render.ComponentNameWindows, ctxFor(operatorv1.ProviderNone, tls, bundle), newObjs()) + out, _ := extensions.ApplyModifiers(render.ComponentNameWindows, ctxFor(operatorv1.ProviderNone, tls, bundle), newObjs(), nil) d := ds(out) Expect(d.Spec.Template.Spec.Volumes).To(ContainElement(tls.Volume())) @@ -152,7 +152,7 @@ var _ = Describe("windows enterprise modifier", func() { It("does nothing for the Calico variant", func() { ctx := extensions.RenderContext{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.Calico}} - out := extensions.ApplyModifiers(render.ComponentNameWindows, ctx, newObjs()) + out, _ := extensions.ApplyModifiers(render.ComponentNameWindows, ctx, newObjs(), nil) _, ok := extensions.FindObject[*corev1.Service](out, render.WindowsNodeMetricsService) Expect(ok).To(BeFalse()) Expect(ds(out).Spec.Template.Spec.Volumes).To(BeEmpty()) diff --git a/pkg/extensions/extension.go b/pkg/extensions/extension.go index 4f57d64dd2..25a43feb13 100644 --- a/pkg/extensions/extension.go +++ b/pkg/extensions/extension.go @@ -35,11 +35,13 @@ type Extension struct { Modify Modifier } -// Modifier post-processes the objects a render component produced. It may mutate -// matched objects and/or append additional objects, and must return the -// (possibly extended) slice. A modifier runs only for the variant it was -// registered under, so it need not re-check the variant. -type Modifier func(ctx RenderContext, objs []client.Object) []client.Object +// Modifier post-processes the objects a render component produced. It receives +// the component's create and delete lists and returns the (possibly extended) +// lists. A modifier may mutate matched objects, append objects to create, and +// append objects to delete (e.g. to clean up resources another variant left +// behind). A modifier runs only for the variant it was registered under, so it +// need not re-check the variant. +type Modifier func(ctx RenderContext, create, delete []client.Object) (newCreate, newDelete []client.Object) type modifierKey struct { variant operatorv1.ProductVariant @@ -63,16 +65,16 @@ func Register(variant operatorv1.ProductVariant, component string, e Extension) } // ApplyModifiers runs the modifier registered for the named component and the -// installation's variant over objs, returning objs unchanged when none is -// registered (or when no installation is set). -func ApplyModifiers(component string, ctx RenderContext, objs []client.Object) []client.Object { +// installation's variant over the create and delete lists, returning them +// unchanged when none is registered (or when no installation is set). +func ApplyModifiers(component string, ctx RenderContext, create, delete []client.Object) ([]client.Object, []client.Object) { if ctx.Installation == nil { - return objs + return create, delete } if fn, ok := modifiers[modifierKey{ctx.Installation.Variant, component}]; ok { - objs = fn(ctx, objs) + create, delete = fn(ctx, create, delete) } - return objs + return create, delete } // FindObject returns the first object of type T with the given name. diff --git a/pkg/extensions/extension_test.go b/pkg/extensions/extension_test.go index c935dac9f0..2c08569355 100644 --- a/pkg/extensions/extension_test.go +++ b/pkg/extensions/extension_test.go @@ -34,16 +34,16 @@ var _ = Describe("extension registry", func() { It("applies a registered modifier to the matching component and variant", func() { extensions.Register(operatorv1.CalicoEnterprise, "test", extensions.Extension{ - Modify: func(ctx extensions.RenderContext, objs []client.Object) []client.Object { + Modify: func(ctx extensions.RenderContext, objs, del []client.Object) ([]client.Object, []client.Object) { cm, ok := extensions.FindObject[*corev1.ConfigMap](objs, "cm") Expect(ok).To(BeTrue()) cm.Data = map[string]string{"k": "v"} - return append(objs, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "extra"}}) + return append(objs, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "extra"}}), del }, }) in := []client.Object{&corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "cm"}}} - out := extensions.ApplyModifiers("test", entCtx, in) + out, _ := extensions.ApplyModifiers("test", entCtx, in, nil) Expect(out).To(HaveLen(2)) cm := out[0].(*corev1.ConfigMap) @@ -51,41 +51,57 @@ var _ = Describe("extension registry", func() { Expect(out[1].GetName()).To(Equal("extra")) }) + It("lets a modifier append to the delete list", func() { + extensions.Register(operatorv1.CalicoEnterprise, "test", extensions.Extension{ + Modify: func(_ extensions.RenderContext, objs, del []client.Object) ([]client.Object, []client.Object) { + return objs, append(del, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "stale"}}) + }, + }) + + in := []client.Object{&corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "cm"}}} + out, del := extensions.ApplyModifiers("test", entCtx, in, nil) + Expect(out).To(Equal(in)) + Expect(del).To(HaveLen(1)) + Expect(del[0].GetName()).To(Equal("stale")) + }) + It("returns objects unchanged when no modifier is registered", func() { in := []client.Object{&corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "cm"}}} - out := extensions.ApplyModifiers("unregistered", entCtx, in) + out, _ := extensions.ApplyModifiers("unregistered", entCtx, in, nil) Expect(out).To(Equal(in)) }) It("does not apply a modifier registered for a different variant", func() { extensions.Register(operatorv1.CalicoEnterprise, "test", extensions.Extension{ - Modify: func(_ extensions.RenderContext, objs []client.Object) []client.Object { - return append(objs, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "extra"}}) + Modify: func(_ extensions.RenderContext, objs, del []client.Object) ([]client.Object, []client.Object) { + return append(objs, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "extra"}}), del }, }) calicoCtx := extensions.RenderContext{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.Calico}} in := []client.Object{&corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "cm"}}} - Expect(extensions.ApplyModifiers("test", calicoCtx, in)).To(Equal(in)) + out, _ := extensions.ApplyModifiers("test", calicoCtx, in, nil) + Expect(out).To(Equal(in)) }) It("returns objects unchanged when no installation is set", func() { in := []client.Object{&corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "cm"}}} - Expect(extensions.ApplyModifiers("test", extensions.RenderContext{}, in)).To(Equal(in)) + out, _ := extensions.ApplyModifiers("test", extensions.RenderContext{}, in, nil) + Expect(out).To(Equal(in)) }) It("replaces rather than stacks when a (variant, component) is registered twice", func() { add := func(name string) extensions.Extension { return extensions.Extension{ - Modify: func(_ extensions.RenderContext, objs []client.Object) []client.Object { - return append(objs, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: name}}) + Modify: func(_ extensions.RenderContext, objs, del []client.Object) ([]client.Object, []client.Object) { + return append(objs, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: name}}), del }, } } extensions.Register(operatorv1.CalicoEnterprise, "test", add("first")) extensions.Register(operatorv1.CalicoEnterprise, "test", add("second")) - out := extensions.ApplyModifiers("test", entCtx, nil) + out, _ := extensions.ApplyModifiers("test", entCtx, nil, nil) Expect(out).To(HaveLen(1)) Expect(out[0].GetName()).To(Equal("second")) }) diff --git a/pkg/render/apiserver.go b/pkg/render/apiserver.go index aed69c6c5f..9702226fd3 100644 --- a/pkg/render/apiserver.go +++ b/pkg/render/apiserver.go @@ -40,7 +40,6 @@ import ( "github.com/tigera/operator/pkg/controller/k8sapi" "github.com/tigera/operator/pkg/render/common/authentication" rcomp "github.com/tigera/operator/pkg/render/common/components" - relasticsearch "github.com/tigera/operator/pkg/render/common/elasticsearch" rmeta "github.com/tigera/operator/pkg/render/common/meta" "github.com/tigera/operator/pkg/render/common/networkpolicy" "github.com/tigera/operator/pkg/render/common/podaffinity" @@ -57,8 +56,9 @@ const ( APIServerPortName = "apiserver" APIServerPolicyName = networkpolicy.CalicoComponentPolicyPrefix + "apiserver-access" - auditLogsVolumeName = "calico-audit-logs" - auditPolicyVolumeName = "calico-audit-policy" + // ComponentNameAPIServer is the extension key under which a variant registers + // its API server modifier and image override. + ComponentNameAPIServer = "apiserver" ) const ( @@ -147,6 +147,12 @@ type APIServerConfiguration struct { // as part of this component. RequiresAggregationServer bool + // Whether or not the API server deployment must run a query server alongside the API + // server. The deployment (and its supporting objects) are rendered when either an + // aggregation API server or a query server is required. The query server itself, and + // the rest of its supporting configuration, is layered on by the variant's modifier. + RequiresQueryServer bool + // When certificate management is enabled, we need a separate init container to create a cert, running // with the same permissions as query server. QueryServerTLSKeyPairCertificateManagementOnly certificatemanagement.KeyPairInterface @@ -159,6 +165,25 @@ type apiServerComponent struct { dikastesImage string } +// APIServerExtensionContext carries the API server's render configuration and resolved +// image to a variant modifier. The modifier uses these to build variant-specific objects +// and to layer additional containers, volumes, and configuration onto the rendered +// deployment. +type APIServerExtensionContext struct { + Config *APIServerConfiguration + CalicoImage string +} + +// ModifierKey implements render.Extensible: the API server's variant-specific objects are +// applied by the modifier registered under this key. +func (c *apiServerComponent) ModifierKey() string { return ComponentNameAPIServer } + +// ExtensionContext implements render.ExtensionContextProvider, handing the modifier the +// config and resolved image it needs. +func (c *apiServerComponent) ExtensionContext() any { + return APIServerExtensionContext{Config: c.cfg, CalicoImage: c.calicoImage} +} + func (c *apiServerComponent) ResolveImages(is *operatorv1.ImageSet) error { reg := c.cfg.Installation.Registry path := c.cfg.Installation.ImagePath @@ -166,15 +191,14 @@ func (c *apiServerComponent) ResolveImages(is *operatorv1.ImageSet) error { var err error errMsgs := []string{} - enterprise := c.cfg.Installation.Variant.IsEnterprise() - if enterprise || c.cfg.RequiresAggregationServer { + if c.cfg.RequiresAggregationServer || c.cfg.RequiresQueryServer { c.calicoImage, err = components.GetReference(components.CombinedCalicoImage(c.cfg.Installation), reg, path, prefix, is) if err != nil { errMsgs = append(errMsgs, err.Error()) } } - if enterprise && c.cfg.IsSidecarInjectionEnabled() { + if c.cfg.IsSidecarInjectionEnabled() { c.l7AdmissionControllerEnvoyImage, err = components.GetReference(components.ComponentEnvoyProxy, reg, path, prefix, is) if err != nil { errMsgs = append(errMsgs, err.Error()) @@ -196,8 +220,8 @@ func (c *apiServerComponent) SupportedOSType() rmeta.OSType { } func (c *apiServerComponent) Objects() ([]client.Object, []client.Object) { - // Start with all of the cluster-scoped resources that are used for both Calico and Calico Enterprise. - // When switching between Calico / Enterprise, these objects are simply updated in-place. + // Cluster-scoped resources used by the API server, independent of variant. Any + // variant-specific objects are layered on by the variant's modifier. globalObjects := []client.Object{ c.calicoCustomResourcesClusterRole(), c.calicoCustomResourcesClusterRoleBinding(), @@ -209,9 +233,6 @@ func (c *apiServerComponent) Objects() ([]client.Object, []client.Object) { } objsToDelete := []client.Object{} - - // Namespaced objects common to both Calico and Calico Enterprise. - // These objects will be updated when switching between the variants. namespacedObjects := []client.Object{} // Add in image pull secrets. @@ -219,8 +240,8 @@ func (c *apiServerComponent) Objects() ([]client.Object, []client.Object) { namespacedObjects = append(namespacedObjects, secret.ToRuntimeObjects(secrets...)...) // The deployment and its supporting objects are needed when running the aggregation API server - // or when running Enterprise (which always needs the queryserver). - if c.cfg.RequiresAggregationServer || c.cfg.Installation.Variant.IsEnterprise() { + // or when a query server runs alongside it (the query server is added by a variant modifier). + if c.cfg.RequiresAggregationServer || c.cfg.RequiresQueryServer { namespacedObjects = append(namespacedObjects, c.apiServerServiceAccount(), c.apiServerDeployment(), @@ -246,16 +267,6 @@ func (c *apiServerComponent) Objects() ([]client.Object, []client.Object) { c.authReaderRoleBinding(), } - if c.cfg.Installation.Variant.IsEnterprise() { - aggregationAPIServerObjects = append(aggregationAPIServerObjects, - c.uiSettingsGroupGetterClusterRole(), - c.kubeControllerManagerUISettingsGroupGetterClusterRoleBinding(), - c.uiSettingsPassthruClusterRole(), - c.uiSettingsPassthruClusterRolebinding(), - c.auditPolicyConfigMap(), - ) - } - // Add in certificates for API server TLS. if !c.cfg.TLSKeyPair.UseCertificateManagement() { aggregationAPIServerObjects = append(aggregationAPIServerObjects, c.apiServiceRegistration(c.cfg.TLSKeyPair.GetCertificatePEM())) @@ -263,73 +274,12 @@ func (c *apiServerComponent) Objects() ([]client.Object, []client.Object) { aggregationAPIServerObjects = append(aggregationAPIServerObjects, c.apiServiceRegistration(c.cfg.Installation.CertificateManagement.CACert)) } - // Global enterprise-only objects. - globalEnterpriseObjects := []client.Object{ - c.tigeraAPIServerClusterRole(), - c.tigeraAPIServerClusterRoleBinding(), - } - - if !c.cfg.MultiTenant { - // These resources are only installed in zero-tenant clusters. Multi-tenant clusters don't use the default - // RBAC resources. - globalEnterpriseObjects = append(globalEnterpriseObjects, - c.tigeraUserClusterRole(), - c.tigeraNetworkAdminClusterRole(), - ) - } - - if c.cfg.ManagementCluster != nil { - globalEnterpriseObjects = append(globalEnterpriseObjects, c.managedClusterWatchClusterRole()) - if c.cfg.MultiTenant { - // Multi-tenant management cluster API servers need access to per-tenant CA secrets in order to sign - // per-tenant guardian certificates when creating ManagedClusters. - globalEnterpriseObjects = append(globalEnterpriseObjects, c.multiTenantSecretsRBAC()...) - // Multi-tenant management cluster components impersonate the single-tenant canonical service account - // in order to retrieve informations from the managed cluster. A cluster role will be created and each - // component will create a role binding in the tenant namespace - globalEnterpriseObjects = append(globalEnterpriseObjects, c.multiTenantManagedClusterAccessClusterRoles()...) - } else { - globalEnterpriseObjects = append(globalEnterpriseObjects, c.secretsRBAC()...) - } - } else { - // If we're not a management cluster, the API server doesn't need permissions to access secrets. - objsToDelete = append(objsToDelete, c.multiTenantSecretsRBAC()...) - objsToDelete = append(objsToDelete, c.secretsRBAC()...) - objsToDelete = append(objsToDelete, c.multiTenantManagedClusterAccessClusterRoles()...) - objsToDelete = append(objsToDelete, c.managedClusterWatchClusterRole()) - } - - // Namespaced enterprise-only objects. - namespacedEnterpriseObjects := []client.Object{} - - if c.cfg.TrustedBundle != nil { - namespacedEnterpriseObjects = append(namespacedEnterpriseObjects, c.cfg.TrustedBundle.ConfigMap(QueryserverNamespace)) - } + // The sidecar mutating webhook is driven by ApplicationLayer configuration, not by variant. if c.cfg.IsSidecarInjectionEnabled() { - namespacedEnterpriseObjects = append(namespacedEnterpriseObjects, c.sidecarMutatingWebhookConfig()) + namespacedObjects = append(namespacedObjects, c.sidecarMutatingWebhookConfig()) } else { objsToDelete = append(objsToDelete, &admregv1.MutatingWebhookConfiguration{ObjectMeta: metav1.ObjectMeta{Name: common.SidecarMutatingWebhookConfigName}}) } - if c.cfg.ManagementClusterConnection != nil { - namespacedEnterpriseObjects = append(namespacedEnterpriseObjects, - c.externalLinseedRoleBinding(), - ) - } - - // Compile the final arrays based on the variant. - if c.cfg.Installation.Variant.IsEnterprise() { - // Create any enterprise specific objects. - globalObjects = append(globalObjects, globalEnterpriseObjects...) - namespacedObjects = append(namespacedObjects, namespacedEnterpriseObjects...) - - // Clean up cluster-scoped resources that were created with the 'tigera' prefix. - // The apiserver now uses consistent resource names with 'calico' prefix across both EE and OSS variants. - objsToDelete = append(objsToDelete, c.deprecatedResources()...) - } else { - // Explicitly delete any global enterprise objects. - // Namespaced objects will be handled by namespace deletion. - objsToDelete = append(objsToDelete, globalEnterpriseObjects...) - } // Clean up deprecated k8s NetworkPolicy, regardless of variant, // avoiding leftovers in the case of switching between variants. @@ -523,9 +473,9 @@ func calicoSystemAPIServerPolicy(cfg *APIServerConfiguration) *v3.NetworkPolicy Action: v3.Pass, }) - apiServerContainerPort := getContainerPort(cfg, APIServerContainerName).ContainerPort - queryServerContainerPort := getContainerPort(cfg, TigeraAPIServerQueryServerContainerName).ContainerPort - l7AdmCtrlContainerPort := getContainerPort(cfg, L7AdmissionControllerContainerName).ContainerPort + apiServerContainerPort := GetContainerPort(cfg, APIServerContainerName).ContainerPort + queryServerContainerPort := GetContainerPort(cfg, TigeraAPIServerQueryServerContainerName).ContainerPort + l7AdmCtrlContainerPort := GetContainerPort(cfg, L7AdmissionControllerContainerName).ContainerPort // The ports Calico Enterprise API Server and Calico Enterprise Query Server are configured to listen on. ingressPorts := networkpolicy.Ports(443, uint16(apiServerContainerPort), uint16(queryServerContainerPort), 10443) @@ -768,17 +718,6 @@ func (c *apiServerComponent) authClusterRole() client.Object { } } -// multiTenantSecretsRBAC provides the tigera API server with the ability to read secrets on the cluster. -// This is needed in multi-tenant management clusters only, in order to read tenant secrets for signing managed cluster certificates. -func (c *apiServerComponent) multiTenantSecretsRBAC() []client.Object { - return TunnelSecretRBAC(APIServerSecretsRBACName, APIServerServiceAccountName, c.cfg.ManagementCluster, true) -} - -// secretsRBAC provides the tigera API server with the ability to read secrets from the API server's namespace. -func (c *apiServerComponent) secretsRBAC() []client.Object { - return TunnelSecretRBAC(APIServerSecretsRBACName, APIServerServiceAccountName, c.cfg.ManagementCluster, false) -} - // authClusterRoleBinding returns a clusterrolebinding to create, and a clusterrolebinding to delete. // // Both Calico and Calico Enterprise, with different names. @@ -854,7 +793,7 @@ func (c *apiServerComponent) webhookReaderClusterRoleBinding() client.Object { } } -func getContainerPort(cfg *APIServerConfiguration, containerName ContainerName) *operatorv1.APIServerDeploymentContainerPort { +func GetContainerPort(cfg *APIServerConfiguration, containerName ContainerName) *operatorv1.APIServerDeploymentContainerPort { // Try to get the override port if cfg != nil && cfg.APIServer != nil && @@ -889,11 +828,11 @@ func getContainerPort(cfg *APIServerConfiguration, containerName ContainerName) return nil } -// apiServerService creates a service backed by the API server and - for enterprise - query server. +// apiServerService creates a service backed by the API server. A variant modifier may add +// additional ports (e.g. the query server port). func (c *apiServerComponent) apiServerService() *corev1.Service { - apiServerTargetPort := getContainerPort(c.cfg, APIServerContainerName) - queryServerTargetPort := getContainerPort(c.cfg, TigeraAPIServerQueryServerContainerName) - l7AdmissionControllerTargetPort := getContainerPort(c.cfg, L7AdmissionControllerContainerName) + apiServerTargetPort := GetContainerPort(c.cfg, APIServerContainerName) + l7AdmissionControllerTargetPort := GetContainerPort(c.cfg, L7AdmissionControllerContainerName) s := &corev1.Service{ TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}, @@ -917,18 +856,6 @@ func (c *apiServerComponent) apiServerService() *corev1.Service { }, } - if c.cfg.Installation.Variant.IsEnterprise() { - // Add port for queryserver if enterprise. - s.Spec.Ports = append(s.Spec.Ports, - corev1.ServicePort{ - Name: QueryServerPortName, - Port: QueryServerPort, - Protocol: corev1.ProtocolTCP, - TargetPort: intstr.FromInt32(queryServerTargetPort.ContainerPort), - }, - ) - } - if c.cfg.IsSidecarInjectionEnabled() { s.Spec.Ports = append(s.Spec.Ports, corev1.ServicePort{ @@ -967,12 +894,13 @@ func (c *apiServerComponent) apiServerDeployment() *appsv1.Deployment { initContainers = append(initContainers, initContainerAPIServer) } - initContainerQueryServer := c.cfg.QueryServerTLSKeyPairCertificateManagementOnly.InitContainer(APIServerNamespace, c.queryServerContainer().SecurityContext) + initContainerQueryServer := c.cfg.QueryServerTLSKeyPairCertificateManagementOnly.InitContainer(APIServerNamespace, securitycontext.NewNonRootContext()) annotations[c.cfg.QueryServerTLSKeyPairCertificateManagementOnly.HashAnnotationKey()] = c.cfg.QueryServerTLSKeyPairCertificateManagementOnly.HashAnnotationValue() initContainers = append(initContainers, initContainerQueryServer) } - // Determine which containers to run. + // Determine which containers to run. A variant modifier may add additional + // containers (e.g. the query server). containers := []corev1.Container{} if c.cfg.RequiresAggregationServer { containers = append(containers, c.apiServerContainer()) @@ -980,9 +908,6 @@ func (c *apiServerComponent) apiServerDeployment() *appsv1.Deployment { if c.cfg.IsSidecarInjectionEnabled() { containers = append(containers, c.l7AdmissionControllerContainer()) } - if c.cfg.Installation.Variant.IsEnterprise() { - containers = append(containers, c.queryServerContainer()) - } d := &appsv1.Deployment{ TypeMeta: metav1.TypeMeta{Kind: "Deployment", APIVersion: "apps/v1"}, @@ -1027,15 +952,6 @@ func (c *apiServerComponent) apiServerDeployment() *appsv1.Deployment { d.Spec.Template.Spec.Affinity = podaffinity.NewPodAntiAffinity(APIServerName, []string{APIServerNamespace, "tigera-system", "calico-apiserver"}) } - if c.cfg.Installation.Variant.IsEnterprise() { - if c.cfg.TrustedBundle != nil { - trustedBundleHashAnnotations := c.cfg.TrustedBundle.HashAnnotations() - for k, v := range trustedBundleHashAnnotations { - d.Spec.Template.Annotations[k] = v - } - } - } - if overrides := c.cfg.APIServer.APIServerDeployment; overrides != nil { rcomp.ApplyDeploymentOverrides(d, overrides) } @@ -1046,7 +962,7 @@ func (c *apiServerComponent) apiServerDeployment() *appsv1.Deployment { // apiServer creates a MutatingWebhookConfiguration for sidecars. func (c *apiServerComponent) sidecarMutatingWebhookConfig() *admregv1.MutatingWebhookConfiguration { var cacert []byte - svcPort := getContainerPort(c.cfg, L7AdmissionControllerContainerName).ContainerPort + svcPort := GetContainerPort(c.cfg, L7AdmissionControllerContainerName).ContainerPort svcpath := "/sidecar-webhook" svcref := admregv1.ServiceReference{ @@ -1103,10 +1019,16 @@ func (c *apiServerComponent) sidecarMutatingWebhookConfig() *admregv1.MutatingWe } func (c *apiServerComponent) hostNetwork() bool { - if c.cfg.ForceHostNetwork { + return HostNetwork(c.cfg) +} + +// HostNetwork reports whether the API server deployment runs on the host network, +// accounting for both the forced setting and the provider-driven requirement. +func HostNetwork(cfg *APIServerConfiguration) bool { + if cfg.ForceHostNetwork { return true } - return HostNetworkRequired(c.cfg.Installation) + return HostNetworkRequired(cfg.Installation) } func HostNetworkRequired(installation *operatorv1.InstallationSpec) bool { @@ -1126,12 +1048,6 @@ func (c *apiServerComponent) apiServerContainer() corev1.Container { volumeMounts := []corev1.VolumeMount{ c.cfg.TLSKeyPair.VolumeMount(c.SupportedOSType()), } - if c.cfg.Installation.Variant.IsEnterprise() { - volumeMounts = append(volumeMounts, - corev1.VolumeMount{Name: auditLogsVolumeName, MountPath: "/var/log/calico/audit"}, - corev1.VolumeMount{Name: auditPolicyVolumeName, MountPath: "/etc/tigera/audit"}, - ) - } env := []corev1.EnvVar{ {Name: "DATASTORE_TYPE", Value: "kubernetes"}, @@ -1161,7 +1077,7 @@ func (c *apiServerComponent) apiServerContainer() corev1.Container { env = append(env, corev1.EnvVar{Name: "MULTI_INTERFACE_MODE", Value: c.cfg.Installation.CalicoNetwork.MultiInterfaceMode.Value()}) } - apiServerTargetPort := getContainerPort(c.cfg, APIServerContainerName).ContainerPort + apiServerTargetPort := GetContainerPort(c.cfg, APIServerContainerName).ContainerPort apiServer := corev1.Container{ Name: string(APIServerContainerName), @@ -1183,19 +1099,13 @@ func (c *apiServerComponent) apiServerContainer() corev1.Container { PeriodSeconds: 60, }, } - // In case of OpenShift, apiserver needs privileged access to write audit logs to host path volume. - // Audit logs are owned by root on hosts so we need to be root user and group. Audit logs are supported only in Enterprise version. - if c.cfg.Installation.Variant.IsEnterprise() { - apiServer.SecurityContext = securitycontext.NewRootContext(c.cfg.OpenShift) - } else { - apiServer.SecurityContext = securitycontext.NewNonRootContext() - } + apiServer.SecurityContext = securitycontext.NewNonRootContext() return apiServer } func (c *apiServerComponent) startUpArgs() []string { - apiServerTargetPort := getContainerPort(c.cfg, APIServerContainerName).ContainerPort + apiServerTargetPort := GetContainerPort(c.cfg, APIServerContainerName).ContainerPort args := []string{ fmt.Sprintf("--secure-port=%d", apiServerTargetPort), @@ -1203,13 +1113,6 @@ func (c *apiServerComponent) startUpArgs() []string { fmt.Sprintf("--tls-cert-file=%s", c.cfg.TLSKeyPair.VolumeMountCertificateFilePath()), } - if c.cfg.Installation.Variant.IsEnterprise() { - args = append(args, - "--audit-policy-file=/etc/tigera/audit/policy.conf", - "--audit-log-path=/var/log/calico/audit/tsee-audit.log", - ) - } - if c.cfg.ManagementCluster != nil { args = append(args, "--enable-managed-clusters-create-api=true") if c.cfg.ManagementCluster.Spec.Address != "" { @@ -1230,122 +1133,6 @@ func (c *apiServerComponent) startUpArgs() []string { return args } -// queryServerContainer creates the query server container. -func (c *apiServerComponent) queryServerContainer() corev1.Container { - queryServerTargetPort := getContainerPort(c.cfg, TigeraAPIServerQueryServerContainerName).ContainerPort - - var tlsSecret certificatemanagement.KeyPairInterface - if c.cfg.QueryServerTLSKeyPairCertificateManagementOnly != nil { - tlsSecret = c.cfg.QueryServerTLSKeyPairCertificateManagementOnly - } else { - tlsSecret = c.cfg.TLSKeyPair - } - env := []corev1.EnvVar{ - {Name: "DATASTORE_TYPE", Value: "kubernetes"}, - {Name: "LISTEN_ADDR", Value: fmt.Sprintf(":%d", queryServerTargetPort)}, - {Name: "TLS_CERT", Value: fmt.Sprintf("/%s/tls.crt", tlsSecret.GetName())}, - {Name: "TLS_KEY", Value: fmt.Sprintf("/%s/tls.key", tlsSecret.GetName())}, - } - if c.cfg.TrustedBundle != nil { - env = append(env, corev1.EnvVar{Name: "TRUSTED_BUNDLE_PATH", Value: c.cfg.TrustedBundle.MountPath()}) - } - - if c.hostNetwork() { - env = append(env, c.cfg.K8SServiceEndpoint.EnvVars()...) - } else { - env = append(env, c.cfg.K8SServiceEndpointPodNetwork.EnvVars()...) - } - - if c.cfg.Installation.CalicoNetwork != nil && c.cfg.Installation.CalicoNetwork.MultiInterfaceMode != nil { - env = append(env, corev1.EnvVar{Name: "MULTI_INTERFACE_MODE", Value: c.cfg.Installation.CalicoNetwork.MultiInterfaceMode.Value()}) - } - - if c.cfg.KeyValidatorConfig != nil { - env = append(env, c.cfg.KeyValidatorConfig.RequiredEnv("")...) - } - - linseedURL := relasticsearch.LinseedEndpoint(c.SupportedOSType(), c.cfg.ClusterDomain, ElasticsearchNamespace, c.cfg.ManagementClusterConnection != nil, false) - env = append(env, - corev1.EnvVar{Name: "LINSEED_URL", Value: linseedURL}, - corev1.EnvVar{Name: "LINSEED_CLIENT_CERT", Value: fmt.Sprintf("/%s/tls.crt", tlsSecret.GetName())}, - corev1.EnvVar{Name: "LINSEED_CLIENT_KEY", Value: fmt.Sprintf("/%s/tls.key", tlsSecret.GetName())}, - ) - if c.cfg.ManagementClusterConnection != nil { - env = append(env, - corev1.EnvVar{Name: "CLUSTER_ID", Value: ""}, - corev1.EnvVar{Name: "LINSEED_TOKEN", Value: GetLinseedTokenPath(true)}, - ) - } - if c.cfg.TrustedBundle != nil { - env = append(env, corev1.EnvVar{Name: "LINSEED_CA", Value: c.cfg.TrustedBundle.MountPath()}) - } - - // set LogLEVEL for queryserver container - if logging := c.cfg.APIServer.Logging; logging != nil && - logging.QueryServerLogging != nil && logging.QueryServerLogging.LogSeverity != nil { - env = append(env, - corev1.EnvVar{Name: "LOGLEVEL", Value: strings.ToLower(string(*logging.QueryServerLogging.LogSeverity))}) - } else { - // set default LOGLEVEL to info when not set by the user - env = append(env, corev1.EnvVar{Name: "LOGLEVEL", Value: "info"}) - } - - volumeMounts := []corev1.VolumeMount{ - tlsSecret.VolumeMount(c.SupportedOSType()), - } - if c.cfg.TrustedBundle != nil { - volumeMounts = append(volumeMounts, c.cfg.TrustedBundle.VolumeMounts(c.SupportedOSType())...) - } - if c.cfg.ManagementClusterConnection != nil { - volumeMounts = append(volumeMounts, corev1.VolumeMount{ - Name: LinseedTokenVolumeName, - MountPath: LinseedVolumeMountPath, - }) - } - - container := corev1.Container{ - Name: string(TigeraAPIServerQueryServerContainerName), - Image: c.calicoImage, - Command: []string{components.CalicoBinaryPath, "component", "queryserver"}, - Env: env, - LivenessProbe: &corev1.Probe{ - ProbeHandler: corev1.ProbeHandler{ - HTTPGet: &corev1.HTTPGetAction{ - Path: "/version", - Port: intstr.FromInt32(queryServerTargetPort), - Scheme: corev1.URISchemeHTTPS, - }, - }, - InitialDelaySeconds: 90, - }, - SecurityContext: securitycontext.NewNonRootContext(), - VolumeMounts: volumeMounts, - } - return container -} - -func (c *apiServerComponent) externalLinseedRoleBinding() *rbacv1.RoleBinding { - return &rbacv1.RoleBinding{ - TypeMeta: metav1.TypeMeta{Kind: "RoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{ - Name: "tigera-linseed", - Namespace: APIServerNamespace, - }, - RoleRef: rbacv1.RoleRef{ - APIGroup: "rbac.authorization.k8s.io", - Kind: "ClusterRole", - Name: TigeraLinseedSecretsClusterRole, - }, - Subjects: []rbacv1.Subject{ - { - Kind: "ServiceAccount", - Name: GuardianServiceAccountName, - Namespace: GuardianNamespace, - }, - }, - } -} - // apiServerVolumes creates the volumes used by the API server deployment. func (c *apiServerComponent) apiServerVolumes() []corev1.Volume { volumes := []corev1.Volume{ @@ -1355,40 +1142,6 @@ func (c *apiServerComponent) apiServerVolumes() []corev1.Volume { volumes = append(volumes, c.cfg.QueryServerTLSKeyPairCertificateManagementOnly.Volume()) } - if c.cfg.Installation.Variant.IsEnterprise() && c.cfg.RequiresAggregationServer { - // Only include these volumes if we're running the aggregation API server, since audit logging is done through the - // main API server otherwise. - volumes = append(volumes, - corev1.Volume{ - Name: auditLogsVolumeName, - VolumeSource: corev1.VolumeSource{ - HostPath: &corev1.HostPathVolumeSource{ - Path: "/var/log/calico/audit", - Type: ptr.To(corev1.HostPathDirectoryOrCreate), - }, - }, - }, - corev1.Volume{ - Name: auditPolicyVolumeName, - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{Name: auditPolicyVolumeName}, - Items: []corev1.KeyToPath{ - { - Key: "config", - Path: "policy.conf", - }, - }, - }, - }, - }, - ) - } - - if c.cfg.Installation.Variant.IsEnterprise() && c.cfg.TrustedBundle != nil { - volumes = append(volumes, c.cfg.TrustedBundle.Volume()) - } - if c.cfg.ManagementClusterConnection != nil { // Optional: the Secret is delivered over the Guardian tunnel, which can't be // established until calico-apiserver is Ready. @@ -1419,119 +1172,6 @@ func (c *apiServerComponent) tolerations() []corev1.Toleration { return tolerations } -// tigeraAPIServerClusterRole creates a clusterrole that gives permissions to access backing CRDs -// -// Calico Enterprise only -func (c *apiServerComponent) tigeraAPIServerClusterRole() *rbacv1.ClusterRole { - rules := []rbacv1.PolicyRule{ - { - // Read access to Linseed policy activity data for queryserver enrichment. - APIGroups: []string{"linseed.tigera.io"}, - Resources: []string{"policyactivity"}, - Verbs: []string{"get"}, - }, - { - // Calico Enterprise backing storage. - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{ - "alertexceptions", - "bfdconfigurations", - "deeppacketinspections", - "deeppacketinspections/status", - "egressgatewaypolicies", - "externalnetworks", - "globalalerts", - "globalalerts/status", - "globalalerttemplates", - "globalreports", - "globalreports/status", - "globalreporttypes", - "globalthreatfeeds", - "globalthreatfeeds/status", - "licensekeys", - "managedclusters", - "managedclusters/status", - "networks", - "packetcaptures", - "packetcaptures/status", - "policyrecommendationscopes", - "policyrecommendationscopes/status", - "remoteclusterconfigurations", - "securityeventwebhooks", - "securityeventwebhooks/status", - "uisettings", - "uisettingsgroups", - }, - Verbs: []string{ - "get", - "list", - "watch", - "create", - "update", - "delete", - "patch", - }, - }, - { - // The queryserver's RBAC calculator needs to list tiers, - // uisettingsgroups, and managedclusters via the aggregated - // API to evaluate user permissions for the /policies endpoint. - APIGroups: []string{"projectcalico.org"}, - Resources: []string{ - "tiers", - "uisettingsgroups", - "managedclusters", - }, - Verbs: []string{"get", "list", "watch"}, - }, - { - // Required by the AuthorizationReview calculator in queryserver to evaluate - // RBAC permissions for users. - APIGroups: []string{"rbac.authorization.k8s.io"}, - Resources: []string{ - "clusterroles", - "clusterrolebindings", - "roles", - "rolebindings", - }, - Verbs: []string{"get", "list", "watch"}, - }, - } - - return &rbacv1.ClusterRole{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{ - Name: APIServerName, - }, - Rules: rules, - } -} - -// tigeraAPIServerClusterRoleBinding creates a clusterrolebinding that applies tigeraAPIServerClusterRole to -// the calico-apiserver service account. -// -// Calico Enterprise only -func (c *apiServerComponent) tigeraAPIServerClusterRoleBinding() *rbacv1.ClusterRoleBinding { - return &rbacv1.ClusterRoleBinding{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{ - Name: APIServerName, - }, - Subjects: []rbacv1.Subject{ - { - Kind: "ServiceAccount", - Name: APIServerServiceAccountName, - Namespace: APIServerNamespace, - }, - }, - RoleRef: rbacv1.RoleRef{ - Kind: "ClusterRole", - Name: APIServerName, - APIGroup: "rbac.authorization.k8s.io", - }, - } -} - // tierGetterClusterRole creates a clusterrole that gives permissions to get tiers. func (c *apiServerComponent) tierGetterClusterRole() *rbacv1.ClusterRole { return &rbacv1.ClusterRole{ @@ -1575,699 +1215,86 @@ func (c *apiServerComponent) kubeControllerMgrTierGetterClusterRoleBinding() *rb } } -// uiSettingsGroupGetterClusterRole creates a clusterrole that gives permissions to get uisettingsgroups. -// -// Calico Enterprise only -func (c *apiServerComponent) uiSettingsGroupGetterClusterRole() *rbacv1.ClusterRole { +// calicoPolicyPassthruClusterRole creates a clusterrole that is used to control the RBAC +// mechanism for Calico tiered policy. +func (c *apiServerComponent) calicoPolicyPassthruClusterRole() *rbacv1.ClusterRole { + resources := []string{"networkpolicies", "globalnetworkpolicies"} + return &rbacv1.ClusterRole{ TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, ObjectMeta: metav1.ObjectMeta{ - Name: "calico-uisettingsgroup-getter", + Name: "calico-tiered-policy-passthrough", }, + // If tiered policy is enabled we allow all authenticated users to access the main tier resource, instead + // restricting access using the tier.xxx resource type. Kubernetes NetworkPolicy and + // StagedKubernetesNetworkPolicy objects are handled using normal (non-tiered) RBAC. Rules: []rbacv1.PolicyRule{ { APIGroups: []string{"projectcalico.org"}, - Resources: []string{ - "uisettingsgroups", - }, - Verbs: []string{"get"}, + Resources: resources, + Verbs: allVerbs, }, }, } } -// kubeControllerManagerUISettingsGroupGetterClusterRoleBinding creates a rolebinding that allows the k8s kube-controller -// manager to get uisettingsgroups. -// -// In k8s 1.15+, cascading resource deletions (for instance pods for a replicaset) failed due to k8s kube-controller -// not having permissions to get tiers. UISettings and UISettingsGroups RBAC works in a similar way to tiered policy -// and so we need similar RBAC for UISettingsGroups. -// -// Calico Enterprise only -func (c *apiServerComponent) kubeControllerManagerUISettingsGroupGetterClusterRoleBinding() *rbacv1.ClusterRoleBinding { +// calicoPolicyPassthruClusterRolebinding creates a clusterrolebinding that applies calicoPolicyPassthruClusterRole to all users. +func (c *apiServerComponent) calicoPolicyPassthruClusterRolebinding() *rbacv1.ClusterRoleBinding { return &rbacv1.ClusterRoleBinding{ TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, ObjectMeta: metav1.ObjectMeta{ - Name: "calico-uisettingsgroup-getter", - }, - RoleRef: rbacv1.RoleRef{ - Kind: "ClusterRole", - Name: "calico-uisettingsgroup-getter", - APIGroup: "rbac.authorization.k8s.io", + Name: "calico-tiered-policy-passthrough", }, Subjects: []rbacv1.Subject{ { - Kind: "User", - Name: "system:kube-controller-manager", + Kind: "Group", + Name: "system:authenticated", APIGroup: "rbac.authorization.k8s.io", }, }, + RoleRef: rbacv1.RoleRef{ + Kind: "ClusterRole", + Name: "calico-tiered-policy-passthrough", + APIGroup: "rbac.authorization.k8s.io", + }, } } -// tigeraUserClusterRole returns a cluster role for a default Calico Enterprise user. -// -// Calico Enterprise only -func (c *apiServerComponent) tigeraUserClusterRole() *rbacv1.ClusterRole { - rules := []rbacv1.PolicyRule{ - // List requests that the Tigera manager needs. - { - APIGroups: []string{ - "projectcalico.org", - "networking.k8s.io", - "extensions", - "", - }, - // Use both the networkpolicies and tier.networkpolicies resource types to ensure identical behavior - // irrespective of the Calico RBAC scheme (see the ClusterRole "calico-tiered-policy-passthrough" for - // more details). Similar for all tiered policy resource types. - Resources: []string{ - "tiers", - "networkpolicies", - "tier.networkpolicies", - "globalnetworkpolicies", - "tier.globalnetworkpolicies", - "namespaces", - "globalnetworksets", - "networksets", - "managedclusters", - "stagedglobalnetworkpolicies", - "tier.stagedglobalnetworkpolicies", - "stagednetworkpolicies", - "tier.stagednetworkpolicies", - "stagedkubernetesnetworkpolicies", - "policyrecommendationscopes", - }, - Verbs: []string{"watch", "list"}, +func (c *apiServerComponent) getDeprecatedResources() []client.Object { + var renamedRscList []client.Object + + // renamed clusterrole tigera-crds to tigera-apiserver + renamedRscList = append(renamedRscList, &rbacv1.ClusterRole{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "tigera-crds", }, - { - APIGroups: []string{"policy.networking.k8s.io"}, - Resources: []string{ - "clusternetworkpolicies", - "adminnetworkpolicies", - "baselineadminnetworkpolicies", - }, - Verbs: []string{"watch", "list"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"packetcaptures/files"}, - Verbs: []string{"get"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"packetcaptures"}, - Verbs: []string{"get", "list", "watch"}, - }, - // Additional "list" requests required to view flows. - { - APIGroups: []string{""}, - Resources: []string{"pods"}, - Verbs: []string{"list"}, - }, - // Additional "list" requests required to view serviceaccount labels. - { - APIGroups: []string{""}, - Resources: []string{"serviceaccounts"}, - Verbs: []string{"list"}, - }, - // Access for WAF API to read in coreruleset configmap - { - APIGroups: []string{""}, - Resources: []string{"configmaps"}, - ResourceNames: []string{"coreruleset-default"}, - Verbs: []string{"get"}, - }, - // Access to statistics. - { - APIGroups: []string{""}, - Resources: []string{"services/proxy"}, - ResourceNames: []string{ - "https:calico-api:8080", "calico-node-prometheus:9090", - }, - Verbs: []string{"get", "create"}, - }, - // Access to policies in all tiers - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"tiers"}, - Verbs: []string{"get"}, - }, - // List and download the reports in the Tigera Secure manager. - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"globalreports"}, - Verbs: []string{"get", "list"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"globalreporttypes"}, - Verbs: []string{"get"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"clusterinformations"}, - Verbs: []string{"get", "list"}, - }, - // Access to hostendpoints from the UI ServiceGraph. - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"hostendpoints"}, - Verbs: []string{"get", "list"}, - }, - // List and view the threat defense configuration - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{ - "alertexceptions", - "globalalerts", - "globalalerts/status", - "globalalerttemplates", - "globalthreatfeeds", - "globalthreatfeeds/status", - "securityeventwebhooks", - }, - Verbs: []string{"get", "watch", "list"}, - }, - // User can: - // - read UISettings in the cluster-settings group - // - read and write UISettings in the user-settings group - // Default settings group and settings are created in manager.go. - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"uisettingsgroups"}, - Verbs: []string{"get"}, - ResourceNames: []string{"cluster-settings", "user-settings"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"uisettingsgroups/data"}, - Verbs: []string{"get", "list", "watch"}, - ResourceNames: []string{"cluster-settings"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"uisettingsgroups/data"}, - Verbs: []string{"*"}, - ResourceNames: []string{"user-settings"}, - }, - // Allow the user to read applicationlayers to detect if WAF is enabled/disabled. - { - APIGroups: []string{"operator.tigera.io"}, - Resources: []string{"applicationlayers", "packetcaptureapis", "compliances", "intrusiondetections"}, - Verbs: []string{"get"}, - }, - { - APIGroups: []string{"apps"}, - Resources: []string{"deployments"}, - Verbs: []string{"get", "list", "watch"}, - }, - // Allow the user to read services to view WAF configuration. - { - APIGroups: []string{""}, - Resources: []string{"services"}, - Verbs: []string{"get", "list", "watch"}, - }, - // Allow the user to read felixconfigurations to detect if wireguard and/or other features are enabled. - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"felixconfigurations"}, - Verbs: []string{"get", "list"}, - }, - // Allow the user to only view securityeventwebhooks. - { - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{"securityeventwebhooks"}, - Verbs: []string{"get", "list"}, - }, - } - - // Privileges for lma.tigera.io have no effect on managed clusters. - if c.cfg.ManagementClusterConnection == nil { - // Access to flow logs, audit logs, and statistics. - // Access to log into Kibana for oidc users. - rules = append(rules, rbacv1.PolicyRule{ - APIGroups: []string{"lma.tigera.io"}, - Resources: []string{"*"}, - ResourceNames: []string{ - "flows", "audit*", "l7", "events", "dns", "waf", "kibana_login", "recommendations", - }, - Verbs: []string{"get"}, - }) - } - - return &rbacv1.ClusterRole{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{ - Name: "tigera-ui-user", - }, - Rules: rules, - } -} - -// tigeraNetworkAdminClusterRole returns a cluster role for a Tigera Secure manager network admin. -// -// Calico Enterprise only -func (c *apiServerComponent) tigeraNetworkAdminClusterRole() *rbacv1.ClusterRole { - rules := []rbacv1.PolicyRule{ - // Full access to all network policies - { - APIGroups: []string{ - "projectcalico.org", - "networking.k8s.io", - "extensions", - }, - // Use both the networkpolicies and tier.networkpolicies resource types to ensure identical behavior - // irrespective of the Calico RBAC scheme (see the ClusterRole "calico-tiered-policy-passthrough" for - // more details). Similar for all tiered policy resource types. - Resources: []string{ - "tiers", - "networkpolicies", - "tier.networkpolicies", - "globalnetworkpolicies", - "tier.globalnetworkpolicies", - "stagedglobalnetworkpolicies", - "tier.stagedglobalnetworkpolicies", - "stagednetworkpolicies", - "tier.stagednetworkpolicies", - "stagedkubernetesnetworkpolicies", - "globalnetworksets", - "networksets", - "managedclusters", - "packetcaptures", - "policyrecommendationscopes", - }, - Verbs: []string{"create", "update", "delete", "patch", "get", "watch", "list"}, - }, - { - APIGroups: []string{ - "policy.networking.k8s.io", - }, - Resources: []string{ - "clusternetworkpolicies", - "adminnetworkpolicies", - "baselineadminnetworkpolicies", - }, - Verbs: []string{"create", "update", "delete", "patch", "get", "watch", "list"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"packetcaptures/files"}, - Verbs: []string{"get", "delete"}, - }, - // Additional "list" requests that the Tigera Secure manager needs - { - APIGroups: []string{""}, - Resources: []string{"namespaces"}, - Verbs: []string{"watch", "list"}, - }, - // Additional "list" requests required to view flows. - { - APIGroups: []string{""}, - Resources: []string{"pods"}, - Verbs: []string{"list"}, - }, - // Additional "list" requests required to view serviceaccount labels. - { - APIGroups: []string{""}, - Resources: []string{"serviceaccounts"}, - Verbs: []string{"list"}, - }, - // Access for WAF API to read in coreruleset configmap - { - APIGroups: []string{""}, - Resources: []string{"configmaps"}, - ResourceNames: []string{"coreruleset-default"}, - Verbs: []string{"get"}, - }, - // Access to statistics. - { - APIGroups: []string{""}, - Resources: []string{"services/proxy"}, - ResourceNames: []string{ - "https:calico-api:8080", "calico-node-prometheus:9090", - }, - Verbs: []string{"get", "create"}, - }, - // Manage globalreport configuration, view report generation status, and list reports in the Tigera Secure manager. - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"globalreports"}, - Verbs: []string{"*"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"globalreports/status"}, - Verbs: []string{"get", "list", "watch"}, - }, - // List and download the reports in the Tigera Secure manager. - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"globalreporttypes"}, - Verbs: []string{"get"}, - }, - // Access to cluster information containing Calico and EE versions from the UI. - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"clusterinformations"}, - Verbs: []string{"get", "list"}, - }, - // Access to hostendpoints from the UI ServiceGraph. - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"hostendpoints"}, - Verbs: []string{"get", "list"}, - }, - // Manage the threat defense configuration - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{ - "alertexceptions", - "globalalerts", - "globalalerts/status", - "globalalerttemplates", - "globalthreatfeeds", - "globalthreatfeeds/status", - "securityeventwebhooks", - }, - Verbs: []string{"create", "update", "delete", "patch", "get", "watch", "list"}, - }, - // User can: - // - read and write UISettings in the cluster-settings group, and rename the group - // - read and write UISettings in the user-settings group, and rename the group - // Default settings group and settings are created in manager.go. - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"uisettingsgroups"}, - Verbs: []string{"get", "patch", "update"}, - ResourceNames: []string{"cluster-settings", "user-settings"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"uisettingsgroups/data"}, - Verbs: []string{"*"}, - ResourceNames: []string{"cluster-settings", "user-settings"}, - }, - // Allow the user to read and write applicationlayers to enable/disable WAF. - { - APIGroups: []string{"operator.tigera.io"}, - Resources: []string{"applicationlayers", "packetcaptureapis", "compliances", "intrusiondetections"}, - Verbs: []string{"get", "update", "patch", "create", "delete"}, - }, - // Allow the user to read deployments to view WAF configuration. - { - APIGroups: []string{"apps"}, - Resources: []string{"deployments"}, - Verbs: []string{"get", "list", "watch", "patch"}, - }, - { - APIGroups: []string{""}, - Resources: []string{"services"}, - Verbs: []string{"get", "list", "watch", "patch"}, - }, - // Allow the user to read felixconfigurations to detect if wireguard and/or other features are enabled. - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"felixconfigurations"}, - Verbs: []string{"get", "list"}, - }, - // Allow the user to perform CRUD operations on securityeventwebhooks. - { - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{"securityeventwebhooks"}, - Verbs: []string{"get", "list", "update", "patch", "create", "delete"}, - }, - // Allow the user to create secrets. - { - APIGroups: []string{""}, - Resources: []string{ - "secrets", - }, - Verbs: []string{"create"}, - }, - // Allow the user to patch webhooks-secret secret. - { - APIGroups: []string{""}, - Resources: []string{ - "secrets", - }, - ResourceNames: []string{ - "webhooks-secret", - }, - Verbs: []string{"patch"}, - }, - } - - // Privileges for lma.tigera.io have no effect on managed clusters. - if c.cfg.ManagementClusterConnection == nil { - // Access to flow logs, audit logs, and statistics. - // Elasticsearch superuser access once logged into Kibana. - rules = append(rules, rbacv1.PolicyRule{ - APIGroups: []string{"lma.tigera.io"}, - Resources: []string{"*"}, - ResourceNames: []string{ - "flows", "audit*", "l7", "events", "dns", "waf", "kibana_login", "elasticsearch_superuser", "recommendations", - }, - Verbs: []string{"get"}, - }) - } - - // In v3 CRD / webhooks mode there is no aggregated apiserver, and the - // calico-uisettings-passthrough ClusterRole that normally grants the broad - // uisettings permission isn't deployed. Grant write verbs here so the - // calico-webhooks UISettings handler (which narrows access via a SAR on - // uisettingsgroups/data) gets invoked instead of being short-circuited by - // kube-apiserver RBAC. - if !c.cfg.RequiresAggregationServer { - rules = append(rules, rbacv1.PolicyRule{ - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"uisettings"}, - Verbs: []string{"create", "update", "delete", "patch"}, - }) - } - - return &rbacv1.ClusterRole{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{ - Name: "tigera-network-admin", - }, - Rules: rules, - } -} - -// calicoPolicyPassthruClusterRole creates a clusterrole that is used to control the RBAC -// mechanism for Calico tiered policy. -func (c *apiServerComponent) calicoPolicyPassthruClusterRole() *rbacv1.ClusterRole { - resources := []string{"networkpolicies", "globalnetworkpolicies"} - - // Append additional resources for enterprise Variant. - if c.cfg.Installation.Variant.IsEnterprise() { - resources = append(resources, "stagednetworkpolicies", "stagedglobalnetworkpolicies") - } - - return &rbacv1.ClusterRole{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{ - Name: "calico-tiered-policy-passthrough", - }, - // If tiered policy is enabled we allow all authenticated users to access the main tier resource, instead - // restricting access using the tier.xxx resource type. Kubernetes NetworkPolicy and - // StagedKubernetesNetworkPolicy objects are handled using normal (non-tiered) RBAC. - Rules: []rbacv1.PolicyRule{ - { - APIGroups: []string{"projectcalico.org"}, - Resources: resources, - Verbs: allVerbs, - }, - }, - } -} + }) -// calicoPolicyPassthruClusterRolebinding creates a clusterrolebinding that applies calicoPolicyPassthruClusterRole to all users. -func (c *apiServerComponent) calicoPolicyPassthruClusterRolebinding() *rbacv1.ClusterRoleBinding { - return &rbacv1.ClusterRoleBinding{ + // renamed clusterrolebinding tigera-apiserver-access-tigera-crds to tigera-apiserver + renamedRscList = append(renamedRscList, &rbacv1.ClusterRoleBinding{ TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, ObjectMeta: metav1.ObjectMeta{ - Name: "calico-tiered-policy-passthrough", - }, - Subjects: []rbacv1.Subject{ - { - Kind: "Group", - Name: "system:authenticated", - APIGroup: "rbac.authorization.k8s.io", - }, - }, - RoleRef: rbacv1.RoleRef{ - Kind: "ClusterRole", - Name: "calico-tiered-policy-passthrough", - APIGroup: "rbac.authorization.k8s.io", - }, - } -} - -// uiSettingsPassthruClusterRole creates a clusterrole that is used to control the RBAC mechanism for Tigera UI Settings. -// RBAC for these is handled within the Tigera API Server which checks uisettingsgroups/data permissions for the user. -// -// Calico Enterprise only -func (c *apiServerComponent) uiSettingsPassthruClusterRole() *rbacv1.ClusterRole { - return &rbacv1.ClusterRole{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{ - Name: "calico-uisettings-passthrough", - }, - Rules: []rbacv1.PolicyRule{ - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"uisettings"}, - Verbs: []string{"*"}, - }, + Name: "tigera-apiserver-access-tigera-crds", }, - } -} + }) -// uiSettingsPassthruClusterRolebinding creates a clusterrolebinding that applies uiSettingsPassthruClusterRole to all -// users. -// -// Calico Enterprise only. -func (c *apiServerComponent) uiSettingsPassthruClusterRolebinding() *rbacv1.ClusterRoleBinding { - return &rbacv1.ClusterRoleBinding{ + // Renamed ClusterRoleBinding tigera-tier-getter to calico-tier-getter since Tier is available in OSS. + // Deleting an object that was never created (e.g. in a fresh OSS install) is a no-op. + renamedRscList = append(renamedRscList, &rbacv1.ClusterRoleBinding{ TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, ObjectMeta: metav1.ObjectMeta{ - Name: "calico-uisettings-passthrough", - }, - Subjects: []rbacv1.Subject{ - { - Kind: "Group", - Name: "system:authenticated", - APIGroup: "rbac.authorization.k8s.io", - }, - }, - RoleRef: rbacv1.RoleRef{ - Kind: "ClusterRole", - Name: "calico-uisettings-passthrough", - APIGroup: "rbac.authorization.k8s.io", - }, - } -} - -// auditPolicyConfigMap returns a configmap with contents to configure audit logging for -// projectcalico.org/v3 APIs. -// -// Calico Enterprise only -func (c *apiServerComponent) auditPolicyConfigMap() *corev1.ConfigMap { - const defaultAuditPolicy = `apiVersion: audit.k8s.io/v1 -kind: Policy -rules: -- level: RequestResponse - omitStages: - - RequestReceived - verbs: - - create - - patch - - update - - delete - resources: - - group: projectcalico.org - resources: - - globalnetworkpolicies - - networkpolicies - - stagedglobalnetworkpolicies - - stagednetworkpolicies - - stagedkubernetesnetworkpolicies - - globalnetworksets - - networksets - - tiers - - hostendpoints` - - return &corev1.ConfigMap{ - TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"}, - ObjectMeta: metav1.ObjectMeta{ - // This object is for Enterprise only, so pass it explicitly. - Namespace: APIServerNamespace, - Name: auditPolicyVolumeName, - }, - Data: map[string]string{ - "config": defaultAuditPolicy, - }, - } -} - -func (c *apiServerComponent) multiTenantManagedClusterAccessClusterRoles() []client.Object { - var objects []client.Object - objects = append(objects, &rbacv1.ClusterRole{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{Name: MultiTenantManagedClustersAccessClusterRoleName}, - Rules: []rbacv1.PolicyRule{ - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"managedclusters"}, - Verbs: []string{ - // The Authentication Proxy in Voltron checks if Enterprise Components (using impersonation headers for - // the service in the canonical namespace) can get a managed clusters before sending the request down the tunnel. - // This ClusterRole will be assigned to each component using a RoleBinding in the canonical or tenant namespace. - "get", - }, - }, + Name: "tigera-tier-getter", }, }) - - return objects -} - -// managedClusterWatchClusterRole creates a ClusterRole for watching the ManagedCluster API -func (c *apiServerComponent) managedClusterWatchClusterRole() client.Object { - return &rbacv1.ClusterRole{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{Name: ManagedClustersWatchClusterRoleName}, - Rules: []rbacv1.PolicyRule{ - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"managedclusters"}, - Verbs: []string{ - "get", "list", "watch", - }, - }, - }, - } -} - -func (c *apiServerComponent) getDeprecatedResources() []client.Object { - var renamedRscList []client.Object - - // renamed clusterrole tigera-crds to tigera-apiserver + // Renamed ClusterRole tigera-tier-getter to calico-tier-getter since Tier is available in OSS renamedRscList = append(renamedRscList, &rbacv1.ClusterRole{ TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, ObjectMeta: metav1.ObjectMeta{ - Name: "tigera-crds", + Name: "tigera-tier-getter", }, }) - // renamed clusterrolebinding tigera-apiserver-access-tigera-crds to tigera-apiserver - renamedRscList = append(renamedRscList, &rbacv1.ClusterRoleBinding{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{ - Name: "tigera-apiserver-access-tigera-crds", - }, - }) - - // The following resources were not present in Calico OSS, so there is no need to clean up in OSS. - if c.cfg.Installation.Variant.IsEnterprise() { - // Renamed ClusterRoleBinging tigera-tier-getter to calico-tier-getter since Tier is available in OSS - renamedRscList = append(renamedRscList, &rbacv1.ClusterRoleBinding{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{ - Name: "tigera-tier-getter", - }, - }) - // Renamed ClusterRole tigera-tier-getter to calico-tier-getter since Tier is available in OSS - renamedRscList = append(renamedRscList, &rbacv1.ClusterRole{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{ - Name: "tigera-tier-getter", - }, - }) - } - renamedRscList = append(renamedRscList, &corev1.Namespace{ TypeMeta: metav1.TypeMeta{Kind: "Namespace", APIVersion: "v1"}, ObjectMeta: metav1.ObjectMeta{ @@ -2302,7 +1329,7 @@ func (c *apiServerComponent) l7AdmissionControllerContainer() corev1.Container { c.cfg.TLSKeyPair.VolumeMount(c.SupportedOSType()), } - l7AdmissionControllerTargetPort := getContainerPort(c.cfg, L7AdmissionControllerContainerName).ContainerPort + l7AdmissionControllerTargetPort := GetContainerPort(c.cfg, L7AdmissionControllerContainerName).ContainerPort dataplane := "iptables" if c.cfg.Installation.IsNftables() { @@ -2353,99 +1380,3 @@ func (c *apiServerComponent) l7AdmissionControllerContainer() corev1.Container { return l7AdmssCtrl } - -// deprecatedResources removes legacy cluster-scoped resources created with the 'tigera' prefix (EE-only). -// Moving forward, both EE and OSS variants standardize on the 'calico' prefix for all shared resources. -// TODO to clean up the below deprecated logic with 14 resources in 3.25+ -func (c *apiServerComponent) deprecatedResources() []client.Object { - return []client.Object{ - &rbacv1.ClusterRole{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{Name: "tigera-extension-apiserver-secrets-access"}, - }, - &rbacv1.ClusterRoleBinding{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{Name: "tigera-extension-apiserver-secrets-access"}, - }, - - // delegateAuthClusterRoleBinding - &rbacv1.ClusterRoleBinding{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{Name: "tigera-apiserver-delegate-auth"}, - }, - - // authClusterRole - &rbacv1.ClusterRole{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{Name: "tigera-extension-apiserver-auth-access"}, - }, - - // authClusterRoleBinding - &rbacv1.ClusterRoleBinding{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{Name: "tigera-extension-apiserver-auth-access"}, - }, - // authReaderRoleBinding - need clean up in diff namespace kube-system - &rbacv1.RoleBinding{ - TypeMeta: metav1.TypeMeta{Kind: "RoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{ - Name: "tigera-auth-reader", - Namespace: "kube-system", - }, - }, - // webhookReaderClusterRole - &rbacv1.ClusterRole{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{Name: "tigera-webhook-reader"}, - }, - - // webhookReaderClusterRoleBinding - &rbacv1.ClusterRoleBinding{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{Name: "tigera-apiserver-webhook-reader"}, - }, - - // calico-apiserver CR and CRB - &rbacv1.ClusterRole{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{Name: "tigera-apiserver"}, - }, - &rbacv1.ClusterRoleBinding{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{Name: "tigera-apiserver"}, - }, - - &rbacv1.ClusterRole{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{Name: "tigera-uisettingsgroup-getter"}, - }, - &rbacv1.ClusterRoleBinding{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{Name: "tigera-uisettingsgroup-getter"}, - }, - - &rbacv1.ClusterRole{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{Name: "tigera-tiered-policy-passthrough"}, - }, - &rbacv1.ClusterRoleBinding{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{Name: "tigera-tiered-policy-passthrough"}, - }, - - &rbacv1.ClusterRole{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{Name: "tigera-uisettings-passthrough"}, - }, - &rbacv1.ClusterRoleBinding{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{Name: "tigera-uisettings-passthrough"}, - }, - - // Clean up legacy secrets in the tigera-operator namespace - &corev1.Secret{ - TypeMeta: metav1.TypeMeta{Kind: "Secret", APIVersion: "v1"}, - ObjectMeta: metav1.ObjectMeta{Name: "tigera-api-cert", Namespace: common.OperatorNamespace()}, - }, - } -} diff --git a/pkg/render/apiserver_test.go b/pkg/render/apiserver_test.go index 4efa14a95a..87317ccc0d 100644 --- a/pkg/render/apiserver_test.go +++ b/pkg/render/apiserver_test.go @@ -38,6 +38,7 @@ import ( "github.com/tigera/operator/pkg/controller/k8sapi" ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" "github.com/tigera/operator/pkg/dns" + "github.com/tigera/operator/pkg/extensions" "github.com/tigera/operator/pkg/render" rmeta "github.com/tigera/operator/pkg/render/common/meta" "github.com/tigera/operator/pkg/render/common/networkpolicy" @@ -60,6 +61,21 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) +// apiServerObjects renders the API server component and applies the registered variant +// modifier the way the componentHandler does, so the Enterprise objects (query server, +// audit logging, Enterprise RBAC) and the Calico cleanup deletes are reflected in the +// returned create and delete lists. +func apiServerObjects(c render.Component) ([]client.Object, []client.Object) { + create, del := c.Objects() + rc := extensions.RenderContext{} + if p, ok := c.(render.ExtensionContextProvider); ok { + ec := p.ExtensionContext().(render.APIServerExtensionContext) + rc.Installation = ec.Config.Installation + rc.Component = ec + } + return extensions.ApplyModifiers(render.ComponentNameAPIServer, rc, create, del) +} + var _ = Describe("API server rendering tests (Calico Enterprise)", func() { apiServerPolicy := testutils.GetExpectedPolicyFromFile("./testutils/expected_policies/apiserver.json") apiServerPolicyForOCP := testutils.GetExpectedPolicyFromFile("./testutils/expected_policies/apiserver_ocp.json") @@ -97,6 +113,7 @@ var _ = Describe("API server rendering tests (Calico Enterprise)", func() { cfg = &render.APIServerConfiguration{ RequiresAggregationServer: true, + RequiresQueryServer: true, K8SServiceEndpoint: k8sapi.ServiceEndpoint{}, Installation: instance, APIServer: apiserver, @@ -149,7 +166,7 @@ var _ = Describe("API server rendering tests (Calico Enterprise)", func() { Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) // Should render the correct resources. // - 1 namespace @@ -389,7 +406,7 @@ var _ = Describe("API server rendering tests (Calico Enterprise)", func() { component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) expectedResources := []client.Object{ &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "tigera-ca-bundle", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"}}, @@ -427,7 +444,7 @@ var _ = Describe("API server rendering tests (Calico Enterprise)", func() { It("should grant the calico-apiserver SA write access to globalreports/status", func() { component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) cr := rtest.GetResource(resources, "calico-apiserver", "", "rbac.authorization.k8s.io", "v1", "ClusterRole").(*rbacv1.ClusterRole) @@ -465,7 +482,7 @@ var _ = Describe("API server rendering tests (Calico Enterprise)", func() { component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) d, ok := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment").(*appsv1.Deployment) Expect(ok).To(BeTrue()) @@ -503,7 +520,7 @@ var _ = Describe("API server rendering tests (Calico Enterprise)", func() { } component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) deploy, ok := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment").(*appsv1.Deployment) Expect(ok).To(BeTrue()) @@ -535,7 +552,7 @@ var _ = Describe("API server rendering tests (Calico Enterprise)", func() { component, err := render.APIServer(cfg) Expect(err).NotTo(HaveOccurred()) Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) role := rtest.GetResource(resources, "calico-extension-apiserver-auth-access", "", "rbac.authorization.k8s.io", "v1", "ClusterRole").(*rbacv1.ClusterRole) Expect(role.Rules).To(ContainElement(rbacv1.PolicyRule{ @@ -585,7 +602,7 @@ var _ = Describe("API server rendering tests (Calico Enterprise)", func() { component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) rtest.ExpectResources(resources, expectedResources) @@ -630,7 +647,7 @@ var _ = Describe("API server rendering tests (Calico Enterprise)", func() { component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) rtest.ExpectResources(resources, expectedResources) @@ -699,7 +716,7 @@ var _ = Describe("API server rendering tests (Calico Enterprise)", func() { component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) rtest.ExpectResources(resources, expectedResources) @@ -720,7 +737,7 @@ var _ = Describe("API server rendering tests (Calico Enterprise)", func() { component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) d := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment").(*appsv1.Deployment) Expect(d.Spec.Template.Spec.Tolerations).To(ContainElements(append(rmeta.TolerateControlPlane, tol))) }) @@ -759,7 +776,7 @@ var _ = Describe("API server rendering tests (Calico Enterprise)", func() { component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) rtest.ExpectResources(resources, expectedResources) @@ -790,7 +807,7 @@ var _ = Describe("API server rendering tests (Calico Enterprise)", func() { component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) deploymentResource := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment") Expect(deploymentResource).ToNot(BeNil()) @@ -803,7 +820,7 @@ var _ = Describe("API server rendering tests (Calico Enterprise)", func() { cfg.ForceHostNetwork = true component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) d := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment").(*appsv1.Deployment) Expect(d.Spec.Strategy.Type).To(Equal(appsv1.RecreateDeploymentStrategyType)) }) @@ -814,7 +831,7 @@ var _ = Describe("API server rendering tests (Calico Enterprise)", func() { cfg.ForceHostNetwork = true component := render.APIServerPolicy(cfg) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) policyName := types.NamespacedName{Name: "calico-system.apiserver-access", Namespace: "calico-system"} policy := testutils.GetCalicoSystemPolicyFromResources(policyName, resources) Expect(policy).ToNot(BeNil()) @@ -836,7 +853,7 @@ var _ = Describe("API server rendering tests (Calico Enterprise)", func() { cfg.ForceHostNetwork = false component := render.APIServerPolicy(cfg) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) policyName := types.NamespacedName{Name: "calico-system.apiserver-access", Namespace: "calico-system"} policy := testutils.GetCalicoSystemPolicyFromResources(policyName, resources) Expect(policy).ToNot(BeNil()) @@ -860,7 +877,7 @@ var _ = Describe("API server rendering tests (Calico Enterprise)", func() { component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) deploymentResource := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment") Expect(deploymentResource).ToNot(BeNil()) @@ -877,7 +894,7 @@ var _ = Describe("API server rendering tests (Calico Enterprise)", func() { component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) deploymentResource := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment") Expect(deploymentResource).ToNot(BeNil()) @@ -892,7 +909,7 @@ var _ = Describe("API server rendering tests (Calico Enterprise)", func() { Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) expectedResources := []client.Object{ &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "calico-audit-policy", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"}}, @@ -958,7 +975,7 @@ var _ = Describe("API server rendering tests (Calico Enterprise)", func() { Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) expected := []client.Object{ &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "calico-audit-policy", Namespace: "calico-system"}}, @@ -1018,7 +1035,7 @@ var _ = Describe("API server rendering tests (Calico Enterprise)", func() { component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) dep := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment") Expect(dep).ToNot(BeNil()) @@ -1035,7 +1052,7 @@ var _ = Describe("API server rendering tests (Calico Enterprise)", func() { component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) dep := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment") Expect(dep).ToNot(BeNil()) @@ -1057,7 +1074,7 @@ var _ = Describe("API server rendering tests (Calico Enterprise)", func() { Expect(err).NotTo(HaveOccurred()) component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) expectedResources := []client.Object{ &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "calico-audit-policy", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"}}, &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "tigera-ca-bundle", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"}}, @@ -1102,7 +1119,7 @@ var _ = Describe("API server rendering tests (Calico Enterprise)", func() { cfg.Installation.ControlPlaneReplicas = ptr.To(int32(1)) component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) deploy, ok := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment").(*appsv1.Deployment) Expect(ok).To(BeTrue()) @@ -1113,7 +1130,7 @@ var _ = Describe("API server rendering tests (Calico Enterprise)", func() { cfg.Installation.ControlPlaneReplicas = ptr.To(int32(2)) component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) deploy, ok := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment").(*appsv1.Deployment) Expect(ok).To(BeTrue()) @@ -1127,7 +1144,7 @@ var _ = Describe("API server rendering tests (Calico Enterprise)", func() { component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) rb, ok := rtest.GetResource(resources, "tigera-linseed", "calico-system", "rbac.authorization.k8s.io", "v1", "RoleBinding").(*rbacv1.RoleBinding) Expect(ok).To(BeTrue(), "expected tigera-linseed RoleBinding in calico-system") @@ -1179,7 +1196,7 @@ var _ = Describe("API server rendering tests (Calico Enterprise)", func() { } component := render.APIServerPolicy(cfg) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) policy := testutils.GetCalicoSystemPolicyFromResources(policyName, resources) expectedPolicy := testutils.SelectPolicyByProvider(scenario, apiServerPolicy, apiServerPolicyForOCP) @@ -1325,7 +1342,7 @@ var _ = Describe("API server rendering tests (Calico Enterprise)", func() { component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) d, ok := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment").(*appsv1.Deployment) Expect(ok).To(BeTrue()) @@ -1438,7 +1455,7 @@ var _ = Describe("API server rendering tests (Calico Enterprise)", func() { component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) d, ok := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment").(*appsv1.Deployment) Expect(ok).To(BeTrue()) // nodeSelectors are merged @@ -1469,7 +1486,7 @@ var _ = Describe("API server rendering tests (Calico Enterprise)", func() { component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) d, ok := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment").(*appsv1.Deployment) Expect(ok).To(BeTrue()) Expect(d.Spec.Template.Spec.Tolerations).To(HaveLen(1)) @@ -1483,7 +1500,7 @@ var _ = Describe("API server rendering tests (Calico Enterprise)", func() { } component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) d := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment").(*appsv1.Deployment) Expect(d.Spec.Template.Spec.Containers[0].Args).To(ConsistOf([]string{ "--secure-port=5443", @@ -1945,7 +1962,7 @@ var _ = Describe("API server rendering tests (Calico)", func() { Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) Expect(component.ResolveImages(nil)).To(BeNil()) - resources, deleteResources := component.Objects() + resources, deleteResources := apiServerObjects(component) rtest.ExpectResources(resources, expectedResources) rtest.ExpectResourceInList(deleteResources, "allow-apiserver", "calico-system", "networking.k8s.io", "v1", "NetworkPolicy") @@ -2041,7 +2058,7 @@ var _ = Describe("API server rendering tests (Calico)", func() { component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) Expect(component.ResolveImages(nil)).To(BeNil()) - resources, deleteResources := component.Objects() + resources, deleteResources := apiServerObjects(component) // Should not include deployment, service, SA, or PDB. Expect(rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment")).To(BeNil()) @@ -2084,7 +2101,7 @@ var _ = Describe("API server rendering tests (Calico)", func() { component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) Expect(component.ResolveImages(nil)).To(BeNil()) - resources, deleteResources := component.Objects() + resources, deleteResources := apiServerObjects(component) // Should render the correct resources. By("Checking each expected resource is actually rendered") @@ -2121,7 +2138,7 @@ var _ = Describe("API server rendering tests (Calico)", func() { component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) d := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment").(*appsv1.Deployment) Expect(d.Spec.Template.Spec.NodeSelector).To(HaveLen(1)) Expect(d.Spec.Template.Spec.NodeSelector).To(HaveKeyWithValue("nodeName", "control01")) @@ -2137,7 +2154,7 @@ var _ = Describe("API server rendering tests (Calico)", func() { cfg.Installation.ControlPlaneTolerations = []corev1.Toleration{tol} component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) d := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment").(*appsv1.Deployment) Expect(d.Spec.Template.Spec.Tolerations).To(ContainElements(append(rmeta.TolerateControlPlane, tol))) }) @@ -2151,7 +2168,7 @@ var _ = Describe("API server rendering tests (Calico)", func() { component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) deploymentResource := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment") Expect(deploymentResource).ToNot(BeNil()) @@ -2164,7 +2181,7 @@ var _ = Describe("API server rendering tests (Calico)", func() { cfg.ForceHostNetwork = true component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) d := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment").(*appsv1.Deployment) Expect(d.Spec.Strategy.Type).To(Equal(appsv1.RecreateDeploymentStrategyType)) }) @@ -2177,7 +2194,7 @@ var _ = Describe("API server rendering tests (Calico)", func() { component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) deploymentResource := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment") Expect(deploymentResource).ToNot(BeNil()) @@ -2194,7 +2211,7 @@ var _ = Describe("API server rendering tests (Calico)", func() { component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) deploymentResource := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment") Expect(deploymentResource).ToNot(BeNil()) @@ -2209,7 +2226,7 @@ var _ = Describe("API server rendering tests (Calico)", func() { component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) deploy, ok := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment").(*appsv1.Deployment) Expect(ok).To(BeTrue()) @@ -2222,7 +2239,7 @@ var _ = Describe("API server rendering tests (Calico)", func() { component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) deploy, ok := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment").(*appsv1.Deployment) Expect(ok).To(BeTrue()) @@ -2236,7 +2253,7 @@ var _ = Describe("API server rendering tests (Calico)", func() { component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) Expect(component.ResolveImages(nil)).To(BeNil()) - _, _ = component.Objects() + _, _ = apiServerObjects(component) }) It("should render host networked with TKG provider", func() { @@ -2247,7 +2264,7 @@ var _ = Describe("API server rendering tests (Calico)", func() { component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) deploy, ok := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment").(*appsv1.Deployment) Expect(ok).To(BeTrue()) @@ -2359,7 +2376,7 @@ var _ = Describe("API server rendering tests (Calico)", func() { component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) d, ok := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment").(*appsv1.Deployment) Expect(ok).To(BeTrue()) @@ -2433,7 +2450,7 @@ var _ = Describe("API server rendering tests (Calico)", func() { component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) d := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment").(*appsv1.Deployment) // nodeSelectors are merged Expect(d.Spec.Template.Spec.NodeSelector).To(HaveLen(2)) @@ -2463,7 +2480,7 @@ var _ = Describe("API server rendering tests (Calico)", func() { component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) d := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment").(*appsv1.Deployment) Expect(d.Spec.Template.Spec.Tolerations).To(HaveLen(1)) Expect(d.Spec.Template.Spec.Tolerations).To(ConsistOf(tol)) @@ -2475,7 +2492,7 @@ var _ = Describe("API server rendering tests (Calico)", func() { component, err := render.APIServer(cfg) Expect(err).NotTo(HaveOccurred(), "Expected APIServer to create successfully %s", err) Expect(component.ResolveImages(nil)).NotTo(HaveOccurred()) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) d := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment").(*appsv1.Deployment) Expect(d).NotTo(BeNil()) Expect(d.Spec.Template.Spec.Tolerations).To(ContainElement(corev1.Toleration{ @@ -2503,7 +2520,7 @@ var _ = Describe("API server rendering tests (Calico)", func() { Expect(err).NotTo(HaveOccurred()) // Expect no UISettings / UISettingsGroups to be installed. - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) obj := rtest.GetResource(resources, "tigera-network-admin", "", "rbac.authorization.k8s.io", "v1", "ClusterRole") Expect(obj).To(BeNil()) obj = rtest.GetResource(resources, "tigera-ui-user", "", "rbac.authorization.k8s.io", "v1", "ClusterRole") @@ -2514,7 +2531,7 @@ var _ = Describe("API server rendering tests (Calico)", func() { component, err := render.APIServer(cfg) Expect(err).NotTo(HaveOccurred()) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) managedClusterAccessRole := rtest.GetResource(resources, render.MultiTenantManagedClustersAccessClusterRoleName, "", rbacv1.GroupName, "v1", "ClusterRole").(*rbacv1.ClusterRole) expectedManagedClusterAccessRules := []rbacv1.PolicyRule{ @@ -2531,7 +2548,7 @@ var _ = Describe("API server rendering tests (Calico)", func() { component, err := render.APIServer(cfg) Expect(err).NotTo(HaveOccurred()) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) managedClusterAccessRole := rtest.GetResource(resources, render.ManagedClustersWatchClusterRoleName, "", rbacv1.GroupName, "v1", "ClusterRole").(*rbacv1.ClusterRole) expectedManagedClusterAccessRules := []rbacv1.PolicyRule{ diff --git a/pkg/render/guardian_test.go b/pkg/render/guardian_test.go index 244e1c6fa5..b28e77f044 100644 --- a/pkg/render/guardian_test.go +++ b/pkg/render/guardian_test.go @@ -54,7 +54,8 @@ func guardianObjects(cfg *render.GuardianConfiguration) []client.Object { if p, ok := g.(render.ExtensionContextProvider); ok { rc.Component = p.ExtensionContext() } - return extensions.ApplyModifiers(render.GuardianName, rc, objs) + out, _ := extensions.ApplyModifiers(render.GuardianName, rc, objs, nil) + return out } var _ = Describe("Rendering tests", func() { @@ -115,7 +116,7 @@ var _ = Describe("Rendering tests", func() { if p, ok := g.(render.ExtensionContextProvider); ok { rc.Component = p.ExtensionContext() } - resources = extensions.ApplyModifiers(render.GuardianName, rc, resources) + resources, _ = extensions.ApplyModifiers(render.GuardianName, rc, resources, nil) } BeforeEach(func() { @@ -351,7 +352,7 @@ var _ = Describe("Rendering tests", func() { if p, ok := g.(render.ExtensionContextProvider); ok { rc.Component = p.ExtensionContext() } - resources = extensions.ApplyModifiers(render.ComponentNameGuardianPolicy, rc, objs) + resources, _ = extensions.ApplyModifiers(render.ComponentNameGuardianPolicy, rc, objs, nil) } Context("policy rendering based on variant and IncludeEgressNetworkPolicy", func() { diff --git a/pkg/render/node_enterprise_test.go b/pkg/render/node_enterprise_test.go index 9e0d9c493b..4b9b6a2f82 100644 --- a/pkg/render/node_enterprise_test.go +++ b/pkg/render/node_enterprise_test.go @@ -110,7 +110,8 @@ var _ = Describe("node enterprise modifier integration", func() { comp := render.Node(cfg) Expect(comp.ResolveImages(nil)).NotTo(HaveOccurred()) objs, _ := comp.Objects() - return extensions.ApplyModifiers(render.ComponentNameNode, renderCtx, objs) + out, _ := extensions.ApplyModifiers(render.ComponentNameNode, renderCtx, objs, nil) + return out } It("appends the node metrics service to the real render output", func() { @@ -166,7 +167,7 @@ var _ = Describe("node enterprise modifier integration", func() { }) Expect(comp.ResolveImages(nil)).NotTo(HaveOccurred()) objs, _ := comp.Objects() - objs = extensions.ApplyModifiers(render.ComponentNameTypha, renderCtx, objs) + objs, _ = extensions.ApplyModifiers(render.ComponentNameTypha, renderCtx, objs, nil) role, ok := extensions.FindObject[*rbacv1.ClusterRole](objs, "calico-typha") Expect(ok).To(BeTrue()) diff --git a/pkg/render/windows_test.go b/pkg/render/windows_test.go index 1eb52c0849..b19063e069 100644 --- a/pkg/render/windows_test.go +++ b/pkg/render/windows_test.go @@ -54,7 +54,8 @@ func renderWindows(cfg *render.WindowsConfiguration) []client.Object { if p, ok := comp.(render.ExtensionContextProvider); ok { rc.Component = p.ExtensionContext() } - return extensions.ApplyModifiers(render.ComponentNameWindows, rc, objs) + out, _ := extensions.ApplyModifiers(render.ComponentNameWindows, rc, objs, nil) + return out } var _ = Describe("Windows rendering tests", func() { From 139aabba3be9d3fdf57c502ac74e4f53d512d668 Mon Sep 17 00:00:00 2001 From: Casey Davenport Date: Wed, 17 Jun 2026 09:27:16 -0700 Subject: [PATCH 25/54] Address review feedback: cleanups Trim over-commenting on the component handler calls and expand the ones threading a render context to one arg per line. Rename RunSetup to BuildContext, drop the nodePrometheusTLS temp var, and log a BUG on a failed per-component context assertion instead of failing silently. --- .../apiserver/apiserver_controller.go | 13 +++++++----- .../clusterconnection_controller.go | 11 ++++++---- .../installation/core_controller.go | 21 +++++++++---------- .../installation/windows_controller.go | 11 ++++++---- pkg/enterprise/apiserver.go | 13 ++++++++++-- pkg/enterprise/guardian.go | 12 +++++++++-- pkg/enterprise/installation_test.go | 6 +++--- pkg/enterprise/windows.go | 7 ++++++- pkg/extensions/doc.go | 2 +- pkg/extensions/setup.go | 7 ++++--- pkg/extensions/setup_test.go | 10 ++++----- 11 files changed, 72 insertions(+), 41 deletions(-) diff --git a/pkg/controller/apiserver/apiserver_controller.go b/pkg/controller/apiserver/apiserver_controller.go index 0e762c1717..402e784920 100644 --- a/pkg/controller/apiserver/apiserver_controller.go +++ b/pkg/controller/apiserver/apiserver_controller.go @@ -474,11 +474,14 @@ func (r *ReconcileAPIServer) Reconcile(ctx context.Context, request reconcile.Re return reconcile.Result{}, err } - // Create a component handler to manage the rendered component. The render context - // carries the installation so the componentHandler applies the variant's API server - // modifier (query server, audit logging, Enterprise RBAC) to the rendered objects. - handler := utils.NewComponentHandler(log, r.client, r.scheme, instance, - utils.WithRenderContext(extensions.RenderContext{Installation: installationSpec})) + // Create a component handler to manage the rendered component. + handler := utils.NewComponentHandler( + log, + r.client, + r.scheme, + instance, + utils.WithRenderContext(extensions.RenderContext{Installation: installationSpec}), + ) // Render the desired objects from the CRD and create or update them. reqLogger.V(3).Info("rendering components") diff --git a/pkg/controller/clusterconnection/clusterconnection_controller.go b/pkg/controller/clusterconnection/clusterconnection_controller.go index b46ad0c205..d796e01638 100644 --- a/pkg/controller/clusterconnection/clusterconnection_controller.go +++ b/pkg/controller/clusterconnection/clusterconnection_controller.go @@ -444,10 +444,13 @@ func (r *ReconcileConnection) Reconcile(ctx context.Context, request reconcile.R return reconcile.Result{}, err } - // The render context carries the installation so registered modifiers gate on - // variant; the guardian component supplies its own per-component context (the - // impersonation config, OpenShift, and CA bundle path) via ExtensionContext. - ch := utils.NewComponentHandler(log, r.cli, r.scheme, managementClusterConnection, utils.WithRenderContext(extensions.RenderContext{Installation: installationSpec})) + ch := utils.NewComponentHandler( + log, + r.cli, + r.scheme, + managementClusterConnection, + utils.WithRenderContext(extensions.RenderContext{Installation: installationSpec}), + ) guardianCfg := &render.GuardianConfiguration{ URL: managementClusterConnection.Spec.ManagementClusterAddr, PodProxies: r.resolvedPodProxies, diff --git a/pkg/controller/installation/core_controller.go b/pkg/controller/installation/core_controller.go index 65ef9def74..d760aed637 100644 --- a/pkg/controller/installation/core_controller.go +++ b/pkg/controller/installation/core_controller.go @@ -1226,13 +1226,7 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile calicoVersion = components.EnterpriseRelease } - // Run the variant setup to build the render context handed to registered - // modifiers. The core operator has no setup, so it gets just the base - // context; an extension's setup additionally does controller-side work for - // its variant - validating config and creating the node-prometheus - // certificate, adding it (and the prometheus/esgw certs) to the trusted - // bundle - and may abort the reconcile by returning an error. - renderCtx, err := extensions.RunSetup(extensions.Inputs{ + renderCtx, err := extensions.BuildContext(extensions.Inputs{ Ctx: ctx, Client: r.client, Installation: &instance.Spec, @@ -1245,7 +1239,6 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile r.status.SetDegraded(operatorv1.ResourceCreateError, "Error preparing installation extension", err, reqLogger) return reconcile.Result{}, err } - nodePrometheusTLS := renderCtx.NodePrometheusTLS kubeControllersMetricsPort, err := utils.GetKubeControllerMetricsPort(ctx, r.client) if err != nil { @@ -1284,7 +1277,13 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile } // Create a component handler to create or update the rendered components. - handler := r.newComponentHandler(log, r.client, r.scheme, instance, utils.WithRenderContext(renderCtx)) + handler := r.newComponentHandler( + log, + r.client, + r.scheme, + instance, + utils.WithRenderContext(renderCtx), + ) // Render namespaces first - this ensures that any other controllers blocked on namespace existence can proceed. namespaceCfg := &render.NamespaceConfiguration{ @@ -1377,7 +1376,7 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile keyPairOptions := []rcertificatemanagement.KeyPairOption{ rcertificatemanagement.NewKeyPairOption(typhaNodeTLS.NodeSecret, true, true), - rcertificatemanagement.NewKeyPairOption(nodePrometheusTLS, true, true), + rcertificatemanagement.NewKeyPairOption(renderCtx.NodePrometheusTLS, true, true), rcertificatemanagement.NewKeyPairOption(typhaNodeTLS.TyphaSecret, true, true), rcertificatemanagement.NewKeyPairOption(typhaNodeTLS.TyphaSecretNonClusterHost, true, true), rcertificatemanagement.NewKeyPairOption(kubeControllerTLS, true, true), @@ -1808,7 +1807,7 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile render.TyphaTLSSecretName: typhaNodeTLS.TyphaSecret, render.NodeTLSSecretName: typhaNodeTLS.NodeSecret, render.TyphaTLSSecretName + render.TyphaNonClusterHostSuffix: typhaNodeTLS.TyphaSecretNonClusterHost, - render.NodePrometheusTLSServerSecret: nodePrometheusTLS, + render.NodePrometheusTLSServerSecret: renderCtx.NodePrometheusTLS, kubecontrollers.KubeControllerPrometheusTLSSecret: kubeControllerTLS, }, r.status) diff --git a/pkg/controller/installation/windows_controller.go b/pkg/controller/installation/windows_controller.go index e3cb1389b2..ca8063ffea 100644 --- a/pkg/controller/installation/windows_controller.go +++ b/pkg/controller/installation/windows_controller.go @@ -407,10 +407,13 @@ func (r *ReconcileWindows) Reconcile(ctx context.Context, request reconcile.Requ } // Create a component handler to create or update the rendered components. - // The render context carries the installation so registered modifiers gate on - // variant; the windows component supplies its own per-component context (the - // reporter port and prometheus keypair) via ExtensionContext. - handler := utils.NewComponentHandler(logw, r.client, r.scheme, instance, utils.WithRenderContext(extensions.RenderContext{Installation: &instance.Spec})) + handler := utils.NewComponentHandler( + logw, + r.client, + r.scheme, + instance, + utils.WithRenderContext(extensions.RenderContext{Installation: &instance.Spec}), + ) if err := handler.CreateOrUpdateOrDelete(ctx, component, nil); err != nil { r.status.SetDegraded(operatorv1.ResourceUpdateError, "Error creating / updating resource", err, reqLogger) return reconcile.Result{}, err diff --git a/pkg/enterprise/apiserver.go b/pkg/enterprise/apiserver.go index d81354a915..cba83f2bfc 100644 --- a/pkg/enterprise/apiserver.go +++ b/pkg/enterprise/apiserver.go @@ -18,6 +18,7 @@ import ( "fmt" "strings" + "github.com/sirupsen/logrus" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" @@ -66,7 +67,11 @@ func registerAPIServer() { // the query server container and its volumes, audit logging on the aggregation API server // container, the Enterprise RBAC objects, and the query server port on the Service. func modifyAPIServer(ctx extensions.RenderContext, create, del []client.Object) ([]client.Object, []client.Object) { - ec, _ := ctx.Component.(render.APIServerExtensionContext) + ec, ok := ctx.Component.(render.APIServerExtensionContext) + if !ok { + logrus.Errorf("BUG: apiserver modifier got %T, want render.APIServerExtensionContext; leaving objects unchanged", ctx.Component) + return create, del + } c := &apiServer{cfg: ec.Config, calicoImage: ec.CalicoImage} if dep, ok := extensions.FindObject[*appsv1.Deployment](create, render.APIServerName); ok { @@ -146,7 +151,11 @@ func modifyAPIServer(ctx extensions.RenderContext, create, del []client.Object) // cleanupAPIServer deletes the Enterprise API server objects when running Calico, so a // cluster switched from Enterprise to Calico does not leave them behind. func cleanupAPIServer(ctx extensions.RenderContext, create, del []client.Object) ([]client.Object, []client.Object) { - ec, _ := ctx.Component.(render.APIServerExtensionContext) + ec, ok := ctx.Component.(render.APIServerExtensionContext) + if !ok { + logrus.Errorf("BUG: apiserver cleanup got %T, want render.APIServerExtensionContext; leaving objects unchanged", ctx.Component) + return create, del + } c := &apiServer{cfg: ec.Config} del = append(del, c.tigeraAPIServerClusterRole(), c.tigeraAPIServerClusterRoleBinding()) diff --git a/pkg/enterprise/guardian.go b/pkg/enterprise/guardian.go index 1d16e13b19..0187166f61 100644 --- a/pkg/enterprise/guardian.go +++ b/pkg/enterprise/guardian.go @@ -52,7 +52,11 @@ func registerGuardian() { // fail (proxy URL parsing); on failure we drop the policy entirely, matching the // core behavior of omitting it rather than installing a partial policy. func modifyGuardianPolicy(ctx extensions.RenderContext, objs, del []client.Object) ([]client.Object, []client.Object) { - gpc, _ := ctx.Component.(render.GuardianPolicyExtensionContext) + gpc, ok := ctx.Component.(render.GuardianPolicyExtensionContext) + if !ok { + logrus.Errorf("BUG: guardian policy modifier got %T, want render.GuardianPolicyExtensionContext; leaving objects unchanged", ctx.Component) + return objs, del + } policy, ok := extensions.FindObject[*v3.NetworkPolicy](objs, render.GuardianPolicyName) if !ok { @@ -212,7 +216,11 @@ func enterpriseGuardianPolicySpec(gpc render.GuardianPolicyExtensionContext) (v3 // elasticsearch/kibana service ports, the management-cluster-request cluster // role rules (which replace the OSS rules), and the CA bundle env vars. func modifyGuardian(ctx extensions.RenderContext, objs, del []client.Object) ([]client.Object, []client.Object) { - gc, _ := ctx.Component.(render.GuardianExtensionContext) + gc, ok := ctx.Component.(render.GuardianExtensionContext) + if !ok { + logrus.Errorf("BUG: guardian modifier got %T, want render.GuardianExtensionContext; leaving objects unchanged", ctx.Component) + return objs, del + } if role, ok := extensions.FindObject[*rbacv1.ClusterRole](objs, render.GuardianClusterRoleName); ok { role.Rules = guardianEnterpriseRules(gc) diff --git a/pkg/enterprise/installation_test.go b/pkg/enterprise/installation_test.go index 1ebf7bf52b..cfab0176bb 100644 --- a/pkg/enterprise/installation_test.go +++ b/pkg/enterprise/installation_test.go @@ -42,18 +42,18 @@ var _ = Describe("installation setup", func() { in.FelixConfiguration = &v3.FelixConfiguration{ Spec: v3.FelixConfigurationSpec{PrometheusReporterPort: &port}, } - _, err := extensions.RunSetup(in) + _, err := extensions.BuildContext(in) Expect(err).To(HaveOccurred()) }) It("creates the node prometheus keypair for the enterprise variant", func() { - rc, err := extensions.RunSetup(newInputs(operatorv1.CalicoEnterprise)) + rc, err := extensions.BuildContext(newInputs(operatorv1.CalicoEnterprise)) Expect(err).NotTo(HaveOccurred()) Expect(rc.NodePrometheusTLS).NotTo(BeNil()) }) It("is a no-op for the Calico variant", func() { - rc, err := extensions.RunSetup(newInputs(operatorv1.Calico)) + rc, err := extensions.BuildContext(newInputs(operatorv1.Calico)) Expect(err).NotTo(HaveOccurred()) Expect(rc.NodePrometheusTLS).To(BeNil()) }) diff --git a/pkg/enterprise/windows.go b/pkg/enterprise/windows.go index 720ea80c20..8a3e086835 100644 --- a/pkg/enterprise/windows.go +++ b/pkg/enterprise/windows.go @@ -17,6 +17,7 @@ package enterprise import ( "fmt" + "github.com/sirupsen/logrus" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -52,7 +53,11 @@ func registerWindows() { // daemonset configuration (flow/DNS log env, prometheus reporter, trusted DNS // servers, the calico log volume, and the prometheus reporter keypair mount). func modifyWindows(ctx extensions.RenderContext, objs, del []client.Object) ([]client.Object, []client.Object) { - wc, _ := ctx.Component.(render.WindowsExtensionContext) + wc, ok := ctx.Component.(render.WindowsExtensionContext) + if !ok { + logrus.Errorf("BUG: windows modifier got %T, want render.WindowsExtensionContext; leaving objects unchanged", ctx.Component) + return objs, del + } if ds, ok := extensions.FindObject[*appsv1.DaemonSet](objs, common.WindowsDaemonSetName); ok { modifyWindowsDaemonSet(ctx, wc, ds) diff --git a/pkg/extensions/doc.go b/pkg/extensions/doc.go index 73adc07fa0..a505f82fa1 100644 --- a/pkg/extensions/doc.go +++ b/pkg/extensions/doc.go @@ -25,7 +25,7 @@ // does the side-effecting work a pure render hook can't: creating certificates, // extending the trusted bundle, validating config. It returns the RenderContext // - the read-only baton passed to the render phase. Register one per variant -// with RegisterSetup; the controller runs it with RunSetup. +// with RegisterSetup; the controller runs it with BuildContext. // // Extension is the render phase: pure, per-component hooks that run after a // component builds its objects. Its Image field overrides the component's image diff --git a/pkg/extensions/setup.go b/pkg/extensions/setup.go index 136863b0ff..a6272b006c 100644 --- a/pkg/extensions/setup.go +++ b/pkg/extensions/setup.go @@ -73,9 +73,10 @@ func RegisterSetup(variant operatorv1.ProductVariant, s Setup) { setups[variant] = s } -// RunSetup runs the setup registered for the installation variant and returns -// its RenderContext, or the base render context when the variant has no setup. -func RunSetup(in Inputs) (RenderContext, error) { +// BuildContext runs the setup registered for the installation variant and +// returns its RenderContext, or the base render context when the variant has no +// setup. +func BuildContext(in Inputs) (RenderContext, error) { if in.Installation != nil { if s, ok := setups[in.Installation.Variant]; ok { return s(in) diff --git a/pkg/extensions/setup_test.go b/pkg/extensions/setup_test.go index 9967bfe209..1ba42e5567 100644 --- a/pkg/extensions/setup_test.go +++ b/pkg/extensions/setup_test.go @@ -29,7 +29,7 @@ var _ = Describe("variant setup", func() { It("returns the base render context when the variant has no setup", func() { install := &operatorv1.InstallationSpec{Variant: operatorv1.Calico} - rc, err := extensions.RunSetup(extensions.Inputs{ + rc, err := extensions.BuildContext(extensions.Inputs{ Installation: install, ClusterDomain: "cluster.local", }) @@ -41,14 +41,14 @@ var _ = Describe("variant setup", func() { It("uses the setup registered for the installation variant", func() { extensions.RegisterSetup(operatorv1.CalicoEnterprise, fakeSetup(nil)) - rc, err := extensions.RunSetup(enterpriseInputs()) + rc, err := extensions.BuildContext(enterpriseInputs()) Expect(err).NotTo(HaveOccurred()) Expect(rc.ClusterDomain).To(Equal("from-fake")) }) It("ignores a setup registered for a different variant", func() { extensions.RegisterSetup(operatorv1.CalicoEnterprise, fakeSetup(nil)) - rc, err := extensions.RunSetup(extensions.Inputs{ + rc, err := extensions.BuildContext(extensions.Inputs{ Installation: &operatorv1.InstallationSpec{Variant: operatorv1.Calico}, ClusterDomain: "real", }) @@ -58,7 +58,7 @@ var _ = Describe("variant setup", func() { It("surfaces the setup error", func() { extensions.RegisterSetup(operatorv1.CalicoEnterprise, fakeSetup(errors.New("boom"))) - _, err := extensions.RunSetup(enterpriseInputs()) + _, err := extensions.BuildContext(enterpriseInputs()) Expect(err).To(MatchError("boom")) }) @@ -67,7 +67,7 @@ var _ = Describe("variant setup", func() { extensions.ResetForTest() in := enterpriseInputs() in.ClusterDomain = "real" - rc, err := extensions.RunSetup(in) + rc, err := extensions.BuildContext(in) Expect(err).NotTo(HaveOccurred()) Expect(rc.ClusterDomain).To(Equal("real")) }) From 36b9881634a8a05b28dd21e4ecaee61ab7504cec Mon Sep 17 00:00:00 2001 From: Casey Davenport Date: Wed, 17 Jun 2026 13:07:07 -0700 Subject: [PATCH 26/54] Inject extensions instead of registering them globally The variant modifiers, image overrides, and setups lived in package-level registries that enterprise populated via an init-style Register() at startup. Replace those with an extensions.Set value that main builds (enterprise.New()) and hands to controllers through ControllerOptions. Each controller threads it into its component handler and the node/windows render configs, so nothing is wired by import side effect and the core operator can run with no Set. A handler that renders an extensible component without being given the Set logs a BUG, so a missing wiring is loud rather than a silent no-op. Rename imageoverride.Registry to imageoverride.Overrides so it doesn't read as the container image registry. --- cmd/main.go | 10 +-- .../apiserver/apiserver_controller.go | 1 + .../apiserver/apiserver_controller_test.go | 21 +++++ .../apiserver/apiserver_suite_test.go | 10 ++- .../clusterconnection_controller.go | 3 + .../clusterconnection_suite_test.go | 6 -- pkg/controller/clusterconnection/shim_test.go | 2 + .../installation/core_controller.go | 6 +- .../installation/core_controller_test.go | 22 +++++ .../installation_controller_suite_test.go | 9 +- .../installation/windows_controller.go | 4 + .../installation/windows_controller_test.go | 3 + pkg/controller/options/options.go | 7 ++ pkg/controller/utils/component.go | 23 +++++- .../utils/component_enterprise_test.go | 5 +- pkg/controller/utils/component_test.go | 9 +- pkg/enterprise/apiserver.go | 6 +- pkg/enterprise/enterprise_suite_test.go | 7 ++ pkg/enterprise/guardian.go | 6 +- pkg/enterprise/guardian_test.go | 13 ++- pkg/enterprise/installation.go | 4 +- pkg/enterprise/installation_test.go | 9 +- pkg/enterprise/node.go | 6 +- pkg/enterprise/node_test.go | 29 +++---- pkg/enterprise/register.go | 25 +++--- pkg/enterprise/typha.go | 4 +- pkg/enterprise/typha_test.go | 11 +-- pkg/enterprise/windows.go | 8 +- pkg/enterprise/windows_test.go | 23 ++---- pkg/extensions/extension.go | 35 ++------ pkg/extensions/extension_test.go | 27 +++--- pkg/extensions/image.go | 9 -- pkg/extensions/image_test.go | 22 ++--- pkg/extensions/set.go | 82 +++++++++++++++++++ pkg/extensions/setup.go | 20 ++--- pkg/extensions/setup_test.go | 26 +++--- pkg/imageoverride/imageoverride.go | 34 ++++---- pkg/render/apiserver_test.go | 2 +- pkg/render/enterprise_setup_test.go | 21 ++--- pkg/render/guardian_test.go | 6 +- pkg/render/node.go | 9 +- pkg/render/node_enterprise_test.go | 4 +- pkg/render/node_test.go | 1 + pkg/render/windows.go | 9 +- pkg/render/windows_test.go | 15 ++-- 45 files changed, 366 insertions(+), 248 deletions(-) create mode 100644 pkg/extensions/set.go diff --git a/cmd/main.go b/cmd/main.go index 346f62a8a1..eb63d9b666 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -522,6 +522,11 @@ If a value other than 'all' is specified, the first CRD with a prefix of the spe ElasticExternal: discovery.UseExternalElastic(bootConfig), UseV3CRDs: v3CRDs, APIDiscovery: apiDiscovery, + // Hand the operator the in-repo Calico Enterprise extensions (modifiers, + // image overrides, and the installation setup). After the monorepo split + // the core operator's main passes none and calico-private's main passes + // its own. + Extensions: enterprise.New(), } // Before we start any controllers, make sure our options are valid. @@ -530,11 +535,6 @@ If a value other than 'all' is specified, the first CRD with a prefix of the spe os.Exit(1) } - // Wire the in-repo Calico Enterprise extensions (the render context factory, - // modifiers, and image overrides) into the operator registries. After the - // monorepo split this call moves to calico-private's main. - enterprise.Register() - // Register a field-selector index on Pod spec.nodeName. The podiprecovery // controller uses this to list operator-managed pods on a specific node // in a single server-side query. Indexes must be registered before the diff --git a/pkg/controller/apiserver/apiserver_controller.go b/pkg/controller/apiserver/apiserver_controller.go index 402e784920..3b8c5e4197 100644 --- a/pkg/controller/apiserver/apiserver_controller.go +++ b/pkg/controller/apiserver/apiserver_controller.go @@ -481,6 +481,7 @@ func (r *ReconcileAPIServer) Reconcile(ctx context.Context, request reconcile.Re r.scheme, instance, utils.WithRenderContext(extensions.RenderContext{Installation: installationSpec}), + utils.WithExtensions(r.opts.Extensions), ) // Render the desired objects from the CRD and create or update them. diff --git a/pkg/controller/apiserver/apiserver_controller_test.go b/pkg/controller/apiserver/apiserver_controller_test.go index b70bd58906..11a4f896de 100644 --- a/pkg/controller/apiserver/apiserver_controller_test.go +++ b/pkg/controller/apiserver/apiserver_controller_test.go @@ -170,6 +170,7 @@ var _ = Describe("apiserver controller tests", func() { tierWatchReady: ready, migrationWatchReady: &utils.ReadyFlag{}, opts: options.ControllerOptions{ + Extensions: testExtensions, EnterpriseCRDExists: true, DetectedProvider: operatorv1.ProviderNone, }, @@ -229,6 +230,7 @@ var _ = Describe("apiserver controller tests", func() { tierWatchReady: ready, migrationWatchReady: &utils.ReadyFlag{}, opts: options.ControllerOptions{ + Extensions: testExtensions, EnterpriseCRDExists: true, DetectedProvider: operatorv1.ProviderNone, }, @@ -282,6 +284,7 @@ var _ = Describe("apiserver controller tests", func() { tierWatchReady: ready, migrationWatchReady: &utils.ReadyFlag{}, opts: options.ControllerOptions{ + Extensions: testExtensions, EnterpriseCRDExists: true, DetectedProvider: operatorv1.ProviderNone, ClusterDomain: dns.DefaultClusterDomain, @@ -307,6 +310,7 @@ var _ = Describe("apiserver controller tests", func() { tierWatchReady: ready, migrationWatchReady: &utils.ReadyFlag{}, opts: options.ControllerOptions{ + Extensions: testExtensions, EnterpriseCRDExists: true, DetectedProvider: operatorv1.ProviderNone, }, @@ -329,6 +333,7 @@ var _ = Describe("apiserver controller tests", func() { tierWatchReady: ready, migrationWatchReady: &utils.ReadyFlag{}, opts: options.ControllerOptions{ + Extensions: testExtensions, EnterpriseCRDExists: true, DetectedProvider: operatorv1.ProviderNone, }, @@ -353,6 +358,7 @@ var _ = Describe("apiserver controller tests", func() { tierWatchReady: ready, migrationWatchReady: &utils.ReadyFlag{}, opts: options.ControllerOptions{ + Extensions: testExtensions, EnterpriseCRDExists: true, DetectedProvider: operatorv1.ProviderNone, }, @@ -375,6 +381,7 @@ var _ = Describe("apiserver controller tests", func() { tierWatchReady: notReady, migrationWatchReady: &utils.ReadyFlag{}, opts: options.ControllerOptions{ + Extensions: testExtensions, EnterpriseCRDExists: true, DetectedProvider: operatorv1.ProviderNone, }, @@ -400,6 +407,7 @@ var _ = Describe("apiserver controller tests", func() { tierWatchReady: ready, migrationWatchReady: &utils.ReadyFlag{}, opts: options.ControllerOptions{ + Extensions: testExtensions, EnterpriseCRDExists: true, DetectedProvider: operatorv1.ProviderNone, }, @@ -427,6 +435,7 @@ var _ = Describe("apiserver controller tests", func() { tierWatchReady: ready, migrationWatchReady: &utils.ReadyFlag{}, opts: options.ControllerOptions{ + Extensions: testExtensions, EnterpriseCRDExists: true, DetectedProvider: operatorv1.ProviderNone, }, @@ -452,6 +461,7 @@ var _ = Describe("apiserver controller tests", func() { tierWatchReady: notReady, migrationWatchReady: &utils.ReadyFlag{}, opts: options.ControllerOptions{ + Extensions: testExtensions, EnterpriseCRDExists: true, DetectedProvider: operatorv1.ProviderNone, }, @@ -478,6 +488,7 @@ var _ = Describe("apiserver controller tests", func() { tierWatchReady: ready, migrationWatchReady: &utils.ReadyFlag{}, opts: options.ControllerOptions{ + Extensions: testExtensions, EnterpriseCRDExists: false, DetectedProvider: operatorv1.ProviderNone, }, @@ -520,6 +531,7 @@ var _ = Describe("apiserver controller tests", func() { tierWatchReady: ready, migrationWatchReady: &utils.ReadyFlag{}, opts: options.ControllerOptions{ + Extensions: testExtensions, EnterpriseCRDExists: true, DetectedProvider: operatorv1.ProviderNone, }, @@ -552,6 +564,7 @@ var _ = Describe("apiserver controller tests", func() { tierWatchReady: ready, migrationWatchReady: &utils.ReadyFlag{}, opts: options.ControllerOptions{ + Extensions: testExtensions, EnterpriseCRDExists: true, DetectedProvider: operatorv1.ProviderNone, }, @@ -604,6 +617,7 @@ var _ = Describe("apiserver controller tests", func() { tierWatchReady: ready, migrationWatchReady: &utils.ReadyFlag{}, opts: options.ControllerOptions{ + Extensions: testExtensions, EnterpriseCRDExists: true, DetectedProvider: operatorv1.ProviderNone, }, @@ -673,6 +687,7 @@ var _ = Describe("apiserver controller tests", func() { tierWatchReady: ready, migrationWatchReady: &utils.ReadyFlag{}, opts: options.ControllerOptions{ + Extensions: testExtensions, EnterpriseCRDExists: true, DetectedProvider: operatorv1.ProviderNone, }, @@ -777,6 +792,7 @@ var _ = Describe("apiserver controller tests", func() { tierWatchReady: ready, migrationWatchReady: &utils.ReadyFlag{}, opts: options.ControllerOptions{ + Extensions: testExtensions, EnterpriseCRDExists: true, DetectedProvider: operatorv1.ProviderNone, }, @@ -806,6 +822,7 @@ var _ = Describe("apiserver controller tests", func() { tierWatchReady: ready, migrationWatchReady: &utils.ReadyFlag{}, opts: options.ControllerOptions{ + Extensions: testExtensions, EnterpriseCRDExists: true, DetectedProvider: operatorv1.ProviderNone, }, @@ -836,6 +853,7 @@ var _ = Describe("apiserver controller tests", func() { tierWatchReady: ready, migrationWatchReady: &utils.ReadyFlag{}, opts: options.ControllerOptions{ + Extensions: testExtensions, EnterpriseCRDExists: true, DetectedProvider: operatorv1.ProviderNone, MultiTenant: true, @@ -883,6 +901,7 @@ var _ = Describe("apiserver controller tests", func() { tierWatchReady: ready, migrationWatchReady: &utils.ReadyFlag{}, opts: options.ControllerOptions{ + Extensions: testExtensions, EnterpriseCRDExists: true, DetectedProvider: operatorv1.ProviderNone, UseV3CRDs: true, @@ -927,6 +946,7 @@ var _ = Describe("apiserver controller tests", func() { tierWatchReady: ready, migrationWatchReady: &utils.ReadyFlag{}, opts: options.ControllerOptions{ + Extensions: testExtensions, EnterpriseCRDExists: false, DetectedProvider: operatorv1.ProviderNone, UseV3CRDs: true, @@ -955,6 +975,7 @@ var _ = Describe("apiserver controller tests", func() { tierWatchReady: ready, migrationWatchReady: &utils.ReadyFlag{}, opts: options.ControllerOptions{ + Extensions: testExtensions, EnterpriseCRDExists: true, DetectedProvider: operatorv1.ProviderNone, UseV3CRDs: false, diff --git a/pkg/controller/apiserver/apiserver_suite_test.go b/pkg/controller/apiserver/apiserver_suite_test.go index 17f3e98c8d..2e1863335b 100644 --- a/pkg/controller/apiserver/apiserver_suite_test.go +++ b/pkg/controller/apiserver/apiserver_suite_test.go @@ -26,12 +26,16 @@ import ( "sigs.k8s.io/controller-runtime/pkg/log/zap" "github.com/tigera/operator/pkg/enterprise" + "github.com/tigera/operator/pkg/extensions" ) +// testExtensions is the enterprise extension Set the API server controller tests +// reconcile with, so the componentHandler applies the API server modifier (query +// server, audit logging, Enterprise RBAC). Reconcilers built in these tests put +// it on their options, mirroring how main wires it in production. +var testExtensions *extensions.Set = enterprise.New() + func TestStatus(t *testing.T) { - // Wire the enterprise extensions so the componentHandler applies the API server - // modifier (query server, audit logging, Enterprise RBAC) during reconciliation. - enterprise.Register() logf.SetLogger(zap.New(zap.WriteTo(ginkgo.GinkgoWriter), zap.UseDevMode(true), zap.Level(uzap.NewAtomicLevelAt(uzap.DebugLevel)))) gomega.RegisterFailHandler(ginkgo.Fail) suiteConfig, reporterConfig := ginkgo.GinkgoConfiguration() diff --git a/pkg/controller/clusterconnection/clusterconnection_controller.go b/pkg/controller/clusterconnection/clusterconnection_controller.go index d796e01638..763743789f 100644 --- a/pkg/controller/clusterconnection/clusterconnection_controller.go +++ b/pkg/controller/clusterconnection/clusterconnection_controller.go @@ -177,6 +177,7 @@ func newReconciler( clusterDomain: opts.ClusterDomain, tierWatchReady: tierWatchReady, clusterInfoWatchReady: clusterInfoWatchReady, + opts: opts, } c.status.Run(opts.ShutdownContext) return c @@ -196,6 +197,7 @@ type ReconcileConnection struct { clusterInfoWatchReady *utils.ReadyFlag resolvedPodProxies []*httpproxy.Config lastAvailabilityTransition metav1.Time + opts options.ControllerOptions } // Reconcile reads that state of the cluster for a ManagementClusterConnection object and makes changes based on the @@ -450,6 +452,7 @@ func (r *ReconcileConnection) Reconcile(ctx context.Context, request reconcile.R r.scheme, managementClusterConnection, utils.WithRenderContext(extensions.RenderContext{Installation: installationSpec}), + utils.WithExtensions(r.opts.Extensions), ) guardianCfg := &render.GuardianConfiguration{ URL: managementClusterConnection.Spec.ManagementClusterAddr, diff --git a/pkg/controller/clusterconnection/clusterconnection_suite_test.go b/pkg/controller/clusterconnection/clusterconnection_suite_test.go index 09a8254690..2e6b6feb43 100644 --- a/pkg/controller/clusterconnection/clusterconnection_suite_test.go +++ b/pkg/controller/clusterconnection/clusterconnection_suite_test.go @@ -22,18 +22,12 @@ import ( "sigs.k8s.io/controller-runtime/pkg/log/zap" logf "sigs.k8s.io/controller-runtime/pkg/log" - - "github.com/tigera/operator/pkg/enterprise" ) func TestStatus(t *testing.T) { logf.SetLogger(zap.New(zap.WriteTo(ginkgo.GinkgoWriter))) gomega.RegisterFailHandler(ginkgo.Fail) - // Wire the enterprise extensions so the guardian modifier runs the way it - // does in the operator binary, which is what these controller tests exercise. - enterprise.Register() - suiteConfig, reporterConfig := ginkgo.GinkgoConfiguration() reporterConfig.JUnitReport = "../../../report/ut/clusterconnection_controller_suite.xml" ginkgo.RunSpecs(t, "pkg/controller/Management Cluster Connection Suite", suiteConfig, reporterConfig) diff --git a/pkg/controller/clusterconnection/shim_test.go b/pkg/controller/clusterconnection/shim_test.go index 7446d229ac..bd5993478d 100644 --- a/pkg/controller/clusterconnection/shim_test.go +++ b/pkg/controller/clusterconnection/shim_test.go @@ -25,6 +25,7 @@ import ( operatorv1 "github.com/tigera/operator/api/v1" "github.com/tigera/operator/pkg/controller/options" "github.com/tigera/operator/pkg/controller/status" + "github.com/tigera/operator/pkg/enterprise" "k8s.io/apimachinery/pkg/runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/reconcile" @@ -40,6 +41,7 @@ func NewReconcilerWithShims( ) reconcile.Reconciler { opts := options.ControllerOptions{ ShutdownContext: context.Background(), + Extensions: enterprise.New(), } return newReconciler(cli, schema, status, provider, tierWatchReady, clusterInfoWatchReady, opts) diff --git a/pkg/controller/installation/core_controller.go b/pkg/controller/installation/core_controller.go index d760aed637..1f241da0a8 100644 --- a/pkg/controller/installation/core_controller.go +++ b/pkg/controller/installation/core_controller.go @@ -355,6 +355,7 @@ func newReconciler(mgr manager.Manager, opts options.ControllerOptions) (*Reconc v3CRDs: opts.UseV3CRDs, kubernetesVersion: opts.KubernetesVersion, apiDiscovery: opts.APIDiscovery, + opts: opts, } r.status.Run(opts.ShutdownContext) r.typhaAutoscaler.start(opts.ShutdownContext) @@ -414,6 +415,7 @@ type ReconcileInstallation struct { v3CRDs bool kubernetesVersion *common.VersionInfo apiDiscovery *discovery.APIDiscovery + opts options.ControllerOptions // newComponentHandler returns a new component handler. Useful stub for unit testing. newComponentHandler func(log logr.Logger, client client.Client, scheme *runtime.Scheme, cr metav1.Object, opts ...utils.ComponentHandlerOption) utils.ComponentHandler @@ -1226,7 +1228,7 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile calicoVersion = components.EnterpriseRelease } - renderCtx, err := extensions.BuildContext(extensions.Inputs{ + renderCtx, err := r.opts.Extensions.BuildContext(extensions.Inputs{ Ctx: ctx, Client: r.client, Installation: &instance.Spec, @@ -1283,6 +1285,7 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile r.scheme, instance, utils.WithRenderContext(renderCtx), + utils.WithExtensions(r.opts.Extensions), ) // Render namespaces first - this ensures that any other controllers blocked on namespace existence can proceed. @@ -1576,6 +1579,7 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile FelixHealthPort: *felixConfiguration.Spec.HealthPort, NodeCgroupV2Path: felixConfiguration.Spec.CgroupV2Path, V3CRDs: r.v3CRDs, + ImageOverrides: r.opts.Extensions.Images(), } if bgpConfiguration.Spec.BindMode != nil { diff --git a/pkg/controller/installation/core_controller_test.go b/pkg/controller/installation/core_controller_test.go index 39e171090a..d2e73fe0ad 100644 --- a/pkg/controller/installation/core_controller_test.go +++ b/pkg/controller/installation/core_controller_test.go @@ -56,6 +56,7 @@ import ( "github.com/tigera/operator/pkg/common/discovery" "github.com/tigera/operator/pkg/components" "github.com/tigera/operator/pkg/controller/certificatemanager" + "github.com/tigera/operator/pkg/controller/options" "github.com/tigera/operator/pkg/controller/status" "github.com/tigera/operator/pkg/controller/utils" ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" @@ -190,6 +191,7 @@ var _ = Describe("Testing core-controller installation", func() { // As the parameters in the client changes, we expect the outcomes of the reconcile loops to change. r = ReconcileInstallation{ + opts: options.ControllerOptions{Extensions: testExtensions}, config: nil, // there is no fake for config client: c, scheme: scheme, @@ -819,6 +821,7 @@ var _ = Describe("Testing core-controller installation", func() { // As the parameters in the client changes, we expect the outcomes of the reconcile loops to change. r = ReconcileInstallation{ + opts: options.ControllerOptions{Extensions: testExtensions}, config: nil, // there is no fake for config client: c, scheme: scheme, @@ -1041,6 +1044,7 @@ var _ = Describe("Testing core-controller installation", func() { // As the parameters in the client changes, we expect the outcomes of the reconcile loops to change. r = ReconcileInstallation{ + opts: options.ControllerOptions{Extensions: testExtensions}, config: nil, // there is no fake for config client: c, scheme: scheme, @@ -2329,6 +2333,7 @@ var _ = Describe("Testing core-controller installation", func() { // As the parameters in the client changes, we expect the outcomes of the reconcile loops to change. r = ReconcileInstallation{ + opts: options.ControllerOptions{Extensions: testExtensions}, config: nil, // there is no fake for config client: c, scheme: scheme, @@ -2466,6 +2471,7 @@ var _ = Describe("Testing core-controller installation", func() { componentHandler = newFakeComponentHandler() r = ReconcileInstallation{ + opts: options.ControllerOptions{Extensions: testExtensions}, config: nil, // there is no fake for config client: c, scheme: scheme, @@ -2629,6 +2635,7 @@ var _ = Describe("updateMutatingAdmissionPolicies", func() { It("should create v1 MAPs when v1 is served", func() { r = ReconcileInstallation{ + opts: options.ControllerOptions{Extensions: testExtensions}, client: clientFor(), scheme: scheme, status: mockStatus, @@ -2660,6 +2667,7 @@ var _ = Describe("updateMutatingAdmissionPolicies", func() { It("should create v1beta1 MAPs when only v1beta1 is served", func() { r = ReconcileInstallation{ + opts: options.ControllerOptions{Extensions: testExtensions}, client: clientFor(), scheme: scheme, status: mockStatus, @@ -2689,6 +2697,7 @@ var _ = Describe("updateMutatingAdmissionPolicies", func() { It("should create v1alpha1 MAPs when only v1alpha1 is served", func() { r = ReconcileInstallation{ + opts: options.ControllerOptions{Extensions: testExtensions}, client: clientFor(), scheme: scheme, status: mockStatus, @@ -2718,6 +2727,7 @@ var _ = Describe("updateMutatingAdmissionPolicies", func() { It("should not create MAPs when no served version exists and should set degraded", func() { r = ReconcileInstallation{ + opts: options.ControllerOptions{Extensions: testExtensions}, client: clientFor(), scheme: scheme, status: mockStatus, @@ -2736,6 +2746,7 @@ var _ = Describe("updateMutatingAdmissionPolicies", func() { It("should not create MAPs when v3CRDs=false", func() { r = ReconcileInstallation{ + opts: options.ControllerOptions{Extensions: testExtensions}, client: clientFor(), scheme: scheme, status: mockStatus, @@ -2753,6 +2764,7 @@ var _ = Describe("updateMutatingAdmissionPolicies", func() { It("should not create MAPs when manageCRDs=false", func() { r = ReconcileInstallation{ + opts: options.ControllerOptions{Extensions: testExtensions}, client: clientFor(), scheme: scheme, status: mockStatus, @@ -2783,6 +2795,7 @@ var _ = Describe("updateMutatingAdmissionPolicies", func() { } r = ReconcileInstallation{ + opts: options.ControllerOptions{Extensions: testExtensions}, client: clientFor(staleMAP, staleMAPB), scheme: scheme, status: mockStatus, @@ -2825,6 +2838,7 @@ var _ = Describe("updateMutatingAdmissionPolicies", func() { } r = ReconcileInstallation{ + opts: options.ControllerOptions{Extensions: testExtensions}, client: clientFor(initial...), scheme: scheme, status: mockStatus, @@ -2843,6 +2857,7 @@ var _ = Describe("updateMutatingAdmissionPolicies", func() { It("should work with Enterprise variant", func() { r = ReconcileInstallation{ + opts: options.ControllerOptions{Extensions: testExtensions}, client: clientFor(), scheme: scheme, status: mockStatus, @@ -2914,6 +2929,7 @@ var _ = Describe("updateValidatingAdmissionPolicies", func() { It("should create v1 VAPs when v1 is served", func() { r = ReconcileInstallation{ + opts: options.ControllerOptions{Extensions: testExtensions}, client: clientFor(), scheme: scheme, status: mockStatus, @@ -2945,6 +2961,7 @@ var _ = Describe("updateValidatingAdmissionPolicies", func() { It("should create v1beta1 VAPs when only v1beta1 is served", func() { r = ReconcileInstallation{ + opts: options.ControllerOptions{Extensions: testExtensions}, client: clientFor(), scheme: scheme, status: mockStatus, @@ -2974,6 +2991,7 @@ var _ = Describe("updateValidatingAdmissionPolicies", func() { It("should create v1alpha1 VAPs when only v1alpha1 is served", func() { r = ReconcileInstallation{ + opts: options.ControllerOptions{Extensions: testExtensions}, client: clientFor(), scheme: scheme, status: mockStatus, @@ -2991,6 +3009,7 @@ var _ = Describe("updateValidatingAdmissionPolicies", func() { It("should skip without degrading when no served version exists", func() { r = ReconcileInstallation{ + opts: options.ControllerOptions{Extensions: testExtensions}, client: clientFor(), scheme: scheme, status: mockStatus, @@ -3009,6 +3028,7 @@ var _ = Describe("updateValidatingAdmissionPolicies", func() { It("should not create VAPs when v3CRDs=false", func() { r = ReconcileInstallation{ + opts: options.ControllerOptions{Extensions: testExtensions}, client: clientFor(), scheme: scheme, status: mockStatus, @@ -3039,6 +3059,7 @@ var _ = Describe("updateValidatingAdmissionPolicies", func() { } r = ReconcileInstallation{ + opts: options.ControllerOptions{Extensions: testExtensions}, client: clientFor(staleVAP, staleVAPB), scheme: scheme, status: mockStatus, @@ -3063,6 +3084,7 @@ var _ = Describe("updateValidatingAdmissionPolicies", func() { It("should work with Enterprise variant", func() { r = ReconcileInstallation{ + opts: options.ControllerOptions{Extensions: testExtensions}, client: clientFor(), scheme: scheme, status: mockStatus, diff --git a/pkg/controller/installation/installation_controller_suite_test.go b/pkg/controller/installation/installation_controller_suite_test.go index 778ee91f8f..f589361c5a 100644 --- a/pkg/controller/installation/installation_controller_suite_test.go +++ b/pkg/controller/installation/installation_controller_suite_test.go @@ -27,11 +27,14 @@ import ( "sigs.k8s.io/controller-runtime/pkg/log/zap" "github.com/tigera/operator/pkg/enterprise" + "github.com/tigera/operator/pkg/extensions" ) -var _ = ginkgo.BeforeSuite(func() { - enterprise.Register() -}) +// testExtensions is the enterprise extension Set the installation controller +// tests reconcile with, mirroring how main wires it in production. Reconcilers +// built in these tests put it on their options so the node image overrides and +// modifiers apply. +var testExtensions *extensions.Set = enterprise.New() func TestInstallation(t *testing.T) { // Disable WatchListClient for tests. In client-go v0.35+, this feature defaults to true and diff --git a/pkg/controller/installation/windows_controller.go b/pkg/controller/installation/windows_controller.go index ca8063ffea..12e31b2a1f 100644 --- a/pkg/controller/installation/windows_controller.go +++ b/pkg/controller/installation/windows_controller.go @@ -182,6 +182,7 @@ type ReconcileWindows struct { enterpriseCRDsExist bool clusterDomain string ipamConfigWatchReady *utils.ReadyFlag + opts options.ControllerOptions } // newWindowsReconciler returns a new reconcile.Reconciler @@ -198,6 +199,7 @@ func newWindowsReconciler(mgr manager.Manager, opts options.ControllerOptions) ( enterpriseCRDsExist: opts.EnterpriseCRDExists, clusterDomain: opts.ClusterDomain, ipamConfigWatchReady: &utils.ReadyFlag{}, + opts: opts, } r.status.Run(opts.ShutdownContext) return r, nil @@ -387,6 +389,7 @@ func (r *ReconcileWindows) Reconcile(ctx context.Context, request reconcile.Requ PrometheusServerTLS: nodePrometheusTLS, NodeReporterMetricsPort: nodeReporterMetricsPort, VXLANVNI: *felixConfiguration.Spec.VXLANVNI, + ImageOverrides: r.opts.Extensions.Images(), } component = render.Windows(&windowsCfg) @@ -413,6 +416,7 @@ func (r *ReconcileWindows) Reconcile(ctx context.Context, request reconcile.Requ r.scheme, instance, utils.WithRenderContext(extensions.RenderContext{Installation: &instance.Spec}), + utils.WithExtensions(r.opts.Extensions), ) if err := handler.CreateOrUpdateOrDelete(ctx, component, nil); err != nil { r.status.SetDegraded(operatorv1.ResourceUpdateError, "Error creating / updating resource", err, reqLogger) diff --git a/pkg/controller/installation/windows_controller_test.go b/pkg/controller/installation/windows_controller_test.go index ae5866bfa5..98121210e7 100644 --- a/pkg/controller/installation/windows_controller_test.go +++ b/pkg/controller/installation/windows_controller_test.go @@ -29,6 +29,7 @@ import ( "github.com/tigera/operator/pkg/common" "github.com/tigera/operator/pkg/components" "github.com/tigera/operator/pkg/controller/certificatemanager" + "github.com/tigera/operator/pkg/controller/options" "github.com/tigera/operator/pkg/controller/status" "github.com/tigera/operator/pkg/controller/utils" ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" @@ -119,6 +120,7 @@ var _ = Describe("windows-controller installation tests", func() { // As the parameters in the client changes, we expect the outcomes of the reconcile loops to change. r = ReconcileWindows{ + opts: options.ControllerOptions{Extensions: testExtensions}, config: nil, // there is no fake for config client: c, scheme: scheme, @@ -609,6 +611,7 @@ var _ = Describe("windows-controller installation tests", func() { // As the parameters in the client changes, we expect the outcomes of the reconcile loops to change. r = ReconcileWindows{ + opts: options.ControllerOptions{Extensions: testExtensions}, config: nil, // there is no fake for config client: c, scheme: scheme, diff --git a/pkg/controller/options/options.go b/pkg/controller/options/options.go index 14acd47230..404c399378 100644 --- a/pkg/controller/options/options.go +++ b/pkg/controller/options/options.go @@ -20,6 +20,7 @@ import ( v1 "github.com/tigera/operator/api/v1" "github.com/tigera/operator/pkg/common" "github.com/tigera/operator/pkg/common/discovery" + "github.com/tigera/operator/pkg/extensions" "k8s.io/client-go/kubernetes" ) @@ -55,4 +56,10 @@ type ControllerOptions struct { // the operator cares about. Populated once at startup so controllers can branch on API // availability without issuing further discovery requests at reconcile time. APIDiscovery *discovery.APIDiscovery + + // Extensions are the variant extensions (modifiers, image overrides, setups) + // the operator runs with. The core operator leaves this nil; an extension + // build (Calico Enterprise) sets it once at startup and controllers thread it + // into their render handlers and component configs. + Extensions *extensions.Set } diff --git a/pkg/controller/utils/component.go b/pkg/controller/utils/component.go index 885ee21da3..76db4ebf44 100644 --- a/pkg/controller/utils/component.go +++ b/pkg/controller/utils/component.go @@ -84,6 +84,13 @@ func WithRenderContext(ctx extensions.RenderContext) ComponentHandlerOption { return func(c *componentHandler) { c.renderCtx = ctx } } +// WithExtensions supplies the operator's extension Set, whose modifiers the +// handler applies to extensible components. A handler that renders an +// extensible component must be given the Set; one that doesn't can omit it. +func WithExtensions(e *extensions.Set) ComponentHandlerOption { + return func(c *componentHandler) { c.extensions = e } +} + // cr is allowed to be nil in the case we don't want to put ownership on a resource, // this is useful for CRD management so that they are not removed automatically. func NewComponentHandler(log logr.Logger, cli client.Client, scheme *runtime.Scheme, cr metav1.Object, opts ...ComponentHandlerOption) ComponentHandler { @@ -108,6 +115,7 @@ type componentHandler struct { createOnly bool apiGroupEnvs []v1.EnvVar renderCtx extensions.RenderContext + extensions *extensions.Set } func (c *componentHandler) SetCreateOnly() { @@ -469,11 +477,18 @@ func (c *componentHandler) CreateOrUpdateOrDelete(ctx context.Context, component objsToCreate, objsToDelete := component.Objects() if ext, ok := component.(render.Extensible); ok { - rc := c.renderCtx - if p, ok := component.(render.ExtensionContextProvider); ok { - rc.Component = p.ExtensionContext() + if c.extensions == nil { + // The component can be extended but this handler was built without an + // extension Set, so any registered modifier silently won't run. That is + // a wiring bug in the controller, not a normal state. + c.log.Info("BUG: extensible component rendered by a handler with no extension Set; modifiers will not be applied", "component", ext.ModifierKey()) + } else { + rc := c.renderCtx + if p, ok := component.(render.ExtensionContextProvider); ok { + rc.Component = p.ExtensionContext() + } + objsToCreate, objsToDelete = c.extensions.ApplyModifiers(ext.ModifierKey(), rc, objsToCreate, objsToDelete) } - objsToCreate, objsToDelete = extensions.ApplyModifiers(ext.ModifierKey(), rc, objsToCreate, objsToDelete) } // Load the InstallationSpec once and reuse it for every object: createOrUpdateObject needs it diff --git a/pkg/controller/utils/component_enterprise_test.go b/pkg/controller/utils/component_enterprise_test.go index a59e1394b6..12d63fa44c 100644 --- a/pkg/controller/utils/component_enterprise_test.go +++ b/pkg/controller/utils/component_enterprise_test.go @@ -43,9 +43,6 @@ import ( // registered modifier must match the real render output by name. If render ever // renames the typha ClusterRole, the modifier silently no-ops and this fails. var _ = Describe("componentHandler enterprise modifier integration", func() { - BeforeEach(func() { enterprise.Register() }) - AfterEach(func() { extensions.ResetForTest() }) - It("applies the enterprise typha modifier to real render output", func() { scheme := runtime.NewScheme() Expect(apis.AddToScheme(scheme, false)).NotTo(HaveOccurred()) @@ -77,7 +74,7 @@ var _ = Describe("componentHandler enterprise modifier integration", func() { }) renderCtx := extensions.RenderContext{Installation: instance} - handler := utils.NewComponentHandler(logf.Log, cli, scheme, nil, utils.WithRenderContext(renderCtx)) + handler := utils.NewComponentHandler(logf.Log, cli, scheme, nil, utils.WithRenderContext(renderCtx), utils.WithExtensions(enterprise.New())) Expect(handler.CreateOrUpdateOrDelete(context.Background(), comp, nil)).NotTo(HaveOccurred()) role := &rbacv1.ClusterRole{} diff --git a/pkg/controller/utils/component_test.go b/pkg/controller/utils/component_test.go index 9790fd5939..d8c227d47b 100644 --- a/pkg/controller/utils/component_test.go +++ b/pkg/controller/utils/component_test.go @@ -2578,12 +2578,9 @@ func (mc *mockClient) SubResource(subResource string) client.SubResourceClient { } var _ = Describe("componentHandler modifier application", func() { - AfterEach(func() { - extensions.ResetForTest() - }) - It("applies registered modifiers to a named component before create", func() { - extensions.Register(operatorv1.CalicoEnterprise, "fake", extensions.Extension{ + ext := extensions.NewSet() + ext.Register(operatorv1.CalicoEnterprise, "fake", extensions.Extension{ Modify: func(ctx extensions.RenderContext, objs, del []client.Object) ([]client.Object, []client.Object) { cm := objs[0].(*corev1.ConfigMap) cm.Data = map[string]string{"patched": "yes"} @@ -2597,7 +2594,7 @@ var _ = Describe("componentHandler modifier application", func() { c := ctrlrfake.DefaultFakeClientBuilder(s).Build() renderCtx := extensions.RenderContext{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.CalicoEnterprise}} - handler := NewComponentHandler(logf.Log, c, s, nil, WithRenderContext(renderCtx)) + handler := NewComponentHandler(logf.Log, c, s, nil, WithRenderContext(renderCtx), WithExtensions(ext)) comp := &namedFakeComponent{name: "fake", obj: &corev1.ConfigMap{ TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"}, ObjectMeta: metav1.ObjectMeta{Name: "cm", Namespace: "default"}, diff --git a/pkg/enterprise/apiserver.go b/pkg/enterprise/apiserver.go index cba83f2bfc..1b7119f239 100644 --- a/pkg/enterprise/apiserver.go +++ b/pkg/enterprise/apiserver.go @@ -52,13 +52,13 @@ type apiServer struct { calicoImage string } -func registerAPIServer() { - extensions.Register(operatorv1.CalicoEnterprise, render.ComponentNameAPIServer, extensions.Extension{ +func registerAPIServer(s *extensions.Set) { + s.Register(operatorv1.CalicoEnterprise, render.ComponentNameAPIServer, extensions.Extension{ Modify: modifyAPIServer, }) // When running Calico, clean up any Enterprise objects left behind by a prior // Enterprise installation. - extensions.Register(operatorv1.Calico, render.ComponentNameAPIServer, extensions.Extension{ + s.Register(operatorv1.Calico, render.ComponentNameAPIServer, extensions.Extension{ Modify: cleanupAPIServer, }) } diff --git a/pkg/enterprise/enterprise_suite_test.go b/pkg/enterprise/enterprise_suite_test.go index e7cfab281b..368107b5a0 100644 --- a/pkg/enterprise/enterprise_suite_test.go +++ b/pkg/enterprise/enterprise_suite_test.go @@ -19,8 +19,15 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + + "github.com/tigera/operator/pkg/enterprise" + "github.com/tigera/operator/pkg/extensions" ) +// ext is the enterprise extension Set under test, shared across the suite. It is +// immutable once built and the specs only read it, so a single instance is safe. +var ext *extensions.Set = enterprise.New() + func TestEnterprise(t *testing.T) { RegisterFailHandler(Fail) RunSpecs(t, "pkg/enterprise Suite") diff --git a/pkg/enterprise/guardian.go b/pkg/enterprise/guardian.go index 0187166f61..cff8185428 100644 --- a/pkg/enterprise/guardian.go +++ b/pkg/enterprise/guardian.go @@ -38,11 +38,11 @@ import ( operatorurl "github.com/tigera/operator/pkg/url" ) -func registerGuardian() { - extensions.Register(operatorv1.CalicoEnterprise, render.GuardianName, extensions.Extension{ +func registerGuardian(s *extensions.Set) { + s.Register(operatorv1.CalicoEnterprise, render.GuardianName, extensions.Extension{ Modify: modifyGuardian, }) - extensions.Register(operatorv1.CalicoEnterprise, render.ComponentNameGuardianPolicy, extensions.Extension{ + s.Register(operatorv1.CalicoEnterprise, render.ComponentNameGuardianPolicy, extensions.Extension{ Modify: modifyGuardianPolicy, }) } diff --git a/pkg/enterprise/guardian_test.go b/pkg/enterprise/guardian_test.go index 9b43423199..5223f72a47 100644 --- a/pkg/enterprise/guardian_test.go +++ b/pkg/enterprise/guardian_test.go @@ -26,14 +26,11 @@ import ( v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" operatorv1 "github.com/tigera/operator/api/v1" - "github.com/tigera/operator/pkg/enterprise" "github.com/tigera/operator/pkg/extensions" "github.com/tigera/operator/pkg/render" ) var _ = Describe("guardian enterprise modifier", func() { - BeforeEach(func() { enterprise.Register() }) - AfterEach(func() { extensions.ResetForTest() }) // newObjs returns the subset of rendered guardian objects the modifier touches. newObjs := func() []client.Object { @@ -60,7 +57,7 @@ var _ = Describe("guardian enterprise modifier", func() { } It("appends the secrets RBAC and UI settings", func() { - out, _ := extensions.ApplyModifiers(render.GuardianName, ctxWith(render.GuardianExtensionContext{}), newObjs(), nil) + out, _ := ext.ApplyModifiers(render.GuardianName, ctxWith(render.GuardianExtensionContext{}), newObjs(), nil) _, ok := extensions.FindObject[*rbacv1.Role](out, render.GuardianSecretsRole) Expect(ok).To(BeTrue()) _, ok = extensions.FindObject[*rbacv1.RoleBinding](out, render.GuardianSecretsRoleBindingName) @@ -70,7 +67,7 @@ var _ = Describe("guardian enterprise modifier", func() { }) It("adds the elasticsearch and kibana service ports", func() { - out, _ := extensions.ApplyModifiers(render.GuardianName, ctxWith(render.GuardianExtensionContext{}), newObjs(), nil) + out, _ := ext.ApplyModifiers(render.GuardianName, ctxWith(render.GuardianExtensionContext{}), newObjs(), nil) svc, _ := extensions.FindObject[*corev1.Service](out, render.GuardianServiceName) names := []string{} for _, p := range svc.Spec.Ports { @@ -83,7 +80,7 @@ var _ = Describe("guardian enterprise modifier", func() { gc := render.GuardianExtensionContext{ Impersonation: &operatorv1.Impersonation{Users: []string{"foo"}, Groups: []string{"bar"}}, } - out, _ := extensions.ApplyModifiers(render.GuardianName, ctxWith(gc), newObjs(), nil) + out, _ := ext.ApplyModifiers(render.GuardianName, ctxWith(gc), newObjs(), nil) role, _ := extensions.FindObject[*rbacv1.ClusterRole](out, render.GuardianClusterRoleName) // The single OSS placeholder rule is gone, replaced by the enterprise set. @@ -94,14 +91,14 @@ var _ = Describe("guardian enterprise modifier", func() { It("adds the CA bundle env to the guardian container", func() { gc := render.GuardianExtensionContext{TrustedBundleMountPath: "/ca/bundle"} - out, _ := extensions.ApplyModifiers(render.GuardianName, ctxWith(gc), newObjs(), nil) + out, _ := ext.ApplyModifiers(render.GuardianName, ctxWith(gc), newObjs(), nil) dep, _ := extensions.FindObject[*appsv1.Deployment](out, render.GuardianDeploymentName) Expect(dep.Spec.Template.Spec.Containers[0].Env).To(ContainElement(corev1.EnvVar{Name: "GUARDIAN_PROMETHEUS_CA_BUNDLE_PATH", Value: "/ca/bundle"})) }) It("does nothing for the Calico variant", func() { ctx := extensions.RenderContext{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.Calico}} - out, _ := extensions.ApplyModifiers(render.GuardianName, ctx, newObjs(), nil) + out, _ := ext.ApplyModifiers(render.GuardianName, ctx, newObjs(), nil) Expect(out).To(HaveLen(len(newObjs()))) role, _ := extensions.FindObject[*rbacv1.ClusterRole](out, render.GuardianClusterRoleName) Expect(role.Rules).To(Equal([]rbacv1.PolicyRule{{Verbs: []string{"get"}}})) diff --git a/pkg/enterprise/installation.go b/pkg/enterprise/installation.go index ffb0dd7c98..30f1c96341 100644 --- a/pkg/enterprise/installation.go +++ b/pkg/enterprise/installation.go @@ -27,8 +27,8 @@ import ( "github.com/tigera/operator/pkg/render/monitor" ) -func registerInstallation() { - extensions.RegisterSetup(operatorv1.CalicoEnterprise, setup) +func registerInstallation(s *extensions.Set) { + s.RegisterSetup(operatorv1.CalicoEnterprise, setup) } // setup is the Calico Enterprise setup phase. It builds the base render context diff --git a/pkg/enterprise/installation_test.go b/pkg/enterprise/installation_test.go index cfab0176bb..bc704a2429 100644 --- a/pkg/enterprise/installation_test.go +++ b/pkg/enterprise/installation_test.go @@ -28,13 +28,10 @@ import ( "github.com/tigera/operator/pkg/common" "github.com/tigera/operator/pkg/controller/certificatemanager" ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" - "github.com/tigera/operator/pkg/enterprise" "github.com/tigera/operator/pkg/extensions" ) var _ = Describe("installation setup", func() { - BeforeEach(func() { enterprise.Register() }) - AfterEach(func() { extensions.ResetForTest() }) It("rejects a zero prometheus reporter port", func() { port := 0 @@ -42,18 +39,18 @@ var _ = Describe("installation setup", func() { in.FelixConfiguration = &v3.FelixConfiguration{ Spec: v3.FelixConfigurationSpec{PrometheusReporterPort: &port}, } - _, err := extensions.BuildContext(in) + _, err := ext.BuildContext(in) Expect(err).To(HaveOccurred()) }) It("creates the node prometheus keypair for the enterprise variant", func() { - rc, err := extensions.BuildContext(newInputs(operatorv1.CalicoEnterprise)) + rc, err := ext.BuildContext(newInputs(operatorv1.CalicoEnterprise)) Expect(err).NotTo(HaveOccurred()) Expect(rc.NodePrometheusTLS).NotTo(BeNil()) }) It("is a no-op for the Calico variant", func() { - rc, err := extensions.BuildContext(newInputs(operatorv1.Calico)) + rc, err := ext.BuildContext(newInputs(operatorv1.Calico)) Expect(err).NotTo(HaveOccurred()) Expect(rc.NodePrometheusTLS).To(BeNil()) }) diff --git a/pkg/enterprise/node.go b/pkg/enterprise/node.go index 39b6ff69c2..752155bd01 100644 --- a/pkg/enterprise/node.go +++ b/pkg/enterprise/node.go @@ -46,8 +46,8 @@ const ( installCNIContainerName = "install-cni" ) -func registerNode() { - extensions.Register(operatorv1.CalicoEnterprise, render.ComponentNameNode, extensions.Extension{ +func registerNode(s *extensions.Set) { + s.Register(operatorv1.CalicoEnterprise, render.ComponentNameNode, extensions.Extension{ Image: func(in *operatorv1.InstallationSpec) components.Component { return components.ComponentTigeraNode }, @@ -56,7 +56,7 @@ func registerNode() { // The node component renders the cni-plugins init container; its image // resolves through its own override key. - extensions.Register(operatorv1.CalicoEnterprise, render.ComponentNameCNIPlugins, extensions.Extension{ + s.Register(operatorv1.CalicoEnterprise, render.ComponentNameCNIPlugins, extensions.Extension{ Image: func(in *operatorv1.InstallationSpec) components.Component { return components.ComponentTigeraCNIPlugins }, diff --git a/pkg/enterprise/node_test.go b/pkg/enterprise/node_test.go index 22a5408754..23ca58c98e 100644 --- a/pkg/enterprise/node_test.go +++ b/pkg/enterprise/node_test.go @@ -28,29 +28,24 @@ import ( operatorv1 "github.com/tigera/operator/api/v1" "github.com/tigera/operator/pkg/common" "github.com/tigera/operator/pkg/components" - "github.com/tigera/operator/pkg/enterprise" "github.com/tigera/operator/pkg/extensions" "github.com/tigera/operator/pkg/render" ) var _ = Describe("node enterprise image override", func() { - BeforeEach(func() { enterprise.Register() }) - AfterEach(func() { extensions.ResetForTest() }) It("selects the enterprise node image for the enterprise variant", func() { ent := &operatorv1.InstallationSpec{Variant: operatorv1.CalicoEnterprise} - Expect(extensions.ResolveImage("node", components.ComponentCalicoNode, ent)).To(Equal(components.ComponentTigeraNode)) + Expect(ext.ResolveImage("node", components.ComponentCalicoNode, ent)).To(Equal(components.ComponentTigeraNode)) }) It("leaves the default in place for the Calico variant", func() { calico := &operatorv1.InstallationSpec{Variant: operatorv1.Calico} - Expect(extensions.ResolveImage("node", components.ComponentCalicoNode, calico)).To(Equal(components.ComponentCalicoNode)) + Expect(ext.ResolveImage("node", components.ComponentCalicoNode, calico)).To(Equal(components.ComponentCalicoNode)) }) }) var _ = Describe("node enterprise modifier", func() { - BeforeEach(func() { enterprise.Register() }) - AfterEach(func() { extensions.ResetForTest() }) // newObjs returns the subset of rendered node objects the modifier touches. newObjs := func() []client.Object { @@ -86,7 +81,7 @@ var _ = Describe("node enterprise modifier", func() { } It("adds the enterprise cluster role rules", func() { - out, _ := extensions.ApplyModifiers(render.ComponentNameNode, entCtx(), newObjs(), nil) + out, _ := ext.ApplyModifiers(render.ComponentNameNode, entCtx(), newObjs(), nil) nodeRole, ok := extensions.FindObject[*rbacv1.ClusterRole](out, render.CalicoNodeObjectName) Expect(ok).To(BeTrue()) @@ -98,7 +93,7 @@ var _ = Describe("node enterprise modifier", func() { }) It("adds the enterprise felix env to the node container", func() { - out, _ := extensions.ApplyModifiers(render.ComponentNameNode, entCtx(), newObjs(), nil) + out, _ := ext.ApplyModifiers(render.ComponentNameNode, entCtx(), newObjs(), nil) ds, _ := extensions.FindObject[*appsv1.DaemonSet](out, common.NodeDaemonSetName) c := nodeContainer(ds) @@ -115,13 +110,13 @@ var _ = Describe("node enterprise modifier", func() { ctx := entCtx() ctx.FelixConfiguration = &v3.FelixConfiguration{Spec: v3.FelixConfigurationSpec{PrometheusReporterPort: &reporter}} - out, _ := extensions.ApplyModifiers(render.ComponentNameNode, ctx, newObjs(), nil) + out, _ := ext.ApplyModifiers(render.ComponentNameNode, ctx, newObjs(), nil) ds, _ := extensions.FindObject[*appsv1.DaemonSet](out, common.NodeDaemonSetName) Expect(nodeContainer(ds).Env).To(ContainElement(corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERPORT", Value: "7081"})) }) It("appends the BGP metrics readiness check when the bird check is present", func() { - out, _ := extensions.ApplyModifiers(render.ComponentNameNode, entCtx(), newObjs(), nil) + out, _ := ext.ApplyModifiers(render.ComponentNameNode, entCtx(), newObjs(), nil) ds, _ := extensions.FindObject[*appsv1.DaemonSet](out, common.NodeDaemonSetName) Expect(nodeContainer(ds).ReadinessProbe.Exec.Command).To(ContainElement("--bgp-metrics-ready")) }) @@ -131,7 +126,7 @@ var _ = Describe("node enterprise modifier", func() { ds := objs[2].(*appsv1.DaemonSet) ds.Spec.Template.Spec.Containers[0].ReadinessProbe.Exec.Command = []string{"/bin/calico-node", "--felix-ready"} - out, _ := extensions.ApplyModifiers(render.ComponentNameNode, entCtx(), objs, nil) + out, _ := ext.ApplyModifiers(render.ComponentNameNode, entCtx(), objs, nil) got, _ := extensions.FindObject[*appsv1.DaemonSet](out, common.NodeDaemonSetName) Expect(nodeContainer(got).ReadinessProbe.Exec.Command).NotTo(ContainElement("--bgp-metrics-ready")) }) @@ -141,7 +136,7 @@ var _ = Describe("node enterprise modifier", func() { ctx := entCtx() ctx.Installation.CalicoNetwork = &operatorv1.CalicoNetworkSpec{MultiInterfaceMode: &mode} - out, _ := extensions.ApplyModifiers(render.ComponentNameNode, ctx, newObjs(), nil) + out, _ := ext.ApplyModifiers(render.ComponentNameNode, ctx, newObjs(), nil) ds, _ := extensions.FindObject[*appsv1.DaemonSet](out, common.NodeDaemonSetName) want := corev1.EnvVar{Name: "MULTI_INTERFACE_MODE", Value: mode.Value()} @@ -150,7 +145,7 @@ var _ = Describe("node enterprise modifier", func() { }) It("appends the node metrics service", func() { - out, _ := extensions.ApplyModifiers(render.ComponentNameNode, entCtx(), newObjs(), nil) + out, _ := ext.ApplyModifiers(render.ComponentNameNode, entCtx(), newObjs(), nil) svc, ok := extensions.FindObject[*corev1.Service](out, render.CalicoNodeMetricsService) Expect(ok).To(BeTrue()) Expect(svc.Spec.Ports).To(HaveLen(2)) @@ -169,7 +164,7 @@ var _ = Describe("node enterprise modifier", func() { PrometheusMetricsEnabled: &enabled, }} - out, _ := extensions.ApplyModifiers(render.ComponentNameNode, ctx, newObjs(), nil) + out, _ := ext.ApplyModifiers(render.ComponentNameNode, ctx, newObjs(), nil) svc, _ := extensions.FindObject[*corev1.Service](out, render.CalicoNodeMetricsService) Expect(svc.Spec.Ports).To(HaveLen(3)) Expect(svc.Spec.Ports[0].Port).To(Equal(int32(7081))) @@ -179,7 +174,7 @@ var _ = Describe("node enterprise modifier", func() { It("is a no-op for the Calico variant", func() { ctx := extensions.RenderContext{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.Calico}} - out, _ := extensions.ApplyModifiers(render.ComponentNameNode, ctx, newObjs(), nil) + out, _ := ext.ApplyModifiers(render.ComponentNameNode, ctx, newObjs(), nil) _, ok := extensions.FindObject[*corev1.Service](out, render.CalicoNodeMetricsService) Expect(ok).To(BeFalse()) @@ -188,7 +183,7 @@ var _ = Describe("node enterprise modifier", func() { }) It("does not panic on a zero RenderContext", func() { - out, _ := extensions.ApplyModifiers(render.ComponentNameNode, extensions.RenderContext{}, newObjs(), nil) + out, _ := ext.ApplyModifiers(render.ComponentNameNode, extensions.RenderContext{}, newObjs(), nil) _, ok := extensions.FindObject[*corev1.Service](out, render.CalicoNodeMetricsService) Expect(ok).To(BeFalse()) }) diff --git a/pkg/enterprise/register.go b/pkg/enterprise/register.go index 8d6bdc0cd8..d1f81451b1 100644 --- a/pkg/enterprise/register.go +++ b/pkg/enterprise/register.go @@ -14,14 +14,19 @@ package enterprise -// Register wires all in-repo enterprise modifiers and controller extensions -// into the operator registries. Called once at process startup. After the -// monorepo split this is what calico-private's main will do instead. -func Register() { - registerTypha() - registerNode() - registerWindows() - registerGuardian() - registerInstallation() - registerAPIServer() +import "github.com/tigera/operator/pkg/extensions" + +// New builds the extension Set for the in-repo Calico Enterprise variant: every +// component modifier, image override, and the installation setup. The operator +// is handed this Set at startup (the core operator is handed none). After the +// monorepo split this is what calico-private's main will construct instead. +func New() *extensions.Set { + s := extensions.NewSet() + registerTypha(s) + registerNode(s) + registerWindows(s) + registerGuardian(s) + registerInstallation(s) + registerAPIServer(s) + return s } diff --git a/pkg/enterprise/typha.go b/pkg/enterprise/typha.go index 38733c871c..4571450365 100644 --- a/pkg/enterprise/typha.go +++ b/pkg/enterprise/typha.go @@ -25,8 +25,8 @@ import ( "github.com/tigera/operator/pkg/render" ) -func registerTypha() { - extensions.Register(operatorv1.CalicoEnterprise, render.ComponentNameTypha, extensions.Extension{ +func registerTypha(s *extensions.Set) { + s.Register(operatorv1.CalicoEnterprise, render.ComponentNameTypha, extensions.Extension{ Modify: modifyTypha, }) } diff --git a/pkg/enterprise/typha_test.go b/pkg/enterprise/typha_test.go index 2ebb853f8c..2b1211c4ec 100644 --- a/pkg/enterprise/typha_test.go +++ b/pkg/enterprise/typha_test.go @@ -24,16 +24,11 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" operatorv1 "github.com/tigera/operator/api/v1" - "github.com/tigera/operator/pkg/enterprise" "github.com/tigera/operator/pkg/extensions" "github.com/tigera/operator/pkg/render" ) var _ = Describe("typha enterprise modifier", func() { - BeforeEach(func() { enterprise.Register() }) - AfterEach(func() { - extensions.ResetForTest() - }) multiMode := operatorv1.MultiInterfaceModeMultus @@ -54,7 +49,7 @@ var _ = Describe("typha enterprise modifier", func() { Variant: operatorv1.CalicoEnterprise, CalicoNetwork: &operatorv1.CalicoNetworkSpec{MultiInterfaceMode: &multiMode}, }} - out, _ := extensions.ApplyModifiers(render.ComponentNameTypha, ctx, newObjs(), nil) + out, _ := ext.ApplyModifiers(render.ComponentNameTypha, ctx, newObjs(), nil) role := out[0].(*rbacv1.ClusterRole) Expect(role.Rules).To(ContainElement(HaveField("Resources", ContainElement("licensekeys")))) @@ -74,14 +69,14 @@ var _ = Describe("typha enterprise modifier", func() { Variant: operatorv1.Calico, CalicoNetwork: &operatorv1.CalicoNetworkSpec{MultiInterfaceMode: &multiMode}, }} - out, _ := extensions.ApplyModifiers(render.ComponentNameTypha, ctx, newObjs(), nil) + out, _ := ext.ApplyModifiers(render.ComponentNameTypha, ctx, newObjs(), nil) Expect(out[0].(*rbacv1.ClusterRole).Rules).To(BeEmpty()) dep := out[1].(*appsv1.Deployment) Expect(dep.Spec.Template.Spec.Containers[0].Env).To(BeEmpty()) }) It("does not panic on a zero Context (nil Installation)", func() { - out, _ := extensions.ApplyModifiers(render.ComponentNameTypha, extensions.RenderContext{}, newObjs(), nil) + out, _ := ext.ApplyModifiers(render.ComponentNameTypha, extensions.RenderContext{}, newObjs(), nil) Expect(out[0].(*rbacv1.ClusterRole).Rules).To(BeEmpty()) }) }) diff --git a/pkg/enterprise/windows.go b/pkg/enterprise/windows.go index 8a3e086835..f4674323b3 100644 --- a/pkg/enterprise/windows.go +++ b/pkg/enterprise/windows.go @@ -36,14 +36,14 @@ import ( // felix env and node volume mounts, so they receive the same enterprise layering. var windowsNodeContainers = map[string]bool{"felix": true, "node": true, "confd": true} -func registerWindows() { - extensions.Register(operatorv1.CalicoEnterprise, render.ComponentNameWindowsNodeImg, extensions.Extension{ +func registerWindows(s *extensions.Set) { + s.Register(operatorv1.CalicoEnterprise, render.ComponentNameWindowsNodeImg, extensions.Extension{ Image: func(*operatorv1.InstallationSpec) components.Component { return components.ComponentTigeraNodeWindows }, }) - extensions.Register(operatorv1.CalicoEnterprise, render.ComponentNameWindowsCNIImg, extensions.Extension{ + s.Register(operatorv1.CalicoEnterprise, render.ComponentNameWindowsCNIImg, extensions.Extension{ Image: func(*operatorv1.InstallationSpec) components.Component { return components.ComponentTigeraCNIWindows }, }) - extensions.Register(operatorv1.CalicoEnterprise, render.ComponentNameWindows, extensions.Extension{ + s.Register(operatorv1.CalicoEnterprise, render.ComponentNameWindows, extensions.Extension{ Modify: modifyWindows, }) } diff --git a/pkg/enterprise/windows_test.go b/pkg/enterprise/windows_test.go index 18668c9079..9bc08e1d52 100644 --- a/pkg/enterprise/windows_test.go +++ b/pkg/enterprise/windows_test.go @@ -30,33 +30,28 @@ import ( "github.com/tigera/operator/pkg/components" "github.com/tigera/operator/pkg/controller/certificatemanager" ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" - "github.com/tigera/operator/pkg/enterprise" "github.com/tigera/operator/pkg/extensions" "github.com/tigera/operator/pkg/render" "github.com/tigera/operator/pkg/tls/certificatemanagement" ) var _ = Describe("windows enterprise image override", func() { - BeforeEach(func() { enterprise.Register() }) - AfterEach(func() { extensions.ResetForTest() }) ent := &operatorv1.InstallationSpec{Variant: operatorv1.CalicoEnterprise} calico := &operatorv1.InstallationSpec{Variant: operatorv1.Calico} It("selects the enterprise windows images for the enterprise variant", func() { - Expect(extensions.ResolveImage(render.ComponentNameWindowsNodeImg, components.ComponentCalicoNodeWindows, ent)).To(Equal(components.ComponentTigeraNodeWindows)) - Expect(extensions.ResolveImage(render.ComponentNameWindowsCNIImg, components.ComponentCalicoCNIWindows, ent)).To(Equal(components.ComponentTigeraCNIWindows)) + Expect(ext.ResolveImage(render.ComponentNameWindowsNodeImg, components.ComponentCalicoNodeWindows, ent)).To(Equal(components.ComponentTigeraNodeWindows)) + Expect(ext.ResolveImage(render.ComponentNameWindowsCNIImg, components.ComponentCalicoCNIWindows, ent)).To(Equal(components.ComponentTigeraCNIWindows)) }) It("leaves the defaults in place for the Calico variant", func() { - Expect(extensions.ResolveImage(render.ComponentNameWindowsNodeImg, components.ComponentCalicoNodeWindows, calico)).To(Equal(components.ComponentCalicoNodeWindows)) - Expect(extensions.ResolveImage(render.ComponentNameWindowsCNIImg, components.ComponentCalicoCNIWindows, calico)).To(Equal(components.ComponentCalicoCNIWindows)) + Expect(ext.ResolveImage(render.ComponentNameWindowsNodeImg, components.ComponentCalicoNodeWindows, calico)).To(Equal(components.ComponentCalicoNodeWindows)) + Expect(ext.ResolveImage(render.ComponentNameWindowsCNIImg, components.ComponentCalicoCNIWindows, calico)).To(Equal(components.ComponentCalicoCNIWindows)) }) }) var _ = Describe("windows enterprise modifier", func() { - BeforeEach(func() { enterprise.Register() }) - AfterEach(func() { extensions.ResetForTest() }) // newObjs returns a windows daemonset with the node containers and the OSS // cni-log-dir mount the modifier swaps out. @@ -102,7 +97,7 @@ var _ = Describe("windows enterprise modifier", func() { } It("appends the node-metrics service", func() { - out, _ := extensions.ApplyModifiers(render.ComponentNameWindows, ctxFor(operatorv1.ProviderNone, nil, nil), newObjs(), nil) + out, _ := ext.ApplyModifiers(render.ComponentNameWindows, ctxFor(operatorv1.ProviderNone, nil, nil), newObjs(), nil) svc, ok := extensions.FindObject[*corev1.Service](out, render.WindowsNodeMetricsService) Expect(ok).To(BeTrue()) Expect(svc.Namespace).To(Equal(common.CalicoNamespace)) @@ -110,7 +105,7 @@ var _ = Describe("windows enterprise modifier", func() { }) It("swaps the cni log mount for the calico log volume and adds enterprise env", func() { - out, _ := extensions.ApplyModifiers(render.ComponentNameWindows, ctxFor(operatorv1.ProviderNone, nil, nil), newObjs(), nil) + out, _ := ext.ApplyModifiers(render.ComponentNameWindows, ctxFor(operatorv1.ProviderNone, nil, nil), newObjs(), nil) d := ds(out) Expect(d.Spec.Template.Spec.Volumes).To(ContainElement(HaveField("Name", "var-log-calico"))) @@ -127,7 +122,7 @@ var _ = Describe("windows enterprise modifier", func() { }) It("sets the trusted DNS server on openshift", func() { - out, _ := extensions.ApplyModifiers(render.ComponentNameWindows, ctxFor(operatorv1.ProviderOpenShift, nil, nil), newObjs(), nil) + out, _ := ext.ApplyModifiers(render.ComponentNameWindows, ctxFor(operatorv1.ProviderOpenShift, nil, nil), newObjs(), nil) Expect(container(ds(out), "node").Env).To(ContainElement(corev1.EnvVar{Name: "FELIX_DNSTRUSTEDSERVERS", Value: "k8s-service:openshift-dns/dns-default"})) }) @@ -141,7 +136,7 @@ var _ = Describe("windows enterprise modifier", func() { Expect(err).NotTo(HaveOccurred()) bundle := cm.CreateTrustedBundle() - out, _ := extensions.ApplyModifiers(render.ComponentNameWindows, ctxFor(operatorv1.ProviderNone, tls, bundle), newObjs(), nil) + out, _ := ext.ApplyModifiers(render.ComponentNameWindows, ctxFor(operatorv1.ProviderNone, tls, bundle), newObjs(), nil) d := ds(out) Expect(d.Spec.Template.Spec.Volumes).To(ContainElement(tls.Volume())) @@ -152,7 +147,7 @@ var _ = Describe("windows enterprise modifier", func() { It("does nothing for the Calico variant", func() { ctx := extensions.RenderContext{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.Calico}} - out, _ := extensions.ApplyModifiers(render.ComponentNameWindows, ctx, newObjs(), nil) + out, _ := ext.ApplyModifiers(render.ComponentNameWindows, ctx, newObjs(), nil) _, ok := extensions.FindObject[*corev1.Service](out, render.WindowsNodeMetricsService) Expect(ok).To(BeFalse()) Expect(ds(out).Spec.Template.Spec.Volumes).To(BeEmpty()) diff --git a/pkg/extensions/extension.go b/pkg/extensions/extension.go index 25a43feb13..346e7347f2 100644 --- a/pkg/extensions/extension.go +++ b/pkg/extensions/extension.go @@ -18,7 +18,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" operatorv1 "github.com/tigera/operator/api/v1" - "github.com/tigera/operator/pkg/imageoverride" ) // Extension is everything a variant layers onto one render component. Every @@ -48,30 +47,16 @@ type modifierKey struct { component string } -var modifiers = map[modifierKey]Modifier{} - -// Register installs e as the extension for the named component under the given -// variant. A (variant, component) pair has at most one extension; registration -// replaces any prior one, so it is idempotent and safe to call more than once. -// The image override lives in the imageoverride leaf (so the render package can -// resolve it without an import cycle); the modifier lives here. -func Register(variant operatorv1.ProductVariant, component string, e Extension) { - if e.Image != nil { - imageoverride.Register(variant, component, e.Image) - } - if e.Modify != nil { - modifiers[modifierKey{variant, component}] = e.Modify - } -} - // ApplyModifiers runs the modifier registered for the named component and the // installation's variant over the create and delete lists, returning them -// unchanged when none is registered (or when no installation is set). -func ApplyModifiers(component string, ctx RenderContext, create, delete []client.Object) ([]client.Object, []client.Object) { - if ctx.Installation == nil { +// unchanged when none is registered (or when no installation is set). Safe to +// call on a nil Set, which is a no-op - the core operator registers no +// modifiers. +func (s *Set) ApplyModifiers(component string, ctx RenderContext, create, delete []client.Object) ([]client.Object, []client.Object) { + if s == nil || ctx.Installation == nil { return create, delete } - if fn, ok := modifiers[modifierKey{ctx.Installation.Variant, component}]; ok { + if fn, ok := s.modifiers[modifierKey{ctx.Installation.Variant, component}]; ok { create, delete = fn(ctx, create, delete) } return create, delete @@ -87,11 +72,3 @@ func FindObject[T client.Object](objs []client.Object, name string) (T, bool) { } return zero, false } - -// ResetForTest clears every registry: modifiers, image overrides, and variant -// setups. Test-only. -func ResetForTest() { - modifiers = map[modifierKey]Modifier{} - setups = map[operatorv1.ProductVariant]Setup{} - imageoverride.ResetForTest() -} diff --git a/pkg/extensions/extension_test.go b/pkg/extensions/extension_test.go index 2c08569355..b02ec3d24d 100644 --- a/pkg/extensions/extension_test.go +++ b/pkg/extensions/extension_test.go @@ -26,14 +26,15 @@ import ( ) var _ = Describe("extension registry", func() { - AfterEach(func() { - extensions.ResetForTest() + var s *extensions.Set + BeforeEach(func() { + s = extensions.NewSet() }) entCtx := extensions.RenderContext{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.CalicoEnterprise}} It("applies a registered modifier to the matching component and variant", func() { - extensions.Register(operatorv1.CalicoEnterprise, "test", extensions.Extension{ + s.Register(operatorv1.CalicoEnterprise, "test", extensions.Extension{ Modify: func(ctx extensions.RenderContext, objs, del []client.Object) ([]client.Object, []client.Object) { cm, ok := extensions.FindObject[*corev1.ConfigMap](objs, "cm") Expect(ok).To(BeTrue()) @@ -43,7 +44,7 @@ var _ = Describe("extension registry", func() { }) in := []client.Object{&corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "cm"}}} - out, _ := extensions.ApplyModifiers("test", entCtx, in, nil) + out, _ := s.ApplyModifiers("test", entCtx, in, nil) Expect(out).To(HaveLen(2)) cm := out[0].(*corev1.ConfigMap) @@ -52,14 +53,14 @@ var _ = Describe("extension registry", func() { }) It("lets a modifier append to the delete list", func() { - extensions.Register(operatorv1.CalicoEnterprise, "test", extensions.Extension{ + s.Register(operatorv1.CalicoEnterprise, "test", extensions.Extension{ Modify: func(_ extensions.RenderContext, objs, del []client.Object) ([]client.Object, []client.Object) { return objs, append(del, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "stale"}}) }, }) in := []client.Object{&corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "cm"}}} - out, del := extensions.ApplyModifiers("test", entCtx, in, nil) + out, del := s.ApplyModifiers("test", entCtx, in, nil) Expect(out).To(Equal(in)) Expect(del).To(HaveLen(1)) Expect(del[0].GetName()).To(Equal("stale")) @@ -67,12 +68,12 @@ var _ = Describe("extension registry", func() { It("returns objects unchanged when no modifier is registered", func() { in := []client.Object{&corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "cm"}}} - out, _ := extensions.ApplyModifiers("unregistered", entCtx, in, nil) + out, _ := s.ApplyModifiers("unregistered", entCtx, in, nil) Expect(out).To(Equal(in)) }) It("does not apply a modifier registered for a different variant", func() { - extensions.Register(operatorv1.CalicoEnterprise, "test", extensions.Extension{ + s.Register(operatorv1.CalicoEnterprise, "test", extensions.Extension{ Modify: func(_ extensions.RenderContext, objs, del []client.Object) ([]client.Object, []client.Object) { return append(objs, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "extra"}}), del }, @@ -80,13 +81,13 @@ var _ = Describe("extension registry", func() { calicoCtx := extensions.RenderContext{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.Calico}} in := []client.Object{&corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "cm"}}} - out, _ := extensions.ApplyModifiers("test", calicoCtx, in, nil) + out, _ := s.ApplyModifiers("test", calicoCtx, in, nil) Expect(out).To(Equal(in)) }) It("returns objects unchanged when no installation is set", func() { in := []client.Object{&corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "cm"}}} - out, _ := extensions.ApplyModifiers("test", extensions.RenderContext{}, in, nil) + out, _ := s.ApplyModifiers("test", extensions.RenderContext{}, in, nil) Expect(out).To(Equal(in)) }) @@ -98,10 +99,10 @@ var _ = Describe("extension registry", func() { }, } } - extensions.Register(operatorv1.CalicoEnterprise, "test", add("first")) - extensions.Register(operatorv1.CalicoEnterprise, "test", add("second")) + s.Register(operatorv1.CalicoEnterprise, "test", add("first")) + s.Register(operatorv1.CalicoEnterprise, "test", add("second")) - out, _ := extensions.ApplyModifiers("test", entCtx, nil, nil) + out, _ := s.ApplyModifiers("test", entCtx, nil, nil) Expect(out).To(HaveLen(1)) Expect(out[0].GetName()).To(Equal("second")) }) diff --git a/pkg/extensions/image.go b/pkg/extensions/image.go index 477c89deef..a551d1e76f 100644 --- a/pkg/extensions/image.go +++ b/pkg/extensions/image.go @@ -15,8 +15,6 @@ package extensions import ( - operatorv1 "github.com/tigera/operator/api/v1" - "github.com/tigera/operator/pkg/components" "github.com/tigera/operator/pkg/imageoverride" ) @@ -24,10 +22,3 @@ import ( // the Image field of an Extension. An override runs only for the variant it was // registered under, so it need not re-check the variant. type ImageOverride = imageoverride.Override - -// ResolveImage returns the override registered for key if it applies to in, -// otherwise def. The render package resolves images through the imageoverride -// leaf directly; this is the same lookup for callers already inside extensions. -func ResolveImage(key string, def components.Component, in *operatorv1.InstallationSpec) components.Component { - return imageoverride.Resolve(key, def, in) -} diff --git a/pkg/extensions/image_test.go b/pkg/extensions/image_test.go index 25926c1a4f..0581d8cc08 100644 --- a/pkg/extensions/image_test.go +++ b/pkg/extensions/image_test.go @@ -24,29 +24,23 @@ import ( ) var _ = Describe("image overrides", func() { - AfterEach(func() { - extensions.ResetForTest() - }) - - It("uses the override registered for the installation variant", func() { - extensions.Register(operatorv1.CalicoEnterprise, "node", extensions.Extension{ + var s *extensions.Set + BeforeEach(func() { + s = extensions.NewSet() + s.Register(operatorv1.CalicoEnterprise, "node", extensions.Extension{ Image: func(in *operatorv1.InstallationSpec) components.Component { return components.ComponentTigeraNode }, }) + }) + It("uses the override registered for the installation variant", func() { ent := &operatorv1.InstallationSpec{Variant: operatorv1.CalicoEnterprise} - Expect(extensions.ResolveImage("node", components.ComponentCalicoNode, ent)).To(Equal(components.ComponentTigeraNode)) + Expect(s.ResolveImage("node", components.ComponentCalicoNode, ent)).To(Equal(components.ComponentTigeraNode)) }) It("falls back to the default for a variant with no override", func() { - extensions.Register(operatorv1.CalicoEnterprise, "node", extensions.Extension{ - Image: func(in *operatorv1.InstallationSpec) components.Component { - return components.ComponentTigeraNode - }, - }) - calico := &operatorv1.InstallationSpec{Variant: operatorv1.Calico} - Expect(extensions.ResolveImage("node", components.ComponentCalicoNode, calico)).To(Equal(components.ComponentCalicoNode)) + Expect(s.ResolveImage("node", components.ComponentCalicoNode, calico)).To(Equal(components.ComponentCalicoNode)) }) }) diff --git a/pkg/extensions/set.go b/pkg/extensions/set.go new file mode 100644 index 0000000000..b638622750 --- /dev/null +++ b/pkg/extensions/set.go @@ -0,0 +1,82 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package extensions + +import ( + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/components" + "github.com/tigera/operator/pkg/imageoverride" +) + +// Set is the collection of variant extensions the operator runs with: the +// per-variant setups, the per-component modifiers, and the image overrides. The +// core operator runs with a nil/empty Set; an extension build (Calico +// Enterprise) constructs a populated one and hands it in through +// options.ControllerOptions. This replaces what used to be package-level +// registries, so nothing is wired by import side effect. +// +// The zero value is not usable; build one with NewSet. The methods that read it +// (BuildContext, ApplyModifiers, ResolveImage, Images) are nil-safe so the core +// operator can pass a nil Set and get base behavior. +type Set struct { + setups map[operatorv1.ProductVariant]Setup + modifiers map[modifierKey]Modifier + images *imageoverride.Overrides +} + +// NewSet returns an empty Set ready to register extensions into. +func NewSet() *Set { + return &Set{ + setups: map[operatorv1.ProductVariant]Setup{}, + modifiers: map[modifierKey]Modifier{}, + images: imageoverride.New(), + } +} + +// Register installs e as the extension for the named component under the given +// variant. A (variant, component) pair has at most one extension; registration +// replaces any prior one. The image override and the modifier are stored +// separately, so a component can set either field or both. +func (s *Set) Register(variant operatorv1.ProductVariant, component string, e Extension) { + if e.Image != nil { + s.images.Register(variant, component, e.Image) + } + if e.Modify != nil { + s.modifiers[modifierKey{variant, component}] = e.Modify + } +} + +// RegisterSetup installs setup as the controller-side setup phase for the given +// variant. Registration replaces any prior setup for that variant. +func (s *Set) RegisterSetup(variant operatorv1.ProductVariant, setup Setup) { + s.setups[variant] = setup +} + +// Images returns the image overrides. The render package resolves a component's +// image through these directly (the imageoverride leaf, so render need not +// import extensions). Safe to call on a nil Set, which returns nil overrides +// that resolve to the default image. +func (s *Set) Images() *imageoverride.Overrides { + if s == nil { + return nil + } + return s.images +} + +// ResolveImage resolves key for the installation through the image overrides, +// returning def when no override applies. Safe to call on a nil Set. +func (s *Set) ResolveImage(key string, def components.Component, in *operatorv1.InstallationSpec) components.Component { + return s.Images().Resolve(key, def, in) +} diff --git a/pkg/extensions/setup.go b/pkg/extensions/setup.go index a6272b006c..2eb0ef1b6b 100644 --- a/pkg/extensions/setup.go +++ b/pkg/extensions/setup.go @@ -64,22 +64,14 @@ func BaseRenderContext(in Inputs) RenderContext { } } -var setups = map[operatorv1.ProductVariant]Setup{} - -// RegisterSetup installs s as the setup for the given variant. Registration -// replaces any prior setup, so it is safe to call more than once. Variants -// without a registered setup get the base render context. -func RegisterSetup(variant operatorv1.ProductVariant, s Setup) { - setups[variant] = s -} - // BuildContext runs the setup registered for the installation variant and // returns its RenderContext, or the base render context when the variant has no -// setup. -func BuildContext(in Inputs) (RenderContext, error) { - if in.Installation != nil { - if s, ok := setups[in.Installation.Variant]; ok { - return s(in) +// setup. Safe to call on a nil Set, which always returns the base context - the +// core operator registers no setups. +func (s *Set) BuildContext(in Inputs) (RenderContext, error) { + if s != nil && in.Installation != nil { + if setup, ok := s.setups[in.Installation.Variant]; ok { + return setup(in) } } return BaseRenderContext(in), nil diff --git a/pkg/extensions/setup_test.go b/pkg/extensions/setup_test.go index 1ba42e5567..9ee58a6648 100644 --- a/pkg/extensions/setup_test.go +++ b/pkg/extensions/setup_test.go @@ -25,11 +25,14 @@ import ( ) var _ = Describe("variant setup", func() { - AfterEach(func() { extensions.ResetForTest() }) + var s *extensions.Set + BeforeEach(func() { + s = extensions.NewSet() + }) It("returns the base render context when the variant has no setup", func() { install := &operatorv1.InstallationSpec{Variant: operatorv1.Calico} - rc, err := extensions.BuildContext(extensions.Inputs{ + rc, err := s.BuildContext(extensions.Inputs{ Installation: install, ClusterDomain: "cluster.local", }) @@ -40,15 +43,15 @@ var _ = Describe("variant setup", func() { }) It("uses the setup registered for the installation variant", func() { - extensions.RegisterSetup(operatorv1.CalicoEnterprise, fakeSetup(nil)) - rc, err := extensions.BuildContext(enterpriseInputs()) + s.RegisterSetup(operatorv1.CalicoEnterprise, fakeSetup(nil)) + rc, err := s.BuildContext(enterpriseInputs()) Expect(err).NotTo(HaveOccurred()) Expect(rc.ClusterDomain).To(Equal("from-fake")) }) It("ignores a setup registered for a different variant", func() { - extensions.RegisterSetup(operatorv1.CalicoEnterprise, fakeSetup(nil)) - rc, err := extensions.BuildContext(extensions.Inputs{ + s.RegisterSetup(operatorv1.CalicoEnterprise, fakeSetup(nil)) + rc, err := s.BuildContext(extensions.Inputs{ Installation: &operatorv1.InstallationSpec{Variant: operatorv1.Calico}, ClusterDomain: "real", }) @@ -57,17 +60,16 @@ var _ = Describe("variant setup", func() { }) It("surfaces the setup error", func() { - extensions.RegisterSetup(operatorv1.CalicoEnterprise, fakeSetup(errors.New("boom"))) - _, err := extensions.BuildContext(enterpriseInputs()) + s.RegisterSetup(operatorv1.CalicoEnterprise, fakeSetup(errors.New("boom"))) + _, err := s.BuildContext(enterpriseInputs()) Expect(err).To(MatchError("boom")) }) - It("restores the base context on reset", func() { - extensions.RegisterSetup(operatorv1.CalicoEnterprise, fakeSetup(nil)) - extensions.ResetForTest() + It("returns the base context for a nil Set", func() { + var nilSet *extensions.Set in := enterpriseInputs() in.ClusterDomain = "real" - rc, err := extensions.BuildContext(in) + rc, err := nilSet.BuildContext(in) Expect(err).NotTo(HaveOccurred()) Expect(rc.ClusterDomain).To(Equal("real")) }) diff --git a/pkg/imageoverride/imageoverride.go b/pkg/imageoverride/imageoverride.go index c3bc471bc2..5c981c0c07 100644 --- a/pkg/imageoverride/imageoverride.go +++ b/pkg/imageoverride/imageoverride.go @@ -13,8 +13,8 @@ // limitations under the License. // Package imageoverride is a leaf package (no render/operator dependencies) -// that holds the image override registry. Both pkg/operator and pkg/render -// import it to avoid the render→operator→render import cycle. +// that holds the image override table. The render package imports it to resolve +// a component's image without depending on pkg/extensions, which would cycle. package imageoverride import ( @@ -30,27 +30,33 @@ type overrideKey struct { key string } -var registry = map[overrideKey]Override{} +// Overrides maps a component (keyed by variant) to the image it should resolve +// to, letting a variant swap a component's image without the render package +// branching on variant. The render component holds one and resolves through it. +type Overrides struct { + m map[overrideKey]Override +} + +// New returns an empty Overrides. +func New() *Overrides { + return &Overrides{m: map[overrideKey]Override{}} +} // Register stores fn under key for the given variant. The key is the render // component's image identifier (e.g. "node"). -func Register(variant operatorv1.ProductVariant, key string, fn Override) { - registry[overrideKey{variant, key}] = fn +func (o *Overrides) Register(variant operatorv1.ProductVariant, key string, fn Override) { + o.m[overrideKey{variant, key}] = fn } // Resolve returns the override registered for key under the installation's -// variant, otherwise def. -func Resolve(key string, def components.Component, in *operatorv1.InstallationSpec) components.Component { - if in == nil { +// variant, otherwise def. It is safe to call on a nil *Overrides (the core +// operator hands render no overrides), which always returns def. +func (o *Overrides) Resolve(key string, def components.Component, in *operatorv1.InstallationSpec) components.Component { + if o == nil || in == nil { return def } - if fn, ok := registry[overrideKey{in.Variant, key}]; ok { + if fn, ok := o.m[overrideKey{in.Variant, key}]; ok { return fn(in) } return def } - -// ResetForTest clears the registry. Test-only. -func ResetForTest() { - registry = map[overrideKey]Override{} -} diff --git a/pkg/render/apiserver_test.go b/pkg/render/apiserver_test.go index 3645e16f33..a81ebb6e01 100644 --- a/pkg/render/apiserver_test.go +++ b/pkg/render/apiserver_test.go @@ -73,7 +73,7 @@ func apiServerObjects(c render.Component) ([]client.Object, []client.Object) { rc.Installation = ec.Config.Installation rc.Component = ec } - return extensions.ApplyModifiers(render.ComponentNameAPIServer, rc, create, del) + return ext.ApplyModifiers(render.ComponentNameAPIServer, rc, create, del) } var _ = Describe("API server rendering tests (Calico Enterprise)", func() { diff --git a/pkg/render/enterprise_setup_test.go b/pkg/render/enterprise_setup_test.go index 13766f7035..e896366124 100644 --- a/pkg/render/enterprise_setup_test.go +++ b/pkg/render/enterprise_setup_test.go @@ -15,20 +15,13 @@ package render_test import ( - . "github.com/onsi/ginkgo/v2" - "github.com/tigera/operator/pkg/enterprise" + "github.com/tigera/operator/pkg/extensions" ) -// Register the enterprise extensions once for the whole render suite. This wires -// two things the suite relies on: -// - the image override, which the Objects()-level render tests pick up through -// ResolveImages (e.g. the enterprise node image), and -// - the modifiers, which node_enterprise_test.go applies explicitly to real -// render output to check they still match it. -// -// The plain Objects()-level tests do not run modifiers - those only run at the -// componentHandler - so registering here does not change their output. -var _ = BeforeSuite(func() { - enterprise.Register() -}) +// ext is the enterprise extension Set the render suite tests against. The +// Objects()-level image tests pass ext.Images() into the node/windows configs to +// pick up the enterprise images, and the enterprise modifier tests apply ext's +// modifiers explicitly to real render output to check they still match it. It is +// immutable once built and specs only read it, so a single instance is safe. +var ext *extensions.Set = enterprise.New() diff --git a/pkg/render/guardian_test.go b/pkg/render/guardian_test.go index b28e77f044..33c04099fd 100644 --- a/pkg/render/guardian_test.go +++ b/pkg/render/guardian_test.go @@ -54,7 +54,7 @@ func guardianObjects(cfg *render.GuardianConfiguration) []client.Object { if p, ok := g.(render.ExtensionContextProvider); ok { rc.Component = p.ExtensionContext() } - out, _ := extensions.ApplyModifiers(render.GuardianName, rc, objs, nil) + out, _ := ext.ApplyModifiers(render.GuardianName, rc, objs, nil) return out } @@ -116,7 +116,7 @@ var _ = Describe("Rendering tests", func() { if p, ok := g.(render.ExtensionContextProvider); ok { rc.Component = p.ExtensionContext() } - resources, _ = extensions.ApplyModifiers(render.GuardianName, rc, resources, nil) + resources, _ = ext.ApplyModifiers(render.GuardianName, rc, resources, nil) } BeforeEach(func() { @@ -352,7 +352,7 @@ var _ = Describe("Rendering tests", func() { if p, ok := g.(render.ExtensionContextProvider); ok { rc.Component = p.ExtensionContext() } - resources, _ = extensions.ApplyModifiers(render.ComponentNameGuardianPolicy, rc, objs, nil) + resources, _ = ext.ApplyModifiers(render.ComponentNameGuardianPolicy, rc, objs, nil) } Context("policy rendering based on variant and IncludeEgressNetworkPolicy", func() { diff --git a/pkg/render/node.go b/pkg/render/node.go index ddcc43e897..4077e43db2 100644 --- a/pkg/render/node.go +++ b/pkg/render/node.go @@ -145,6 +145,11 @@ type NodeConfiguration struct { BindMode string V3CRDs bool + + // ImageOverrides lets a variant swap the node and cni-plugins images. The + // controller wires in the operator's image overrides; nil resolves to the + // core images. + ImageOverrides *imageoverride.Overrides } // Node creates the node daemonset and other resources for the daemonset to operate normally. @@ -179,10 +184,10 @@ func (c *nodeComponent) ResolveImages(is *operatorv1.ImageSet) error { } c.calicoImage = appendIfErr(components.GetReference(components.CombinedCalicoImage(c.cfg.Installation), reg, path, prefix, is)) - nodeImage := imageoverride.Resolve(ComponentNameNode, components.ComponentCalicoNode, c.cfg.Installation) + nodeImage := c.cfg.ImageOverrides.Resolve(ComponentNameNode, components.ComponentCalicoNode, c.cfg.Installation) c.nodeImage = appendIfErr(components.GetReference(nodeImage, reg, path, prefix, is)) if c.installUpstreamPlugins() { - cniPluginsImage := imageoverride.Resolve(ComponentNameCNIPlugins, components.ComponentCalicoCNIPlugins, c.cfg.Installation) + cniPluginsImage := c.cfg.ImageOverrides.Resolve(ComponentNameCNIPlugins, components.ComponentCalicoCNIPlugins, c.cfg.Installation) c.cniPluginsImage = appendIfErr(components.GetReference(cniPluginsImage, reg, path, prefix, is)) } diff --git a/pkg/render/node_enterprise_test.go b/pkg/render/node_enterprise_test.go index 4b9b6a2f82..7b11bbf6eb 100644 --- a/pkg/render/node_enterprise_test.go +++ b/pkg/render/node_enterprise_test.go @@ -110,7 +110,7 @@ var _ = Describe("node enterprise modifier integration", func() { comp := render.Node(cfg) Expect(comp.ResolveImages(nil)).NotTo(HaveOccurred()) objs, _ := comp.Objects() - out, _ := extensions.ApplyModifiers(render.ComponentNameNode, renderCtx, objs, nil) + out, _ := ext.ApplyModifiers(render.ComponentNameNode, renderCtx, objs, nil) return out } @@ -167,7 +167,7 @@ var _ = Describe("node enterprise modifier integration", func() { }) Expect(comp.ResolveImages(nil)).NotTo(HaveOccurred()) objs, _ := comp.Objects() - objs, _ = extensions.ApplyModifiers(render.ComponentNameTypha, renderCtx, objs, nil) + objs, _ = ext.ApplyModifiers(render.ComponentNameTypha, renderCtx, objs, nil) role, ok := extensions.FindObject[*rbacv1.ClusterRole](objs, "calico-typha") Expect(ok).To(BeTrue()) diff --git a/pkg/render/node_test.go b/pkg/render/node_test.go index 3b2b487d53..b052be0ef4 100644 --- a/pkg/render/node_test.go +++ b/pkg/render/node_test.go @@ -140,6 +140,7 @@ var _ = Describe("Node rendering tests", func() { ClusterDomain: defaultClusterDomain, FelixHealthPort: 9099, IPPools: defaultInstance.CalicoNetwork.IPPools, + ImageOverrides: ext.Images(), } }) diff --git a/pkg/render/windows.go b/pkg/render/windows.go index f27120979b..7392094458 100644 --- a/pkg/render/windows.go +++ b/pkg/render/windows.go @@ -57,6 +57,11 @@ type WindowsConfiguration struct { PrometheusServerTLS certificatemanagement.KeyPairInterface NodeReporterMetricsPort int VXLANVNI int + + // ImageOverrides lets a variant swap the windows node and CNI images. The + // controller wires in the operator's image overrides; nil resolves to the + // core images. + ImageOverrides *imageoverride.Overrides } type windowsComponent struct { @@ -77,8 +82,8 @@ func (c *windowsComponent) ResolveImages(is *operatorv1.ImageSet) error { return imageName } - cniImage := imageoverride.Resolve(ComponentNameWindowsCNIImg, components.ComponentCalicoCNIWindows, c.cfg.Installation) - nodeImage := imageoverride.Resolve(ComponentNameWindowsNodeImg, components.ComponentCalicoNodeWindows, c.cfg.Installation) + cniImage := c.cfg.ImageOverrides.Resolve(ComponentNameWindowsCNIImg, components.ComponentCalicoCNIWindows, c.cfg.Installation) + nodeImage := c.cfg.ImageOverrides.Resolve(ComponentNameWindowsNodeImg, components.ComponentCalicoNodeWindows, c.cfg.Installation) c.cniImage = appendIfErr(components.GetReference(cniImage, reg, path, prefix, is)) c.nodeImage = appendIfErr(components.GetReference(nodeImage, reg, path, prefix, is)) diff --git a/pkg/render/windows_test.go b/pkg/render/windows_test.go index 5e845b31d5..e92bc86478 100644 --- a/pkg/render/windows_test.go +++ b/pkg/render/windows_test.go @@ -54,7 +54,7 @@ func renderWindows(cfg *render.WindowsConfiguration) []client.Object { if p, ok := comp.(render.ExtensionContextProvider); ok { rc.Component = p.ExtensionContext() } - out, _ := extensions.ApplyModifiers(render.ComponentNameWindows, rc, objs, nil) + out, _ := ext.ApplyModifiers(render.ComponentNameWindows, rc, objs, nil) return out } @@ -124,12 +124,13 @@ var _ = Describe("Windows rendering tests", func() { // Create a default configuration. cfg = render.WindowsConfiguration{ - K8sServiceEp: k8sServiceEp, - K8sDNSServers: []string{"10.96.0.10"}, - Installation: defaultInstance, - ClusterDomain: defaultClusterDomain, - TLS: typhaNodeTLS, - VXLANVNI: 4096, + K8sServiceEp: k8sServiceEp, + K8sDNSServers: []string{"10.96.0.10"}, + Installation: defaultInstance, + ClusterDomain: defaultClusterDomain, + TLS: typhaNodeTLS, + VXLANVNI: 4096, + ImageOverrides: ext.Images(), } }) From cc48b455e7109407d9053211f0f1e73d47f4c8a8 Mon Sep 17 00:00:00 2001 From: Casey Davenport Date: Wed, 17 Jun 2026 13:45:54 -0700 Subject: [PATCH 27/54] Use r.opts for controller options instead of copied fields The installation, windows, and clusterconnection reconcilers copied a handful of ControllerOptions values into their own struct fields. Drop those and read them off r.opts so the options live in one place. Also removes the dead kubernetesVersion field on the installation reconciler. --- .../clusterconnection_controller.go | 6 +- .../installation/core_controller.go | 89 ++-- .../installation/core_controller_test.go | 400 ++++++++++-------- .../installation/windows_controller.go | 18 +- .../installation/windows_controller_test.go | 22 +- 5 files changed, 277 insertions(+), 258 deletions(-) diff --git a/pkg/controller/clusterconnection/clusterconnection_controller.go b/pkg/controller/clusterconnection/clusterconnection_controller.go index 763743789f..c51bc19238 100644 --- a/pkg/controller/clusterconnection/clusterconnection_controller.go +++ b/pkg/controller/clusterconnection/clusterconnection_controller.go @@ -174,7 +174,6 @@ func newReconciler( scheme: schema, provider: p, status: statusMgr, - clusterDomain: opts.ClusterDomain, tierWatchReady: tierWatchReady, clusterInfoWatchReady: clusterInfoWatchReady, opts: opts, @@ -192,7 +191,6 @@ type ReconcileConnection struct { scheme *runtime.Scheme provider operatorv1.Provider status status.StatusManager - clusterDomain string tierWatchReady *utils.ReadyFlag clusterInfoWatchReady *utils.ReadyFlag resolvedPodProxies []*httpproxy.Config @@ -286,7 +284,7 @@ func (r *ReconcileConnection) Reconcile(ctx context.Context, request reconcile.R log.V(2).Info("Loaded ManagementClusterConnection config", "config", managementClusterConnection) - certificateManager, err := certificatemanager.Create(r.cli, installationSpec, r.clusterDomain, common.OperatorNamespace(), certificatemanager.WithLogger(reqLogger)) + certificateManager, err := certificatemanager.Create(r.cli, installationSpec, r.opts.ClusterDomain, common.OperatorNamespace(), certificatemanager.WithLogger(reqLogger)) if err != nil { r.status.SetDegraded(operatorv1.ResourceCreateError, "Unable to create the Tigera CA", err, reqLogger) return reconcile.Result{}, err @@ -310,7 +308,7 @@ func (r *ReconcileConnection) Reconcile(ctx context.Context, request reconcile.R var guardianKeyPair certificatemanagement.KeyPairInterface if !variant.IsEnterprise() { - guardianCertificateNames := dns.GetServiceDNSNames("guardian", render.GuardianNamespace, r.clusterDomain) + guardianCertificateNames := dns.GetServiceDNSNames("guardian", render.GuardianNamespace, r.opts.ClusterDomain) guardianCertificateNames = append(guardianCertificateNames, "localhost", "127.0.0.1") guardianKeyPair, err = certificateManager.GetOrCreateKeyPair(r.cli, render.GuardianKeyPairSecret, whisker.WhiskerNamespace, guardianCertificateNames) if err != nil { diff --git a/pkg/controller/installation/core_controller.go b/pkg/controller/installation/core_controller.go index 1f241da0a8..08444f08e8 100644 --- a/pkg/controller/installation/core_controller.go +++ b/pkg/controller/installation/core_controller.go @@ -43,7 +43,6 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" - "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "k8s.io/client-go/tools/cache" @@ -336,26 +335,17 @@ func newReconciler(mgr manager.Manager, opts options.ControllerOptions) (*Reconc typhaScaler := newTyphaAutoscaler(opts.K8sClientset, nodeIndexInformer, typhaListWatch, statusManager) r := &ReconcileInstallation{ - config: mgr.GetConfig(), - client: mgr.GetClient(), - clientset: opts.K8sClientset, - scheme: mgr.GetScheme(), - shutdownContext: opts.ShutdownContext, - watches: make(map[runtime.Object]struct{}), - autoDetectedProvider: opts.DetectedProvider, - status: statusManager, - typhaAutoscaler: typhaScaler, - namespaceMigration: nm, - enterpriseCRDsExist: opts.EnterpriseCRDExists, - clusterDomain: opts.ClusterDomain, - manageCRDs: opts.ManageCRDs, - tierWatchReady: &utils.ReadyFlag{}, - migrationWatchReady: &utils.ReadyFlag{}, - newComponentHandler: utils.NewComponentHandler, - v3CRDs: opts.UseV3CRDs, - kubernetesVersion: opts.KubernetesVersion, - apiDiscovery: opts.APIDiscovery, - opts: opts, + config: mgr.GetConfig(), + client: mgr.GetClient(), + scheme: mgr.GetScheme(), + watches: make(map[runtime.Object]struct{}), + status: statusManager, + typhaAutoscaler: typhaScaler, + namespaceMigration: nm, + tierWatchReady: &utils.ReadyFlag{}, + migrationWatchReady: &utils.ReadyFlag{}, + newComponentHandler: utils.NewComponentHandler, + opts: opts, } r.status.Run(opts.ShutdownContext) r.typhaAutoscaler.start(opts.ShutdownContext) @@ -397,24 +387,15 @@ type ReconcileInstallation struct { // that reads objects from the cache and writes to the apiserver config *rest.Config client client.Client - clientset *kubernetes.Clientset scheme *runtime.Scheme - shutdownContext context.Context watches map[runtime.Object]struct{} - autoDetectedProvider operatorv1.Provider status status.StatusManager typhaAutoscaler *typhaAutoscaler typhaAutoscalerNonClusterHost *typhaAutoscaler namespaceMigration migration.NamespaceMigration - enterpriseCRDsExist bool migrationChecked bool - clusterDomain string - manageCRDs bool tierWatchReady *utils.ReadyFlag migrationWatchReady *utils.ReadyFlag - v3CRDs bool - kubernetesVersion *common.VersionInfo - apiDiscovery *discovery.APIDiscovery opts options.ControllerOptions // newComponentHandler returns a new component handler. Useful stub for unit testing. @@ -859,7 +840,7 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile } // update Installation with defaults - if err := updateInstallationWithDefaults(ctx, r.client, instance, r.autoDetectedProvider); err != nil { + if err := updateInstallationWithDefaults(ctx, r.client, instance, r.opts.DetectedProvider); err != nil { r.status.SetDegraded(operatorv1.ResourceReadError, "Error querying installation", err, reqLogger) return reconcile.Result{}, err } @@ -1023,10 +1004,10 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile // The operator supports running in a "Calico only" mode so that it doesn't need to run enterprise-specific controllers. // If we are switching from this mode to one that enables enterprise, we need to restart the operator to enable the other controllers. - if !r.enterpriseCRDsExist && instance.Spec.Variant.IsEnterprise() { + if !r.opts.EnterpriseCRDExists && instance.Spec.Variant.IsEnterprise() { // Perform an API discovery to determine if the necessary APIs exist. If they do, we can reboot into enterprise mode. // if they do not, we need to notify the user that the requested configuration is invalid. - b, err := discovery.RequiresTigeraSecure(r.clientset) + b, err := discovery.RequiresTigeraSecure(r.opts.K8sClientset) if b { log.Info("Rebooting to enable TigeraSecure controllers") os.Exit(0) @@ -1052,7 +1033,7 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile var managementCluster *operatorv1.ManagementCluster var managementClusterConnection *operatorv1.ManagementClusterConnection var logCollector *operatorv1.LogCollector - if r.enterpriseCRDsExist { + if r.opts.EnterpriseCRDExists { logCollector, err = utils.GetLogCollector(ctx, r.client) if logCollector != nil { if err != nil { @@ -1098,7 +1079,7 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile } } - certificateManager, err := certificatemanager.Create(r.client, &instance.Spec, r.clusterDomain, common.OperatorNamespace(), certificatemanager.WithLogger(reqLogger)) + certificateManager, err := certificatemanager.Create(r.client, &instance.Spec, r.opts.ClusterDomain, common.OperatorNamespace(), certificatemanager.WithLogger(reqLogger)) if err != nil { r.status.SetDegraded(operatorv1.ResourceCreateError, "Unable to create the Tigera CA", err, reqLogger) return reconcile.Result{}, err @@ -1235,7 +1216,7 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile FelixConfiguration: felixConfiguration, CertificateManager: certificateManager, TrustedBundle: typhaNodeTLS.TrustedBundle, - ClusterDomain: r.clusterDomain, + ClusterDomain: r.opts.ClusterDomain, }) if err != nil { r.status.SetDegraded(operatorv1.ResourceCreateError, "Error preparing installation extension", err, reqLogger) @@ -1256,7 +1237,7 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile r.client, kubecontrollers.KubeControllerPrometheusTLSSecret, common.OperatorNamespace(), - dns.GetServiceDNSNames(kubecontrollers.KubeControllerMetrics, common.CalicoNamespace, r.clusterDomain)) + dns.GetServiceDNSNames(kubecontrollers.KubeControllerMetrics, common.CalicoNamespace, r.opts.ClusterDomain)) if err != nil { r.status.SetDegraded(operatorv1.ResourceReadError, "Error finding or creating TLS certificate kube controllers metric", err, reqLogger) return reconcile.Result{}, err @@ -1370,7 +1351,7 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile r.client, applicationlayer.WAFWebhookServerTLSSecretName, common.OperatorNamespace(), - dns.GetServiceDNSNames(applicationlayer.WAFWebhookServiceName, common.CalicoNamespace, r.clusterDomain)) + dns.GetServiceDNSNames(applicationlayer.WAFWebhookServiceName, common.CalicoNamespace, r.opts.ClusterDomain)) if err != nil { r.status.SetDegraded(operatorv1.ResourceCreateError, "Error creating WAF admission webhook TLS certificate", err, reqLogger) return reconcile.Result{}, err @@ -1429,11 +1410,11 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile hepListWatch := cache.NewListWatchFromClient(calicoClient.ProjectcalicoV3().RESTClient(), "hostendpoints", corev1.NamespaceAll, fields.Everything()) hepIndexInformer := cache.NewSharedIndexInformer(hepListWatch, &v3.HostEndpoint{}, 0, cache.Indexers{}) - go hepIndexInformer.Run(r.shutdownContext.Done()) + go hepIndexInformer.Run(r.opts.ShutdownContext.Done()) - typhaNonClusterHostWatch := cache.NewListWatchFromClient(r.clientset.AppsV1().RESTClient(), "deployments", "calico-system", fields.OneTermEqualSelector("metadata.name", "calico-typha"+render.TyphaNonClusterHostSuffix)) - r.typhaAutoscalerNonClusterHost = newTyphaAutoscaler(r.clientset, hepIndexInformer, typhaNonClusterHostWatch, r.status, typhaAutoscalerOptionNonclusterHost(true)) - r.typhaAutoscalerNonClusterHost.start(r.shutdownContext) + typhaNonClusterHostWatch := cache.NewListWatchFromClient(r.opts.K8sClientset.AppsV1().RESTClient(), "deployments", "calico-system", fields.OneTermEqualSelector("metadata.name", "calico-typha"+render.TyphaNonClusterHostSuffix)) + r.typhaAutoscalerNonClusterHost = newTyphaAutoscaler(r.opts.K8sClientset, hepIndexInformer, typhaNonClusterHostWatch, r.status, typhaAutoscalerOptionNonclusterHost(true)) + r.typhaAutoscalerNonClusterHost.start(r.opts.ShutdownContext) } } } @@ -1445,7 +1426,7 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile Installation: &instance.Spec, TLS: typhaNodeTLS, MigrateNamespaces: needsNamespaceMigration, - ClusterDomain: r.clusterDomain, + ClusterDomain: r.opts.ClusterDomain, NonClusterHost: nonclusterhost, FelixHealthPort: *felixConfiguration.Spec.HealthPort, } @@ -1568,7 +1549,7 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile LogCollector: logCollector, BirdTemplates: birdTemplates, TLS: typhaNodeTLS, - ClusterDomain: r.clusterDomain, + ClusterDomain: r.opts.ClusterDomain, DefaultDNSPolicy: defaultDNSPolicy, DefaultDNSConfig: defaultDNSConfig, GoldmaneIP: goldmaneIP, @@ -1578,7 +1559,7 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile CanRemoveCNIFinalizer: canRemoveCNI, FelixHealthPort: *felixConfiguration.Spec.HealthPort, NodeCgroupV2Path: felixConfiguration.Spec.CgroupV2Path, - V3CRDs: r.v3CRDs, + V3CRDs: r.opts.UseV3CRDs, ImageOverrides: r.opts.Extensions.Images(), } @@ -1656,7 +1637,7 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile Installation: &instance.Spec, ManagementCluster: managementCluster, ManagementClusterConnection: managementClusterConnection, - ClusterDomain: r.clusterDomain, + ClusterDomain: r.opts.ClusterDomain, MetricsPort: kubeControllersMetricsPort, Terminating: installationMarkedForDeletion, MetricsServerTLS: kubeControllerTLS, @@ -2314,10 +2295,10 @@ func (r *ReconcileInstallation) checkActive(log logr.Logger) (*corev1.ConfigMap, } func (r *ReconcileInstallation) updateCRDs(ctx context.Context, variant operatorv1.ProductVariant, log logr.Logger) error { - if !r.manageCRDs { + if !r.opts.ManageCRDs { return nil } - crdComponent := render.NewCreationPassthrough(crds.ToRuntimeObjects(crds.GetCRDs(variant, r.v3CRDs)...)...) + crdComponent := render.NewCreationPassthrough(crds.ToRuntimeObjects(crds.GetCRDs(variant, r.opts.UseV3CRDs)...)...) // Specify nil for the CR so no ownership is put on the CRDs. We do this so removing the // Installation CR will not remove the CRDs. handler := r.newComponentHandler(log, r.client, r.scheme, nil) @@ -2329,19 +2310,19 @@ func (r *ReconcileInstallation) updateCRDs(ctx context.Context, variant operator } func (r *ReconcileInstallation) updateMutatingAdmissionPolicies(ctx context.Context, install *operatorv1.Installation, log logr.Logger) error { - if !r.manageCRDs || !r.v3CRDs { + if !r.opts.ManageCRDs || !r.opts.UseV3CRDs { return nil } // MutatingAdmissionPolicy served version was discovered once at startup (v1 was promoted to GA // in k8s 1.36 and v1beta1 (introduced in 1.32) is scheduled for removal in 1.37). - mapAPIVersion := r.apiDiscovery.ServedVersion(admission.APIGroup, admission.KindPolicy) + mapAPIVersion := r.opts.APIDiscovery.ServedVersion(admission.APIGroup, admission.KindPolicy) if mapAPIVersion == "" { r.status.SetDegraded(operatorv1.ResourceNotReady, "Kubernetes cluster does not serve MutatingAdmissionPolicy (requires v1.32+); policy defaulting will not be available", nil, log) return nil } - desired := admission.GetMutatingAdmissionPolicies(install.Spec.Variant, r.v3CRDs, mapAPIVersion) + desired := admission.GetMutatingAdmissionPolicies(install.Spec.Variant, r.opts.UseV3CRDs, mapAPIVersion) existingMAPs, existingMAPBs, err := admission.ListManaged(ctx, r.client, mapAPIVersion) if err != nil { r.status.SetDegraded(operatorv1.ResourceReadError, "Error listing managed MutatingAdmissionPolicy resources", err, log) @@ -2352,20 +2333,20 @@ func (r *ReconcileInstallation) updateMutatingAdmissionPolicies(ctx context.Cont } func (r *ReconcileInstallation) updateValidatingAdmissionPolicies(ctx context.Context, install *operatorv1.Installation, log logr.Logger) error { - if !r.manageCRDs || !r.v3CRDs { + if !r.opts.ManageCRDs || !r.opts.UseV3CRDs { return nil } // ValidatingAdmissionPolicy reached GA (v1) well before MutatingAdmissionPolicy, so it has its own // served version and is reconciled independently of whether the cluster serves MAPs. If the cluster // doesn't serve it at all there's nothing to do, so skip rather than degrade. - vapAPIVersion := r.apiDiscovery.ServedVersion(admission.APIGroup, admission.KindValidatingPolicy) + vapAPIVersion := r.opts.APIDiscovery.ServedVersion(admission.APIGroup, admission.KindValidatingPolicy) if vapAPIVersion == "" { log.Info("Kubernetes cluster does not serve ValidatingAdmissionPolicy, skipping") return nil } - desired := admission.GetValidatingAdmissionPolicies(install.Spec.Variant, r.v3CRDs, vapAPIVersion) + desired := admission.GetValidatingAdmissionPolicies(install.Spec.Variant, r.opts.UseV3CRDs, vapAPIVersion) existingVAPs, existingVAPBs, err := admission.ListManagedValidating(ctx, r.client, vapAPIVersion) if err != nil { r.status.SetDegraded(operatorv1.ResourceReadError, "Error listing managed ValidatingAdmissionPolicy resources", err, log) diff --git a/pkg/controller/installation/core_controller_test.go b/pkg/controller/installation/core_controller_test.go index d2e73fe0ad..3fcd559551 100644 --- a/pkg/controller/installation/core_controller_test.go +++ b/pkg/controller/installation/core_controller_test.go @@ -191,19 +191,21 @@ var _ = Describe("Testing core-controller installation", func() { // As the parameters in the client changes, we expect the outcomes of the reconcile loops to change. r = ReconcileInstallation{ - opts: options.ControllerOptions{Extensions: testExtensions}, - config: nil, // there is no fake for config - client: c, - scheme: scheme, - autoDetectedProvider: operator.ProviderNone, - status: mockStatus, - typhaAutoscaler: newTyphaAutoscaler(cs, nodeIndexInformer, test.NewTyphaListWatch(cs), mockStatus), - namespaceMigration: &fakeNamespaceMigration{}, - enterpriseCRDsExist: true, - migrationChecked: true, - tierWatchReady: ready, - migrationWatchReady: &utils.ReadyFlag{}, - newComponentHandler: utils.NewComponentHandler, + opts: options.ControllerOptions{ + Extensions: testExtensions, + DetectedProvider: operator.ProviderNone, + EnterpriseCRDExists: true, + }, + config: nil, // there is no fake for config + client: c, + scheme: scheme, + status: mockStatus, + typhaAutoscaler: newTyphaAutoscaler(cs, nodeIndexInformer, test.NewTyphaListWatch(cs), mockStatus), + namespaceMigration: &fakeNamespaceMigration{}, + migrationChecked: true, + tierWatchReady: ready, + migrationWatchReady: &utils.ReadyFlag{}, + newComponentHandler: utils.NewComponentHandler, } r.typhaAutoscaler.start(ctx) @@ -821,20 +823,22 @@ var _ = Describe("Testing core-controller installation", func() { // As the parameters in the client changes, we expect the outcomes of the reconcile loops to change. r = ReconcileInstallation{ - opts: options.ControllerOptions{Extensions: testExtensions}, - config: nil, // there is no fake for config - client: c, - scheme: scheme, - autoDetectedProvider: operator.ProviderNone, - status: mockStatus, - typhaAutoscaler: newTyphaAutoscaler(cs, nodeIndexInformer, test.NewTyphaListWatch(cs), mockStatus), - namespaceMigration: &fakeNamespaceMigration{}, - enterpriseCRDsExist: true, - migrationChecked: true, - clusterDomain: dns.DefaultClusterDomain, - tierWatchReady: ready, - migrationWatchReady: &utils.ReadyFlag{}, - newComponentHandler: utils.NewComponentHandler, + opts: options.ControllerOptions{ + Extensions: testExtensions, + DetectedProvider: operator.ProviderNone, + EnterpriseCRDExists: true, + ClusterDomain: dns.DefaultClusterDomain, + }, + config: nil, // there is no fake for config + client: c, + scheme: scheme, + status: mockStatus, + typhaAutoscaler: newTyphaAutoscaler(cs, nodeIndexInformer, test.NewTyphaListWatch(cs), mockStatus), + namespaceMigration: &fakeNamespaceMigration{}, + migrationChecked: true, + tierWatchReady: ready, + migrationWatchReady: &utils.ReadyFlag{}, + newComponentHandler: utils.NewComponentHandler, } r.typhaAutoscaler.start(ctx) @@ -1044,19 +1048,21 @@ var _ = Describe("Testing core-controller installation", func() { // As the parameters in the client changes, we expect the outcomes of the reconcile loops to change. r = ReconcileInstallation{ - opts: options.ControllerOptions{Extensions: testExtensions}, - config: nil, // there is no fake for config - client: c, - scheme: scheme, - autoDetectedProvider: operator.ProviderNone, - status: mockStatus, - typhaAutoscaler: newTyphaAutoscaler(cs, nodeIndexInformer, test.NewTyphaListWatch(cs), mockStatus), - namespaceMigration: &fakeNamespaceMigration{}, - enterpriseCRDsExist: true, - migrationChecked: true, - tierWatchReady: ready, - migrationWatchReady: &utils.ReadyFlag{}, - newComponentHandler: utils.NewComponentHandler, + opts: options.ControllerOptions{ + Extensions: testExtensions, + DetectedProvider: operator.ProviderNone, + EnterpriseCRDExists: true, + }, + config: nil, // there is no fake for config + client: c, + scheme: scheme, + status: mockStatus, + typhaAutoscaler: newTyphaAutoscaler(cs, nodeIndexInformer, test.NewTyphaListWatch(cs), mockStatus), + namespaceMigration: &fakeNamespaceMigration{}, + migrationChecked: true, + tierWatchReady: ready, + migrationWatchReady: &utils.ReadyFlag{}, + newComponentHandler: utils.NewComponentHandler, } r.typhaAutoscaler.start(ctx) @@ -2221,7 +2227,7 @@ var _ = Describe("Testing core-controller installation", func() { cr.Spec.Variant = operator.Calico cr.Status.Variant = operator.Calico Expect(c.Create(ctx, cr)).NotTo(HaveOccurred()) - r.enterpriseCRDsExist = false + r.opts.EnterpriseCRDExists = false Expect(c.Delete(ctx, &v3.Tier{ObjectMeta: metav1.ObjectMeta{Name: "calico-system"}})).NotTo(HaveOccurred()) _, err := r.Reconcile(ctx, reconcile.Request{}) @@ -2333,20 +2339,22 @@ var _ = Describe("Testing core-controller installation", func() { // As the parameters in the client changes, we expect the outcomes of the reconcile loops to change. r = ReconcileInstallation{ - opts: options.ControllerOptions{Extensions: testExtensions}, - config: nil, // there is no fake for config - client: c, - scheme: scheme, - autoDetectedProvider: operator.ProviderNone, - status: mockStatus, - typhaAutoscaler: newTyphaAutoscaler(cs, nodeIndexInformer, test.NewTyphaListWatch(cs), mockStatus), - namespaceMigration: &fakeNamespaceMigration{}, - enterpriseCRDsExist: true, - migrationChecked: true, - clusterDomain: dns.DefaultClusterDomain, - tierWatchReady: ready, - migrationWatchReady: &utils.ReadyFlag{}, - newComponentHandler: utils.NewComponentHandler, + opts: options.ControllerOptions{ + Extensions: testExtensions, + DetectedProvider: operator.ProviderNone, + EnterpriseCRDExists: true, + ClusterDomain: dns.DefaultClusterDomain, + }, + config: nil, // there is no fake for config + client: c, + scheme: scheme, + status: mockStatus, + typhaAutoscaler: newTyphaAutoscaler(cs, nodeIndexInformer, test.NewTyphaListWatch(cs), mockStatus), + namespaceMigration: &fakeNamespaceMigration{}, + migrationChecked: true, + tierWatchReady: ready, + migrationWatchReady: &utils.ReadyFlag{}, + newComponentHandler: utils.NewComponentHandler, } r.typhaAutoscaler.start(ctx) @@ -2471,18 +2479,20 @@ var _ = Describe("Testing core-controller installation", func() { componentHandler = newFakeComponentHandler() r = ReconcileInstallation{ - opts: options.ControllerOptions{Extensions: testExtensions}, - config: nil, // there is no fake for config - client: c, - scheme: scheme, - autoDetectedProvider: operator.ProviderNone, - status: mockStatus, - typhaAutoscaler: newTyphaAutoscaler(cs, nodeIndexInformer, test.NewTyphaListWatch(cs), mockStatus), - namespaceMigration: &fakeNamespaceMigration{}, - enterpriseCRDsExist: true, - migrationChecked: true, - tierWatchReady: ready, - migrationWatchReady: &utils.ReadyFlag{}, + opts: options.ControllerOptions{ + Extensions: testExtensions, + DetectedProvider: operator.ProviderNone, + EnterpriseCRDExists: true, + }, + config: nil, // there is no fake for config + client: c, + scheme: scheme, + status: mockStatus, + typhaAutoscaler: newTyphaAutoscaler(cs, nodeIndexInformer, test.NewTyphaListWatch(cs), mockStatus), + namespaceMigration: &fakeNamespaceMigration{}, + migrationChecked: true, + tierWatchReady: ready, + migrationWatchReady: &utils.ReadyFlag{}, newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object, ...utils.ComponentHandlerOption) utils.ComponentHandler { return componentHandler }, @@ -2635,13 +2645,15 @@ var _ = Describe("updateMutatingAdmissionPolicies", func() { It("should create v1 MAPs when v1 is served", func() { r = ReconcileInstallation{ - opts: options.ControllerOptions{Extensions: testExtensions}, - client: clientFor(), - scheme: scheme, - status: mockStatus, - manageCRDs: true, - v3CRDs: true, - apiDiscovery: discoveryFor(admission.VersionV1), + opts: options.ControllerOptions{ + Extensions: testExtensions, + ManageCRDs: true, + UseV3CRDs: true, + APIDiscovery: discoveryFor(admission.VersionV1), + }, + client: clientFor(), + scheme: scheme, + status: mockStatus, newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object, ...utils.ComponentHandlerOption) utils.ComponentHandler { return componentHandler }, @@ -2667,13 +2679,15 @@ var _ = Describe("updateMutatingAdmissionPolicies", func() { It("should create v1beta1 MAPs when only v1beta1 is served", func() { r = ReconcileInstallation{ - opts: options.ControllerOptions{Extensions: testExtensions}, - client: clientFor(), - scheme: scheme, - status: mockStatus, - manageCRDs: true, - v3CRDs: true, - apiDiscovery: discoveryFor(admission.VersionV1Beta1), + opts: options.ControllerOptions{ + Extensions: testExtensions, + ManageCRDs: true, + UseV3CRDs: true, + APIDiscovery: discoveryFor(admission.VersionV1Beta1), + }, + client: clientFor(), + scheme: scheme, + status: mockStatus, newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object, ...utils.ComponentHandlerOption) utils.ComponentHandler { return componentHandler }, @@ -2697,13 +2711,15 @@ var _ = Describe("updateMutatingAdmissionPolicies", func() { It("should create v1alpha1 MAPs when only v1alpha1 is served", func() { r = ReconcileInstallation{ - opts: options.ControllerOptions{Extensions: testExtensions}, - client: clientFor(), - scheme: scheme, - status: mockStatus, - manageCRDs: true, - v3CRDs: true, - apiDiscovery: discoveryFor(admission.VersionV1Alpha1), + opts: options.ControllerOptions{ + Extensions: testExtensions, + ManageCRDs: true, + UseV3CRDs: true, + APIDiscovery: discoveryFor(admission.VersionV1Alpha1), + }, + client: clientFor(), + scheme: scheme, + status: mockStatus, newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object, ...utils.ComponentHandlerOption) utils.ComponentHandler { return componentHandler }, @@ -2727,13 +2743,15 @@ var _ = Describe("updateMutatingAdmissionPolicies", func() { It("should not create MAPs when no served version exists and should set degraded", func() { r = ReconcileInstallation{ - opts: options.ControllerOptions{Extensions: testExtensions}, - client: clientFor(), - scheme: scheme, - status: mockStatus, - manageCRDs: true, - v3CRDs: true, - apiDiscovery: discoveryFor(""), + opts: options.ControllerOptions{ + Extensions: testExtensions, + ManageCRDs: true, + UseV3CRDs: true, + APIDiscovery: discoveryFor(""), + }, + client: clientFor(), + scheme: scheme, + status: mockStatus, newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object, ...utils.ComponentHandlerOption) utils.ComponentHandler { return componentHandler }, @@ -2746,13 +2764,15 @@ var _ = Describe("updateMutatingAdmissionPolicies", func() { It("should not create MAPs when v3CRDs=false", func() { r = ReconcileInstallation{ - opts: options.ControllerOptions{Extensions: testExtensions}, - client: clientFor(), - scheme: scheme, - status: mockStatus, - manageCRDs: true, - v3CRDs: false, - apiDiscovery: discoveryFor(admission.VersionV1), + opts: options.ControllerOptions{ + Extensions: testExtensions, + ManageCRDs: true, + UseV3CRDs: false, + APIDiscovery: discoveryFor(admission.VersionV1), + }, + client: clientFor(), + scheme: scheme, + status: mockStatus, newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object, ...utils.ComponentHandlerOption) utils.ComponentHandler { return componentHandler }, @@ -2764,13 +2784,15 @@ var _ = Describe("updateMutatingAdmissionPolicies", func() { It("should not create MAPs when manageCRDs=false", func() { r = ReconcileInstallation{ - opts: options.ControllerOptions{Extensions: testExtensions}, - client: clientFor(), - scheme: scheme, - status: mockStatus, - manageCRDs: false, - v3CRDs: true, - apiDiscovery: discoveryFor(admission.VersionV1), + opts: options.ControllerOptions{ + Extensions: testExtensions, + ManageCRDs: false, + UseV3CRDs: true, + APIDiscovery: discoveryFor(admission.VersionV1), + }, + client: clientFor(), + scheme: scheme, + status: mockStatus, newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object, ...utils.ComponentHandlerOption) utils.ComponentHandler { return componentHandler }, @@ -2795,13 +2817,15 @@ var _ = Describe("updateMutatingAdmissionPolicies", func() { } r = ReconcileInstallation{ - opts: options.ControllerOptions{Extensions: testExtensions}, - client: clientFor(staleMAP, staleMAPB), - scheme: scheme, - status: mockStatus, - manageCRDs: true, - v3CRDs: true, - apiDiscovery: discoveryFor(admission.VersionV1), + opts: options.ControllerOptions{ + Extensions: testExtensions, + ManageCRDs: true, + UseV3CRDs: true, + APIDiscovery: discoveryFor(admission.VersionV1), + }, + client: clientFor(staleMAP, staleMAPB), + scheme: scheme, + status: mockStatus, newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object, ...utils.ComponentHandlerOption) utils.ComponentHandler { return componentHandler }, @@ -2838,13 +2862,15 @@ var _ = Describe("updateMutatingAdmissionPolicies", func() { } r = ReconcileInstallation{ - opts: options.ControllerOptions{Extensions: testExtensions}, - client: clientFor(initial...), - scheme: scheme, - status: mockStatus, - manageCRDs: true, - v3CRDs: true, - apiDiscovery: discoveryFor(admission.VersionV1), + opts: options.ControllerOptions{ + Extensions: testExtensions, + ManageCRDs: true, + UseV3CRDs: true, + APIDiscovery: discoveryFor(admission.VersionV1), + }, + client: clientFor(initial...), + scheme: scheme, + status: mockStatus, newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object, ...utils.ComponentHandlerOption) utils.ComponentHandler { return componentHandler }, @@ -2857,13 +2883,15 @@ var _ = Describe("updateMutatingAdmissionPolicies", func() { It("should work with Enterprise variant", func() { r = ReconcileInstallation{ - opts: options.ControllerOptions{Extensions: testExtensions}, - client: clientFor(), - scheme: scheme, - status: mockStatus, - manageCRDs: true, - v3CRDs: true, - apiDiscovery: discoveryFor(admission.VersionV1), + opts: options.ControllerOptions{ + Extensions: testExtensions, + ManageCRDs: true, + UseV3CRDs: true, + APIDiscovery: discoveryFor(admission.VersionV1), + }, + client: clientFor(), + scheme: scheme, + status: mockStatus, newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object, ...utils.ComponentHandlerOption) utils.ComponentHandler { return componentHandler }, @@ -2929,13 +2957,15 @@ var _ = Describe("updateValidatingAdmissionPolicies", func() { It("should create v1 VAPs when v1 is served", func() { r = ReconcileInstallation{ - opts: options.ControllerOptions{Extensions: testExtensions}, - client: clientFor(), - scheme: scheme, - status: mockStatus, - manageCRDs: true, - v3CRDs: true, - apiDiscovery: discoveryFor(admission.VersionV1), + opts: options.ControllerOptions{ + Extensions: testExtensions, + ManageCRDs: true, + UseV3CRDs: true, + APIDiscovery: discoveryFor(admission.VersionV1), + }, + client: clientFor(), + scheme: scheme, + status: mockStatus, newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object, ...utils.ComponentHandlerOption) utils.ComponentHandler { return componentHandler }, @@ -2961,13 +2991,15 @@ var _ = Describe("updateValidatingAdmissionPolicies", func() { It("should create v1beta1 VAPs when only v1beta1 is served", func() { r = ReconcileInstallation{ - opts: options.ControllerOptions{Extensions: testExtensions}, - client: clientFor(), - scheme: scheme, - status: mockStatus, - manageCRDs: true, - v3CRDs: true, - apiDiscovery: discoveryFor(admission.VersionV1Beta1), + opts: options.ControllerOptions{ + Extensions: testExtensions, + ManageCRDs: true, + UseV3CRDs: true, + APIDiscovery: discoveryFor(admission.VersionV1Beta1), + }, + client: clientFor(), + scheme: scheme, + status: mockStatus, newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object, ...utils.ComponentHandlerOption) utils.ComponentHandler { return componentHandler }, @@ -2991,13 +3023,15 @@ var _ = Describe("updateValidatingAdmissionPolicies", func() { It("should create v1alpha1 VAPs when only v1alpha1 is served", func() { r = ReconcileInstallation{ - opts: options.ControllerOptions{Extensions: testExtensions}, - client: clientFor(), - scheme: scheme, - status: mockStatus, - manageCRDs: true, - v3CRDs: true, - apiDiscovery: discoveryFor(admission.VersionV1Alpha1), + opts: options.ControllerOptions{ + Extensions: testExtensions, + ManageCRDs: true, + UseV3CRDs: true, + APIDiscovery: discoveryFor(admission.VersionV1Alpha1), + }, + client: clientFor(), + scheme: scheme, + status: mockStatus, newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object, ...utils.ComponentHandlerOption) utils.ComponentHandler { return componentHandler }, @@ -3009,13 +3043,15 @@ var _ = Describe("updateValidatingAdmissionPolicies", func() { It("should skip without degrading when no served version exists", func() { r = ReconcileInstallation{ - opts: options.ControllerOptions{Extensions: testExtensions}, - client: clientFor(), - scheme: scheme, - status: mockStatus, - manageCRDs: true, - v3CRDs: true, - apiDiscovery: discoveryFor(""), + opts: options.ControllerOptions{ + Extensions: testExtensions, + ManageCRDs: true, + UseV3CRDs: true, + APIDiscovery: discoveryFor(""), + }, + client: clientFor(), + scheme: scheme, + status: mockStatus, newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object, ...utils.ComponentHandlerOption) utils.ComponentHandler { return componentHandler }, @@ -3028,13 +3064,15 @@ var _ = Describe("updateValidatingAdmissionPolicies", func() { It("should not create VAPs when v3CRDs=false", func() { r = ReconcileInstallation{ - opts: options.ControllerOptions{Extensions: testExtensions}, - client: clientFor(), - scheme: scheme, - status: mockStatus, - manageCRDs: true, - v3CRDs: false, - apiDiscovery: discoveryFor(admission.VersionV1), + opts: options.ControllerOptions{ + Extensions: testExtensions, + ManageCRDs: true, + UseV3CRDs: false, + APIDiscovery: discoveryFor(admission.VersionV1), + }, + client: clientFor(), + scheme: scheme, + status: mockStatus, newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object, ...utils.ComponentHandlerOption) utils.ComponentHandler { return componentHandler }, @@ -3059,13 +3097,15 @@ var _ = Describe("updateValidatingAdmissionPolicies", func() { } r = ReconcileInstallation{ - opts: options.ControllerOptions{Extensions: testExtensions}, - client: clientFor(staleVAP, staleVAPB), - scheme: scheme, - status: mockStatus, - manageCRDs: true, - v3CRDs: true, - apiDiscovery: discoveryFor(admission.VersionV1), + opts: options.ControllerOptions{ + Extensions: testExtensions, + ManageCRDs: true, + UseV3CRDs: true, + APIDiscovery: discoveryFor(admission.VersionV1), + }, + client: clientFor(staleVAP, staleVAPB), + scheme: scheme, + status: mockStatus, newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object, ...utils.ComponentHandlerOption) utils.ComponentHandler { return componentHandler }, @@ -3084,13 +3124,15 @@ var _ = Describe("updateValidatingAdmissionPolicies", func() { It("should work with Enterprise variant", func() { r = ReconcileInstallation{ - opts: options.ControllerOptions{Extensions: testExtensions}, - client: clientFor(), - scheme: scheme, - status: mockStatus, - manageCRDs: true, - v3CRDs: true, - apiDiscovery: discoveryFor(admission.VersionV1), + opts: options.ControllerOptions{ + Extensions: testExtensions, + ManageCRDs: true, + UseV3CRDs: true, + APIDiscovery: discoveryFor(admission.VersionV1), + }, + client: clientFor(), + scheme: scheme, + status: mockStatus, newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object, ...utils.ComponentHandlerOption) utils.ComponentHandler { return componentHandler }, diff --git a/pkg/controller/installation/windows_controller.go b/pkg/controller/installation/windows_controller.go index 12e31b2a1f..ad758dbbce 100644 --- a/pkg/controller/installation/windows_controller.go +++ b/pkg/controller/installation/windows_controller.go @@ -83,7 +83,7 @@ func AddWindowsController(mgr manager.Manager, opts options.ControllerOptions) e return fmt.Errorf("tigera-windows-controller failed to watch calico Tigerastatus: %w", err) } - if ri.autoDetectedProvider.IsOpenShift() { + if ri.opts.DetectedProvider.IsOpenShift() { // Watch for openshift network configuration as well. If we're running in OpenShift, we need to // merge this configuration with our own and the write back the status object. err = c.WatchObject(&configv1.Network{}, &handler.EnqueueRequestForObject{}) @@ -151,7 +151,7 @@ func AddWindowsController(mgr manager.Manager, opts options.ControllerOptions) e // Watch for changes to IPAMConfiguration. go utils.WaitToAddResourceWatch(c, opts.K8sClientset, logw, ri.ipamConfigWatchReady, []client.Object{&v3.IPAMConfiguration{TypeMeta: metav1.TypeMeta{Kind: v3.KindIPAMConfiguration}}}) - if ri.enterpriseCRDsExist { + if ri.opts.EnterpriseCRDExists { for _, ns := range []string{common.CalicoNamespace, common.OperatorNamespace()} { if err = utils.AddSecretsWatch(c, render.NodePrometheusTLSServerSecret, ns); err != nil { return fmt.Errorf("tigera-windows-controller failed to watch secret '%s' in '%s' namespace: %w", render.NodePrometheusTLSServerSecret, ns, err) @@ -177,10 +177,7 @@ type ReconcileWindows struct { client client.Client scheme *runtime.Scheme watches map[runtime.Object]struct{} - autoDetectedProvider operatorv1.Provider status status.StatusManager - enterpriseCRDsExist bool - clusterDomain string ipamConfigWatchReady *utils.ReadyFlag opts options.ControllerOptions } @@ -194,10 +191,7 @@ func newWindowsReconciler(mgr manager.Manager, opts options.ControllerOptions) ( client: mgr.GetClient(), scheme: mgr.GetScheme(), watches: make(map[runtime.Object]struct{}), - autoDetectedProvider: opts.DetectedProvider, status: statusManager, - enterpriseCRDsExist: opts.EnterpriseCRDExists, - clusterDomain: opts.ClusterDomain, ipamConfigWatchReady: &utils.ReadyFlag{}, opts: opts, } @@ -291,7 +285,7 @@ func (r *ReconcileWindows) Reconcile(ctx context.Context, request reconcile.Requ return reconcile.Result{}, err } - certificateManager, err := certificatemanager.Create(r.client, &instance.Spec, r.clusterDomain, common.OperatorNamespace()) + certificateManager, err := certificatemanager.Create(r.client, &instance.Spec, r.opts.ClusterDomain, common.OperatorNamespace()) if err != nil { r.status.SetDegraded(operatorv1.ResourceCreateError, "Unable to create the Tigera CA", err, reqLogger) return reconcile.Result{}, err @@ -351,7 +345,7 @@ func (r *ReconcileWindows) Reconcile(ctx context.Context, request reconcile.Requ } // The key pair is created by the core controller, so if it isn't set, requeue to wait until it is - nodePrometheusTLS, err = certificateManager.GetKeyPair(r.client, render.NodePrometheusTLSServerSecret, common.OperatorNamespace(), dns.GetServiceDNSNames(render.WindowsNodeMetricsService, common.CalicoNamespace, r.clusterDomain)) + nodePrometheusTLS, err = certificateManager.GetKeyPair(r.client, render.NodePrometheusTLSServerSecret, common.OperatorNamespace(), dns.GetServiceDNSNames(render.WindowsNodeMetricsService, common.CalicoNamespace, r.opts.ClusterDomain)) if err != nil { r.status.SetDegraded(operatorv1.ResourceCreateError, "Error getting TLS certificate", err, reqLogger) return reconcile.Result{}, err @@ -360,7 +354,7 @@ func (r *ReconcileWindows) Reconcile(ctx context.Context, request reconcile.Requ var component render.Component - kubeDNSServiceName := utils.GetDNSServiceName(r.autoDetectedProvider) + kubeDNSServiceName := utils.GetDNSServiceName(r.opts.DetectedProvider) kubeDNSService := &corev1.Service{} err = r.client.Get(ctx, kubeDNSServiceName, kubeDNSService) if err != nil { @@ -384,7 +378,7 @@ func (r *ReconcileWindows) Reconcile(ctx context.Context, request reconcile.Requ K8sServiceEp: k8sapi.Endpoint, K8sDNSServers: kubeDNSIPs, Installation: &instance.Spec, - ClusterDomain: r.clusterDomain, + ClusterDomain: r.opts.ClusterDomain, TLS: typhaNodeTLS, PrometheusServerTLS: nodePrometheusTLS, NodeReporterMetricsPort: nodeReporterMetricsPort, diff --git a/pkg/controller/installation/windows_controller_test.go b/pkg/controller/installation/windows_controller_test.go index 98121210e7..6fa1d2cf0d 100644 --- a/pkg/controller/installation/windows_controller_test.go +++ b/pkg/controller/installation/windows_controller_test.go @@ -120,13 +120,15 @@ var _ = Describe("windows-controller installation tests", func() { // As the parameters in the client changes, we expect the outcomes of the reconcile loops to change. r = ReconcileWindows{ - opts: options.ControllerOptions{Extensions: testExtensions}, + opts: options.ControllerOptions{ + Extensions: testExtensions, + DetectedProvider: operator.ProviderNone, + EnterpriseCRDExists: true, + }, config: nil, // there is no fake for config client: c, scheme: scheme, - autoDetectedProvider: operator.ProviderNone, status: mockStatus, - enterpriseCRDsExist: true, ipamConfigWatchReady: &utils.ReadyFlag{}, } r.ipamConfigWatchReady.MarkAsReady() @@ -157,7 +159,7 @@ var _ = Describe("windows-controller installation tests", func() { }, }, } - Expect(updateInstallationWithDefaults(ctx, r.client, cr, r.autoDetectedProvider)).NotTo(HaveOccurred()) + Expect(updateInstallationWithDefaults(ctx, r.client, cr, r.opts.DetectedProvider)).NotTo(HaveOccurred()) certificateManager, err := certificatemanager.Create(c, nil, "", common.OperatorNamespace(), certificatemanager.AllowCACreation()) Expect(err).NotTo(HaveOccurred()) prometheusTLS, err := certificateManager.GetOrCreateKeyPair(c, monitor.PrometheusClientTLSSecretName, common.OperatorNamespace(), []string{monitor.PrometheusClientTLSSecretName}) @@ -196,7 +198,7 @@ var _ = Describe("windows-controller installation tests", func() { cr.Status = operator.InstallationStatus{ Variant: operator.Calico, } - Expect(updateInstallationWithDefaults(ctx, r.client, cr, r.autoDetectedProvider)).NotTo(HaveOccurred()) + Expect(updateInstallationWithDefaults(ctx, r.client, cr, r.opts.DetectedProvider)).NotTo(HaveOccurred()) // Set serviceCIDRs in the installation (required for Calico for Windows) cr.Spec.ServiceCIDRs = []string{"10.96.0.0/12"} @@ -611,13 +613,15 @@ var _ = Describe("windows-controller installation tests", func() { // As the parameters in the client changes, we expect the outcomes of the reconcile loops to change. r = ReconcileWindows{ - opts: options.ControllerOptions{Extensions: testExtensions}, + opts: options.ControllerOptions{ + Extensions: testExtensions, + DetectedProvider: operator.ProviderNone, + EnterpriseCRDExists: true, + }, config: nil, // there is no fake for config client: c, scheme: scheme, - autoDetectedProvider: operator.ProviderNone, status: mockStatus, - enterpriseCRDsExist: true, ipamConfigWatchReady: &utils.ReadyFlag{}, } r.ipamConfigWatchReady.MarkAsReady() @@ -666,7 +670,7 @@ var _ = Describe("windows-controller installation tests", func() { }, }, } - Expect(updateInstallationWithDefaults(ctx, r.client, instance, r.autoDetectedProvider)).NotTo(HaveOccurred()) + Expect(updateInstallationWithDefaults(ctx, r.client, instance, r.opts.DetectedProvider)).NotTo(HaveOccurred()) Expect(c.Create(ctx, instance)).NotTo(HaveOccurred()) }) AfterEach(func() { From 8b5349e365f984a4764f46e6d4a6f0dd64d5a0c0 Mon Sep 17 00:00:00 2001 From: Casey Davenport Date: Thu, 18 Jun 2026 09:00:37 -0700 Subject: [PATCH 28/54] Apply variant extensions by decorating the rendered component Set.Decorate wraps a component so its objects pass through the registered modifier, and the wrapped value is itself a render.Component the handler renders like any other. Rename Extension to ComponentExtension. --- pkg/controller/utils/component.go | 26 +++++----- pkg/controller/utils/component_test.go | 2 +- pkg/enterprise/apiserver.go | 4 +- pkg/enterprise/decorate_helpers_test.go | 65 +++++++++++++++++++++++++ pkg/enterprise/guardian.go | 4 +- pkg/enterprise/guardian_test.go | 10 ++-- pkg/enterprise/node.go | 4 +- pkg/enterprise/node_test.go | 20 ++++---- pkg/enterprise/typha.go | 2 +- pkg/enterprise/typha_test.go | 6 +-- pkg/enterprise/windows.go | 6 +-- pkg/enterprise/windows_test.go | 10 ++-- pkg/extensions/decorate_helpers_test.go | 65 +++++++++++++++++++++++++ pkg/extensions/extension.go | 55 +++++++++++++++------ pkg/extensions/extension_test.go | 22 ++++----- pkg/extensions/image_test.go | 2 +- pkg/extensions/set.go | 4 +- pkg/render/apiserver_test.go | 2 +- pkg/render/decorate_helpers_test.go | 65 +++++++++++++++++++++++++ pkg/render/guardian_test.go | 6 +-- pkg/render/node_enterprise_test.go | 4 +- pkg/render/windows_test.go | 2 +- 22 files changed, 303 insertions(+), 83 deletions(-) create mode 100644 pkg/enterprise/decorate_helpers_test.go create mode 100644 pkg/extensions/decorate_helpers_test.go create mode 100644 pkg/render/decorate_helpers_test.go diff --git a/pkg/controller/utils/component.go b/pkg/controller/utils/component.go index 76db4ebf44..f805aef659 100644 --- a/pkg/controller/utils/component.go +++ b/pkg/controller/utils/component.go @@ -458,6 +458,18 @@ func resetMetadataForCreate(obj client.Object) { } func (c *componentHandler) CreateOrUpdateOrDelete(ctx context.Context, component render.Component, status status.StatusManager) error { + // Decorate the component with any registered variant extension before doing + // anything with it, so Ready, SupportedOSType, and Objects all reflect the + // extended component. Decorate is a no-op for components with no registered + // extension. + if ext, ok := component.(render.Extensible); ok && c.extensions == nil { + // The component can be extended but this handler was built without an + // extension Set, so any registered extension silently won't run. That is + // a wiring bug in the controller, not a normal state. + c.log.Info("BUG: extensible component rendered by a handler with no extension Set; extensions will not be applied", "component", ext.ModifierKey()) + } + component = c.extensions.Decorate(component, c.renderCtx) + // Before creating the component, make sure that it is ready. This provides a hook to do // dependency checking for the component. cmpLog := c.log.WithValues("component", reflect.TypeOf(component)) @@ -476,20 +488,6 @@ func (c *componentHandler) CreateOrUpdateOrDelete(ctx context.Context, component var cronJobs []types.NamespacedName objsToCreate, objsToDelete := component.Objects() - if ext, ok := component.(render.Extensible); ok { - if c.extensions == nil { - // The component can be extended but this handler was built without an - // extension Set, so any registered modifier silently won't run. That is - // a wiring bug in the controller, not a normal state. - c.log.Info("BUG: extensible component rendered by a handler with no extension Set; modifiers will not be applied", "component", ext.ModifierKey()) - } else { - rc := c.renderCtx - if p, ok := component.(render.ExtensionContextProvider); ok { - rc.Component = p.ExtensionContext() - } - objsToCreate, objsToDelete = c.extensions.ApplyModifiers(ext.ModifierKey(), rc, objsToCreate, objsToDelete) - } - } // Load the InstallationSpec once and reuse it for every object: createOrUpdateObject needs it // for image pull policy and TLS ciphers, and we use it here to decide whether the user has diff --git a/pkg/controller/utils/component_test.go b/pkg/controller/utils/component_test.go index d8c227d47b..ec369e2ac3 100644 --- a/pkg/controller/utils/component_test.go +++ b/pkg/controller/utils/component_test.go @@ -2580,7 +2580,7 @@ func (mc *mockClient) SubResource(subResource string) client.SubResourceClient { var _ = Describe("componentHandler modifier application", func() { It("applies registered modifiers to a named component before create", func() { ext := extensions.NewSet() - ext.Register(operatorv1.CalicoEnterprise, "fake", extensions.Extension{ + ext.Register(operatorv1.CalicoEnterprise, "fake", extensions.ComponentExtension{ Modify: func(ctx extensions.RenderContext, objs, del []client.Object) ([]client.Object, []client.Object) { cm := objs[0].(*corev1.ConfigMap) cm.Data = map[string]string{"patched": "yes"} diff --git a/pkg/enterprise/apiserver.go b/pkg/enterprise/apiserver.go index 1b7119f239..913391cb16 100644 --- a/pkg/enterprise/apiserver.go +++ b/pkg/enterprise/apiserver.go @@ -53,12 +53,12 @@ type apiServer struct { } func registerAPIServer(s *extensions.Set) { - s.Register(operatorv1.CalicoEnterprise, render.ComponentNameAPIServer, extensions.Extension{ + s.Register(operatorv1.CalicoEnterprise, render.ComponentNameAPIServer, extensions.ComponentExtension{ Modify: modifyAPIServer, }) // When running Calico, clean up any Enterprise objects left behind by a prior // Enterprise installation. - s.Register(operatorv1.Calico, render.ComponentNameAPIServer, extensions.Extension{ + s.Register(operatorv1.Calico, render.ComponentNameAPIServer, extensions.ComponentExtension{ Modify: cleanupAPIServer, }) } diff --git a/pkg/enterprise/decorate_helpers_test.go b/pkg/enterprise/decorate_helpers_test.go new file mode 100644 index 0000000000..5bade86c8f --- /dev/null +++ b/pkg/enterprise/decorate_helpers_test.go @@ -0,0 +1,65 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package enterprise_test + +import ( + client "sigs.k8s.io/controller-runtime/pkg/client" + + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/extensions" + rmeta "github.com/tigera/operator/pkg/render/common/meta" +) + +// stubExtComponent adapts raw object lists to a render.Component so a registered +// extension can be exercised through Set.Decorate, the same seam the component +// handler uses. key selects the extension; extCtx is delivered as the per-component +// ExtensionContext. +type stubExtComponent struct { + key string + extCtx any + create, delete []client.Object +} + +func (s stubExtComponent) ResolveImages(*operatorv1.ImageSet) error { + return nil +} + +func (s stubExtComponent) Objects() ([]client.Object, []client.Object) { + return s.create, s.delete +} + +func (s stubExtComponent) Ready() bool { + return true +} + +func (s stubExtComponent) SupportedOSType() rmeta.OSType { + return rmeta.OSTypeAny +} + +func (s stubExtComponent) ModifierKey() string { + return s.key +} + +func (s stubExtComponent) ExtensionContext() any { + return s.extCtx +} + +// applyExtensions decorates a stub component holding the given objects with the +// extension registered under key in s, then renders it - returning the decorated +// create and delete lists. +func applyExtensions(s *extensions.Set, key string, ctx extensions.RenderContext, create, del []client.Object) ([]client.Object, []client.Object) { + stub := stubExtComponent{key: key, extCtx: ctx.Component, create: create, delete: del} + return s.Decorate(stub, ctx).Objects() +} diff --git a/pkg/enterprise/guardian.go b/pkg/enterprise/guardian.go index cff8185428..0d81779c8f 100644 --- a/pkg/enterprise/guardian.go +++ b/pkg/enterprise/guardian.go @@ -39,10 +39,10 @@ import ( ) func registerGuardian(s *extensions.Set) { - s.Register(operatorv1.CalicoEnterprise, render.GuardianName, extensions.Extension{ + s.Register(operatorv1.CalicoEnterprise, render.GuardianName, extensions.ComponentExtension{ Modify: modifyGuardian, }) - s.Register(operatorv1.CalicoEnterprise, render.ComponentNameGuardianPolicy, extensions.Extension{ + s.Register(operatorv1.CalicoEnterprise, render.ComponentNameGuardianPolicy, extensions.ComponentExtension{ Modify: modifyGuardianPolicy, }) } diff --git a/pkg/enterprise/guardian_test.go b/pkg/enterprise/guardian_test.go index 5223f72a47..87f5207ee2 100644 --- a/pkg/enterprise/guardian_test.go +++ b/pkg/enterprise/guardian_test.go @@ -57,7 +57,7 @@ var _ = Describe("guardian enterprise modifier", func() { } It("appends the secrets RBAC and UI settings", func() { - out, _ := ext.ApplyModifiers(render.GuardianName, ctxWith(render.GuardianExtensionContext{}), newObjs(), nil) + out, _ := applyExtensions(ext, render.GuardianName, ctxWith(render.GuardianExtensionContext{}), newObjs(), nil) _, ok := extensions.FindObject[*rbacv1.Role](out, render.GuardianSecretsRole) Expect(ok).To(BeTrue()) _, ok = extensions.FindObject[*rbacv1.RoleBinding](out, render.GuardianSecretsRoleBindingName) @@ -67,7 +67,7 @@ var _ = Describe("guardian enterprise modifier", func() { }) It("adds the elasticsearch and kibana service ports", func() { - out, _ := ext.ApplyModifiers(render.GuardianName, ctxWith(render.GuardianExtensionContext{}), newObjs(), nil) + out, _ := applyExtensions(ext, render.GuardianName, ctxWith(render.GuardianExtensionContext{}), newObjs(), nil) svc, _ := extensions.FindObject[*corev1.Service](out, render.GuardianServiceName) names := []string{} for _, p := range svc.Spec.Ports { @@ -80,7 +80,7 @@ var _ = Describe("guardian enterprise modifier", func() { gc := render.GuardianExtensionContext{ Impersonation: &operatorv1.Impersonation{Users: []string{"foo"}, Groups: []string{"bar"}}, } - out, _ := ext.ApplyModifiers(render.GuardianName, ctxWith(gc), newObjs(), nil) + out, _ := applyExtensions(ext, render.GuardianName, ctxWith(gc), newObjs(), nil) role, _ := extensions.FindObject[*rbacv1.ClusterRole](out, render.GuardianClusterRoleName) // The single OSS placeholder rule is gone, replaced by the enterprise set. @@ -91,14 +91,14 @@ var _ = Describe("guardian enterprise modifier", func() { It("adds the CA bundle env to the guardian container", func() { gc := render.GuardianExtensionContext{TrustedBundleMountPath: "/ca/bundle"} - out, _ := ext.ApplyModifiers(render.GuardianName, ctxWith(gc), newObjs(), nil) + out, _ := applyExtensions(ext, render.GuardianName, ctxWith(gc), newObjs(), nil) dep, _ := extensions.FindObject[*appsv1.Deployment](out, render.GuardianDeploymentName) Expect(dep.Spec.Template.Spec.Containers[0].Env).To(ContainElement(corev1.EnvVar{Name: "GUARDIAN_PROMETHEUS_CA_BUNDLE_PATH", Value: "/ca/bundle"})) }) It("does nothing for the Calico variant", func() { ctx := extensions.RenderContext{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.Calico}} - out, _ := ext.ApplyModifiers(render.GuardianName, ctx, newObjs(), nil) + out, _ := applyExtensions(ext, render.GuardianName, ctx, newObjs(), nil) Expect(out).To(HaveLen(len(newObjs()))) role, _ := extensions.FindObject[*rbacv1.ClusterRole](out, render.GuardianClusterRoleName) Expect(role.Rules).To(Equal([]rbacv1.PolicyRule{{Verbs: []string{"get"}}})) diff --git a/pkg/enterprise/node.go b/pkg/enterprise/node.go index 752155bd01..d0ca3e2135 100644 --- a/pkg/enterprise/node.go +++ b/pkg/enterprise/node.go @@ -47,7 +47,7 @@ const ( ) func registerNode(s *extensions.Set) { - s.Register(operatorv1.CalicoEnterprise, render.ComponentNameNode, extensions.Extension{ + s.Register(operatorv1.CalicoEnterprise, render.ComponentNameNode, extensions.ComponentExtension{ Image: func(in *operatorv1.InstallationSpec) components.Component { return components.ComponentTigeraNode }, @@ -56,7 +56,7 @@ func registerNode(s *extensions.Set) { // The node component renders the cni-plugins init container; its image // resolves through its own override key. - s.Register(operatorv1.CalicoEnterprise, render.ComponentNameCNIPlugins, extensions.Extension{ + s.Register(operatorv1.CalicoEnterprise, render.ComponentNameCNIPlugins, extensions.ComponentExtension{ Image: func(in *operatorv1.InstallationSpec) components.Component { return components.ComponentTigeraCNIPlugins }, diff --git a/pkg/enterprise/node_test.go b/pkg/enterprise/node_test.go index 23ca58c98e..4257766122 100644 --- a/pkg/enterprise/node_test.go +++ b/pkg/enterprise/node_test.go @@ -81,7 +81,7 @@ var _ = Describe("node enterprise modifier", func() { } It("adds the enterprise cluster role rules", func() { - out, _ := ext.ApplyModifiers(render.ComponentNameNode, entCtx(), newObjs(), nil) + out, _ := applyExtensions(ext, render.ComponentNameNode, entCtx(), newObjs(), nil) nodeRole, ok := extensions.FindObject[*rbacv1.ClusterRole](out, render.CalicoNodeObjectName) Expect(ok).To(BeTrue()) @@ -93,7 +93,7 @@ var _ = Describe("node enterprise modifier", func() { }) It("adds the enterprise felix env to the node container", func() { - out, _ := ext.ApplyModifiers(render.ComponentNameNode, entCtx(), newObjs(), nil) + out, _ := applyExtensions(ext, render.ComponentNameNode, entCtx(), newObjs(), nil) ds, _ := extensions.FindObject[*appsv1.DaemonSet](out, common.NodeDaemonSetName) c := nodeContainer(ds) @@ -110,13 +110,13 @@ var _ = Describe("node enterprise modifier", func() { ctx := entCtx() ctx.FelixConfiguration = &v3.FelixConfiguration{Spec: v3.FelixConfigurationSpec{PrometheusReporterPort: &reporter}} - out, _ := ext.ApplyModifiers(render.ComponentNameNode, ctx, newObjs(), nil) + out, _ := applyExtensions(ext, render.ComponentNameNode, ctx, newObjs(), nil) ds, _ := extensions.FindObject[*appsv1.DaemonSet](out, common.NodeDaemonSetName) Expect(nodeContainer(ds).Env).To(ContainElement(corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERPORT", Value: "7081"})) }) It("appends the BGP metrics readiness check when the bird check is present", func() { - out, _ := ext.ApplyModifiers(render.ComponentNameNode, entCtx(), newObjs(), nil) + out, _ := applyExtensions(ext, render.ComponentNameNode, entCtx(), newObjs(), nil) ds, _ := extensions.FindObject[*appsv1.DaemonSet](out, common.NodeDaemonSetName) Expect(nodeContainer(ds).ReadinessProbe.Exec.Command).To(ContainElement("--bgp-metrics-ready")) }) @@ -126,7 +126,7 @@ var _ = Describe("node enterprise modifier", func() { ds := objs[2].(*appsv1.DaemonSet) ds.Spec.Template.Spec.Containers[0].ReadinessProbe.Exec.Command = []string{"/bin/calico-node", "--felix-ready"} - out, _ := ext.ApplyModifiers(render.ComponentNameNode, entCtx(), objs, nil) + out, _ := applyExtensions(ext, render.ComponentNameNode, entCtx(), objs, nil) got, _ := extensions.FindObject[*appsv1.DaemonSet](out, common.NodeDaemonSetName) Expect(nodeContainer(got).ReadinessProbe.Exec.Command).NotTo(ContainElement("--bgp-metrics-ready")) }) @@ -136,7 +136,7 @@ var _ = Describe("node enterprise modifier", func() { ctx := entCtx() ctx.Installation.CalicoNetwork = &operatorv1.CalicoNetworkSpec{MultiInterfaceMode: &mode} - out, _ := ext.ApplyModifiers(render.ComponentNameNode, ctx, newObjs(), nil) + out, _ := applyExtensions(ext, render.ComponentNameNode, ctx, newObjs(), nil) ds, _ := extensions.FindObject[*appsv1.DaemonSet](out, common.NodeDaemonSetName) want := corev1.EnvVar{Name: "MULTI_INTERFACE_MODE", Value: mode.Value()} @@ -145,7 +145,7 @@ var _ = Describe("node enterprise modifier", func() { }) It("appends the node metrics service", func() { - out, _ := ext.ApplyModifiers(render.ComponentNameNode, entCtx(), newObjs(), nil) + out, _ := applyExtensions(ext, render.ComponentNameNode, entCtx(), newObjs(), nil) svc, ok := extensions.FindObject[*corev1.Service](out, render.CalicoNodeMetricsService) Expect(ok).To(BeTrue()) Expect(svc.Spec.Ports).To(HaveLen(2)) @@ -164,7 +164,7 @@ var _ = Describe("node enterprise modifier", func() { PrometheusMetricsEnabled: &enabled, }} - out, _ := ext.ApplyModifiers(render.ComponentNameNode, ctx, newObjs(), nil) + out, _ := applyExtensions(ext, render.ComponentNameNode, ctx, newObjs(), nil) svc, _ := extensions.FindObject[*corev1.Service](out, render.CalicoNodeMetricsService) Expect(svc.Spec.Ports).To(HaveLen(3)) Expect(svc.Spec.Ports[0].Port).To(Equal(int32(7081))) @@ -174,7 +174,7 @@ var _ = Describe("node enterprise modifier", func() { It("is a no-op for the Calico variant", func() { ctx := extensions.RenderContext{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.Calico}} - out, _ := ext.ApplyModifiers(render.ComponentNameNode, ctx, newObjs(), nil) + out, _ := applyExtensions(ext, render.ComponentNameNode, ctx, newObjs(), nil) _, ok := extensions.FindObject[*corev1.Service](out, render.CalicoNodeMetricsService) Expect(ok).To(BeFalse()) @@ -183,7 +183,7 @@ var _ = Describe("node enterprise modifier", func() { }) It("does not panic on a zero RenderContext", func() { - out, _ := ext.ApplyModifiers(render.ComponentNameNode, extensions.RenderContext{}, newObjs(), nil) + out, _ := applyExtensions(ext, render.ComponentNameNode, extensions.RenderContext{}, newObjs(), nil) _, ok := extensions.FindObject[*corev1.Service](out, render.CalicoNodeMetricsService) Expect(ok).To(BeFalse()) }) diff --git a/pkg/enterprise/typha.go b/pkg/enterprise/typha.go index 4571450365..ce3a434bbe 100644 --- a/pkg/enterprise/typha.go +++ b/pkg/enterprise/typha.go @@ -26,7 +26,7 @@ import ( ) func registerTypha(s *extensions.Set) { - s.Register(operatorv1.CalicoEnterprise, render.ComponentNameTypha, extensions.Extension{ + s.Register(operatorv1.CalicoEnterprise, render.ComponentNameTypha, extensions.ComponentExtension{ Modify: modifyTypha, }) } diff --git a/pkg/enterprise/typha_test.go b/pkg/enterprise/typha_test.go index 2b1211c4ec..cfc60b2b62 100644 --- a/pkg/enterprise/typha_test.go +++ b/pkg/enterprise/typha_test.go @@ -49,7 +49,7 @@ var _ = Describe("typha enterprise modifier", func() { Variant: operatorv1.CalicoEnterprise, CalicoNetwork: &operatorv1.CalicoNetworkSpec{MultiInterfaceMode: &multiMode}, }} - out, _ := ext.ApplyModifiers(render.ComponentNameTypha, ctx, newObjs(), nil) + out, _ := applyExtensions(ext, render.ComponentNameTypha, ctx, newObjs(), nil) role := out[0].(*rbacv1.ClusterRole) Expect(role.Rules).To(ContainElement(HaveField("Resources", ContainElement("licensekeys")))) @@ -69,14 +69,14 @@ var _ = Describe("typha enterprise modifier", func() { Variant: operatorv1.Calico, CalicoNetwork: &operatorv1.CalicoNetworkSpec{MultiInterfaceMode: &multiMode}, }} - out, _ := ext.ApplyModifiers(render.ComponentNameTypha, ctx, newObjs(), nil) + out, _ := applyExtensions(ext, render.ComponentNameTypha, ctx, newObjs(), nil) Expect(out[0].(*rbacv1.ClusterRole).Rules).To(BeEmpty()) dep := out[1].(*appsv1.Deployment) Expect(dep.Spec.Template.Spec.Containers[0].Env).To(BeEmpty()) }) It("does not panic on a zero Context (nil Installation)", func() { - out, _ := ext.ApplyModifiers(render.ComponentNameTypha, extensions.RenderContext{}, newObjs(), nil) + out, _ := applyExtensions(ext, render.ComponentNameTypha, extensions.RenderContext{}, newObjs(), nil) Expect(out[0].(*rbacv1.ClusterRole).Rules).To(BeEmpty()) }) }) diff --git a/pkg/enterprise/windows.go b/pkg/enterprise/windows.go index f4674323b3..7883429e41 100644 --- a/pkg/enterprise/windows.go +++ b/pkg/enterprise/windows.go @@ -37,13 +37,13 @@ import ( var windowsNodeContainers = map[string]bool{"felix": true, "node": true, "confd": true} func registerWindows(s *extensions.Set) { - s.Register(operatorv1.CalicoEnterprise, render.ComponentNameWindowsNodeImg, extensions.Extension{ + s.Register(operatorv1.CalicoEnterprise, render.ComponentNameWindowsNodeImg, extensions.ComponentExtension{ Image: func(*operatorv1.InstallationSpec) components.Component { return components.ComponentTigeraNodeWindows }, }) - s.Register(operatorv1.CalicoEnterprise, render.ComponentNameWindowsCNIImg, extensions.Extension{ + s.Register(operatorv1.CalicoEnterprise, render.ComponentNameWindowsCNIImg, extensions.ComponentExtension{ Image: func(*operatorv1.InstallationSpec) components.Component { return components.ComponentTigeraCNIWindows }, }) - s.Register(operatorv1.CalicoEnterprise, render.ComponentNameWindows, extensions.Extension{ + s.Register(operatorv1.CalicoEnterprise, render.ComponentNameWindows, extensions.ComponentExtension{ Modify: modifyWindows, }) } diff --git a/pkg/enterprise/windows_test.go b/pkg/enterprise/windows_test.go index 9bc08e1d52..b6c6b0c2d0 100644 --- a/pkg/enterprise/windows_test.go +++ b/pkg/enterprise/windows_test.go @@ -97,7 +97,7 @@ var _ = Describe("windows enterprise modifier", func() { } It("appends the node-metrics service", func() { - out, _ := ext.ApplyModifiers(render.ComponentNameWindows, ctxFor(operatorv1.ProviderNone, nil, nil), newObjs(), nil) + out, _ := applyExtensions(ext, render.ComponentNameWindows, ctxFor(operatorv1.ProviderNone, nil, nil), newObjs(), nil) svc, ok := extensions.FindObject[*corev1.Service](out, render.WindowsNodeMetricsService) Expect(ok).To(BeTrue()) Expect(svc.Namespace).To(Equal(common.CalicoNamespace)) @@ -105,7 +105,7 @@ var _ = Describe("windows enterprise modifier", func() { }) It("swaps the cni log mount for the calico log volume and adds enterprise env", func() { - out, _ := ext.ApplyModifiers(render.ComponentNameWindows, ctxFor(operatorv1.ProviderNone, nil, nil), newObjs(), nil) + out, _ := applyExtensions(ext, render.ComponentNameWindows, ctxFor(operatorv1.ProviderNone, nil, nil), newObjs(), nil) d := ds(out) Expect(d.Spec.Template.Spec.Volumes).To(ContainElement(HaveField("Name", "var-log-calico"))) @@ -122,7 +122,7 @@ var _ = Describe("windows enterprise modifier", func() { }) It("sets the trusted DNS server on openshift", func() { - out, _ := ext.ApplyModifiers(render.ComponentNameWindows, ctxFor(operatorv1.ProviderOpenShift, nil, nil), newObjs(), nil) + out, _ := applyExtensions(ext, render.ComponentNameWindows, ctxFor(operatorv1.ProviderOpenShift, nil, nil), newObjs(), nil) Expect(container(ds(out), "node").Env).To(ContainElement(corev1.EnvVar{Name: "FELIX_DNSTRUSTEDSERVERS", Value: "k8s-service:openshift-dns/dns-default"})) }) @@ -136,7 +136,7 @@ var _ = Describe("windows enterprise modifier", func() { Expect(err).NotTo(HaveOccurred()) bundle := cm.CreateTrustedBundle() - out, _ := ext.ApplyModifiers(render.ComponentNameWindows, ctxFor(operatorv1.ProviderNone, tls, bundle), newObjs(), nil) + out, _ := applyExtensions(ext, render.ComponentNameWindows, ctxFor(operatorv1.ProviderNone, tls, bundle), newObjs(), nil) d := ds(out) Expect(d.Spec.Template.Spec.Volumes).To(ContainElement(tls.Volume())) @@ -147,7 +147,7 @@ var _ = Describe("windows enterprise modifier", func() { It("does nothing for the Calico variant", func() { ctx := extensions.RenderContext{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.Calico}} - out, _ := ext.ApplyModifiers(render.ComponentNameWindows, ctx, newObjs(), nil) + out, _ := applyExtensions(ext, render.ComponentNameWindows, ctx, newObjs(), nil) _, ok := extensions.FindObject[*corev1.Service](out, render.WindowsNodeMetricsService) Expect(ok).To(BeFalse()) Expect(ds(out).Spec.Template.Spec.Volumes).To(BeEmpty()) diff --git a/pkg/extensions/decorate_helpers_test.go b/pkg/extensions/decorate_helpers_test.go new file mode 100644 index 0000000000..227c4f552a --- /dev/null +++ b/pkg/extensions/decorate_helpers_test.go @@ -0,0 +1,65 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package extensions_test + +import ( + client "sigs.k8s.io/controller-runtime/pkg/client" + + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/extensions" + rmeta "github.com/tigera/operator/pkg/render/common/meta" +) + +// stubExtComponent adapts raw object lists to a render.Component so a registered +// extension can be exercised through Set.Decorate, the same seam the component +// handler uses. key selects the extension; extCtx is delivered as the per-component +// ExtensionContext. +type stubExtComponent struct { + key string + extCtx any + create, delete []client.Object +} + +func (s stubExtComponent) ResolveImages(*operatorv1.ImageSet) error { + return nil +} + +func (s stubExtComponent) Objects() ([]client.Object, []client.Object) { + return s.create, s.delete +} + +func (s stubExtComponent) Ready() bool { + return true +} + +func (s stubExtComponent) SupportedOSType() rmeta.OSType { + return rmeta.OSTypeAny +} + +func (s stubExtComponent) ModifierKey() string { + return s.key +} + +func (s stubExtComponent) ExtensionContext() any { + return s.extCtx +} + +// applyExtensions decorates a stub component holding the given objects with the +// extension registered under key in s, then renders it - returning the decorated +// create and delete lists. +func applyExtensions(s *extensions.Set, key string, ctx extensions.RenderContext, create, del []client.Object) ([]client.Object, []client.Object) { + stub := stubExtComponent{key: key, extCtx: ctx.Component, create: create, delete: del} + return s.Decorate(stub, ctx).Objects() +} diff --git a/pkg/extensions/extension.go b/pkg/extensions/extension.go index 346e7347f2..e9ef998a6b 100644 --- a/pkg/extensions/extension.go +++ b/pkg/extensions/extension.go @@ -18,14 +18,15 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/render" ) -// Extension is everything a variant layers onto one render component. Every -// field is optional: a component that only needs a different image sets Image -// and leaves Modify nil, and vice versa. This is the single registration a +// ComponentExtension is everything a variant layers onto one render component. +// Every field is optional: a component that only needs a different image sets +// Image and leaves Modify nil, and vice versa. This is the single registration a // variant makes per component, so all of that component's variance lives in one // place. -type Extension struct { +type ComponentExtension struct { // Image overrides the component's image. Resolved during ResolveImages, in // the render package, via the imageoverride leaf. Image ImageOverride @@ -47,19 +48,45 @@ type modifierKey struct { component string } -// ApplyModifiers runs the modifier registered for the named component and the -// installation's variant over the create and delete lists, returning them -// unchanged when none is registered (or when no installation is set). Safe to -// call on a nil Set, which is a no-op - the core operator registers no -// modifiers. -func (s *Set) ApplyModifiers(component string, ctx RenderContext, create, delete []client.Object) ([]client.Object, []client.Object) { +// Decorate wraps component with the variant extension registered for it, so +// that when the handler renders the component its objects are post-processed by +// the modifier registered for the component and the installation's variant. A +// decorated component is itself a render.Component, so it flows through the +// component handler exactly like any other. Returns component unchanged when it +// exposes no extension point, when no modifier is registered for it, when no +// installation is set, or on a nil Set (the core operator registers no +// extensions). +func (s *Set) Decorate(component render.Component, ctx RenderContext) render.Component { if s == nil || ctx.Installation == nil { - return create, delete + return component } - if fn, ok := s.modifiers[modifierKey{ctx.Installation.Variant, component}]; ok { - create, delete = fn(ctx, create, delete) + ext, ok := component.(render.Extensible) + if !ok { + return component } - return create, delete + modify, ok := s.modifiers[modifierKey{ctx.Installation.Variant, ext.ModifierKey()}] + if !ok { + return component + } + if p, ok := component.(render.ExtensionContextProvider); ok { + ctx.Component = p.ExtensionContext() + } + return &decoratedComponent{Component: component, ctx: ctx, modify: modify} +} + +// decoratedComponent is the render.Component produced by Decorate: it renders +// its embedded base component and then runs the variant modifier over the +// result. It embeds the base render.Component, so ResolveImages, SupportedOSType, +// and Ready delegate to the base; only Objects is augmented. +type decoratedComponent struct { + render.Component + ctx RenderContext + modify Modifier +} + +func (d *decoratedComponent) Objects() ([]client.Object, []client.Object) { + create, del := d.Component.Objects() + return d.modify(d.ctx, create, del) } // FindObject returns the first object of type T with the given name. diff --git a/pkg/extensions/extension_test.go b/pkg/extensions/extension_test.go index b02ec3d24d..8cbb0a7c89 100644 --- a/pkg/extensions/extension_test.go +++ b/pkg/extensions/extension_test.go @@ -34,7 +34,7 @@ var _ = Describe("extension registry", func() { entCtx := extensions.RenderContext{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.CalicoEnterprise}} It("applies a registered modifier to the matching component and variant", func() { - s.Register(operatorv1.CalicoEnterprise, "test", extensions.Extension{ + s.Register(operatorv1.CalicoEnterprise, "test", extensions.ComponentExtension{ Modify: func(ctx extensions.RenderContext, objs, del []client.Object) ([]client.Object, []client.Object) { cm, ok := extensions.FindObject[*corev1.ConfigMap](objs, "cm") Expect(ok).To(BeTrue()) @@ -44,7 +44,7 @@ var _ = Describe("extension registry", func() { }) in := []client.Object{&corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "cm"}}} - out, _ := s.ApplyModifiers("test", entCtx, in, nil) + out, _ := applyExtensions(s, "test", entCtx, in, nil) Expect(out).To(HaveLen(2)) cm := out[0].(*corev1.ConfigMap) @@ -53,14 +53,14 @@ var _ = Describe("extension registry", func() { }) It("lets a modifier append to the delete list", func() { - s.Register(operatorv1.CalicoEnterprise, "test", extensions.Extension{ + s.Register(operatorv1.CalicoEnterprise, "test", extensions.ComponentExtension{ Modify: func(_ extensions.RenderContext, objs, del []client.Object) ([]client.Object, []client.Object) { return objs, append(del, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "stale"}}) }, }) in := []client.Object{&corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "cm"}}} - out, del := s.ApplyModifiers("test", entCtx, in, nil) + out, del := applyExtensions(s, "test", entCtx, in, nil) Expect(out).To(Equal(in)) Expect(del).To(HaveLen(1)) Expect(del[0].GetName()).To(Equal("stale")) @@ -68,12 +68,12 @@ var _ = Describe("extension registry", func() { It("returns objects unchanged when no modifier is registered", func() { in := []client.Object{&corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "cm"}}} - out, _ := s.ApplyModifiers("unregistered", entCtx, in, nil) + out, _ := applyExtensions(s, "unregistered", entCtx, in, nil) Expect(out).To(Equal(in)) }) It("does not apply a modifier registered for a different variant", func() { - s.Register(operatorv1.CalicoEnterprise, "test", extensions.Extension{ + s.Register(operatorv1.CalicoEnterprise, "test", extensions.ComponentExtension{ Modify: func(_ extensions.RenderContext, objs, del []client.Object) ([]client.Object, []client.Object) { return append(objs, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "extra"}}), del }, @@ -81,19 +81,19 @@ var _ = Describe("extension registry", func() { calicoCtx := extensions.RenderContext{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.Calico}} in := []client.Object{&corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "cm"}}} - out, _ := s.ApplyModifiers("test", calicoCtx, in, nil) + out, _ := applyExtensions(s, "test", calicoCtx, in, nil) Expect(out).To(Equal(in)) }) It("returns objects unchanged when no installation is set", func() { in := []client.Object{&corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "cm"}}} - out, _ := s.ApplyModifiers("test", extensions.RenderContext{}, in, nil) + out, _ := applyExtensions(s, "test", extensions.RenderContext{}, in, nil) Expect(out).To(Equal(in)) }) It("replaces rather than stacks when a (variant, component) is registered twice", func() { - add := func(name string) extensions.Extension { - return extensions.Extension{ + add := func(name string) extensions.ComponentExtension { + return extensions.ComponentExtension{ Modify: func(_ extensions.RenderContext, objs, del []client.Object) ([]client.Object, []client.Object) { return append(objs, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: name}}), del }, @@ -102,7 +102,7 @@ var _ = Describe("extension registry", func() { s.Register(operatorv1.CalicoEnterprise, "test", add("first")) s.Register(operatorv1.CalicoEnterprise, "test", add("second")) - out, _ := s.ApplyModifiers("test", entCtx, nil, nil) + out, _ := applyExtensions(s, "test", entCtx, nil, nil) Expect(out).To(HaveLen(1)) Expect(out[0].GetName()).To(Equal("second")) }) diff --git a/pkg/extensions/image_test.go b/pkg/extensions/image_test.go index 0581d8cc08..dcc4876fdd 100644 --- a/pkg/extensions/image_test.go +++ b/pkg/extensions/image_test.go @@ -27,7 +27,7 @@ var _ = Describe("image overrides", func() { var s *extensions.Set BeforeEach(func() { s = extensions.NewSet() - s.Register(operatorv1.CalicoEnterprise, "node", extensions.Extension{ + s.Register(operatorv1.CalicoEnterprise, "node", extensions.ComponentExtension{ Image: func(in *operatorv1.InstallationSpec) components.Component { return components.ComponentTigeraNode }, diff --git a/pkg/extensions/set.go b/pkg/extensions/set.go index b638622750..08aa52f259 100644 --- a/pkg/extensions/set.go +++ b/pkg/extensions/set.go @@ -28,7 +28,7 @@ import ( // registries, so nothing is wired by import side effect. // // The zero value is not usable; build one with NewSet. The methods that read it -// (BuildContext, ApplyModifiers, ResolveImage, Images) are nil-safe so the core +// (BuildContext, Decorate, ResolveImage, Images) are nil-safe so the core // operator can pass a nil Set and get base behavior. type Set struct { setups map[operatorv1.ProductVariant]Setup @@ -49,7 +49,7 @@ func NewSet() *Set { // variant. A (variant, component) pair has at most one extension; registration // replaces any prior one. The image override and the modifier are stored // separately, so a component can set either field or both. -func (s *Set) Register(variant operatorv1.ProductVariant, component string, e Extension) { +func (s *Set) Register(variant operatorv1.ProductVariant, component string, e ComponentExtension) { if e.Image != nil { s.images.Register(variant, component, e.Image) } diff --git a/pkg/render/apiserver_test.go b/pkg/render/apiserver_test.go index a81ebb6e01..31b15779ee 100644 --- a/pkg/render/apiserver_test.go +++ b/pkg/render/apiserver_test.go @@ -73,7 +73,7 @@ func apiServerObjects(c render.Component) ([]client.Object, []client.Object) { rc.Installation = ec.Config.Installation rc.Component = ec } - return ext.ApplyModifiers(render.ComponentNameAPIServer, rc, create, del) + return applyExtensions(ext, render.ComponentNameAPIServer, rc, create, del) } var _ = Describe("API server rendering tests (Calico Enterprise)", func() { diff --git a/pkg/render/decorate_helpers_test.go b/pkg/render/decorate_helpers_test.go new file mode 100644 index 0000000000..750abb8df4 --- /dev/null +++ b/pkg/render/decorate_helpers_test.go @@ -0,0 +1,65 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package render_test + +import ( + client "sigs.k8s.io/controller-runtime/pkg/client" + + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/extensions" + rmeta "github.com/tigera/operator/pkg/render/common/meta" +) + +// stubExtComponent adapts raw object lists to a render.Component so a registered +// extension can be exercised through Set.Decorate, the same seam the component +// handler uses. key selects the extension; extCtx is delivered as the per-component +// ExtensionContext. +type stubExtComponent struct { + key string + extCtx any + create, delete []client.Object +} + +func (s stubExtComponent) ResolveImages(*operatorv1.ImageSet) error { + return nil +} + +func (s stubExtComponent) Objects() ([]client.Object, []client.Object) { + return s.create, s.delete +} + +func (s stubExtComponent) Ready() bool { + return true +} + +func (s stubExtComponent) SupportedOSType() rmeta.OSType { + return rmeta.OSTypeAny +} + +func (s stubExtComponent) ModifierKey() string { + return s.key +} + +func (s stubExtComponent) ExtensionContext() any { + return s.extCtx +} + +// applyExtensions decorates a stub component holding the given objects with the +// extension registered under key in s, then renders it - returning the decorated +// create and delete lists. +func applyExtensions(s *extensions.Set, key string, ctx extensions.RenderContext, create, del []client.Object) ([]client.Object, []client.Object) { + stub := stubExtComponent{key: key, extCtx: ctx.Component, create: create, delete: del} + return s.Decorate(stub, ctx).Objects() +} diff --git a/pkg/render/guardian_test.go b/pkg/render/guardian_test.go index 33c04099fd..44e4e2fe4a 100644 --- a/pkg/render/guardian_test.go +++ b/pkg/render/guardian_test.go @@ -54,7 +54,7 @@ func guardianObjects(cfg *render.GuardianConfiguration) []client.Object { if p, ok := g.(render.ExtensionContextProvider); ok { rc.Component = p.ExtensionContext() } - out, _ := ext.ApplyModifiers(render.GuardianName, rc, objs, nil) + out, _ := applyExtensions(ext, render.GuardianName, rc, objs, nil) return out } @@ -116,7 +116,7 @@ var _ = Describe("Rendering tests", func() { if p, ok := g.(render.ExtensionContextProvider); ok { rc.Component = p.ExtensionContext() } - resources, _ = ext.ApplyModifiers(render.GuardianName, rc, resources, nil) + resources, _ = applyExtensions(ext, render.GuardianName, rc, resources, nil) } BeforeEach(func() { @@ -352,7 +352,7 @@ var _ = Describe("Rendering tests", func() { if p, ok := g.(render.ExtensionContextProvider); ok { rc.Component = p.ExtensionContext() } - resources, _ = ext.ApplyModifiers(render.ComponentNameGuardianPolicy, rc, objs, nil) + resources, _ = applyExtensions(ext, render.ComponentNameGuardianPolicy, rc, objs, nil) } Context("policy rendering based on variant and IncludeEgressNetworkPolicy", func() { diff --git a/pkg/render/node_enterprise_test.go b/pkg/render/node_enterprise_test.go index 7b11bbf6eb..c9c32b69fd 100644 --- a/pkg/render/node_enterprise_test.go +++ b/pkg/render/node_enterprise_test.go @@ -110,7 +110,7 @@ var _ = Describe("node enterprise modifier integration", func() { comp := render.Node(cfg) Expect(comp.ResolveImages(nil)).NotTo(HaveOccurred()) objs, _ := comp.Objects() - out, _ := ext.ApplyModifiers(render.ComponentNameNode, renderCtx, objs, nil) + out, _ := applyExtensions(ext, render.ComponentNameNode, renderCtx, objs, nil) return out } @@ -167,7 +167,7 @@ var _ = Describe("node enterprise modifier integration", func() { }) Expect(comp.ResolveImages(nil)).NotTo(HaveOccurred()) objs, _ := comp.Objects() - objs, _ = ext.ApplyModifiers(render.ComponentNameTypha, renderCtx, objs, nil) + objs, _ = applyExtensions(ext, render.ComponentNameTypha, renderCtx, objs, nil) role, ok := extensions.FindObject[*rbacv1.ClusterRole](objs, "calico-typha") Expect(ok).To(BeTrue()) diff --git a/pkg/render/windows_test.go b/pkg/render/windows_test.go index e92bc86478..f15e185eda 100644 --- a/pkg/render/windows_test.go +++ b/pkg/render/windows_test.go @@ -54,7 +54,7 @@ func renderWindows(cfg *render.WindowsConfiguration) []client.Object { if p, ok := comp.(render.ExtensionContextProvider); ok { rc.Component = p.ExtensionContext() } - out, _ := ext.ApplyModifiers(render.ComponentNameWindows, rc, objs, nil) + out, _ := applyExtensions(ext, render.ComponentNameWindows, rc, objs, nil) return out } From 8ef2965b65e710debec4c586bc1d827112d159a0 Mon Sep 17 00:00:00 2001 From: Casey Davenport Date: Thu, 18 Jun 2026 10:33:07 -0700 Subject: [PATCH 29/54] Restructure extensions into per-variant bundles with a typed controller hook A Set holds one Variant bundle per product variant; the controller selects one from the installation variant, so each component has at most one extension and a modifier never re-checks the variant. ControllerExtension (Validate, ExtendContext) replaces the setup func as the controller-side hook. ControllerContext embeds RenderContext and carries the cluster-access deps, so a modifier given a RenderContext can't do I/O. Component modifiers register with RegisterModifier, which hands them typed config and removes RenderContext.Component. --- cmd/main.go | 6 +- .../installation/core_controller.go | 19 ++- pkg/controller/utils/component_test.go | 10 +- pkg/enterprise/apiserver.go | 34 ++--- pkg/enterprise/decorate_helpers_test.go | 16 +- pkg/enterprise/guardian.go | 27 +--- pkg/enterprise/guardian_test.go | 15 +- pkg/enterprise/installation.go | 43 +++--- pkg/enterprise/installation_test.go | 27 ++-- pkg/enterprise/node.go | 16 +- pkg/enterprise/register.go | 34 +++-- pkg/enterprise/typha.go | 7 +- pkg/enterprise/windows.go | 23 +-- pkg/enterprise/windows_test.go | 23 +-- pkg/extensions/controllerextension.go | 58 ++++++++ pkg/extensions/controllerextension_test.go | 104 +++++++++++++ pkg/extensions/decorate_helpers_test.go | 16 +- pkg/extensions/doc.go | 38 +++-- pkg/extensions/extension.go | 69 +-------- pkg/extensions/extension_test.go | 40 ++--- pkg/extensions/image_test.go | 6 +- pkg/extensions/rendercontext.go | 27 ++-- pkg/extensions/set.go | 97 ++++++++---- pkg/extensions/setup.go | 78 ---------- pkg/extensions/setup_test.go | 89 ----------- pkg/extensions/variant.go | 138 ++++++++++++++++++ pkg/render/apiserver_test.go | 5 +- pkg/render/decorate_helpers_test.go | 16 +- pkg/render/guardian_test.go | 15 +- pkg/render/windows_test.go | 5 +- 30 files changed, 593 insertions(+), 508 deletions(-) create mode 100644 pkg/extensions/controllerextension.go create mode 100644 pkg/extensions/controllerextension_test.go delete mode 100644 pkg/extensions/setup.go delete mode 100644 pkg/extensions/setup_test.go create mode 100644 pkg/extensions/variant.go diff --git a/cmd/main.go b/cmd/main.go index eb63d9b666..60560c9e46 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -522,11 +522,7 @@ If a value other than 'all' is specified, the first CRD with a prefix of the spe ElasticExternal: discovery.UseExternalElastic(bootConfig), UseV3CRDs: v3CRDs, APIDiscovery: apiDiscovery, - // Hand the operator the in-repo Calico Enterprise extensions (modifiers, - // image overrides, and the installation setup). After the monorepo split - // the core operator's main passes none and calico-private's main passes - // its own. - Extensions: enterprise.New(), + Extensions: enterprise.New(), } // Before we start any controllers, make sure our options are valid. diff --git a/pkg/controller/installation/core_controller.go b/pkg/controller/installation/core_controller.go index 08444f08e8..baeb11364e 100644 --- a/pkg/controller/installation/core_controller.go +++ b/pkg/controller/installation/core_controller.go @@ -1209,15 +1209,22 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile calicoVersion = components.EnterpriseRelease } - renderCtx, err := r.opts.Extensions.BuildContext(extensions.Inputs{ + cc := extensions.ControllerContext{ + RenderContext: extensions.RenderContext{ + Installation: &instance.Spec, + FelixConfiguration: felixConfiguration, + ClusterDomain: r.opts.ClusterDomain, + TrustedBundle: typhaNodeTLS.TrustedBundle, + }, Ctx: ctx, Client: r.client, - Installation: &instance.Spec, - FelixConfiguration: felixConfiguration, CertificateManager: certificateManager, - TrustedBundle: typhaNodeTLS.TrustedBundle, - ClusterDomain: r.opts.ClusterDomain, - }) + } + if err := r.opts.Extensions.Validate(cc); err != nil { + r.status.SetDegraded(operatorv1.ResourceValidationError, "Invalid installation configuration for variant", err, reqLogger) + return reconcile.Result{}, err + } + renderCtx, err := r.opts.Extensions.ExtendContext(cc) if err != nil { r.status.SetDegraded(operatorv1.ResourceCreateError, "Error preparing installation extension", err, reqLogger) return reconcile.Result{}, err diff --git a/pkg/controller/utils/component_test.go b/pkg/controller/utils/component_test.go index ec369e2ac3..776ef6b4a5 100644 --- a/pkg/controller/utils/component_test.go +++ b/pkg/controller/utils/component_test.go @@ -2580,12 +2580,10 @@ func (mc *mockClient) SubResource(subResource string) client.SubResourceClient { var _ = Describe("componentHandler modifier application", func() { It("applies registered modifiers to a named component before create", func() { ext := extensions.NewSet() - ext.Register(operatorv1.CalicoEnterprise, "fake", extensions.ComponentExtension{ - Modify: func(ctx extensions.RenderContext, objs, del []client.Object) ([]client.Object, []client.Object) { - cm := objs[0].(*corev1.ConfigMap) - cm.Data = map[string]string{"patched": "yes"} - return objs, del - }, + ext.Variant(operatorv1.CalicoEnterprise).Modify("fake", func(ctx extensions.RenderContext, objs, del []client.Object) ([]client.Object, []client.Object) { + cm := objs[0].(*corev1.ConfigMap) + cm.Data = map[string]string{"patched": "yes"} + return objs, del }) s := runtime.NewScheme() diff --git a/pkg/enterprise/apiserver.go b/pkg/enterprise/apiserver.go index 913391cb16..7a265877d1 100644 --- a/pkg/enterprise/apiserver.go +++ b/pkg/enterprise/apiserver.go @@ -18,7 +18,6 @@ import ( "fmt" "strings" - "github.com/sirupsen/logrus" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" @@ -27,7 +26,6 @@ import ( "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" - operatorv1 "github.com/tigera/operator/api/v1" "github.com/tigera/operator/pkg/common" "github.com/tigera/operator/pkg/components" "github.com/tigera/operator/pkg/extensions" @@ -52,26 +50,21 @@ type apiServer struct { calicoImage string } -func registerAPIServer(s *extensions.Set) { - s.Register(operatorv1.CalicoEnterprise, render.ComponentNameAPIServer, extensions.ComponentExtension{ - Modify: modifyAPIServer, - }) - // When running Calico, clean up any Enterprise objects left behind by a prior - // Enterprise installation. - s.Register(operatorv1.Calico, render.ComponentNameAPIServer, extensions.ComponentExtension{ - Modify: cleanupAPIServer, - }) +func registerAPIServer(v *extensions.Variant) { + extensions.RegisterModifier(v, render.ComponentNameAPIServer, modifyAPIServer) +} + +// registerAPIServerCleanup registers, for the Calico variant, the cleanup that +// deletes the Enterprise API server objects left behind by a prior Enterprise +// installation. +func registerAPIServerCleanup(v *extensions.Variant) { + extensions.RegisterModifier(v, render.ComponentNameAPIServer, cleanupAPIServer) } // modifyAPIServer layers Calico Enterprise behavior onto the rendered API server objects: // the query server container and its volumes, audit logging on the aggregation API server // container, the Enterprise RBAC objects, and the query server port on the Service. -func modifyAPIServer(ctx extensions.RenderContext, create, del []client.Object) ([]client.Object, []client.Object) { - ec, ok := ctx.Component.(render.APIServerExtensionContext) - if !ok { - logrus.Errorf("BUG: apiserver modifier got %T, want render.APIServerExtensionContext; leaving objects unchanged", ctx.Component) - return create, del - } +func modifyAPIServer(ctx extensions.RenderContext, ec render.APIServerExtensionContext, create, del []client.Object) ([]client.Object, []client.Object) { c := &apiServer{cfg: ec.Config, calicoImage: ec.CalicoImage} if dep, ok := extensions.FindObject[*appsv1.Deployment](create, render.APIServerName); ok { @@ -150,12 +143,7 @@ func modifyAPIServer(ctx extensions.RenderContext, create, del []client.Object) // cleanupAPIServer deletes the Enterprise API server objects when running Calico, so a // cluster switched from Enterprise to Calico does not leave them behind. -func cleanupAPIServer(ctx extensions.RenderContext, create, del []client.Object) ([]client.Object, []client.Object) { - ec, ok := ctx.Component.(render.APIServerExtensionContext) - if !ok { - logrus.Errorf("BUG: apiserver cleanup got %T, want render.APIServerExtensionContext; leaving objects unchanged", ctx.Component) - return create, del - } +func cleanupAPIServer(ctx extensions.RenderContext, ec render.APIServerExtensionContext, create, del []client.Object) ([]client.Object, []client.Object) { c := &apiServer{cfg: ec.Config} del = append(del, c.tigeraAPIServerClusterRole(), c.tigeraAPIServerClusterRoleBinding()) diff --git a/pkg/enterprise/decorate_helpers_test.go b/pkg/enterprise/decorate_helpers_test.go index 5bade86c8f..184a801a71 100644 --- a/pkg/enterprise/decorate_helpers_test.go +++ b/pkg/enterprise/decorate_helpers_test.go @@ -24,8 +24,8 @@ import ( // stubExtComponent adapts raw object lists to a render.Component so a registered // extension can be exercised through Set.Decorate, the same seam the component -// handler uses. key selects the extension; extCtx is delivered as the per-component -// ExtensionContext. +// handler uses. key selects the extension; extCtx is delivered as the component's +// ExtensionContext (the typed config a RegisterModifier modifier reads). type stubExtComponent struct { key string extCtx any @@ -57,9 +57,15 @@ func (s stubExtComponent) ExtensionContext() any { } // applyExtensions decorates a stub component holding the given objects with the -// extension registered under key in s, then renders it - returning the decorated -// create and delete lists. +// extension registered under key, then renders it. For a modifier that needs the +// component's typed config, use applyExtensionsWithContext. func applyExtensions(s *extensions.Set, key string, ctx extensions.RenderContext, create, del []client.Object) ([]client.Object, []client.Object) { - stub := stubExtComponent{key: key, extCtx: ctx.Component, create: create, delete: del} + return applyExtensionsWithContext(s, key, ctx, nil, create, del) +} + +// applyExtensionsWithContext is applyExtensions for a modifier that reads the +// component's typed config: extCtx is delivered as the stub's ExtensionContext. +func applyExtensionsWithContext(s *extensions.Set, key string, ctx extensions.RenderContext, extCtx any, create, del []client.Object) ([]client.Object, []client.Object) { + stub := stubExtComponent{key: key, extCtx: extCtx, create: create, delete: del} return s.Decorate(stub, ctx).Objects() } diff --git a/pkg/enterprise/guardian.go b/pkg/enterprise/guardian.go index 0d81779c8f..11895b0b29 100644 --- a/pkg/enterprise/guardian.go +++ b/pkg/enterprise/guardian.go @@ -29,7 +29,6 @@ import ( v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" "github.com/tigera/api/pkg/lib/numorstring" - operatorv1 "github.com/tigera/operator/api/v1" "github.com/tigera/operator/pkg/common" "github.com/tigera/operator/pkg/extensions" "github.com/tigera/operator/pkg/render" @@ -38,26 +37,16 @@ import ( operatorurl "github.com/tigera/operator/pkg/url" ) -func registerGuardian(s *extensions.Set) { - s.Register(operatorv1.CalicoEnterprise, render.GuardianName, extensions.ComponentExtension{ - Modify: modifyGuardian, - }) - s.Register(operatorv1.CalicoEnterprise, render.ComponentNameGuardianPolicy, extensions.ComponentExtension{ - Modify: modifyGuardianPolicy, - }) +func registerGuardian(v *extensions.Variant) { + extensions.RegisterModifier(v, render.GuardianName, modifyGuardian) + extensions.RegisterModifier(v, render.ComponentNameGuardianPolicy, modifyGuardianPolicy) } // modifyGuardianPolicy replaces the core OSS guardian network policy with the // enterprise management-cluster policy. Building the enterprise egress rules can // fail (proxy URL parsing); on failure we drop the policy entirely, matching the // core behavior of omitting it rather than installing a partial policy. -func modifyGuardianPolicy(ctx extensions.RenderContext, objs, del []client.Object) ([]client.Object, []client.Object) { - gpc, ok := ctx.Component.(render.GuardianPolicyExtensionContext) - if !ok { - logrus.Errorf("BUG: guardian policy modifier got %T, want render.GuardianPolicyExtensionContext; leaving objects unchanged", ctx.Component) - return objs, del - } - +func modifyGuardianPolicy(ctx extensions.RenderContext, gpc render.GuardianPolicyExtensionContext, objs, del []client.Object) ([]client.Object, []client.Object) { policy, ok := extensions.FindObject[*v3.NetworkPolicy](objs, render.GuardianPolicyName) if !ok { return objs, del @@ -215,13 +204,7 @@ func enterpriseGuardianPolicySpec(gpc render.GuardianPolicyExtensionContext) (v3 // objects: the secrets Role/RoleBinding and default UI settings, the // elasticsearch/kibana service ports, the management-cluster-request cluster // role rules (which replace the OSS rules), and the CA bundle env vars. -func modifyGuardian(ctx extensions.RenderContext, objs, del []client.Object) ([]client.Object, []client.Object) { - gc, ok := ctx.Component.(render.GuardianExtensionContext) - if !ok { - logrus.Errorf("BUG: guardian modifier got %T, want render.GuardianExtensionContext; leaving objects unchanged", ctx.Component) - return objs, del - } - +func modifyGuardian(ctx extensions.RenderContext, gc render.GuardianExtensionContext, objs, del []client.Object) ([]client.Object, []client.Object) { if role, ok := extensions.FindObject[*rbacv1.ClusterRole](objs, render.GuardianClusterRoleName); ok { role.Rules = guardianEnterpriseRules(gc) } diff --git a/pkg/enterprise/guardian_test.go b/pkg/enterprise/guardian_test.go index 87f5207ee2..07e6694904 100644 --- a/pkg/enterprise/guardian_test.go +++ b/pkg/enterprise/guardian_test.go @@ -49,15 +49,10 @@ var _ = Describe("guardian enterprise modifier", func() { } } - ctxWith := func(c render.GuardianExtensionContext) extensions.RenderContext { - return extensions.RenderContext{ - Installation: &operatorv1.InstallationSpec{Variant: operatorv1.CalicoEnterprise}, - Component: c, - } - } + entCtx := extensions.RenderContext{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.CalicoEnterprise}} It("appends the secrets RBAC and UI settings", func() { - out, _ := applyExtensions(ext, render.GuardianName, ctxWith(render.GuardianExtensionContext{}), newObjs(), nil) + out, _ := applyExtensionsWithContext(ext, render.GuardianName, entCtx, render.GuardianExtensionContext{}, newObjs(), nil) _, ok := extensions.FindObject[*rbacv1.Role](out, render.GuardianSecretsRole) Expect(ok).To(BeTrue()) _, ok = extensions.FindObject[*rbacv1.RoleBinding](out, render.GuardianSecretsRoleBindingName) @@ -67,7 +62,7 @@ var _ = Describe("guardian enterprise modifier", func() { }) It("adds the elasticsearch and kibana service ports", func() { - out, _ := applyExtensions(ext, render.GuardianName, ctxWith(render.GuardianExtensionContext{}), newObjs(), nil) + out, _ := applyExtensionsWithContext(ext, render.GuardianName, entCtx, render.GuardianExtensionContext{}, newObjs(), nil) svc, _ := extensions.FindObject[*corev1.Service](out, render.GuardianServiceName) names := []string{} for _, p := range svc.Spec.Ports { @@ -80,7 +75,7 @@ var _ = Describe("guardian enterprise modifier", func() { gc := render.GuardianExtensionContext{ Impersonation: &operatorv1.Impersonation{Users: []string{"foo"}, Groups: []string{"bar"}}, } - out, _ := applyExtensions(ext, render.GuardianName, ctxWith(gc), newObjs(), nil) + out, _ := applyExtensionsWithContext(ext, render.GuardianName, entCtx, gc, newObjs(), nil) role, _ := extensions.FindObject[*rbacv1.ClusterRole](out, render.GuardianClusterRoleName) // The single OSS placeholder rule is gone, replaced by the enterprise set. @@ -91,7 +86,7 @@ var _ = Describe("guardian enterprise modifier", func() { It("adds the CA bundle env to the guardian container", func() { gc := render.GuardianExtensionContext{TrustedBundleMountPath: "/ca/bundle"} - out, _ := applyExtensions(ext, render.GuardianName, ctxWith(gc), newObjs(), nil) + out, _ := applyExtensionsWithContext(ext, render.GuardianName, entCtx, gc, newObjs(), nil) dep, _ := extensions.FindObject[*appsv1.Deployment](out, render.GuardianDeploymentName) Expect(dep.Spec.Template.Spec.Containers[0].Env).To(ContainElement(corev1.EnvVar{Name: "GUARDIAN_PROMETHEUS_CA_BUNDLE_PATH", Value: "/ca/bundle"})) }) diff --git a/pkg/enterprise/installation.go b/pkg/enterprise/installation.go index 30f1c96341..57ad67476c 100644 --- a/pkg/enterprise/installation.go +++ b/pkg/enterprise/installation.go @@ -18,7 +18,6 @@ import ( "errors" "fmt" - operatorv1 "github.com/tigera/operator/api/v1" "github.com/tigera/operator/pkg/common" "github.com/tigera/operator/pkg/dns" "github.com/tigera/operator/pkg/extensions" @@ -27,50 +26,54 @@ import ( "github.com/tigera/operator/pkg/render/monitor" ) -func registerInstallation(s *extensions.Set) { - s.RegisterSetup(operatorv1.CalicoEnterprise, setup) -} - -// setup is the Calico Enterprise setup phase. It builds the base render context -// and then does the controller-side work the modifiers can't: validating config -// and creating/fetching the certificates that feed the trusted bundle. -func setup(in extensions.Inputs) (extensions.RenderContext, error) { - rc := extensions.BaseRenderContext(in) +// controllerExtension is the Calico Enterprise controller-side hook for the +// installation controller. +type controllerExtension struct{} +// Validate rejects installation config Calico Enterprise does not support. +func (controllerExtension) Validate(cc extensions.ControllerContext) error { // Reject the unsupported zero reporter port. The port value itself is derived // in the node modifier; only this validation lives here. - if in.FelixConfiguration.Spec.PrometheusReporterPort != nil && *in.FelixConfiguration.Spec.PrometheusReporterPort == 0 { - return rc, errors.New("felixConfiguration prometheusReporterPort=0 not supported") + if cc.FelixConfiguration.Spec.PrometheusReporterPort != nil && *cc.FelixConfiguration.Spec.PrometheusReporterPort == 0 { + return errors.New("felixConfiguration prometheusReporterPort=0 not supported") } + return nil +} + +// ExtendContext does the controller-side work the modifiers can't: creating and +// fetching the certificates that feed the trusted bundle, returning the render +// context with the produced node prometheus keypair layered on. +func (controllerExtension) ExtendContext(cc extensions.ControllerContext) (extensions.RenderContext, error) { + rc := cc.RenderContext - nodePrometheusTLS, err := in.CertificateManager.GetOrCreateKeyPair( - in.Client, + nodePrometheusTLS, err := cc.CertificateManager.GetOrCreateKeyPair( + cc.Client, render.NodePrometheusTLSServerSecret, common.OperatorNamespace(), - dns.GetServiceDNSNames(render.CalicoNodeMetricsService, common.CalicoNamespace, in.ClusterDomain), + dns.GetServiceDNSNames(render.CalicoNodeMetricsService, common.CalicoNamespace, cc.ClusterDomain), ) if err != nil { return rc, fmt.Errorf("error creating node prometheus TLS certificate: %w", err) } if nodePrometheusTLS != nil { - in.TrustedBundle.AddCertificates(nodePrometheusTLS) + cc.TrustedBundle.AddCertificates(nodePrometheusTLS) } rc.NodePrometheusTLS = nodePrometheusTLS - prometheusClientCert, err := in.CertificateManager.GetCertificate(in.Client, monitor.PrometheusClientTLSSecretName, common.OperatorNamespace()) + prometheusClientCert, err := cc.CertificateManager.GetCertificate(cc.Client, monitor.PrometheusClientTLSSecretName, common.OperatorNamespace()) if err != nil { return rc, fmt.Errorf("unable to fetch prometheus certificate: %w", err) } if prometheusClientCert != nil { - in.TrustedBundle.AddCertificates(prometheusClientCert) + cc.TrustedBundle.AddCertificates(prometheusClientCert) } - esgwCertificate, err := in.CertificateManager.GetCertificate(in.Client, relasticsearch.PublicCertSecret, common.OperatorNamespace()) + esgwCertificate, err := cc.CertificateManager.GetCertificate(cc.Client, relasticsearch.PublicCertSecret, common.OperatorNamespace()) if err != nil { return rc, fmt.Errorf("failed to retrieve / validate %s: %w", relasticsearch.PublicCertSecret, err) } if esgwCertificate != nil { - in.TrustedBundle.AddCertificates(esgwCertificate) + cc.TrustedBundle.AddCertificates(esgwCertificate) } return rc, nil diff --git a/pkg/enterprise/installation_test.go b/pkg/enterprise/installation_test.go index bc704a2429..f968306b96 100644 --- a/pkg/enterprise/installation_test.go +++ b/pkg/enterprise/installation_test.go @@ -31,32 +31,31 @@ import ( "github.com/tigera/operator/pkg/extensions" ) -var _ = Describe("installation setup", func() { +var _ = Describe("installation controller extension", func() { It("rejects a zero prometheus reporter port", func() { port := 0 - in := newInputs(operatorv1.CalicoEnterprise) - in.FelixConfiguration = &v3.FelixConfiguration{ + cc := newControllerContext(operatorv1.CalicoEnterprise) + cc.FelixConfiguration = &v3.FelixConfiguration{ Spec: v3.FelixConfigurationSpec{PrometheusReporterPort: &port}, } - _, err := ext.BuildContext(in) - Expect(err).To(HaveOccurred()) + Expect(ext.Validate(cc)).To(HaveOccurred()) }) It("creates the node prometheus keypair for the enterprise variant", func() { - rc, err := ext.BuildContext(newInputs(operatorv1.CalicoEnterprise)) + rc, err := ext.ExtendContext(newControllerContext(operatorv1.CalicoEnterprise)) Expect(err).NotTo(HaveOccurred()) Expect(rc.NodePrometheusTLS).NotTo(BeNil()) }) It("is a no-op for the Calico variant", func() { - rc, err := ext.BuildContext(newInputs(operatorv1.Calico)) + rc, err := ext.ExtendContext(newControllerContext(operatorv1.Calico)) Expect(err).NotTo(HaveOccurred()) Expect(rc.NodePrometheusTLS).To(BeNil()) }) }) -func newInputs(variant operatorv1.ProductVariant) extensions.Inputs { +func newControllerContext(variant operatorv1.ProductVariant) extensions.ControllerContext { scheme := runtime.NewScheme() Expect(apis.AddToScheme(scheme, false)).NotTo(HaveOccurred()) c := ctrlrfake.DefaultFakeClientBuilder(scheme).Build() @@ -65,13 +64,15 @@ func newInputs(variant operatorv1.ProductVariant) extensions.Inputs { Expect(err).NotTo(HaveOccurred()) trustedBundle := certManager.CreateTrustedBundle() - return extensions.Inputs{ + return extensions.ControllerContext{ + RenderContext: extensions.RenderContext{ + Installation: &operatorv1.InstallationSpec{Variant: variant}, + FelixConfiguration: &v3.FelixConfiguration{}, + TrustedBundle: trustedBundle, + ClusterDomain: "cluster.local", + }, Ctx: context.Background(), Client: c, - Installation: &operatorv1.InstallationSpec{Variant: variant}, - FelixConfiguration: &v3.FelixConfiguration{}, CertificateManager: certManager, - TrustedBundle: trustedBundle, - ClusterDomain: "cluster.local", } } diff --git a/pkg/enterprise/node.go b/pkg/enterprise/node.go index d0ca3e2135..60b02ccdff 100644 --- a/pkg/enterprise/node.go +++ b/pkg/enterprise/node.go @@ -46,20 +46,16 @@ const ( installCNIContainerName = "install-cni" ) -func registerNode(s *extensions.Set) { - s.Register(operatorv1.CalicoEnterprise, render.ComponentNameNode, extensions.ComponentExtension{ - Image: func(in *operatorv1.InstallationSpec) components.Component { - return components.ComponentTigeraNode - }, - Modify: modifyNode, +func registerNode(v *extensions.Variant) { + v.Image(render.ComponentNameNode, func(in *operatorv1.InstallationSpec) components.Component { + return components.ComponentTigeraNode }) + v.Modify(render.ComponentNameNode, modifyNode) // The node component renders the cni-plugins init container; its image // resolves through its own override key. - s.Register(operatorv1.CalicoEnterprise, render.ComponentNameCNIPlugins, extensions.ComponentExtension{ - Image: func(in *operatorv1.InstallationSpec) components.Component { - return components.ComponentTigeraCNIPlugins - }, + v.Image(render.ComponentNameCNIPlugins, func(in *operatorv1.InstallationSpec) components.Component { + return components.ComponentTigeraCNIPlugins }) } diff --git a/pkg/enterprise/register.go b/pkg/enterprise/register.go index d1f81451b1..e0d0532b31 100644 --- a/pkg/enterprise/register.go +++ b/pkg/enterprise/register.go @@ -14,19 +14,31 @@ package enterprise -import "github.com/tigera/operator/pkg/extensions" +import ( + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/extensions" +) -// New builds the extension Set for the in-repo Calico Enterprise variant: every -// component modifier, image override, and the installation setup. The operator -// is handed this Set at startup (the core operator is handed none). After the -// monorepo split this is what calico-private's main will construct instead. +// New builds the extension Set for the in-repo Calico Enterprise variant: the +// controller extension, every component modifier, and the image overrides. The +// operator is handed this Set at startup (the core operator is handed none). +// After the monorepo split this is what calico-private's main will construct +// instead. func New() *extensions.Set { s := extensions.NewSet() - registerTypha(s) - registerNode(s) - registerWindows(s) - registerGuardian(s) - registerInstallation(s) - registerAPIServer(s) + + ent := s.Variant(operatorv1.CalicoEnterprise) + ent.Controller(controllerExtension{}) + registerTypha(ent) + registerNode(ent) + registerWindows(ent) + registerGuardian(ent) + registerAPIServer(ent) + + // When the enterprise operator manages a Calico installation, clean up the + // Enterprise objects left behind by a prior Enterprise installation. + cal := s.Variant(operatorv1.Calico) + registerAPIServerCleanup(cal) + return s } diff --git a/pkg/enterprise/typha.go b/pkg/enterprise/typha.go index ce3a434bbe..77ce79000c 100644 --- a/pkg/enterprise/typha.go +++ b/pkg/enterprise/typha.go @@ -20,15 +20,12 @@ import ( rbacv1 "k8s.io/api/rbac/v1" "sigs.k8s.io/controller-runtime/pkg/client" - operatorv1 "github.com/tigera/operator/api/v1" "github.com/tigera/operator/pkg/extensions" "github.com/tigera/operator/pkg/render" ) -func registerTypha(s *extensions.Set) { - s.Register(operatorv1.CalicoEnterprise, render.ComponentNameTypha, extensions.ComponentExtension{ - Modify: modifyTypha, - }) +func registerTypha(v *extensions.Variant) { + v.Modify(render.ComponentNameTypha, modifyTypha) } func modifyTypha(ctx extensions.RenderContext, objs, del []client.Object) ([]client.Object, []client.Object) { diff --git a/pkg/enterprise/windows.go b/pkg/enterprise/windows.go index 7883429e41..2b2d43753a 100644 --- a/pkg/enterprise/windows.go +++ b/pkg/enterprise/windows.go @@ -17,7 +17,6 @@ package enterprise import ( "fmt" - "github.com/sirupsen/logrus" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -36,29 +35,21 @@ import ( // felix env and node volume mounts, so they receive the same enterprise layering. var windowsNodeContainers = map[string]bool{"felix": true, "node": true, "confd": true} -func registerWindows(s *extensions.Set) { - s.Register(operatorv1.CalicoEnterprise, render.ComponentNameWindowsNodeImg, extensions.ComponentExtension{ - Image: func(*operatorv1.InstallationSpec) components.Component { return components.ComponentTigeraNodeWindows }, +func registerWindows(v *extensions.Variant) { + v.Image(render.ComponentNameWindowsNodeImg, func(*operatorv1.InstallationSpec) components.Component { + return components.ComponentTigeraNodeWindows }) - s.Register(operatorv1.CalicoEnterprise, render.ComponentNameWindowsCNIImg, extensions.ComponentExtension{ - Image: func(*operatorv1.InstallationSpec) components.Component { return components.ComponentTigeraCNIWindows }, - }) - s.Register(operatorv1.CalicoEnterprise, render.ComponentNameWindows, extensions.ComponentExtension{ - Modify: modifyWindows, + v.Image(render.ComponentNameWindowsCNIImg, func(*operatorv1.InstallationSpec) components.Component { + return components.ComponentTigeraCNIWindows }) + extensions.RegisterModifier(v, render.ComponentNameWindows, modifyWindows) } // modifyWindows layers Calico Enterprise behavior onto the rendered // calico-node-windows objects: the node-metrics Service and the Enterprise // daemonset configuration (flow/DNS log env, prometheus reporter, trusted DNS // servers, the calico log volume, and the prometheus reporter keypair mount). -func modifyWindows(ctx extensions.RenderContext, objs, del []client.Object) ([]client.Object, []client.Object) { - wc, ok := ctx.Component.(render.WindowsExtensionContext) - if !ok { - logrus.Errorf("BUG: windows modifier got %T, want render.WindowsExtensionContext; leaving objects unchanged", ctx.Component) - return objs, del - } - +func modifyWindows(ctx extensions.RenderContext, wc render.WindowsExtensionContext, objs, del []client.Object) ([]client.Object, []client.Object) { if ds, ok := extensions.FindObject[*appsv1.DaemonSet](objs, common.WindowsDaemonSetName); ok { modifyWindowsDaemonSet(ctx, wc, ds) } diff --git a/pkg/enterprise/windows_test.go b/pkg/enterprise/windows_test.go index b6c6b0c2d0..d0c11c6642 100644 --- a/pkg/enterprise/windows_test.go +++ b/pkg/enterprise/windows_test.go @@ -85,19 +85,22 @@ var _ = Describe("windows enterprise modifier", func() { return nil } - ctxFor := func(provider operatorv1.Provider, tls certificatemanagement.KeyPairInterface, bundle certificatemanagement.TrustedBundleRO) extensions.RenderContext { + ctxFor := func(provider operatorv1.Provider) extensions.RenderContext { return extensions.RenderContext{ Installation: &operatorv1.InstallationSpec{Variant: operatorv1.CalicoEnterprise, KubernetesProvider: provider}, - Component: render.WindowsExtensionContext{ - NodeReporterMetricsPort: 9081, - PrometheusServerTLS: tls, - TrustedBundle: bundle, - }, + } + } + + wcFor := func(tls certificatemanagement.KeyPairInterface, bundle certificatemanagement.TrustedBundleRO) render.WindowsExtensionContext { + return render.WindowsExtensionContext{ + NodeReporterMetricsPort: 9081, + PrometheusServerTLS: tls, + TrustedBundle: bundle, } } It("appends the node-metrics service", func() { - out, _ := applyExtensions(ext, render.ComponentNameWindows, ctxFor(operatorv1.ProviderNone, nil, nil), newObjs(), nil) + out, _ := applyExtensionsWithContext(ext, render.ComponentNameWindows, ctxFor(operatorv1.ProviderNone), wcFor(nil, nil), newObjs(), nil) svc, ok := extensions.FindObject[*corev1.Service](out, render.WindowsNodeMetricsService) Expect(ok).To(BeTrue()) Expect(svc.Namespace).To(Equal(common.CalicoNamespace)) @@ -105,7 +108,7 @@ var _ = Describe("windows enterprise modifier", func() { }) It("swaps the cni log mount for the calico log volume and adds enterprise env", func() { - out, _ := applyExtensions(ext, render.ComponentNameWindows, ctxFor(operatorv1.ProviderNone, nil, nil), newObjs(), nil) + out, _ := applyExtensionsWithContext(ext, render.ComponentNameWindows, ctxFor(operatorv1.ProviderNone), wcFor(nil, nil), newObjs(), nil) d := ds(out) Expect(d.Spec.Template.Spec.Volumes).To(ContainElement(HaveField("Name", "var-log-calico"))) @@ -122,7 +125,7 @@ var _ = Describe("windows enterprise modifier", func() { }) It("sets the trusted DNS server on openshift", func() { - out, _ := applyExtensions(ext, render.ComponentNameWindows, ctxFor(operatorv1.ProviderOpenShift, nil, nil), newObjs(), nil) + out, _ := applyExtensionsWithContext(ext, render.ComponentNameWindows, ctxFor(operatorv1.ProviderOpenShift), wcFor(nil, nil), newObjs(), nil) Expect(container(ds(out), "node").Env).To(ContainElement(corev1.EnvVar{Name: "FELIX_DNSTRUSTEDSERVERS", Value: "k8s-service:openshift-dns/dns-default"})) }) @@ -136,7 +139,7 @@ var _ = Describe("windows enterprise modifier", func() { Expect(err).NotTo(HaveOccurred()) bundle := cm.CreateTrustedBundle() - out, _ := applyExtensions(ext, render.ComponentNameWindows, ctxFor(operatorv1.ProviderNone, tls, bundle), newObjs(), nil) + out, _ := applyExtensionsWithContext(ext, render.ComponentNameWindows, ctxFor(operatorv1.ProviderNone), wcFor(tls, bundle), newObjs(), nil) d := ds(out) Expect(d.Spec.Template.Spec.Volumes).To(ContainElement(tls.Volume())) diff --git a/pkg/extensions/controllerextension.go b/pkg/extensions/controllerextension.go new file mode 100644 index 0000000000..ae22f5fd40 --- /dev/null +++ b/pkg/extensions/controllerextension.go @@ -0,0 +1,58 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package extensions + +import ( + "context" + + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/tigera/operator/pkg/controller/certificatemanager" +) + +// ControllerExtension is a variant's controller-side reconcile hook. The +// installation controller calls it to do the work core can't: reject +// unsupported configuration (Validate) and create the controller-side artifacts +// - certificates, trusted bundle additions - that feed the render context +// (ExtendContext). A variant registers at most one; the core operator registers +// none and runs with the base behavior. +type ControllerExtension interface { + // Validate rejects configuration the variant does not support, before any + // rendering happens. + Validate(cc ControllerContext) error + + // ExtendContext does the controller-side work the render modifiers can't + // (creating certificates, extending the trusted bundle) and returns the + // RenderContext those modifiers read, or an error that aborts the reconcile. + ExtendContext(cc ControllerContext) (RenderContext, error) +} + +// ControllerContext is the controller-phase context, the corollary to the +// render-phase RenderContext. It is the embedded RenderContext (the same data +// the render phase sees) plus the controller-side machinery a ControllerExtension +// needs to produce artifacts: a client, a certificate manager, a context. Those +// deps live here, not on RenderContext, so the modifiers that read RenderContext +// can't do I/O - they only transform objects. +// +// The controller fills the embedded RenderContext's data fields and the deps; +// ExtendContext does its work, sets the produced artifacts (e.g. +// NodePrometheusTLS) on the embedded context, and returns it. +type ControllerContext struct { + RenderContext + + Ctx context.Context + Client client.Client + CertificateManager certificatemanager.CertificateManager +} diff --git a/pkg/extensions/controllerextension_test.go b/pkg/extensions/controllerextension_test.go new file mode 100644 index 0000000000..a0fc51275b --- /dev/null +++ b/pkg/extensions/controllerextension_test.go @@ -0,0 +1,104 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package extensions_test + +import ( + "errors" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/extensions" +) + +var _ = Describe("controller extension", func() { + var s *extensions.Set + BeforeEach(func() { + s = extensions.NewSet() + }) + + It("returns the base render context when the variant has no extension", func() { + install := &operatorv1.InstallationSpec{Variant: operatorv1.Calico} + rc, err := s.ExtendContext(extensions.ControllerContext{ + RenderContext: extensions.RenderContext{Installation: install, ClusterDomain: "cluster.local"}, + }) + Expect(err).NotTo(HaveOccurred()) + Expect(rc.Installation).To(BeIdenticalTo(install)) + Expect(rc.ClusterDomain).To(Equal("cluster.local")) + Expect(rc.NodePrometheusTLS).To(BeNil()) + }) + + It("runs the extension registered for the installation variant", func() { + s.Variant(operatorv1.CalicoEnterprise).Controller(fakeController{}) + rc, err := s.ExtendContext(enterpriseContext()) + Expect(err).NotTo(HaveOccurred()) + Expect(rc.ClusterDomain).To(Equal("from-fake")) + }) + + It("ignores an extension registered for a different variant", func() { + s.Variant(operatorv1.CalicoEnterprise).Controller(fakeController{}) + rc, err := s.ExtendContext(extensions.ControllerContext{ + RenderContext: extensions.RenderContext{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.Calico}, ClusterDomain: "real"}, + }) + Expect(err).NotTo(HaveOccurred()) + Expect(rc.ClusterDomain).To(Equal("real")) + }) + + It("surfaces the extension error", func() { + s.Variant(operatorv1.CalicoEnterprise).Controller(fakeController{err: errors.New("boom")}) + _, err := s.ExtendContext(enterpriseContext()) + Expect(err).To(MatchError("boom")) + }) + + It("runs the extension's validation", func() { + s.Variant(operatorv1.CalicoEnterprise).Controller(fakeController{validateErr: errors.New("invalid")}) + Expect(s.Validate(enterpriseContext())).To(MatchError("invalid")) + }) + + It("returns the base context and no validation error for a nil Set", func() { + var nilSet *extensions.Set + cc := enterpriseContext() + cc.ClusterDomain = "real" + rc, err := nilSet.ExtendContext(cc) + Expect(err).NotTo(HaveOccurred()) + Expect(rc.ClusterDomain).To(Equal("real")) + Expect(nilSet.Validate(cc)).NotTo(HaveOccurred()) + }) +}) + +func enterpriseContext() extensions.ControllerContext { + return extensions.ControllerContext{ + RenderContext: extensions.RenderContext{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.CalicoEnterprise}}, + } +} + +// fakeController is a ControllerExtension whose Validate and ExtendContext return +// configurable results. +type fakeController struct { + err error + validateErr error +} + +func (f fakeController) Validate(_ extensions.ControllerContext) error { + return f.validateErr +} + +func (f fakeController) ExtendContext(_ extensions.ControllerContext) (extensions.RenderContext, error) { + if f.err != nil { + return extensions.RenderContext{}, f.err + } + return extensions.RenderContext{ClusterDomain: "from-fake"}, nil +} diff --git a/pkg/extensions/decorate_helpers_test.go b/pkg/extensions/decorate_helpers_test.go index 227c4f552a..86afb71df0 100644 --- a/pkg/extensions/decorate_helpers_test.go +++ b/pkg/extensions/decorate_helpers_test.go @@ -24,8 +24,8 @@ import ( // stubExtComponent adapts raw object lists to a render.Component so a registered // extension can be exercised through Set.Decorate, the same seam the component -// handler uses. key selects the extension; extCtx is delivered as the per-component -// ExtensionContext. +// handler uses. key selects the extension; extCtx is delivered as the component's +// ExtensionContext (the typed config a RegisterModifier modifier reads). type stubExtComponent struct { key string extCtx any @@ -57,9 +57,15 @@ func (s stubExtComponent) ExtensionContext() any { } // applyExtensions decorates a stub component holding the given objects with the -// extension registered under key in s, then renders it - returning the decorated -// create and delete lists. +// extension registered under key, then renders it. For a modifier that needs the +// component's typed config, use applyExtensionsWithContext. func applyExtensions(s *extensions.Set, key string, ctx extensions.RenderContext, create, del []client.Object) ([]client.Object, []client.Object) { - stub := stubExtComponent{key: key, extCtx: ctx.Component, create: create, delete: del} + return applyExtensionsWithContext(s, key, ctx, nil, create, del) +} + +// applyExtensionsWithContext is applyExtensions for a modifier that reads the +// component's typed config: extCtx is delivered as the stub's ExtensionContext. +func applyExtensionsWithContext(s *extensions.Set, key string, ctx extensions.RenderContext, extCtx any, create, del []client.Object) ([]client.Object, []client.Object) { + stub := stubExtComponent{key: key, extCtx: extCtx, create: create, delete: del} return s.Decorate(stub, ctx).Objects() } diff --git a/pkg/extensions/doc.go b/pkg/extensions/doc.go index a505f82fa1..ec06d9597b 100644 --- a/pkg/extensions/doc.go +++ b/pkg/extensions/doc.go @@ -16,23 +16,29 @@ // Enterprise) use to layer variant-specific behavior onto the core operator's // render output, so core code never branches on variant. // -// Everything keys off the installation Variant, and registration is per -// variant, so a registered hook only ever runs for its own variant and never -// re-checks it. There are two phases: +// A Set holds the extensions for every variant. Per reconcile the controller +// selects one Variant from the installation's variant, so a registered hook only +// ever runs for its own variant and never re-checks it. A Variant bundles two +// kinds of extension: // -// Setup is the controller-side phase. It runs once per reconcile in the -// installation controller, has cluster access (Client, CertificateManager), and -// does the side-effecting work a pure render hook can't: creating certificates, -// extending the trusted bundle, validating config. It returns the RenderContext -// - the read-only baton passed to the render phase. Register one per variant -// with RegisterSetup; the controller runs it with BuildContext. +// A ControllerExtension is the controller-side hook. It runs once per reconcile +// in the installation controller, has cluster access (Client, +// CertificateManager) via the ControllerContext, and does the side-effecting +// work a pure render hook can't: rejecting unsupported config (Validate) and +// creating certificates / extending the trusted bundle (ExtendContext). It +// returns the RenderContext, the read-only baton passed to the render phase. // -// Extension is the render phase: pure, per-component hooks that run after a -// component builds its objects. Its Image field overrides the component's image -// (resolved during ResolveImages), and its Modify field post-processes the -// rendered objects (run at the componentHandler). Register one per component -// with Register. +// Per-component modifiers are the render phase: pure hooks that run after a +// component builds its objects. An image override swaps the component's image +// (resolved during ResolveImages); a Modifier post-processes the rendered +// objects (run at the componentHandler, which renders the decorated component). +// Register a modifier with Variant.Modify, or with RegisterModifier when it +// needs the component's own typed config. // -// A variant wires up its setup and extensions in one place at startup - see -// pkg/enterprise. +// ControllerContext (controller phase) and RenderContext (render phase) are a +// pair: ControllerContext embeds RenderContext and adds the cluster-access deps, +// which is why modifiers, given only a RenderContext, can't do I/O. +// +// A variant wires up its controller extension and modifiers in one place at +// startup - see pkg/enterprise. package extensions diff --git a/pkg/extensions/extension.go b/pkg/extensions/extension.go index e9ef998a6b..3e00227624 100644 --- a/pkg/extensions/extension.go +++ b/pkg/extensions/extension.go @@ -16,79 +16,16 @@ package extensions import ( "sigs.k8s.io/controller-runtime/pkg/client" - - operatorv1 "github.com/tigera/operator/api/v1" - "github.com/tigera/operator/pkg/render" ) -// ComponentExtension is everything a variant layers onto one render component. -// Every field is optional: a component that only needs a different image sets -// Image and leaves Modify nil, and vice versa. This is the single registration a -// variant makes per component, so all of that component's variance lives in one -// place. -type ComponentExtension struct { - // Image overrides the component's image. Resolved during ResolveImages, in - // the render package, via the imageoverride leaf. - Image ImageOverride - - // Modify post-processes the component's rendered objects, after Objects(). - Modify Modifier -} - // Modifier post-processes the objects a render component produced. It receives // the component's create and delete lists and returns the (possibly extended) // lists. A modifier may mutate matched objects, append objects to create, and -// append objects to delete (e.g. to clean up resources another variant left -// behind). A modifier runs only for the variant it was registered under, so it -// need not re-check the variant. +// append objects to delete (e.g. to clean up resources a prior variant left +// behind). It runs only for the variant it is registered under, so it need not +// re-check the variant. type Modifier func(ctx RenderContext, create, delete []client.Object) (newCreate, newDelete []client.Object) -type modifierKey struct { - variant operatorv1.ProductVariant - component string -} - -// Decorate wraps component with the variant extension registered for it, so -// that when the handler renders the component its objects are post-processed by -// the modifier registered for the component and the installation's variant. A -// decorated component is itself a render.Component, so it flows through the -// component handler exactly like any other. Returns component unchanged when it -// exposes no extension point, when no modifier is registered for it, when no -// installation is set, or on a nil Set (the core operator registers no -// extensions). -func (s *Set) Decorate(component render.Component, ctx RenderContext) render.Component { - if s == nil || ctx.Installation == nil { - return component - } - ext, ok := component.(render.Extensible) - if !ok { - return component - } - modify, ok := s.modifiers[modifierKey{ctx.Installation.Variant, ext.ModifierKey()}] - if !ok { - return component - } - if p, ok := component.(render.ExtensionContextProvider); ok { - ctx.Component = p.ExtensionContext() - } - return &decoratedComponent{Component: component, ctx: ctx, modify: modify} -} - -// decoratedComponent is the render.Component produced by Decorate: it renders -// its embedded base component and then runs the variant modifier over the -// result. It embeds the base render.Component, so ResolveImages, SupportedOSType, -// and Ready delegate to the base; only Objects is augmented. -type decoratedComponent struct { - render.Component - ctx RenderContext - modify Modifier -} - -func (d *decoratedComponent) Objects() ([]client.Object, []client.Object) { - create, del := d.Component.Objects() - return d.modify(d.ctx, create, del) -} - // FindObject returns the first object of type T with the given name. func FindObject[T client.Object](objs []client.Object, name string) (T, bool) { var zero T diff --git a/pkg/extensions/extension_test.go b/pkg/extensions/extension_test.go index 8cbb0a7c89..0ea3359345 100644 --- a/pkg/extensions/extension_test.go +++ b/pkg/extensions/extension_test.go @@ -34,13 +34,11 @@ var _ = Describe("extension registry", func() { entCtx := extensions.RenderContext{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.CalicoEnterprise}} It("applies a registered modifier to the matching component and variant", func() { - s.Register(operatorv1.CalicoEnterprise, "test", extensions.ComponentExtension{ - Modify: func(ctx extensions.RenderContext, objs, del []client.Object) ([]client.Object, []client.Object) { - cm, ok := extensions.FindObject[*corev1.ConfigMap](objs, "cm") - Expect(ok).To(BeTrue()) - cm.Data = map[string]string{"k": "v"} - return append(objs, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "extra"}}), del - }, + s.Variant(operatorv1.CalicoEnterprise).Modify("test", func(ctx extensions.RenderContext, objs, del []client.Object) ([]client.Object, []client.Object) { + cm, ok := extensions.FindObject[*corev1.ConfigMap](objs, "cm") + Expect(ok).To(BeTrue()) + cm.Data = map[string]string{"k": "v"} + return append(objs, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "extra"}}), del }) in := []client.Object{&corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "cm"}}} @@ -53,10 +51,8 @@ var _ = Describe("extension registry", func() { }) It("lets a modifier append to the delete list", func() { - s.Register(operatorv1.CalicoEnterprise, "test", extensions.ComponentExtension{ - Modify: func(_ extensions.RenderContext, objs, del []client.Object) ([]client.Object, []client.Object) { - return objs, append(del, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "stale"}}) - }, + s.Variant(operatorv1.CalicoEnterprise).Modify("test", func(_ extensions.RenderContext, objs, del []client.Object) ([]client.Object, []client.Object) { + return objs, append(del, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "stale"}}) }) in := []client.Object{&corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "cm"}}} @@ -73,10 +69,8 @@ var _ = Describe("extension registry", func() { }) It("does not apply a modifier registered for a different variant", func() { - s.Register(operatorv1.CalicoEnterprise, "test", extensions.ComponentExtension{ - Modify: func(_ extensions.RenderContext, objs, del []client.Object) ([]client.Object, []client.Object) { - return append(objs, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "extra"}}), del - }, + s.Variant(operatorv1.CalicoEnterprise).Modify("test", func(_ extensions.RenderContext, objs, del []client.Object) ([]client.Object, []client.Object) { + return append(objs, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "extra"}}), del }) calicoCtx := extensions.RenderContext{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.Calico}} @@ -91,16 +85,14 @@ var _ = Describe("extension registry", func() { Expect(out).To(Equal(in)) }) - It("replaces rather than stacks when a (variant, component) is registered twice", func() { - add := func(name string) extensions.ComponentExtension { - return extensions.ComponentExtension{ - Modify: func(_ extensions.RenderContext, objs, del []client.Object) ([]client.Object, []client.Object) { - return append(objs, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: name}}), del - }, - } + It("replaces rather than stacks when a component modifier is registered twice", func() { + add := func(name string) { + s.Variant(operatorv1.CalicoEnterprise).Modify("test", func(_ extensions.RenderContext, objs, del []client.Object) ([]client.Object, []client.Object) { + return append(objs, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: name}}), del + }) } - s.Register(operatorv1.CalicoEnterprise, "test", add("first")) - s.Register(operatorv1.CalicoEnterprise, "test", add("second")) + add("first") + add("second") out, _ := applyExtensions(s, "test", entCtx, nil, nil) Expect(out).To(HaveLen(1)) diff --git a/pkg/extensions/image_test.go b/pkg/extensions/image_test.go index dcc4876fdd..03d4abbce2 100644 --- a/pkg/extensions/image_test.go +++ b/pkg/extensions/image_test.go @@ -27,10 +27,8 @@ var _ = Describe("image overrides", func() { var s *extensions.Set BeforeEach(func() { s = extensions.NewSet() - s.Register(operatorv1.CalicoEnterprise, "node", extensions.ComponentExtension{ - Image: func(in *operatorv1.InstallationSpec) components.Component { - return components.ComponentTigeraNode - }, + s.Variant(operatorv1.CalicoEnterprise).Image("node", func(in *operatorv1.InstallationSpec) components.Component { + return components.ComponentTigeraNode }) }) diff --git a/pkg/extensions/rendercontext.go b/pkg/extensions/rendercontext.go index c03389c153..95daec43aa 100644 --- a/pkg/extensions/rendercontext.go +++ b/pkg/extensions/rendercontext.go @@ -22,14 +22,15 @@ import ( // RenderContext carries reconcile-derived inputs from controllers into render // modifiers. Core operator code never reads these fields - only registered -// modifiers do. -// Three kinds of value live here: +// modifiers do. Two kinds of value live here: // - raw cluster state gathered generically (Installation, FelixConfiguration, -// ClusterDomain) that modifiers derive their own values from, +// ClusterDomain) that modifiers derive their own values from, and // - controller-produced artifacts (TrustedBundle, NodePrometheusTLS) that can -// only be created controller-side because they have cluster side effects, and -// - Component, the per-component context the component being modified supplies -// for config a modifier can't derive from the fields above. +// only be created controller-side because they have cluster side effects. +// +// Per-component config a modifier needs but can't derive from these fields is +// not carried here; it flows to the modifier as a typed argument (see +// RegisterModifier), supplied by the component via render.ExtensionContextProvider. type RenderContext struct { Installation *operatorv1.InstallationSpec FelixConfiguration *v3.FelixConfiguration @@ -38,17 +39,9 @@ type RenderContext struct { // TrustedBundle is the shared CA bundle for the calico-system namespace. TrustedBundle certificatemanagement.TrustedBundle - // NodePrometheusTLS is created by the enterprise setup (it has cluster side - // effects, so it can't be built in a modifier). The node modifier is its only - // consumer: it mounts the keypair onto the daemonset and sets the + // NodePrometheusTLS is created by the enterprise controller extension (it has + // cluster side effects, so it can't be built in a modifier). The node modifier + // is its only consumer: it mounts the keypair onto the daemonset and sets the // FELIX_PROMETHEUSREPORTER* certificate env vars. NodePrometheusTLS certificatemanagement.KeyPairInterface - - // Component is per-component context that the component being modified supplies - // via render.ExtensionContextProvider - config a modifier needs but can't - // derive from the fields above (e.g. a keypair the component's own controller - // created, or a CR field only that controller reads). The componentHandler - // sets it per component before applying the modifier; a modifier type-asserts - // it to the component's own context type. Nil when the component supplies none. - Component any } diff --git a/pkg/extensions/set.go b/pkg/extensions/set.go index 08aa52f259..42a76267ad 100644 --- a/pkg/extensions/set.go +++ b/pkg/extensions/set.go @@ -18,56 +18,89 @@ import ( operatorv1 "github.com/tigera/operator/api/v1" "github.com/tigera/operator/pkg/components" "github.com/tigera/operator/pkg/imageoverride" + "github.com/tigera/operator/pkg/render" ) -// Set is the collection of variant extensions the operator runs with: the -// per-variant setups, the per-component modifiers, and the image overrides. The -// core operator runs with a nil/empty Set; an extension build (Calico -// Enterprise) constructs a populated one and hands it in through +// Set is all the variant extensions the operator runs with, indexed by product +// variant. The core operator runs with a nil Set; an extension build (e.g. +// Calico Enterprise) constructs a populated one and hands it in through // options.ControllerOptions. This replaces what used to be package-level // registries, so nothing is wired by import side effect. // -// The zero value is not usable; build one with NewSet. The methods that read it -// (BuildContext, Decorate, ResolveImage, Images) are nil-safe so the core -// operator can pass a nil Set and get base behavior. +// Per reconcile the controller selects one Variant from the installation's +// variant. The methods the controller calls (Decorate, Validate, ExtendContext, +// Images, ResolveImage) are nil-safe, so the core operator's nil Set yields base +// behavior. type Set struct { - setups map[operatorv1.ProductVariant]Setup - modifiers map[modifierKey]Modifier - images *imageoverride.Overrides + variants map[operatorv1.ProductVariant]*Variant + images *imageoverride.Overrides } -// NewSet returns an empty Set ready to register extensions into. +// NewSet returns an empty Set ready to register variant extensions into. func NewSet() *Set { return &Set{ - setups: map[operatorv1.ProductVariant]Setup{}, - modifiers: map[modifierKey]Modifier{}, - images: imageoverride.New(), + variants: map[operatorv1.ProductVariant]*Variant{}, + images: imageoverride.New(), } } -// Register installs e as the extension for the named component under the given -// variant. A (variant, component) pair has at most one extension; registration -// replaces any prior one. The image override and the modifier are stored -// separately, so a component can set either field or both. -func (s *Set) Register(variant operatorv1.ProductVariant, component string, e ComponentExtension) { - if e.Image != nil { - s.images.Register(variant, component, e.Image) +// Variant returns the extension bundle for v, creating an empty one if needed. +// Used at registration time to build up a variant's extensions. +func (s *Set) Variant(v operatorv1.ProductVariant) *Variant { + if s.variants[v] == nil { + s.variants[v] = &Variant{ + variant: v, + modifiers: map[string]decorator{}, + images: s.images, + } } - if e.Modify != nil { - s.modifiers[modifierKey{variant, component}] = e.Modify + return s.variants[v] +} + +// variant looks up the bundle for v, returning nil when none is registered. +// Nil-safe. +func (s *Set) variant(v operatorv1.ProductVariant) *Variant { + if s == nil { + return nil + } + return s.variants[v] +} + +// Decorate wraps component with the extension registered for it under the +// installation's variant, so that when the handler renders the component its +// objects are post-processed by that modifier. A decorated component is itself a +// render.Component, so it flows through the component handler like any other. +// Returns component unchanged when no extension applies. Nil-safe. +func (s *Set) Decorate(component render.Component, ctx RenderContext) render.Component { + if ctx.Installation == nil { + return component + } + return s.variant(ctx.Installation.Variant).decorate(component, ctx) +} + +// Validate runs the controller extension's validation for the installation's +// variant, or returns nil when no extension is registered. Nil-safe. +func (s *Set) Validate(cc ControllerContext) error { + if cc.Installation == nil { + return nil } + return s.variant(cc.Installation.Variant).validate(cc) } -// RegisterSetup installs setup as the controller-side setup phase for the given -// variant. Registration replaces any prior setup for that variant. -func (s *Set) RegisterSetup(variant operatorv1.ProductVariant, setup Setup) { - s.setups[variant] = setup +// ExtendContext runs the controller extension for the installation's variant and +// returns the resulting RenderContext, or the base render context when no +// extension is registered. Nil-safe. +func (s *Set) ExtendContext(cc ControllerContext) (RenderContext, error) { + if cc.Installation == nil { + return cc.RenderContext, nil + } + return s.variant(cc.Installation.Variant).extendContext(cc) } -// Images returns the image overrides. The render package resolves a component's -// image through these directly (the imageoverride leaf, so render need not -// import extensions). Safe to call on a nil Set, which returns nil overrides -// that resolve to the default image. +// Images returns the shared image override table. The render package resolves a +// component's image through it directly (the imageoverride leaf, so render need +// not import extensions). Nil-safe, returning nil overrides that resolve to the +// default image. func (s *Set) Images() *imageoverride.Overrides { if s == nil { return nil @@ -76,7 +109,7 @@ func (s *Set) Images() *imageoverride.Overrides { } // ResolveImage resolves key for the installation through the image overrides, -// returning def when no override applies. Safe to call on a nil Set. +// returning def when no override applies. Nil-safe. func (s *Set) ResolveImage(key string, def components.Component, in *operatorv1.InstallationSpec) components.Component { return s.Images().Resolve(key, def, in) } diff --git a/pkg/extensions/setup.go b/pkg/extensions/setup.go deleted file mode 100644 index 2eb0ef1b6b..0000000000 --- a/pkg/extensions/setup.go +++ /dev/null @@ -1,78 +0,0 @@ -// Copyright (c) 2026 Tigera, Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package extensions - -import ( - "context" - - v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" - "sigs.k8s.io/controller-runtime/pkg/client" - - operatorv1 "github.com/tigera/operator/api/v1" - "github.com/tigera/operator/pkg/controller/certificatemanager" - "github.com/tigera/operator/pkg/tls/certificatemanagement" -) - -// Inputs is the reconcile state a Setup builds a RenderContext from. The -// installation controller populates it directly. It carries both the values -// that flow straight into the RenderContext and the side-effecting dependencies -// (Client, CertificateManager) a setup needs to produce controller-side -// artifacts. -type Inputs struct { - Ctx context.Context - Client client.Client - Installation *operatorv1.InstallationSpec - FelixConfiguration *v3.FelixConfiguration - CertificateManager certificatemanager.CertificateManager - TrustedBundle certificatemanagement.TrustedBundle - ClusterDomain string -} - -// Setup is a variant's controller-side reconcile phase. It performs the work -// modifiers can't (creating certificates, extending the trusted bundle, -// validating config) and returns the RenderContext that is then handed to that -// variant's modifiers - or an error that aborts the reconcile. -// -// This is the generic seam controllers use to extend base operator behavior; -// its first consumer is Calico Enterprise, but nothing here is enterprise -// specific. A setup runs only for the variant it was registered under, so it -// need not re-check the variant. -type Setup func(in Inputs) (RenderContext, error) - -// BaseRenderContext maps the generically-gathered inputs onto a RenderContext. -// Every setup builds on it, so the base fields are assembled in exactly one -// place. A setup layers its side-effect artifacts (e.g. NodePrometheusTLS) on -// top of the returned value. -func BaseRenderContext(in Inputs) RenderContext { - return RenderContext{ - Installation: in.Installation, - FelixConfiguration: in.FelixConfiguration, - ClusterDomain: in.ClusterDomain, - TrustedBundle: in.TrustedBundle, - } -} - -// BuildContext runs the setup registered for the installation variant and -// returns its RenderContext, or the base render context when the variant has no -// setup. Safe to call on a nil Set, which always returns the base context - the -// core operator registers no setups. -func (s *Set) BuildContext(in Inputs) (RenderContext, error) { - if s != nil && in.Installation != nil { - if setup, ok := s.setups[in.Installation.Variant]; ok { - return setup(in) - } - } - return BaseRenderContext(in), nil -} diff --git a/pkg/extensions/setup_test.go b/pkg/extensions/setup_test.go deleted file mode 100644 index 9ee58a6648..0000000000 --- a/pkg/extensions/setup_test.go +++ /dev/null @@ -1,89 +0,0 @@ -// Copyright (c) 2026 Tigera, Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package extensions_test - -import ( - "errors" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - - operatorv1 "github.com/tigera/operator/api/v1" - "github.com/tigera/operator/pkg/extensions" -) - -var _ = Describe("variant setup", func() { - var s *extensions.Set - BeforeEach(func() { - s = extensions.NewSet() - }) - - It("returns the base render context when the variant has no setup", func() { - install := &operatorv1.InstallationSpec{Variant: operatorv1.Calico} - rc, err := s.BuildContext(extensions.Inputs{ - Installation: install, - ClusterDomain: "cluster.local", - }) - Expect(err).NotTo(HaveOccurred()) - Expect(rc.Installation).To(BeIdenticalTo(install)) - Expect(rc.ClusterDomain).To(Equal("cluster.local")) - Expect(rc.NodePrometheusTLS).To(BeNil()) - }) - - It("uses the setup registered for the installation variant", func() { - s.RegisterSetup(operatorv1.CalicoEnterprise, fakeSetup(nil)) - rc, err := s.BuildContext(enterpriseInputs()) - Expect(err).NotTo(HaveOccurred()) - Expect(rc.ClusterDomain).To(Equal("from-fake")) - }) - - It("ignores a setup registered for a different variant", func() { - s.RegisterSetup(operatorv1.CalicoEnterprise, fakeSetup(nil)) - rc, err := s.BuildContext(extensions.Inputs{ - Installation: &operatorv1.InstallationSpec{Variant: operatorv1.Calico}, - ClusterDomain: "real", - }) - Expect(err).NotTo(HaveOccurred()) - Expect(rc.ClusterDomain).To(Equal("real")) - }) - - It("surfaces the setup error", func() { - s.RegisterSetup(operatorv1.CalicoEnterprise, fakeSetup(errors.New("boom"))) - _, err := s.BuildContext(enterpriseInputs()) - Expect(err).To(MatchError("boom")) - }) - - It("returns the base context for a nil Set", func() { - var nilSet *extensions.Set - in := enterpriseInputs() - in.ClusterDomain = "real" - rc, err := nilSet.BuildContext(in) - Expect(err).NotTo(HaveOccurred()) - Expect(rc.ClusterDomain).To(Equal("real")) - }) -}) - -func enterpriseInputs() extensions.Inputs { - return extensions.Inputs{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.CalicoEnterprise}} -} - -func fakeSetup(err error) extensions.Setup { - return func(_ extensions.Inputs) (extensions.RenderContext, error) { - if err != nil { - return extensions.RenderContext{}, err - } - return extensions.RenderContext{ClusterDomain: "from-fake"}, nil - } -} diff --git a/pkg/extensions/variant.go b/pkg/extensions/variant.go new file mode 100644 index 0000000000..175463d4e8 --- /dev/null +++ b/pkg/extensions/variant.go @@ -0,0 +1,138 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package extensions + +import ( + "github.com/sirupsen/logrus" + "sigs.k8s.io/controller-runtime/pkg/client" + + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/imageoverride" + "github.com/tigera/operator/pkg/render" +) + +// Variant bundles everything that extends the core operator for one product +// variant: the controller-side hook, the per-component modifiers, and the image +// overrides. The Set selects one Variant per reconcile from the installation's +// variant, so within a Variant there is at most one extension per component and +// nothing here is itself keyed by variant. +type Variant struct { + variant operatorv1.ProductVariant + controller ControllerExtension + modifiers map[string]decorator + images *imageoverride.Overrides // shared with the owning Set +} + +// decorator wraps a base component, returning one whose Objects() are augmented +// by a registered modifier. +type decorator func(base render.Component, ctx RenderContext) render.Component + +// Controller registers the variant's controller-side extension. A variant has +// at most one; registering replaces any prior one. +func (v *Variant) Controller(c ControllerExtension) { + v.controller = c +} + +// Image registers an image override for the named component. +func (v *Variant) Image(component string, fn ImageOverride) { + v.images.Register(v.variant, component, fn) +} + +// Modify registers a modifier for a component that needs no per-component +// config. For components whose modifier needs the component's own typed config, +// use RegisterModifier. +func (v *Variant) Modify(component string, m Modifier) { + v.modifiers[component] = func(base render.Component, ctx RenderContext) render.Component { + return &decoratedComponent{Component: base, ctx: ctx, modify: m} + } +} + +// RegisterModifier registers a modifier for component whose modifier needs the +// component's own typed config. The component supplies it via +// render.ExtensionContextProvider; RegisterModifier asserts it to Cfg once, here, +// and hands the typed value to modify - so the modifier body needs no assertion. +// It is a free function because Go has no generic methods. +func RegisterModifier[Cfg any](v *Variant, component string, + modify func(ctx RenderContext, cfg Cfg, create, delete []client.Object) ([]client.Object, []client.Object), +) { + v.modifiers[component] = func(base render.Component, ctx RenderContext) render.Component { + provider, ok := base.(render.ExtensionContextProvider) + if !ok { + logrus.Errorf("BUG: component %q has a registered modifier but provides no extension context; leaving it unmodified", component) + return base + } + cfg, ok := provider.ExtensionContext().(Cfg) + if !ok { + var want Cfg + logrus.Errorf("BUG: component %q extension context is %T, want %T; leaving it unmodified", component, provider.ExtensionContext(), want) + return base + } + bound := func(ctx RenderContext, create, delete []client.Object) ([]client.Object, []client.Object) { + return modify(ctx, cfg, create, delete) + } + return &decoratedComponent{Component: base, ctx: ctx, modify: bound} + } +} + +// decorate wraps component with the modifier registered for its extension key, +// or returns it unchanged when the component exposes no extension point or none +// is registered. Nil-safe. +func (v *Variant) decorate(component render.Component, ctx RenderContext) render.Component { + if v == nil { + return component + } + ext, ok := component.(render.Extensible) + if !ok { + return component + } + build, ok := v.modifiers[ext.ModifierKey()] + if !ok { + return component + } + return build(component, ctx) +} + +// validate runs the controller extension's validation, or nil when the variant +// has none. Nil-safe. +func (v *Variant) validate(cc ControllerContext) error { + if v == nil || v.controller == nil { + return nil + } + return v.controller.Validate(cc) +} + +// extendContext runs the controller extension, or returns the base render +// context when the variant has none. Nil-safe. +func (v *Variant) extendContext(cc ControllerContext) (RenderContext, error) { + if v == nil || v.controller == nil { + return cc.RenderContext, nil + } + return v.controller.ExtendContext(cc) +} + +// decoratedComponent is the render.Component produced by decorate: it renders +// its embedded base component and then runs the variant modifier over the +// result. It embeds the base render.Component, so ResolveImages, SupportedOSType, +// and Ready delegate to the base; only Objects is augmented. +type decoratedComponent struct { + render.Component + ctx RenderContext + modify Modifier +} + +func (d *decoratedComponent) Objects() ([]client.Object, []client.Object) { + create, del := d.Component.Objects() + return d.modify(d.ctx, create, del) +} diff --git a/pkg/render/apiserver_test.go b/pkg/render/apiserver_test.go index 31b15779ee..57eeb83e66 100644 --- a/pkg/render/apiserver_test.go +++ b/pkg/render/apiserver_test.go @@ -68,12 +68,13 @@ import ( func apiServerObjects(c render.Component) ([]client.Object, []client.Object) { create, del := c.Objects() rc := extensions.RenderContext{} + var extCtx any if p, ok := c.(render.ExtensionContextProvider); ok { ec := p.ExtensionContext().(render.APIServerExtensionContext) rc.Installation = ec.Config.Installation - rc.Component = ec + extCtx = ec } - return applyExtensions(ext, render.ComponentNameAPIServer, rc, create, del) + return applyExtensionsWithContext(ext, render.ComponentNameAPIServer, rc, extCtx, create, del) } var _ = Describe("API server rendering tests (Calico Enterprise)", func() { diff --git a/pkg/render/decorate_helpers_test.go b/pkg/render/decorate_helpers_test.go index 750abb8df4..c748f56295 100644 --- a/pkg/render/decorate_helpers_test.go +++ b/pkg/render/decorate_helpers_test.go @@ -24,8 +24,8 @@ import ( // stubExtComponent adapts raw object lists to a render.Component so a registered // extension can be exercised through Set.Decorate, the same seam the component -// handler uses. key selects the extension; extCtx is delivered as the per-component -// ExtensionContext. +// handler uses. key selects the extension; extCtx is delivered as the component's +// ExtensionContext (the typed config a RegisterModifier modifier reads). type stubExtComponent struct { key string extCtx any @@ -57,9 +57,15 @@ func (s stubExtComponent) ExtensionContext() any { } // applyExtensions decorates a stub component holding the given objects with the -// extension registered under key in s, then renders it - returning the decorated -// create and delete lists. +// extension registered under key, then renders it. For a modifier that needs the +// component's typed config, use applyExtensionsWithContext. func applyExtensions(s *extensions.Set, key string, ctx extensions.RenderContext, create, del []client.Object) ([]client.Object, []client.Object) { - stub := stubExtComponent{key: key, extCtx: ctx.Component, create: create, delete: del} + return applyExtensionsWithContext(s, key, ctx, nil, create, del) +} + +// applyExtensionsWithContext is applyExtensions for a modifier that reads the +// component's typed config: extCtx is delivered as the stub's ExtensionContext. +func applyExtensionsWithContext(s *extensions.Set, key string, ctx extensions.RenderContext, extCtx any, create, del []client.Object) ([]client.Object, []client.Object) { + stub := stubExtComponent{key: key, extCtx: extCtx, create: create, delete: del} return s.Decorate(stub, ctx).Objects() } diff --git a/pkg/render/guardian_test.go b/pkg/render/guardian_test.go index 44e4e2fe4a..16f7653031 100644 --- a/pkg/render/guardian_test.go +++ b/pkg/render/guardian_test.go @@ -51,10 +51,11 @@ func guardianObjects(cfg *render.GuardianConfiguration) []client.Object { ExpectWithOffset(1, g.ResolveImages(nil)).To(BeNil()) objs, _ := g.Objects() rc := extensions.RenderContext{Installation: cfg.Installation} + var extCtx any if p, ok := g.(render.ExtensionContextProvider); ok { - rc.Component = p.ExtensionContext() + extCtx = p.ExtensionContext() } - out, _ := applyExtensions(ext, render.GuardianName, rc, objs, nil) + out, _ := applyExtensionsWithContext(ext, render.GuardianName, rc, extCtx, objs, nil) return out } @@ -113,10 +114,11 @@ var _ = Describe("Rendering tests", func() { // Apply the registered enterprise modifier the way the componentHandler // does, so these enterprise tests exercise the integrated output. rc := extensions.RenderContext{Installation: cfg.Installation} + var extCtx any if p, ok := g.(render.ExtensionContextProvider); ok { - rc.Component = p.ExtensionContext() + extCtx = p.ExtensionContext() } - resources, _ = applyExtensions(ext, render.GuardianName, rc, resources, nil) + resources, _ = applyExtensionsWithContext(ext, render.GuardianName, rc, extCtx, resources, nil) } BeforeEach(func() { @@ -349,10 +351,11 @@ var _ = Describe("Rendering tests", func() { // does, so the enterprise policy is exercised. For the Calico variant the // modifier is a no-op and the OSS policy is returned. rc := extensions.RenderContext{Installation: cfg.Installation} + var extCtx any if p, ok := g.(render.ExtensionContextProvider); ok { - rc.Component = p.ExtensionContext() + extCtx = p.ExtensionContext() } - resources, _ = applyExtensions(ext, render.ComponentNameGuardianPolicy, rc, objs, nil) + resources, _ = applyExtensionsWithContext(ext, render.ComponentNameGuardianPolicy, rc, extCtx, objs, nil) } Context("policy rendering based on variant and IncludeEgressNetworkPolicy", func() { diff --git a/pkg/render/windows_test.go b/pkg/render/windows_test.go index f15e185eda..903fb3349b 100644 --- a/pkg/render/windows_test.go +++ b/pkg/render/windows_test.go @@ -51,10 +51,11 @@ func renderWindows(cfg *render.WindowsConfiguration) []client.Object { ExpectWithOffset(1, comp.ResolveImages(nil)).To(BeNil()) objs, _ := comp.Objects() rc := extensions.RenderContext{Installation: cfg.Installation} + var extCtx any if p, ok := comp.(render.ExtensionContextProvider); ok { - rc.Component = p.ExtensionContext() + extCtx = p.ExtensionContext() } - out, _ := applyExtensions(ext, render.ComponentNameWindows, rc, objs, nil) + out, _ := applyExtensionsWithContext(ext, render.ComponentNameWindows, rc, extCtx, objs, nil) return out } From 9ae30122ff167c5f47e7abd99d98c15db1e26df0 Mon Sep 17 00:00:00 2001 From: Casey Davenport Date: Thu, 18 Jun 2026 11:21:26 -0700 Subject: [PATCH 30/54] Clean up the extensions API per review Image overrides are plain components.Component values instead of funcs - an override only picks which image (registry, path, and FIPS handling are applied downstream in render), so registration reads v.Image(name, image) and the ImageOverride alias is gone. Rename the modifier RenderContext param from ctx to rc, split the RegisterModifier signature one arg per line, use slices.Contains over the local helpers, and trim the over-comments in the component handler and the ControllerExtension docs. --- .../installation/core_controller.go | 2 +- pkg/controller/utils/component.go | 7 -- pkg/enterprise/apiserver.go | 16 ++--- pkg/enterprise/decorate_helpers_test.go | 8 +-- pkg/enterprise/guardian.go | 4 +- pkg/enterprise/node.go | 64 ++++++++----------- pkg/enterprise/typha.go | 4 +- pkg/enterprise/windows.go | 20 +++--- pkg/extensions/controllerextension.go | 18 +++--- pkg/extensions/decorate_helpers_test.go | 8 +-- pkg/extensions/extension.go | 2 +- pkg/extensions/image.go | 24 ------- pkg/extensions/image_test.go | 4 +- pkg/extensions/variant.go | 33 +++++----- pkg/imageoverride/imageoverride.go | 19 +++--- pkg/render/decorate_helpers_test.go | 8 +-- 16 files changed, 92 insertions(+), 149 deletions(-) delete mode 100644 pkg/extensions/image.go diff --git a/pkg/controller/installation/core_controller.go b/pkg/controller/installation/core_controller.go index baeb11364e..47fdea89c1 100644 --- a/pkg/controller/installation/core_controller.go +++ b/pkg/controller/installation/core_controller.go @@ -1221,7 +1221,7 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile CertificateManager: certificateManager, } if err := r.opts.Extensions.Validate(cc); err != nil { - r.status.SetDegraded(operatorv1.ResourceValidationError, "Invalid installation configuration for variant", err, reqLogger) + r.status.SetDegraded(operatorv1.ResourceValidationError, "Invalid installation configuration", err, reqLogger) return reconcile.Result{}, err } renderCtx, err := r.opts.Extensions.ExtendContext(cc) diff --git a/pkg/controller/utils/component.go b/pkg/controller/utils/component.go index f805aef659..52eb2dc374 100644 --- a/pkg/controller/utils/component.go +++ b/pkg/controller/utils/component.go @@ -458,14 +458,7 @@ func resetMetadataForCreate(obj client.Object) { } func (c *componentHandler) CreateOrUpdateOrDelete(ctx context.Context, component render.Component, status status.StatusManager) error { - // Decorate the component with any registered variant extension before doing - // anything with it, so Ready, SupportedOSType, and Objects all reflect the - // extended component. Decorate is a no-op for components with no registered - // extension. if ext, ok := component.(render.Extensible); ok && c.extensions == nil { - // The component can be extended but this handler was built without an - // extension Set, so any registered extension silently won't run. That is - // a wiring bug in the controller, not a normal state. c.log.Info("BUG: extensible component rendered by a handler with no extension Set; extensions will not be applied", "component", ext.ModifierKey()) } component = c.extensions.Decorate(component, c.renderCtx) diff --git a/pkg/enterprise/apiserver.go b/pkg/enterprise/apiserver.go index 7a265877d1..b20f3579e5 100644 --- a/pkg/enterprise/apiserver.go +++ b/pkg/enterprise/apiserver.go @@ -16,6 +16,7 @@ package enterprise import ( "fmt" + "slices" "strings" appsv1 "k8s.io/api/apps/v1" @@ -64,7 +65,7 @@ func registerAPIServerCleanup(v *extensions.Variant) { // modifyAPIServer layers Calico Enterprise behavior onto the rendered API server objects: // the query server container and its volumes, audit logging on the aggregation API server // container, the Enterprise RBAC objects, and the query server port on the Service. -func modifyAPIServer(ctx extensions.RenderContext, ec render.APIServerExtensionContext, create, del []client.Object) ([]client.Object, []client.Object) { +func modifyAPIServer(rc extensions.RenderContext, ec render.APIServerExtensionContext, create, del []client.Object) ([]client.Object, []client.Object) { c := &apiServer{cfg: ec.Config, calicoImage: ec.CalicoImage} if dep, ok := extensions.FindObject[*appsv1.Deployment](create, render.APIServerName); ok { @@ -76,7 +77,7 @@ func modifyAPIServer(ctx extensions.RenderContext, ec render.APIServerExtensionC // Enterprise serves staged policies through the tiered-policy passthrough role. if role, ok := extensions.FindObject[*rbacv1.ClusterRole](create, "calico-tiered-policy-passthrough"); ok { for i := range role.Rules { - if contains(role.Rules[i].Resources, "networkpolicies") { + if slices.Contains(role.Rules[i].Resources, "networkpolicies") { role.Rules[i].Resources = append(role.Rules[i].Resources, "stagednetworkpolicies", "stagedglobalnetworkpolicies") } } @@ -143,7 +144,7 @@ func modifyAPIServer(ctx extensions.RenderContext, ec render.APIServerExtensionC // cleanupAPIServer deletes the Enterprise API server objects when running Calico, so a // cluster switched from Enterprise to Calico does not leave them behind. -func cleanupAPIServer(ctx extensions.RenderContext, ec render.APIServerExtensionContext, create, del []client.Object) ([]client.Object, []client.Object) { +func cleanupAPIServer(rc extensions.RenderContext, ec render.APIServerExtensionContext, create, del []client.Object) ([]client.Object, []client.Object) { c := &apiServer{cfg: ec.Config} del = append(del, c.tigeraAPIServerClusterRole(), c.tigeraAPIServerClusterRoleBinding()) @@ -235,15 +236,6 @@ func (c *apiServer) addQueryServerPort(s *corev1.Service) { }) } -func contains(s []string, v string) bool { - for _, x := range s { - if x == v { - return true - } - } - return false -} - func (c *apiServer) multiTenantSecretsRBAC() []client.Object { return render.TunnelSecretRBAC(render.APIServerSecretsRBACName, render.APIServerServiceAccountName, c.cfg.ManagementCluster, true) } diff --git a/pkg/enterprise/decorate_helpers_test.go b/pkg/enterprise/decorate_helpers_test.go index 184a801a71..bf9c6a72d7 100644 --- a/pkg/enterprise/decorate_helpers_test.go +++ b/pkg/enterprise/decorate_helpers_test.go @@ -59,13 +59,13 @@ func (s stubExtComponent) ExtensionContext() any { // applyExtensions decorates a stub component holding the given objects with the // extension registered under key, then renders it. For a modifier that needs the // component's typed config, use applyExtensionsWithContext. -func applyExtensions(s *extensions.Set, key string, ctx extensions.RenderContext, create, del []client.Object) ([]client.Object, []client.Object) { - return applyExtensionsWithContext(s, key, ctx, nil, create, del) +func applyExtensions(s *extensions.Set, key string, rc extensions.RenderContext, create, del []client.Object) ([]client.Object, []client.Object) { + return applyExtensionsWithContext(s, key, rc, nil, create, del) } // applyExtensionsWithContext is applyExtensions for a modifier that reads the // component's typed config: extCtx is delivered as the stub's ExtensionContext. -func applyExtensionsWithContext(s *extensions.Set, key string, ctx extensions.RenderContext, extCtx any, create, del []client.Object) ([]client.Object, []client.Object) { +func applyExtensionsWithContext(s *extensions.Set, key string, rc extensions.RenderContext, extCtx any, create, del []client.Object) ([]client.Object, []client.Object) { stub := stubExtComponent{key: key, extCtx: extCtx, create: create, delete: del} - return s.Decorate(stub, ctx).Objects() + return s.Decorate(stub, rc).Objects() } diff --git a/pkg/enterprise/guardian.go b/pkg/enterprise/guardian.go index 11895b0b29..4886889a29 100644 --- a/pkg/enterprise/guardian.go +++ b/pkg/enterprise/guardian.go @@ -46,7 +46,7 @@ func registerGuardian(v *extensions.Variant) { // enterprise management-cluster policy. Building the enterprise egress rules can // fail (proxy URL parsing); on failure we drop the policy entirely, matching the // core behavior of omitting it rather than installing a partial policy. -func modifyGuardianPolicy(ctx extensions.RenderContext, gpc render.GuardianPolicyExtensionContext, objs, del []client.Object) ([]client.Object, []client.Object) { +func modifyGuardianPolicy(rc extensions.RenderContext, gpc render.GuardianPolicyExtensionContext, objs, del []client.Object) ([]client.Object, []client.Object) { policy, ok := extensions.FindObject[*v3.NetworkPolicy](objs, render.GuardianPolicyName) if !ok { return objs, del @@ -204,7 +204,7 @@ func enterpriseGuardianPolicySpec(gpc render.GuardianPolicyExtensionContext) (v3 // objects: the secrets Role/RoleBinding and default UI settings, the // elasticsearch/kibana service ports, the management-cluster-request cluster // role rules (which replace the OSS rules), and the CA bundle env vars. -func modifyGuardian(ctx extensions.RenderContext, gc render.GuardianExtensionContext, objs, del []client.Object) ([]client.Object, []client.Object) { +func modifyGuardian(rc extensions.RenderContext, gc render.GuardianExtensionContext, objs, del []client.Object) ([]client.Object, []client.Object) { if role, ok := extensions.FindObject[*rbacv1.ClusterRole](objs, render.GuardianClusterRoleName); ok { role.Rules = guardianEnterpriseRules(gc) } diff --git a/pkg/enterprise/node.go b/pkg/enterprise/node.go index 60b02ccdff..43969b12a6 100644 --- a/pkg/enterprise/node.go +++ b/pkg/enterprise/node.go @@ -16,6 +16,7 @@ package enterprise import ( "fmt" + "slices" v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" appsv1 "k8s.io/api/apps/v1" @@ -47,23 +48,19 @@ const ( ) func registerNode(v *extensions.Variant) { - v.Image(render.ComponentNameNode, func(in *operatorv1.InstallationSpec) components.Component { - return components.ComponentTigeraNode - }) + v.Image(render.ComponentNameNode, components.ComponentTigeraNode) v.Modify(render.ComponentNameNode, modifyNode) // The node component renders the cni-plugins init container; its image // resolves through its own override key. - v.Image(render.ComponentNameCNIPlugins, func(in *operatorv1.InstallationSpec) components.Component { - return components.ComponentTigeraCNIPlugins - }) + v.Image(render.ComponentNameCNIPlugins, components.ComponentTigeraCNIPlugins) } // modifyNode layers Calico Enterprise behavior onto the rendered calico/node // objects: the extra RBAC rules, the node-metrics Service, and the Enterprise // daemonset configuration (flow/DNS log env, prometheus reporter, BGP metrics // readiness check, multi-interface mode, and the calico log volume). -func modifyNode(ctx extensions.RenderContext, objs, del []client.Object) ([]client.Object, []client.Object) { +func modifyNode(rc extensions.RenderContext, objs, del []client.Object) ([]client.Object, []client.Object) { if role, ok := extensions.FindObject[*rbacv1.ClusterRole](objs, render.CalicoNodeObjectName); ok { role.Rules = append(role.Rules, nodeEnterpriseRules()...) } @@ -78,10 +75,10 @@ func modifyNode(ctx extensions.RenderContext, objs, del []client.Object) ([]clie } if ds, ok := extensions.FindObject[*appsv1.DaemonSet](objs, common.NodeDaemonSetName); ok { - modifyNodeDaemonSet(ctx, ds) + modifyNodeDaemonSet(rc, ds) } - return append(objs, nodeMetricsService(ctx)), del + return append(objs, nodeMetricsService(rc)), del } // nodeEnterpriseRules are the additional cluster role rules calico/node needs in @@ -119,10 +116,10 @@ func nodeEnterpriseRules() []rbacv1.PolicyRule { // BGP metrics readiness check, and the prometheus reporter keypair mount. The // calico log volume is mounted by the base render for both variants, so it is // not handled here. -func modifyNodeDaemonSet(ctx extensions.RenderContext, ds *appsv1.DaemonSet) { +func modifyNodeDaemonSet(rc extensions.RenderContext, ds *appsv1.DaemonSet) { spec := &ds.Spec.Template.Spec - multiInterfaceMode := multiInterfaceModeEnv(ctx.Installation) + multiInterfaceMode := multiInterfaceModeEnv(rc.Installation) for i := range spec.InitContainers { if spec.InitContainers[i].Name == installCNIContainerName && multiInterfaceMode != nil { @@ -136,30 +133,30 @@ func modifyNodeDaemonSet(ctx extensions.RenderContext, ds *appsv1.DaemonSet) { continue } - c.Env = append(c.Env, nodeEnterpriseEnv(ctx)...) + c.Env = append(c.Env, nodeEnterpriseEnv(rc)...) // Add the BGP metrics readiness check, but only when the base render kept // the bird readiness check (i.e. BGP is in use and we're not on VPP). - if c.ReadinessProbe != nil && c.ReadinessProbe.Exec != nil && containsString(c.ReadinessProbe.Exec.Command, "--bird-ready") { + if c.ReadinessProbe != nil && c.ReadinessProbe.Exec != nil && slices.Contains(c.ReadinessProbe.Exec.Command, "--bird-ready") { c.ReadinessProbe.Exec.Command = append(c.ReadinessProbe.Exec.Command, "--bgp-metrics-ready") } } - mountNodePrometheusTLS(ctx, ds) + mountNodePrometheusTLS(rc, ds) } // mountNodePrometheusTLS mounts the node prometheus reporter keypair onto the // daemonset: the volume, the calico-node volume mount, the cert-management init // container (when in use), and the pod hash annotation that rolls the pods on // cert rotation. The keypair has cluster side effects, so the enterprise setup -// creates it and hands it in via ctx rather than the modifier building it. In +// creates it and hands it in via rc rather than the modifier building it. In // core (calico) the keypair is never created, so the base node render carries // no prometheus mount at all. -func mountNodePrometheusTLS(ctx extensions.RenderContext, ds *appsv1.DaemonSet) { - if ctx.NodePrometheusTLS == nil { +func mountNodePrometheusTLS(rc extensions.RenderContext, ds *appsv1.DaemonSet) { + if rc.NodePrometheusTLS == nil { return } - tls := ctx.NodePrometheusTLS + tls := rc.NodePrometheusTLS spec := &ds.Spec.Template.Spec spec.Volumes = append(spec.Volumes, tls.Volume()) @@ -183,10 +180,10 @@ func mountNodePrometheusTLS(ctx extensions.RenderContext, ds *appsv1.DaemonSet) // nodeEnterpriseEnv is the Enterprise felix configuration added to the // calico/node container. -func nodeEnterpriseEnv(ctx extensions.RenderContext) []corev1.EnvVar { +func nodeEnterpriseEnv(rc extensions.RenderContext) []corev1.EnvVar { env := []corev1.EnvVar{ {Name: "FELIX_PROMETHEUSREPORTERENABLED", Value: "true"}, - {Name: "FELIX_PROMETHEUSREPORTERPORT", Value: fmt.Sprintf("%d", nodeReporterPort(ctx.FelixConfiguration))}, + {Name: "FELIX_PROMETHEUSREPORTERPORT", Value: fmt.Sprintf("%d", nodeReporterPort(rc.FelixConfiguration))}, {Name: "FELIX_FLOWLOGSFILEENABLED", Value: "true"}, {Name: "FELIX_FLOWLOGSFILEINCLUDELABELS", Value: "true"}, {Name: "FELIX_FLOWLOGSFILEINCLUDEPOLICIES", Value: "true"}, @@ -197,15 +194,15 @@ func nodeEnterpriseEnv(ctx extensions.RenderContext) []corev1.EnvVar { {Name: "FELIX_DNSLOGSFILEPERNODELIMIT", Value: "1000"}, } - if mode := multiInterfaceModeEnv(ctx.Installation); mode != nil { + if mode := multiInterfaceModeEnv(rc.Installation); mode != nil { env = append(env, *mode) } - if ctx.NodePrometheusTLS != nil && ctx.TrustedBundle != nil { + if rc.NodePrometheusTLS != nil && rc.TrustedBundle != nil { env = append(env, - corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERCERTFILE", Value: ctx.NodePrometheusTLS.VolumeMountCertificateFilePath()}, - corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERKEYFILE", Value: ctx.NodePrometheusTLS.VolumeMountKeyFilePath()}, - corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERCAFILE", Value: ctx.TrustedBundle.MountPath()}, + corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERCERTFILE", Value: rc.NodePrometheusTLS.VolumeMountCertificateFilePath()}, + corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERKEYFILE", Value: rc.NodePrometheusTLS.VolumeMountKeyFilePath()}, + corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERCAFILE", Value: rc.TrustedBundle.MountPath()}, ) } @@ -222,10 +219,10 @@ func multiInterfaceModeEnv(install *operatorv1.InstallationSpec) *corev1.EnvVar } // nodeMetricsService builds the enterprise-only calico-node-metrics Service. -func nodeMetricsService(ctx extensions.RenderContext) *corev1.Service { - reporterPort := nodeReporterPort(ctx.FelixConfiguration) - felixPort := felixMetricsPort(ctx.FelixConfiguration) - felixEnabled := ctx.FelixConfiguration != nil && utils.IsFelixPrometheusMetricsEnabled(ctx.FelixConfiguration) +func nodeMetricsService(rc extensions.RenderContext) *corev1.Service { + reporterPort := nodeReporterPort(rc.FelixConfiguration) + felixPort := felixMetricsPort(rc.FelixConfiguration) + felixEnabled := rc.FelixConfiguration != nil && utils.IsFelixPrometheusMetricsEnabled(rc.FelixConfiguration) ports := []corev1.ServicePort{ { @@ -283,12 +280,3 @@ func felixMetricsPort(fc *v3.FelixConfiguration) int { } return defaultFelixMetricsPort } - -func containsString(s []string, v string) bool { - for _, x := range s { - if x == v { - return true - } - } - return false -} diff --git a/pkg/enterprise/typha.go b/pkg/enterprise/typha.go index 77ce79000c..3786e7be07 100644 --- a/pkg/enterprise/typha.go +++ b/pkg/enterprise/typha.go @@ -28,7 +28,7 @@ func registerTypha(v *extensions.Variant) { v.Modify(render.ComponentNameTypha, modifyTypha) } -func modifyTypha(ctx extensions.RenderContext, objs, del []client.Object) ([]client.Object, []client.Object) { +func modifyTypha(rc extensions.RenderContext, objs, del []client.Object) ([]client.Object, []client.Object) { if role, ok := extensions.FindObject[*rbacv1.ClusterRole](objs, "calico-typha"); ok { role.Rules = append(role.Rules, rbacv1.PolicyRule{ APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, @@ -47,7 +47,7 @@ func modifyTypha(ctx extensions.RenderContext, objs, del []client.Object) ([]cli } if dep, ok := extensions.FindObject[*appsv1.Deployment](objs, "calico-typha"); ok { - net := ctx.Installation.CalicoNetwork + net := rc.Installation.CalicoNetwork if net != nil && net.MultiInterfaceMode != nil { for i := range dep.Spec.Template.Spec.Containers { if dep.Spec.Template.Spec.Containers[i].Name == render.TyphaContainerName { diff --git a/pkg/enterprise/windows.go b/pkg/enterprise/windows.go index 2b2d43753a..d00e7da88d 100644 --- a/pkg/enterprise/windows.go +++ b/pkg/enterprise/windows.go @@ -36,12 +36,8 @@ import ( var windowsNodeContainers = map[string]bool{"felix": true, "node": true, "confd": true} func registerWindows(v *extensions.Variant) { - v.Image(render.ComponentNameWindowsNodeImg, func(*operatorv1.InstallationSpec) components.Component { - return components.ComponentTigeraNodeWindows - }) - v.Image(render.ComponentNameWindowsCNIImg, func(*operatorv1.InstallationSpec) components.Component { - return components.ComponentTigeraCNIWindows - }) + v.Image(render.ComponentNameWindowsNodeImg, components.ComponentTigeraNodeWindows) + v.Image(render.ComponentNameWindowsCNIImg, components.ComponentTigeraCNIWindows) extensions.RegisterModifier(v, render.ComponentNameWindows, modifyWindows) } @@ -49,15 +45,15 @@ func registerWindows(v *extensions.Variant) { // calico-node-windows objects: the node-metrics Service and the Enterprise // daemonset configuration (flow/DNS log env, prometheus reporter, trusted DNS // servers, the calico log volume, and the prometheus reporter keypair mount). -func modifyWindows(ctx extensions.RenderContext, wc render.WindowsExtensionContext, objs, del []client.Object) ([]client.Object, []client.Object) { +func modifyWindows(rc extensions.RenderContext, wc render.WindowsExtensionContext, objs, del []client.Object) ([]client.Object, []client.Object) { if ds, ok := extensions.FindObject[*appsv1.DaemonSet](objs, common.WindowsDaemonSetName); ok { - modifyWindowsDaemonSet(ctx, wc, ds) + modifyWindowsDaemonSet(rc, wc, ds) } return append(objs, windowsNodeMetricsService(wc)), del } -func modifyWindowsDaemonSet(ctx extensions.RenderContext, wc render.WindowsExtensionContext, ds *appsv1.DaemonSet) { +func modifyWindowsDaemonSet(rc extensions.RenderContext, wc render.WindowsExtensionContext, ds *appsv1.DaemonSet) { dirOrCreate := corev1.HostPathDirectoryOrCreate spec := &ds.Spec.Template.Spec @@ -72,7 +68,7 @@ func modifyWindowsDaemonSet(ctx extensions.RenderContext, wc render.WindowsExten continue } - c.Env = append(c.Env, windowsEnterpriseEnv(ctx, wc)...) + c.Env = append(c.Env, windowsEnterpriseEnv(rc, wc)...) // Enterprise mounts the calico log directory in place of the OSS CNI log // directory, so drop the OSS mount before adding the enterprise one. @@ -85,7 +81,7 @@ func modifyWindowsDaemonSet(ctx extensions.RenderContext, wc render.WindowsExten // windowsEnterpriseEnv is the Enterprise felix configuration added to the // calico-node-windows containers. -func windowsEnterpriseEnv(ctx extensions.RenderContext, wc render.WindowsExtensionContext) []corev1.EnvVar { +func windowsEnterpriseEnv(rc extensions.RenderContext, wc render.WindowsExtensionContext) []corev1.EnvVar { env := []corev1.EnvVar{ {Name: "FELIX_PROMETHEUSREPORTERENABLED", Value: "true"}, {Name: "FELIX_PROMETHEUSREPORTERPORT", Value: fmt.Sprintf("%d", wc.NodeReporterMetricsPort)}, @@ -108,7 +104,7 @@ func windowsEnterpriseEnv(ctx extensions.RenderContext, wc render.WindowsExtensi } // Providers without a kube-dns service need a non-default trusted DNS server. - switch ctx.Installation.KubernetesProvider { + switch rc.Installation.KubernetesProvider { case operatorv1.ProviderOpenShift: env = append(env, corev1.EnvVar{Name: "FELIX_DNSTRUSTEDSERVERS", Value: "k8s-service:openshift-dns/dns-default"}) case operatorv1.ProviderRKE2: diff --git a/pkg/extensions/controllerextension.go b/pkg/extensions/controllerextension.go index ae22f5fd40..fada677689 100644 --- a/pkg/extensions/controllerextension.go +++ b/pkg/extensions/controllerextension.go @@ -22,20 +22,18 @@ import ( "github.com/tigera/operator/pkg/controller/certificatemanager" ) -// ControllerExtension is a variant's controller-side reconcile hook. The -// installation controller calls it to do the work core can't: reject -// unsupported configuration (Validate) and create the controller-side artifacts -// - certificates, trusted bundle additions - that feed the render context -// (ExtendContext). A variant registers at most one; the core operator registers -// none and runs with the base behavior. +// ControllerExtension extends a controller's reconcile: it validates the +// configuration and builds the RenderContext the render phase consumes. The core +// operator registers none and runs with the base behavior; an extension build +// registers one. type ControllerExtension interface { - // Validate rejects configuration the variant does not support, before any + // Validate rejects configuration the extension does not support, before any // rendering happens. Validate(cc ControllerContext) error - // ExtendContext does the controller-side work the render modifiers can't - // (creating certificates, extending the trusted bundle) and returns the - // RenderContext those modifiers read, or an error that aborts the reconcile. + // ExtendContext does the controller-side reconcile work the render phase + // cannot, returning the RenderContext the render phase consumes, or an error + // that aborts the reconcile. ExtendContext(cc ControllerContext) (RenderContext, error) } diff --git a/pkg/extensions/decorate_helpers_test.go b/pkg/extensions/decorate_helpers_test.go index 86afb71df0..82dc31b601 100644 --- a/pkg/extensions/decorate_helpers_test.go +++ b/pkg/extensions/decorate_helpers_test.go @@ -59,13 +59,13 @@ func (s stubExtComponent) ExtensionContext() any { // applyExtensions decorates a stub component holding the given objects with the // extension registered under key, then renders it. For a modifier that needs the // component's typed config, use applyExtensionsWithContext. -func applyExtensions(s *extensions.Set, key string, ctx extensions.RenderContext, create, del []client.Object) ([]client.Object, []client.Object) { - return applyExtensionsWithContext(s, key, ctx, nil, create, del) +func applyExtensions(s *extensions.Set, key string, rc extensions.RenderContext, create, del []client.Object) ([]client.Object, []client.Object) { + return applyExtensionsWithContext(s, key, rc, nil, create, del) } // applyExtensionsWithContext is applyExtensions for a modifier that reads the // component's typed config: extCtx is delivered as the stub's ExtensionContext. -func applyExtensionsWithContext(s *extensions.Set, key string, ctx extensions.RenderContext, extCtx any, create, del []client.Object) ([]client.Object, []client.Object) { +func applyExtensionsWithContext(s *extensions.Set, key string, rc extensions.RenderContext, extCtx any, create, del []client.Object) ([]client.Object, []client.Object) { stub := stubExtComponent{key: key, extCtx: extCtx, create: create, delete: del} - return s.Decorate(stub, ctx).Objects() + return s.Decorate(stub, rc).Objects() } diff --git a/pkg/extensions/extension.go b/pkg/extensions/extension.go index 3e00227624..106f1ee8bb 100644 --- a/pkg/extensions/extension.go +++ b/pkg/extensions/extension.go @@ -24,7 +24,7 @@ import ( // append objects to delete (e.g. to clean up resources a prior variant left // behind). It runs only for the variant it is registered under, so it need not // re-check the variant. -type Modifier func(ctx RenderContext, create, delete []client.Object) (newCreate, newDelete []client.Object) +type Modifier func(rc RenderContext, create, delete []client.Object) (newCreate, newDelete []client.Object) // FindObject returns the first object of type T with the given name. func FindObject[T client.Object](objs []client.Object, name string) (T, bool) { diff --git a/pkg/extensions/image.go b/pkg/extensions/image.go deleted file mode 100644 index a551d1e76f..0000000000 --- a/pkg/extensions/image.go +++ /dev/null @@ -1,24 +0,0 @@ -// Copyright (c) 2026 Tigera, Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package extensions - -import ( - "github.com/tigera/operator/pkg/imageoverride" -) - -// ImageOverride returns the component image to use for an installation. It is -// the Image field of an Extension. An override runs only for the variant it was -// registered under, so it need not re-check the variant. -type ImageOverride = imageoverride.Override diff --git a/pkg/extensions/image_test.go b/pkg/extensions/image_test.go index 03d4abbce2..736f110ca3 100644 --- a/pkg/extensions/image_test.go +++ b/pkg/extensions/image_test.go @@ -27,9 +27,7 @@ var _ = Describe("image overrides", func() { var s *extensions.Set BeforeEach(func() { s = extensions.NewSet() - s.Variant(operatorv1.CalicoEnterprise).Image("node", func(in *operatorv1.InstallationSpec) components.Component { - return components.ComponentTigeraNode - }) + s.Variant(operatorv1.CalicoEnterprise).Image("node", components.ComponentTigeraNode) }) It("uses the override registered for the installation variant", func() { diff --git a/pkg/extensions/variant.go b/pkg/extensions/variant.go index 175463d4e8..36d4d8b96c 100644 --- a/pkg/extensions/variant.go +++ b/pkg/extensions/variant.go @@ -19,6 +19,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/components" "github.com/tigera/operator/pkg/imageoverride" "github.com/tigera/operator/pkg/render" ) @@ -37,7 +38,7 @@ type Variant struct { // decorator wraps a base component, returning one whose Objects() are augmented // by a registered modifier. -type decorator func(base render.Component, ctx RenderContext) render.Component +type decorator func(base render.Component, rc RenderContext) render.Component // Controller registers the variant's controller-side extension. A variant has // at most one; registering replaces any prior one. @@ -46,16 +47,16 @@ func (v *Variant) Controller(c ControllerExtension) { } // Image registers an image override for the named component. -func (v *Variant) Image(component string, fn ImageOverride) { - v.images.Register(v.variant, component, fn) +func (v *Variant) Image(component string, image components.Component) { + v.images.Register(v.variant, component, image) } // Modify registers a modifier for a component that needs no per-component // config. For components whose modifier needs the component's own typed config, // use RegisterModifier. func (v *Variant) Modify(component string, m Modifier) { - v.modifiers[component] = func(base render.Component, ctx RenderContext) render.Component { - return &decoratedComponent{Component: base, ctx: ctx, modify: m} + v.modifiers[component] = func(base render.Component, rc RenderContext) render.Component { + return &decoratedComponent{Component: base, rc: rc, modify: m} } } @@ -64,10 +65,12 @@ func (v *Variant) Modify(component string, m Modifier) { // render.ExtensionContextProvider; RegisterModifier asserts it to Cfg once, here, // and hands the typed value to modify - so the modifier body needs no assertion. // It is a free function because Go has no generic methods. -func RegisterModifier[Cfg any](v *Variant, component string, - modify func(ctx RenderContext, cfg Cfg, create, delete []client.Object) ([]client.Object, []client.Object), +func RegisterModifier[Cfg any]( + v *Variant, + component string, + modify func(rc RenderContext, cfg Cfg, create, delete []client.Object) ([]client.Object, []client.Object), ) { - v.modifiers[component] = func(base render.Component, ctx RenderContext) render.Component { + v.modifiers[component] = func(base render.Component, rc RenderContext) render.Component { provider, ok := base.(render.ExtensionContextProvider) if !ok { logrus.Errorf("BUG: component %q has a registered modifier but provides no extension context; leaving it unmodified", component) @@ -79,17 +82,17 @@ func RegisterModifier[Cfg any](v *Variant, component string, logrus.Errorf("BUG: component %q extension context is %T, want %T; leaving it unmodified", component, provider.ExtensionContext(), want) return base } - bound := func(ctx RenderContext, create, delete []client.Object) ([]client.Object, []client.Object) { - return modify(ctx, cfg, create, delete) + bound := func(rc RenderContext, create, delete []client.Object) ([]client.Object, []client.Object) { + return modify(rc, cfg, create, delete) } - return &decoratedComponent{Component: base, ctx: ctx, modify: bound} + return &decoratedComponent{Component: base, rc: rc, modify: bound} } } // decorate wraps component with the modifier registered for its extension key, // or returns it unchanged when the component exposes no extension point or none // is registered. Nil-safe. -func (v *Variant) decorate(component render.Component, ctx RenderContext) render.Component { +func (v *Variant) decorate(component render.Component, rc RenderContext) render.Component { if v == nil { return component } @@ -101,7 +104,7 @@ func (v *Variant) decorate(component render.Component, ctx RenderContext) render if !ok { return component } - return build(component, ctx) + return build(component, rc) } // validate runs the controller extension's validation, or nil when the variant @@ -128,11 +131,11 @@ func (v *Variant) extendContext(cc ControllerContext) (RenderContext, error) { // and Ready delegate to the base; only Objects is augmented. type decoratedComponent struct { render.Component - ctx RenderContext + rc RenderContext modify Modifier } func (d *decoratedComponent) Objects() ([]client.Object, []client.Object) { create, del := d.Component.Objects() - return d.modify(d.ctx, create, del) + return d.modify(d.rc, create, del) } diff --git a/pkg/imageoverride/imageoverride.go b/pkg/imageoverride/imageoverride.go index 5c981c0c07..ee4c48d559 100644 --- a/pkg/imageoverride/imageoverride.go +++ b/pkg/imageoverride/imageoverride.go @@ -22,9 +22,6 @@ import ( "github.com/tigera/operator/pkg/components" ) -// Override selects the component image to use for an installation. -type Override func(in *operatorv1.InstallationSpec) components.Component - type overrideKey struct { variant operatorv1.ProductVariant key string @@ -33,19 +30,21 @@ type overrideKey struct { // Overrides maps a component (keyed by variant) to the image it should resolve // to, letting a variant swap a component's image without the render package // branching on variant. The render component holds one and resolves through it. +// Registry, image path, and FIPS handling are applied downstream in the render +// package, so an override only picks which component. type Overrides struct { - m map[overrideKey]Override + m map[overrideKey]components.Component } // New returns an empty Overrides. func New() *Overrides { - return &Overrides{m: map[overrideKey]Override{}} + return &Overrides{m: map[overrideKey]components.Component{}} } -// Register stores fn under key for the given variant. The key is the render +// Register stores image under key for the given variant. The key is the render // component's image identifier (e.g. "node"). -func (o *Overrides) Register(variant operatorv1.ProductVariant, key string, fn Override) { - o.m[overrideKey{variant, key}] = fn +func (o *Overrides) Register(variant operatorv1.ProductVariant, key string, image components.Component) { + o.m[overrideKey{variant, key}] = image } // Resolve returns the override registered for key under the installation's @@ -55,8 +54,8 @@ func (o *Overrides) Resolve(key string, def components.Component, in *operatorv1 if o == nil || in == nil { return def } - if fn, ok := o.m[overrideKey{in.Variant, key}]; ok { - return fn(in) + if image, ok := o.m[overrideKey{in.Variant, key}]; ok { + return image } return def } diff --git a/pkg/render/decorate_helpers_test.go b/pkg/render/decorate_helpers_test.go index c748f56295..75f0587276 100644 --- a/pkg/render/decorate_helpers_test.go +++ b/pkg/render/decorate_helpers_test.go @@ -59,13 +59,13 @@ func (s stubExtComponent) ExtensionContext() any { // applyExtensions decorates a stub component holding the given objects with the // extension registered under key, then renders it. For a modifier that needs the // component's typed config, use applyExtensionsWithContext. -func applyExtensions(s *extensions.Set, key string, ctx extensions.RenderContext, create, del []client.Object) ([]client.Object, []client.Object) { - return applyExtensionsWithContext(s, key, ctx, nil, create, del) +func applyExtensions(s *extensions.Set, key string, rc extensions.RenderContext, create, del []client.Object) ([]client.Object, []client.Object) { + return applyExtensionsWithContext(s, key, rc, nil, create, del) } // applyExtensionsWithContext is applyExtensions for a modifier that reads the // component's typed config: extCtx is delivered as the stub's ExtensionContext. -func applyExtensionsWithContext(s *extensions.Set, key string, ctx extensions.RenderContext, extCtx any, create, del []client.Object) ([]client.Object, []client.Object) { +func applyExtensionsWithContext(s *extensions.Set, key string, rc extensions.RenderContext, extCtx any, create, del []client.Object) ([]client.Object, []client.Object) { stub := stubExtComponent{key: key, extCtx: extCtx, create: create, delete: del} - return s.Decorate(stub, ctx).Objects() + return s.Decorate(stub, rc).Objects() } From 76eefc3cc7b0d88af8a4d2801bb0661e5e39d70d Mon Sep 17 00:00:00 2001 From: Casey Davenport Date: Thu, 18 Jun 2026 11:48:27 -0700 Subject: [PATCH 31/54] Carry extension-produced data in an opaque RenderContext slot RenderContext no longer names the enterprise node prometheus keypair. It has an opaque Extension slot the controller extension fills and its own modifiers assert back out. The installation extension stashes the keypair there for the node modifier and returns it as one the controller should manage, so the controller no longer references the enterprise keypair in its cert-management and warning wiring. --- .../installation/core_controller.go | 16 +++++--- pkg/enterprise/installation.go | 36 +++++++++++++---- pkg/enterprise/installation_test.go | 10 +++-- pkg/enterprise/node.go | 11 ++--- pkg/extensions/controllerextension.go | 9 +++-- pkg/extensions/controllerextension_test.go | 19 ++++----- pkg/extensions/rendercontext.go | 20 +++++----- pkg/extensions/set.go | 8 ++-- pkg/extensions/variant.go | 7 ++-- pkg/render/node_enterprise_test.go | 40 +++++++++++++------ 10 files changed, 113 insertions(+), 63 deletions(-) diff --git a/pkg/controller/installation/core_controller.go b/pkg/controller/installation/core_controller.go index 47fdea89c1..4dd3d8554c 100644 --- a/pkg/controller/installation/core_controller.go +++ b/pkg/controller/installation/core_controller.go @@ -1224,7 +1224,7 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile r.status.SetDegraded(operatorv1.ResourceValidationError, "Invalid installation configuration", err, reqLogger) return reconcile.Result{}, err } - renderCtx, err := r.opts.Extensions.ExtendContext(cc) + renderCtx, managedKeyPairs, err := r.opts.Extensions.ExtendContext(cc) if err != nil { r.status.SetDegraded(operatorv1.ResourceCreateError, "Error preparing installation extension", err, reqLogger) return reconcile.Result{}, err @@ -1367,7 +1367,6 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile keyPairOptions := []rcertificatemanagement.KeyPairOption{ rcertificatemanagement.NewKeyPairOption(typhaNodeTLS.NodeSecret, true, true), - rcertificatemanagement.NewKeyPairOption(renderCtx.NodePrometheusTLS, true, true), rcertificatemanagement.NewKeyPairOption(typhaNodeTLS.TyphaSecret, true, true), rcertificatemanagement.NewKeyPairOption(typhaNodeTLS.TyphaSecretNonClusterHost, true, true), rcertificatemanagement.NewKeyPairOption(kubeControllerTLS, true, true), @@ -1375,6 +1374,10 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile // render skips nil key pairs. rcertificatemanagement.NewKeyPairOption(wafWebhookTLS, true, true), } + // Manage any key pairs the variant extension created controller-side. + for _, kp := range managedKeyPairs { + keyPairOptions = append(keyPairOptions, rcertificatemanagement.NewKeyPairOption(kp, true, true)) + } components = append(components, rcertificatemanagement.CertificateManagement(&rcertificatemanagement.Config{ @@ -1795,13 +1798,16 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile r.status.ReadyToMonitor() // Check BYO certificate expiry warnings and propagate them to the status manager. - certificatemanagement.CheckKeyPairWarnings(map[string]certificatemanagement.KeyPairInterface{ + keyPairWarnings := map[string]certificatemanagement.KeyPairInterface{ render.TyphaTLSSecretName: typhaNodeTLS.TyphaSecret, render.NodeTLSSecretName: typhaNodeTLS.NodeSecret, render.TyphaTLSSecretName + render.TyphaNonClusterHostSuffix: typhaNodeTLS.TyphaSecretNonClusterHost, - render.NodePrometheusTLSServerSecret: renderCtx.NodePrometheusTLS, kubecontrollers.KubeControllerPrometheusTLSSecret: kubeControllerTLS, - }, r.status) + } + for _, kp := range managedKeyPairs { + keyPairWarnings[kp.GetName()] = kp + } + certificatemanagement.CheckKeyPairWarnings(keyPairWarnings, r.status) // We can clear the degraded state now since as far as we know everything is in order. r.status.ClearDegraded() diff --git a/pkg/enterprise/installation.go b/pkg/enterprise/installation.go index 57ad67476c..4eb89c2282 100644 --- a/pkg/enterprise/installation.go +++ b/pkg/enterprise/installation.go @@ -24,12 +24,27 @@ import ( "github.com/tigera/operator/pkg/render" relasticsearch "github.com/tigera/operator/pkg/render/common/elasticsearch" "github.com/tigera/operator/pkg/render/monitor" + "github.com/tigera/operator/pkg/tls/certificatemanagement" ) // controllerExtension is the Calico Enterprise controller-side hook for the // installation controller. type controllerExtension struct{} +// installationRenderData is the controller-produced data the installation +// extension hands to its modifiers through RenderContext.Extension. The node +// modifier type-asserts it back out. +type installationRenderData struct { + nodePrometheusTLS certificatemanagement.KeyPairInterface +} + +// installationData pulls the installation extension's render data back out of the +// render context, returning the zero value when none is set. +func installationData(rc extensions.RenderContext) installationRenderData { + data, _ := rc.Extension.(installationRenderData) + return data +} + // Validate rejects installation config Calico Enterprise does not support. func (controllerExtension) Validate(cc extensions.ControllerContext) error { // Reject the unsupported zero reporter port. The port value itself is derived @@ -41,9 +56,10 @@ func (controllerExtension) Validate(cc extensions.ControllerContext) error { } // ExtendContext does the controller-side work the modifiers can't: creating and -// fetching the certificates that feed the trusted bundle, returning the render -// context with the produced node prometheus keypair layered on. -func (controllerExtension) ExtendContext(cc extensions.ControllerContext) (extensions.RenderContext, error) { +// fetching the certificates that feed the trusted bundle. It returns the render +// context carrying the produced node prometheus keypair, and that keypair as one +// the controller should manage. +func (controllerExtension) ExtendContext(cc extensions.ControllerContext) (extensions.RenderContext, []certificatemanagement.KeyPairInterface, error) { rc := cc.RenderContext nodePrometheusTLS, err := cc.CertificateManager.GetOrCreateKeyPair( @@ -53,16 +69,16 @@ func (controllerExtension) ExtendContext(cc extensions.ControllerContext) (exten dns.GetServiceDNSNames(render.CalicoNodeMetricsService, common.CalicoNamespace, cc.ClusterDomain), ) if err != nil { - return rc, fmt.Errorf("error creating node prometheus TLS certificate: %w", err) + return rc, nil, fmt.Errorf("error creating node prometheus TLS certificate: %w", err) } if nodePrometheusTLS != nil { cc.TrustedBundle.AddCertificates(nodePrometheusTLS) } - rc.NodePrometheusTLS = nodePrometheusTLS + rc.Extension = installationRenderData{nodePrometheusTLS: nodePrometheusTLS} prometheusClientCert, err := cc.CertificateManager.GetCertificate(cc.Client, monitor.PrometheusClientTLSSecretName, common.OperatorNamespace()) if err != nil { - return rc, fmt.Errorf("unable to fetch prometheus certificate: %w", err) + return rc, nil, fmt.Errorf("unable to fetch prometheus certificate: %w", err) } if prometheusClientCert != nil { cc.TrustedBundle.AddCertificates(prometheusClientCert) @@ -70,11 +86,15 @@ func (controllerExtension) ExtendContext(cc extensions.ControllerContext) (exten esgwCertificate, err := cc.CertificateManager.GetCertificate(cc.Client, relasticsearch.PublicCertSecret, common.OperatorNamespace()) if err != nil { - return rc, fmt.Errorf("failed to retrieve / validate %s: %w", relasticsearch.PublicCertSecret, err) + return rc, nil, fmt.Errorf("failed to retrieve / validate %s: %w", relasticsearch.PublicCertSecret, err) } if esgwCertificate != nil { cc.TrustedBundle.AddCertificates(esgwCertificate) } - return rc, nil + var managed []certificatemanagement.KeyPairInterface + if nodePrometheusTLS != nil { + managed = append(managed, nodePrometheusTLS) + } + return rc, managed, nil } diff --git a/pkg/enterprise/installation_test.go b/pkg/enterprise/installation_test.go index f968306b96..bd790bc161 100644 --- a/pkg/enterprise/installation_test.go +++ b/pkg/enterprise/installation_test.go @@ -29,6 +29,7 @@ import ( "github.com/tigera/operator/pkg/controller/certificatemanager" ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" "github.com/tigera/operator/pkg/extensions" + "github.com/tigera/operator/pkg/render" ) var _ = Describe("installation controller extension", func() { @@ -43,15 +44,16 @@ var _ = Describe("installation controller extension", func() { }) It("creates the node prometheus keypair for the enterprise variant", func() { - rc, err := ext.ExtendContext(newControllerContext(operatorv1.CalicoEnterprise)) + _, managed, err := ext.ExtendContext(newControllerContext(operatorv1.CalicoEnterprise)) Expect(err).NotTo(HaveOccurred()) - Expect(rc.NodePrometheusTLS).NotTo(BeNil()) + Expect(managed).To(HaveLen(1), "expected the node prometheus keypair to be managed") + Expect(managed[0].GetName()).To(Equal(render.NodePrometheusTLSServerSecret)) }) It("is a no-op for the Calico variant", func() { - rc, err := ext.ExtendContext(newControllerContext(operatorv1.Calico)) + _, managed, err := ext.ExtendContext(newControllerContext(operatorv1.Calico)) Expect(err).NotTo(HaveOccurred()) - Expect(rc.NodePrometheusTLS).To(BeNil()) + Expect(managed).To(BeEmpty()) }) }) diff --git a/pkg/enterprise/node.go b/pkg/enterprise/node.go index 43969b12a6..08f2ecf7f8 100644 --- a/pkg/enterprise/node.go +++ b/pkg/enterprise/node.go @@ -153,10 +153,10 @@ func modifyNodeDaemonSet(rc extensions.RenderContext, ds *appsv1.DaemonSet) { // core (calico) the keypair is never created, so the base node render carries // no prometheus mount at all. func mountNodePrometheusTLS(rc extensions.RenderContext, ds *appsv1.DaemonSet) { - if rc.NodePrometheusTLS == nil { + tls := installationData(rc).nodePrometheusTLS + if tls == nil { return } - tls := rc.NodePrometheusTLS spec := &ds.Spec.Template.Spec spec.Volumes = append(spec.Volumes, tls.Volume()) @@ -181,6 +181,7 @@ func mountNodePrometheusTLS(rc extensions.RenderContext, ds *appsv1.DaemonSet) { // nodeEnterpriseEnv is the Enterprise felix configuration added to the // calico/node container. func nodeEnterpriseEnv(rc extensions.RenderContext) []corev1.EnvVar { + tls := installationData(rc).nodePrometheusTLS env := []corev1.EnvVar{ {Name: "FELIX_PROMETHEUSREPORTERENABLED", Value: "true"}, {Name: "FELIX_PROMETHEUSREPORTERPORT", Value: fmt.Sprintf("%d", nodeReporterPort(rc.FelixConfiguration))}, @@ -198,10 +199,10 @@ func nodeEnterpriseEnv(rc extensions.RenderContext) []corev1.EnvVar { env = append(env, *mode) } - if rc.NodePrometheusTLS != nil && rc.TrustedBundle != nil { + if tls != nil && rc.TrustedBundle != nil { env = append(env, - corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERCERTFILE", Value: rc.NodePrometheusTLS.VolumeMountCertificateFilePath()}, - corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERKEYFILE", Value: rc.NodePrometheusTLS.VolumeMountKeyFilePath()}, + corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERCERTFILE", Value: tls.VolumeMountCertificateFilePath()}, + corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERKEYFILE", Value: tls.VolumeMountKeyFilePath()}, corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERCAFILE", Value: rc.TrustedBundle.MountPath()}, ) } diff --git a/pkg/extensions/controllerextension.go b/pkg/extensions/controllerextension.go index fada677689..05a176bdd9 100644 --- a/pkg/extensions/controllerextension.go +++ b/pkg/extensions/controllerextension.go @@ -20,6 +20,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "github.com/tigera/operator/pkg/controller/certificatemanager" + "github.com/tigera/operator/pkg/tls/certificatemanagement" ) // ControllerExtension extends a controller's reconcile: it validates the @@ -32,9 +33,11 @@ type ControllerExtension interface { Validate(cc ControllerContext) error // ExtendContext does the controller-side reconcile work the render phase - // cannot, returning the RenderContext the render phase consumes, or an error - // that aborts the reconcile. - ExtendContext(cc ControllerContext) (RenderContext, error) + // cannot, returning the RenderContext the render phase consumes plus any + // keypairs the extension created that the controller should manage (add to + // certificate management and BYO-expiry warnings), or an error that aborts the + // reconcile. + ExtendContext(cc ControllerContext) (RenderContext, []certificatemanagement.KeyPairInterface, error) } // ControllerContext is the controller-phase context, the corollary to the diff --git a/pkg/extensions/controllerextension_test.go b/pkg/extensions/controllerextension_test.go index a0fc51275b..c38d4e5e02 100644 --- a/pkg/extensions/controllerextension_test.go +++ b/pkg/extensions/controllerextension_test.go @@ -22,6 +22,7 @@ import ( operatorv1 "github.com/tigera/operator/api/v1" "github.com/tigera/operator/pkg/extensions" + "github.com/tigera/operator/pkg/tls/certificatemanagement" ) var _ = Describe("controller extension", func() { @@ -32,25 +33,25 @@ var _ = Describe("controller extension", func() { It("returns the base render context when the variant has no extension", func() { install := &operatorv1.InstallationSpec{Variant: operatorv1.Calico} - rc, err := s.ExtendContext(extensions.ControllerContext{ + rc, _, err := s.ExtendContext(extensions.ControllerContext{ RenderContext: extensions.RenderContext{Installation: install, ClusterDomain: "cluster.local"}, }) Expect(err).NotTo(HaveOccurred()) Expect(rc.Installation).To(BeIdenticalTo(install)) Expect(rc.ClusterDomain).To(Equal("cluster.local")) - Expect(rc.NodePrometheusTLS).To(BeNil()) + Expect(rc.Extension).To(BeNil()) }) It("runs the extension registered for the installation variant", func() { s.Variant(operatorv1.CalicoEnterprise).Controller(fakeController{}) - rc, err := s.ExtendContext(enterpriseContext()) + rc, _, err := s.ExtendContext(enterpriseContext()) Expect(err).NotTo(HaveOccurred()) Expect(rc.ClusterDomain).To(Equal("from-fake")) }) It("ignores an extension registered for a different variant", func() { s.Variant(operatorv1.CalicoEnterprise).Controller(fakeController{}) - rc, err := s.ExtendContext(extensions.ControllerContext{ + rc, _, err := s.ExtendContext(extensions.ControllerContext{ RenderContext: extensions.RenderContext{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.Calico}, ClusterDomain: "real"}, }) Expect(err).NotTo(HaveOccurred()) @@ -59,7 +60,7 @@ var _ = Describe("controller extension", func() { It("surfaces the extension error", func() { s.Variant(operatorv1.CalicoEnterprise).Controller(fakeController{err: errors.New("boom")}) - _, err := s.ExtendContext(enterpriseContext()) + _, _, err := s.ExtendContext(enterpriseContext()) Expect(err).To(MatchError("boom")) }) @@ -72,7 +73,7 @@ var _ = Describe("controller extension", func() { var nilSet *extensions.Set cc := enterpriseContext() cc.ClusterDomain = "real" - rc, err := nilSet.ExtendContext(cc) + rc, _, err := nilSet.ExtendContext(cc) Expect(err).NotTo(HaveOccurred()) Expect(rc.ClusterDomain).To(Equal("real")) Expect(nilSet.Validate(cc)).NotTo(HaveOccurred()) @@ -96,9 +97,9 @@ func (f fakeController) Validate(_ extensions.ControllerContext) error { return f.validateErr } -func (f fakeController) ExtendContext(_ extensions.ControllerContext) (extensions.RenderContext, error) { +func (f fakeController) ExtendContext(_ extensions.ControllerContext) (extensions.RenderContext, []certificatemanagement.KeyPairInterface, error) { if f.err != nil { - return extensions.RenderContext{}, f.err + return extensions.RenderContext{}, nil, f.err } - return extensions.RenderContext{ClusterDomain: "from-fake"}, nil + return extensions.RenderContext{ClusterDomain: "from-fake"}, nil, nil } diff --git a/pkg/extensions/rendercontext.go b/pkg/extensions/rendercontext.go index 95daec43aa..3e092bf18c 100644 --- a/pkg/extensions/rendercontext.go +++ b/pkg/extensions/rendercontext.go @@ -22,11 +22,10 @@ import ( // RenderContext carries reconcile-derived inputs from controllers into render // modifiers. Core operator code never reads these fields - only registered -// modifiers do. Two kinds of value live here: -// - raw cluster state gathered generically (Installation, FelixConfiguration, -// ClusterDomain) that modifiers derive their own values from, and -// - controller-produced artifacts (TrustedBundle, NodePrometheusTLS) that can -// only be created controller-side because they have cluster side effects. +// modifiers do. It carries raw cluster state gathered generically (Installation, +// FelixConfiguration, ClusterDomain) that modifiers derive their own values from, +// the shared TrustedBundle, and an opaque Extension slot for controller-produced +// data specific to one extension. // // Per-component config a modifier needs but can't derive from these fields is // not carried here; it flows to the modifier as a typed argument (see @@ -39,9 +38,10 @@ type RenderContext struct { // TrustedBundle is the shared CA bundle for the calico-system namespace. TrustedBundle certificatemanagement.TrustedBundle - // NodePrometheusTLS is created by the enterprise controller extension (it has - // cluster side effects, so it can't be built in a modifier). The node modifier - // is its only consumer: it mounts the keypair onto the daemonset and sets the - // FELIX_PROMETHEUSREPORTER* certificate env vars. - NodePrometheusTLS certificatemanagement.KeyPairInterface + // Extension is opaque, extension-owned data that the controller extension + // produced for its own modifiers - typically an artifact that can only be + // created controller-side because it has cluster side effects (e.g. a keypair). + // The extension that set it type-asserts it back out in its modifiers; core + // code never reads it. Nil when no extension is active. + Extension any } diff --git a/pkg/extensions/set.go b/pkg/extensions/set.go index 42a76267ad..ac97f821d5 100644 --- a/pkg/extensions/set.go +++ b/pkg/extensions/set.go @@ -19,6 +19,7 @@ import ( "github.com/tigera/operator/pkg/components" "github.com/tigera/operator/pkg/imageoverride" "github.com/tigera/operator/pkg/render" + "github.com/tigera/operator/pkg/tls/certificatemanagement" ) // Set is all the variant extensions the operator runs with, indexed by product @@ -88,11 +89,12 @@ func (s *Set) Validate(cc ControllerContext) error { } // ExtendContext runs the controller extension for the installation's variant and -// returns the resulting RenderContext, or the base render context when no +// returns the resulting RenderContext plus any keypairs the extension wants the +// controller to manage, or the base render context and no keypairs when no // extension is registered. Nil-safe. -func (s *Set) ExtendContext(cc ControllerContext) (RenderContext, error) { +func (s *Set) ExtendContext(cc ControllerContext) (RenderContext, []certificatemanagement.KeyPairInterface, error) { if cc.Installation == nil { - return cc.RenderContext, nil + return cc.RenderContext, nil, nil } return s.variant(cc.Installation.Variant).extendContext(cc) } diff --git a/pkg/extensions/variant.go b/pkg/extensions/variant.go index 36d4d8b96c..08c920bbc5 100644 --- a/pkg/extensions/variant.go +++ b/pkg/extensions/variant.go @@ -22,6 +22,7 @@ import ( "github.com/tigera/operator/pkg/components" "github.com/tigera/operator/pkg/imageoverride" "github.com/tigera/operator/pkg/render" + "github.com/tigera/operator/pkg/tls/certificatemanagement" ) // Variant bundles everything that extends the core operator for one product @@ -117,10 +118,10 @@ func (v *Variant) validate(cc ControllerContext) error { } // extendContext runs the controller extension, or returns the base render -// context when the variant has none. Nil-safe. -func (v *Variant) extendContext(cc ControllerContext) (RenderContext, error) { +// context and no managed keypairs when the variant has none. Nil-safe. +func (v *Variant) extendContext(cc ControllerContext) (RenderContext, []certificatemanagement.KeyPairInterface, error) { if v == nil || v.controller == nil { - return cc.RenderContext, nil + return cc.RenderContext, nil, nil } return v.controller.ExtendContext(cc) } diff --git a/pkg/render/node_enterprise_test.go b/pkg/render/node_enterprise_test.go index c9c32b69fd..9ea5db8d2e 100644 --- a/pkg/render/node_enterprise_test.go +++ b/pkg/render/node_enterprise_test.go @@ -15,6 +15,8 @@ package render_test import ( + "context" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -34,6 +36,7 @@ import ( "github.com/tigera/operator/pkg/extensions" "github.com/tigera/operator/pkg/render" rmeta "github.com/tigera/operator/pkg/render/common/meta" + "github.com/tigera/operator/pkg/tls/certificatemanagement" ) // These tests run the real node/typha render output through the registered @@ -43,11 +46,12 @@ import ( // matching because render renamed an object or container. var _ = Describe("node enterprise modifier integration", func() { var ( - cli client.Client - certManager certificatemanager.CertificateManager - typhaNodeTLS *render.TyphaNodeTLS - instance *operatorv1.InstallationSpec - renderCtx extensions.RenderContext + cli client.Client + certManager certificatemanager.CertificateManager + typhaNodeTLS *render.TyphaNodeTLS + instance *operatorv1.InstallationSpec + renderCtx extensions.RenderContext + nodePrometheusTLS certificatemanagement.KeyPairInterface ) nodeContainer := func(ds *appsv1.DaemonSet) *corev1.Container { @@ -69,7 +73,7 @@ var _ = Describe("node enterprise modifier integration", func() { Expect(err).NotTo(HaveOccurred()) typhaNodeTLS = getTyphaNodeTLS(cli, certManager) - nodePrometheusTLS, err := certManager.GetOrCreateKeyPair(cli, render.NodePrometheusTLSServerSecret, common.OperatorNamespace(), []string{"calico-node-metrics"}) + nodePrometheusTLS, err = certManager.GetOrCreateKeyPair(cli, render.NodePrometheusTLSServerSecret, common.OperatorNamespace(), []string{"calico-node-metrics"}) Expect(err).NotTo(HaveOccurred()) typhaNodeTLS.TrustedBundle.AddCertificates(nodePrometheusTLS) @@ -89,11 +93,21 @@ var _ = Describe("node enterprise modifier integration", func() { }, } - renderCtx = extensions.RenderContext{ - Installation: instance, - TrustedBundle: typhaNodeTLS.TrustedBundle, - NodePrometheusTLS: nodePrometheusTLS, + // Build the render context the way the controller does: run the enterprise + // controller extension, which stashes the node prometheus keypair in the + // context for the node modifier to read. + cc := extensions.ControllerContext{ + RenderContext: extensions.RenderContext{ + Installation: instance, + TrustedBundle: typhaNodeTLS.TrustedBundle, + ClusterDomain: dns.DefaultClusterDomain, + }, + Ctx: context.Background(), + Client: cli, + CertificateManager: certManager, } + renderCtx, _, err = ext.ExtendContext(cc) + Expect(err).NotTo(HaveOccurred()) }) // renderNodeObjects renders the real node component and applies the registered @@ -148,9 +162,9 @@ var _ = Describe("node enterprise modifier integration", func() { // The reporter cert env is wired from the NodePrometheusTLS keypair the // builder creates, and the modifier mounts that keypair onto the daemonset. Expect(c.Env).To(ContainElement(HaveField("Name", "FELIX_PROMETHEUSREPORTERCERTFILE"))) - Expect(ds.Spec.Template.Spec.Volumes).To(ContainElement(renderCtx.NodePrometheusTLS.Volume())) - Expect(c.VolumeMounts).To(ContainElement(renderCtx.NodePrometheusTLS.VolumeMount(rmeta.OSTypeLinux))) - Expect(ds.Spec.Template.Annotations).To(HaveKey(renderCtx.NodePrometheusTLS.HashAnnotationKey())) + Expect(ds.Spec.Template.Spec.Volumes).To(ContainElement(nodePrometheusTLS.Volume())) + Expect(c.VolumeMounts).To(ContainElement(nodePrometheusTLS.VolumeMount(rmeta.OSTypeLinux))) + Expect(ds.Spec.Template.Annotations).To(HaveKey(nodePrometheusTLS.HashAnnotationKey())) // BGP is enabled, so the bird readiness check is present and the modifier // adds the BGP metrics check. From 1d831fd9ac000768b0b553c8db8bf9dfb81501e3 Mon Sep 17 00:00:00 2001 From: Casey Davenport Date: Thu, 18 Jun 2026 12:42:00 -0700 Subject: [PATCH 32/54] Decouple the windows controller from enterprise via a per-controller hook Controller extensions are now registered per controller (a ControllerName plus constants) and selected by ControllerContext.Controller, so each controller runs its own hook. The windows controller runs its hook to fetch the node prometheus keypair into the render context's Extension slot; its IsEnterprise branch is gone and WindowsConfiguration no longer carries the enterprise reporter port or keypair (the modifier derives the port from FelixConfiguration and reads the keypair from the slot). Rename the installation hook to coreControllerExtension. --- .../installation/core_controller.go | 7 +- .../installation/windows_controller.go | 72 ++++++++--------- pkg/enterprise/installation.go | 16 ++-- pkg/enterprise/installation_test.go | 1 + pkg/enterprise/node.go | 10 +++ pkg/enterprise/register.go | 3 +- pkg/enterprise/windows.go | 81 ++++++++++++++----- pkg/enterprise/windows_test.go | 39 ++++++--- pkg/extensions/controllerextension.go | 22 ++++- pkg/extensions/controllerextension_test.go | 11 ++- pkg/extensions/set.go | 15 ++-- pkg/extensions/variant.go | 32 ++++---- pkg/render/node_enterprise_test.go | 1 + pkg/render/render_test.go | 32 ++++---- pkg/render/windows.go | 34 ++------ pkg/render/windows_test.go | 10 +-- 16 files changed, 211 insertions(+), 175 deletions(-) diff --git a/pkg/controller/installation/core_controller.go b/pkg/controller/installation/core_controller.go index 4dd3d8554c..29fafcccf6 100644 --- a/pkg/controller/installation/core_controller.go +++ b/pkg/controller/installation/core_controller.go @@ -91,12 +91,6 @@ import ( const ( techPreviewFeatureSeccompApparmor = "tech-preview.operator.tigera.io/node-apparmor-profile" - - // defaultNodeReporterPort is the default port calico/node uses to report Calico - // Enterprise internal metrics. The Linux node path derives this in the - // enterprise node modifier; this copy serves the Windows controller, which - // still carries its enterprise logic inline. - defaultNodeReporterPort = 9081 ) const InstallationName string = "calico" @@ -1216,6 +1210,7 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile ClusterDomain: r.opts.ClusterDomain, TrustedBundle: typhaNodeTLS.TrustedBundle, }, + Controller: extensions.InstallationController, Ctx: ctx, Client: r.client, CertificateManager: certificateManager, diff --git a/pkg/controller/installation/windows_controller.go b/pkg/controller/installation/windows_controller.go index ad758dbbce..0e8152d85b 100644 --- a/pkg/controller/installation/windows_controller.go +++ b/pkg/controller/installation/windows_controller.go @@ -16,7 +16,6 @@ package installation import ( "context" - "errors" "fmt" "reflect" @@ -50,11 +49,9 @@ import ( "github.com/tigera/operator/pkg/controller/utils" "github.com/tigera/operator/pkg/controller/utils/imageset" "github.com/tigera/operator/pkg/ctrlruntime" - "github.com/tigera/operator/pkg/dns" "github.com/tigera/operator/pkg/extensions" "github.com/tigera/operator/pkg/render" "github.com/tigera/operator/pkg/render/monitor" - "github.com/tigera/operator/pkg/tls/certificatemanagement" ) var logw = logf.Log.WithName("controller_windows") @@ -325,33 +322,6 @@ func (r *ReconcileWindows) Reconcile(ctx context.Context, request reconcile.Requ } } - // nodeReporterMetricsPort is a port used in Enterprise to host internal metrics. - // Operator is responsible for creating a service which maps to that port. - // Here, we'll check the default felixconfiguration to see if the user is specifying - // a non-default port, and use that value if they are. - nodeReporterMetricsPort := defaultNodeReporterPort - var nodePrometheusTLS certificatemanagement.KeyPairInterface - if instance.Spec.Variant.IsEnterprise() { - - // Determine the port to use for nodeReporter metrics. - if felixConfiguration.Spec.PrometheusReporterPort != nil { - nodeReporterMetricsPort = *felixConfiguration.Spec.PrometheusReporterPort - } - - if nodeReporterMetricsPort == 0 { - err := errors.New("felixConfiguration prometheusReporterPort=0 not supported") - r.status.SetDegraded(operatorv1.InvalidConfigurationError, "invalid metrics port", err, reqLogger) - return reconcile.Result{}, err - } - - // The key pair is created by the core controller, so if it isn't set, requeue to wait until it is - nodePrometheusTLS, err = certificateManager.GetKeyPair(r.client, render.NodePrometheusTLSServerSecret, common.OperatorNamespace(), dns.GetServiceDNSNames(render.WindowsNodeMetricsService, common.CalicoNamespace, r.opts.ClusterDomain)) - if err != nil { - r.status.SetDegraded(operatorv1.ResourceCreateError, "Error getting TLS certificate", err, reqLogger) - return reconcile.Result{}, err - } - } - var component render.Component kubeDNSServiceName := utils.GetDNSServiceName(r.opts.DetectedProvider) @@ -374,16 +344,38 @@ func (r *ReconcileWindows) Reconcile(ctx context.Context, request reconcile.Requ return reconcile.Result{}, err } + // Run the variant's windows controller extension to build the render context + // (creating no enterprise artifacts in core). + cc := extensions.ControllerContext{ + RenderContext: extensions.RenderContext{ + Installation: &instance.Spec, + FelixConfiguration: felixConfiguration, + ClusterDomain: r.opts.ClusterDomain, + TrustedBundle: typhaNodeTLS.TrustedBundle, + }, + Controller: extensions.WindowsController, + Ctx: ctx, + Client: r.client, + CertificateManager: certificateManager, + } + if err := r.opts.Extensions.Validate(cc); err != nil { + r.status.SetDegraded(operatorv1.ResourceValidationError, "Invalid installation configuration", err, reqLogger) + return reconcile.Result{}, err + } + renderCtx, _, err := r.opts.Extensions.ExtendContext(cc) + if err != nil { + r.status.SetDegraded(operatorv1.ResourceCreateError, "Error preparing windows extension", err, reqLogger) + return reconcile.Result{}, err + } + windowsCfg := render.WindowsConfiguration{ - K8sServiceEp: k8sapi.Endpoint, - K8sDNSServers: kubeDNSIPs, - Installation: &instance.Spec, - ClusterDomain: r.opts.ClusterDomain, - TLS: typhaNodeTLS, - PrometheusServerTLS: nodePrometheusTLS, - NodeReporterMetricsPort: nodeReporterMetricsPort, - VXLANVNI: *felixConfiguration.Spec.VXLANVNI, - ImageOverrides: r.opts.Extensions.Images(), + K8sServiceEp: k8sapi.Endpoint, + K8sDNSServers: kubeDNSIPs, + Installation: &instance.Spec, + ClusterDomain: r.opts.ClusterDomain, + TLS: typhaNodeTLS, + VXLANVNI: *felixConfiguration.Spec.VXLANVNI, + ImageOverrides: r.opts.Extensions.Images(), } component = render.Windows(&windowsCfg) @@ -409,7 +401,7 @@ func (r *ReconcileWindows) Reconcile(ctx context.Context, request reconcile.Requ r.client, r.scheme, instance, - utils.WithRenderContext(extensions.RenderContext{Installation: &instance.Spec}), + utils.WithRenderContext(renderCtx), utils.WithExtensions(r.opts.Extensions), ) if err := handler.CreateOrUpdateOrDelete(ctx, component, nil); err != nil { diff --git a/pkg/enterprise/installation.go b/pkg/enterprise/installation.go index 4eb89c2282..6b3658f62a 100644 --- a/pkg/enterprise/installation.go +++ b/pkg/enterprise/installation.go @@ -15,7 +15,6 @@ package enterprise import ( - "errors" "fmt" "github.com/tigera/operator/pkg/common" @@ -27,9 +26,9 @@ import ( "github.com/tigera/operator/pkg/tls/certificatemanagement" ) -// controllerExtension is the Calico Enterprise controller-side hook for the +// coreControllerExtension is the Calico Enterprise controller-side hook for the // installation controller. -type controllerExtension struct{} +type coreControllerExtension struct{} // installationRenderData is the controller-produced data the installation // extension hands to its modifiers through RenderContext.Extension. The node @@ -46,20 +45,15 @@ func installationData(rc extensions.RenderContext) installationRenderData { } // Validate rejects installation config Calico Enterprise does not support. -func (controllerExtension) Validate(cc extensions.ControllerContext) error { - // Reject the unsupported zero reporter port. The port value itself is derived - // in the node modifier; only this validation lives here. - if cc.FelixConfiguration.Spec.PrometheusReporterPort != nil && *cc.FelixConfiguration.Spec.PrometheusReporterPort == 0 { - return errors.New("felixConfiguration prometheusReporterPort=0 not supported") - } - return nil +func (coreControllerExtension) Validate(cc extensions.ControllerContext) error { + return validateReporterPort(cc.FelixConfiguration) } // ExtendContext does the controller-side work the modifiers can't: creating and // fetching the certificates that feed the trusted bundle. It returns the render // context carrying the produced node prometheus keypair, and that keypair as one // the controller should manage. -func (controllerExtension) ExtendContext(cc extensions.ControllerContext) (extensions.RenderContext, []certificatemanagement.KeyPairInterface, error) { +func (coreControllerExtension) ExtendContext(cc extensions.ControllerContext) (extensions.RenderContext, []certificatemanagement.KeyPairInterface, error) { rc := cc.RenderContext nodePrometheusTLS, err := cc.CertificateManager.GetOrCreateKeyPair( diff --git a/pkg/enterprise/installation_test.go b/pkg/enterprise/installation_test.go index bd790bc161..2c0f917984 100644 --- a/pkg/enterprise/installation_test.go +++ b/pkg/enterprise/installation_test.go @@ -73,6 +73,7 @@ func newControllerContext(variant operatorv1.ProductVariant) extensions.Controll TrustedBundle: trustedBundle, ClusterDomain: "cluster.local", }, + Controller: extensions.InstallationController, Ctx: context.Background(), Client: c, CertificateManager: certManager, diff --git a/pkg/enterprise/node.go b/pkg/enterprise/node.go index 08f2ecf7f8..ad4469a8b7 100644 --- a/pkg/enterprise/node.go +++ b/pkg/enterprise/node.go @@ -15,6 +15,7 @@ package enterprise import ( + "errors" "fmt" "slices" @@ -263,6 +264,15 @@ func nodeMetricsService(rc extensions.RenderContext) *corev1.Service { } } +// validateReporterPort rejects the unsupported zero prometheus reporter port. +// The node and windows controller extensions share it. +func validateReporterPort(fc *v3.FelixConfiguration) error { + if fc != nil && fc.Spec.PrometheusReporterPort != nil && *fc.Spec.PrometheusReporterPort == 0 { + return errors.New("felixConfiguration prometheusReporterPort=0 not supported") + } + return nil +} + // nodeReporterPort returns the reporter metrics port from the FelixConfiguration, // falling back to the default. The node-metrics Service and the // FELIX_PROMETHEUSREPORTERPORT env var both derive from here so they can't drift. diff --git a/pkg/enterprise/register.go b/pkg/enterprise/register.go index e0d0532b31..4d533a0e33 100644 --- a/pkg/enterprise/register.go +++ b/pkg/enterprise/register.go @@ -28,7 +28,8 @@ func New() *extensions.Set { s := extensions.NewSet() ent := s.Variant(operatorv1.CalicoEnterprise) - ent.Controller(controllerExtension{}) + ent.Controller(extensions.InstallationController, coreControllerExtension{}) + ent.Controller(extensions.WindowsController, windowsControllerExtension{}) registerTypha(ent) registerNode(ent) registerWindows(ent) diff --git a/pkg/enterprise/windows.go b/pkg/enterprise/windows.go index d00e7da88d..bffb7ec1d9 100644 --- a/pkg/enterprise/windows.go +++ b/pkg/enterprise/windows.go @@ -26,9 +26,11 @@ import ( operatorv1 "github.com/tigera/operator/api/v1" "github.com/tigera/operator/pkg/common" "github.com/tigera/operator/pkg/components" + "github.com/tigera/operator/pkg/dns" "github.com/tigera/operator/pkg/extensions" "github.com/tigera/operator/pkg/render" rmeta "github.com/tigera/operator/pkg/render/common/meta" + "github.com/tigera/operator/pkg/tls/certificatemanagement" ) // windowsNodeContainers are the calico-node-windows containers that share the @@ -38,22 +40,61 @@ var windowsNodeContainers = map[string]bool{"felix": true, "node": true, "confd" func registerWindows(v *extensions.Variant) { v.Image(render.ComponentNameWindowsNodeImg, components.ComponentTigeraNodeWindows) v.Image(render.ComponentNameWindowsCNIImg, components.ComponentTigeraCNIWindows) - extensions.RegisterModifier(v, render.ComponentNameWindows, modifyWindows) + v.Modify(render.ComponentNameWindows, modifyWindows) +} + +// windowsControllerExtension is the Calico Enterprise controller-side hook for the +// windows controller. +type windowsControllerExtension struct{} + +// windowsRenderData is the controller-produced data the windows extension hands to +// its modifier through RenderContext.Extension. +type windowsRenderData struct { + prometheusServerTLS certificatemanagement.KeyPairInterface +} + +// windowsData pulls the windows extension's render data back out of the render +// context, returning the zero value when none is set. +func windowsData(rc extensions.RenderContext) windowsRenderData { + data, _ := rc.Extension.(windowsRenderData) + return data +} + +// Validate rejects windows installation config Calico Enterprise does not support. +func (windowsControllerExtension) Validate(cc extensions.ControllerContext) error { + return validateReporterPort(cc.FelixConfiguration) +} + +// ExtendContext fetches the node prometheus keypair the installation controller +// created and stashes it in the render context for the windows modifier. +func (windowsControllerExtension) ExtendContext(cc extensions.ControllerContext) (extensions.RenderContext, []certificatemanagement.KeyPairInterface, error) { + rc := cc.RenderContext + tls, err := cc.CertificateManager.GetKeyPair( + cc.Client, + render.NodePrometheusTLSServerSecret, + common.OperatorNamespace(), + dns.GetServiceDNSNames(render.WindowsNodeMetricsService, common.CalicoNamespace, cc.ClusterDomain), + ) + if err != nil { + return rc, nil, fmt.Errorf("error getting node prometheus TLS certificate: %w", err) + } + rc.Extension = windowsRenderData{prometheusServerTLS: tls} + return rc, nil, nil } // modifyWindows layers Calico Enterprise behavior onto the rendered // calico-node-windows objects: the node-metrics Service and the Enterprise // daemonset configuration (flow/DNS log env, prometheus reporter, trusted DNS // servers, the calico log volume, and the prometheus reporter keypair mount). -func modifyWindows(rc extensions.RenderContext, wc render.WindowsExtensionContext, objs, del []client.Object) ([]client.Object, []client.Object) { +func modifyWindows(rc extensions.RenderContext, objs, del []client.Object) ([]client.Object, []client.Object) { if ds, ok := extensions.FindObject[*appsv1.DaemonSet](objs, common.WindowsDaemonSetName); ok { - modifyWindowsDaemonSet(rc, wc, ds) + modifyWindowsDaemonSet(rc, ds) } - return append(objs, windowsNodeMetricsService(wc)), del + return append(objs, windowsNodeMetricsService(rc)), del } -func modifyWindowsDaemonSet(rc extensions.RenderContext, wc render.WindowsExtensionContext, ds *appsv1.DaemonSet) { +func modifyWindowsDaemonSet(rc extensions.RenderContext, ds *appsv1.DaemonSet) { dirOrCreate := corev1.HostPathDirectoryOrCreate spec := &ds.Spec.Template.Spec @@ -68,7 +109,7 @@ func modifyWindowsDaemonSet(rc extensions.RenderContext, wc render.WindowsExtens continue } - c.Env = append(c.Env, windowsEnterpriseEnv(rc, wc)...) + c.Env = append(c.Env, windowsEnterpriseEnv(rc)...) // Enterprise mounts the calico log directory in place of the OSS CNI log // directory, so drop the OSS mount before adding the enterprise one. @@ -76,15 +117,16 @@ func modifyWindowsDaemonSet(rc extensions.RenderContext, wc render.WindowsExtens c.VolumeMounts = append(c.VolumeMounts, corev1.VolumeMount{MountPath: "/var/log/calico", Name: "var-log-calico"}) } - mountWindowsPrometheusTLS(wc, ds) + mountWindowsPrometheusTLS(rc, ds) } // windowsEnterpriseEnv is the Enterprise felix configuration added to the // calico-node-windows containers. -func windowsEnterpriseEnv(rc extensions.RenderContext, wc render.WindowsExtensionContext) []corev1.EnvVar { +func windowsEnterpriseEnv(rc extensions.RenderContext) []corev1.EnvVar { + tls := windowsData(rc).prometheusServerTLS env := []corev1.EnvVar{ {Name: "FELIX_PROMETHEUSREPORTERENABLED", Value: "true"}, - {Name: "FELIX_PROMETHEUSREPORTERPORT", Value: fmt.Sprintf("%d", wc.NodeReporterMetricsPort)}, + {Name: "FELIX_PROMETHEUSREPORTERPORT", Value: fmt.Sprintf("%d", nodeReporterPort(rc.FelixConfiguration))}, {Name: "FELIX_FLOWLOGSFILEENABLED", Value: "true"}, {Name: "FELIX_FLOWLOGSFILEINCLUDELABELS", Value: "true"}, {Name: "FELIX_FLOWLOGSFILEINCLUDEPOLICIES", Value: "true"}, @@ -95,11 +137,11 @@ func windowsEnterpriseEnv(rc extensions.RenderContext, wc render.WindowsExtensio {Name: "FELIX_DNSLOGSFILEPERNODELIMIT", Value: "1000"}, } - if wc.PrometheusServerTLS != nil && wc.TrustedBundle != nil { + if tls != nil && rc.TrustedBundle != nil { env = append(env, - corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERCERTFILE", Value: wc.PrometheusServerTLS.VolumeMountCertificateFilePath()}, - corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERKEYFILE", Value: wc.PrometheusServerTLS.VolumeMountKeyFilePath()}, - corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERCAFILE", Value: wc.TrustedBundle.MountPath()}, + corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERCERTFILE", Value: tls.VolumeMountCertificateFilePath()}, + corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERKEYFILE", Value: tls.VolumeMountKeyFilePath()}, + corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERCAFILE", Value: rc.TrustedBundle.MountPath()}, ) } @@ -117,11 +159,11 @@ func windowsEnterpriseEnv(rc extensions.RenderContext, wc render.WindowsExtensio // mountWindowsPrometheusTLS mounts the node prometheus reporter keypair onto the // windows daemonset: the volume, the volume mount on each node container, and // the pod hash annotation that rolls the pods on cert rotation. -func mountWindowsPrometheusTLS(wc render.WindowsExtensionContext, ds *appsv1.DaemonSet) { - if wc.PrometheusServerTLS == nil { +func mountWindowsPrometheusTLS(rc extensions.RenderContext, ds *appsv1.DaemonSet) { + tls := windowsData(rc).prometheusServerTLS + if tls == nil { return } - tls := wc.PrometheusServerTLS spec := &ds.Spec.Template.Spec spec.Volumes = append(spec.Volumes, tls.Volume()) @@ -141,7 +183,8 @@ func mountWindowsPrometheusTLS(wc render.WindowsExtensionContext, ds *appsv1.Dae // windowsNodeMetricsService builds the enterprise-only calico-node-metrics-windows // Service. -func windowsNodeMetricsService(wc render.WindowsExtensionContext) *corev1.Service { +func windowsNodeMetricsService(rc extensions.RenderContext) *corev1.Service { + reporterPort := nodeReporterPort(rc.FelixConfiguration) return &corev1.Service{ TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}, ObjectMeta: metav1.ObjectMeta{ @@ -155,8 +198,8 @@ func windowsNodeMetricsService(wc render.WindowsExtensionContext) *corev1.Servic Ports: []corev1.ServicePort{ { Name: "calico-metrics-port", - Port: int32(wc.NodeReporterMetricsPort), - TargetPort: intstr.FromInt(wc.NodeReporterMetricsPort), + Port: int32(reporterPort), + TargetPort: intstr.FromInt(reporterPort), Protocol: corev1.ProtocolTCP, }, { diff --git a/pkg/enterprise/windows_test.go b/pkg/enterprise/windows_test.go index d0c11c6642..d4c7967b9b 100644 --- a/pkg/enterprise/windows_test.go +++ b/pkg/enterprise/windows_test.go @@ -15,6 +15,8 @@ package enterprise_test import ( + "context" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -30,9 +32,9 @@ import ( "github.com/tigera/operator/pkg/components" "github.com/tigera/operator/pkg/controller/certificatemanager" ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" + "github.com/tigera/operator/pkg/dns" "github.com/tigera/operator/pkg/extensions" "github.com/tigera/operator/pkg/render" - "github.com/tigera/operator/pkg/tls/certificatemanagement" ) var _ = Describe("windows enterprise image override", func() { @@ -91,16 +93,8 @@ var _ = Describe("windows enterprise modifier", func() { } } - wcFor := func(tls certificatemanagement.KeyPairInterface, bundle certificatemanagement.TrustedBundleRO) render.WindowsExtensionContext { - return render.WindowsExtensionContext{ - NodeReporterMetricsPort: 9081, - PrometheusServerTLS: tls, - TrustedBundle: bundle, - } - } - It("appends the node-metrics service", func() { - out, _ := applyExtensionsWithContext(ext, render.ComponentNameWindows, ctxFor(operatorv1.ProviderNone), wcFor(nil, nil), newObjs(), nil) + out, _ := applyExtensions(ext, render.ComponentNameWindows, ctxFor(operatorv1.ProviderNone), newObjs(), nil) svc, ok := extensions.FindObject[*corev1.Service](out, render.WindowsNodeMetricsService) Expect(ok).To(BeTrue()) Expect(svc.Namespace).To(Equal(common.CalicoNamespace)) @@ -108,7 +102,7 @@ var _ = Describe("windows enterprise modifier", func() { }) It("swaps the cni log mount for the calico log volume and adds enterprise env", func() { - out, _ := applyExtensionsWithContext(ext, render.ComponentNameWindows, ctxFor(operatorv1.ProviderNone), wcFor(nil, nil), newObjs(), nil) + out, _ := applyExtensions(ext, render.ComponentNameWindows, ctxFor(operatorv1.ProviderNone), newObjs(), nil) d := ds(out) Expect(d.Spec.Template.Spec.Volumes).To(ContainElement(HaveField("Name", "var-log-calico"))) @@ -125,7 +119,7 @@ var _ = Describe("windows enterprise modifier", func() { }) It("sets the trusted DNS server on openshift", func() { - out, _ := applyExtensionsWithContext(ext, render.ComponentNameWindows, ctxFor(operatorv1.ProviderOpenShift), wcFor(nil, nil), newObjs(), nil) + out, _ := applyExtensions(ext, render.ComponentNameWindows, ctxFor(operatorv1.ProviderOpenShift), newObjs(), nil) Expect(container(ds(out), "node").Env).To(ContainElement(corev1.EnvVar{Name: "FELIX_DNSTRUSTEDSERVERS", Value: "k8s-service:openshift-dns/dns-default"})) }) @@ -137,9 +131,28 @@ var _ = Describe("windows enterprise modifier", func() { Expect(err).NotTo(HaveOccurred()) tls, err := cm.GetOrCreateKeyPair(cli, render.NodePrometheusTLSServerSecret, common.OperatorNamespace(), []string{"calico-node-metrics-windows"}) Expect(err).NotTo(HaveOccurred()) + // The installation controller persists the secret; do the same here so the + // windows extension's GetKeyPair finds it. + Expect(cli.Create(context.Background(), tls.Secret(common.OperatorNamespace()))).NotTo(HaveOccurred()) bundle := cm.CreateTrustedBundle() - out, _ := applyExtensionsWithContext(ext, render.ComponentNameWindows, ctxFor(operatorv1.ProviderNone), wcFor(tls, bundle), newObjs(), nil) + // Build the render context the way the windows controller does: run the + // windows extension, which fetches the keypair into the context. + cc := extensions.ControllerContext{ + RenderContext: extensions.RenderContext{ + Installation: ctxFor(operatorv1.ProviderNone).Installation, + TrustedBundle: bundle, + ClusterDomain: dns.DefaultClusterDomain, + }, + Controller: extensions.WindowsController, + Ctx: context.Background(), + Client: cli, + CertificateManager: cm, + } + rc, _, err := ext.ExtendContext(cc) + Expect(err).NotTo(HaveOccurred()) + + out, _ := applyExtensions(ext, render.ComponentNameWindows, rc, newObjs(), nil) d := ds(out) Expect(d.Spec.Template.Spec.Volumes).To(ContainElement(tls.Volume())) diff --git a/pkg/extensions/controllerextension.go b/pkg/extensions/controllerextension.go index 05a176bdd9..ce5b713dfd 100644 --- a/pkg/extensions/controllerextension.go +++ b/pkg/extensions/controllerextension.go @@ -23,10 +23,20 @@ import ( "github.com/tigera/operator/pkg/tls/certificatemanagement" ) +// ControllerName identifies the controller a ControllerExtension extends, so a +// variant can register a different hook per controller. Use the constants below +// rather than bare strings so registration and lookup stay in sync. +type ControllerName string + +const ( + InstallationController ControllerName = "installation" + WindowsController ControllerName = "windows" +) + // ControllerExtension extends a controller's reconcile: it validates the // configuration and builds the RenderContext the render phase consumes. The core // operator registers none and runs with the base behavior; an extension build -// registers one. +// registers one per controller it extends. type ControllerExtension interface { // Validate rejects configuration the extension does not support, before any // rendering happens. @@ -47,12 +57,16 @@ type ControllerExtension interface { // deps live here, not on RenderContext, so the modifiers that read RenderContext // can't do I/O - they only transform objects. // -// The controller fills the embedded RenderContext's data fields and the deps; -// ExtendContext does its work, sets the produced artifacts (e.g. -// NodePrometheusTLS) on the embedded context, and returns it. +// Controller names which controller is reconciling, selecting that controller's +// extension hook. The controller fills the embedded RenderContext's data fields, +// the deps, and Controller; ExtendContext does its work, sets the produced +// artifacts on the embedded context, and returns it. type ControllerContext struct { RenderContext + // Controller identifies the reconciling controller, selecting its hook. + Controller ControllerName + Ctx context.Context Client client.Client CertificateManager certificatemanager.CertificateManager diff --git a/pkg/extensions/controllerextension_test.go b/pkg/extensions/controllerextension_test.go index c38d4e5e02..f8808e860a 100644 --- a/pkg/extensions/controllerextension_test.go +++ b/pkg/extensions/controllerextension_test.go @@ -35,6 +35,7 @@ var _ = Describe("controller extension", func() { install := &operatorv1.InstallationSpec{Variant: operatorv1.Calico} rc, _, err := s.ExtendContext(extensions.ControllerContext{ RenderContext: extensions.RenderContext{Installation: install, ClusterDomain: "cluster.local"}, + Controller: extensions.InstallationController, }) Expect(err).NotTo(HaveOccurred()) Expect(rc.Installation).To(BeIdenticalTo(install)) @@ -43,29 +44,30 @@ var _ = Describe("controller extension", func() { }) It("runs the extension registered for the installation variant", func() { - s.Variant(operatorv1.CalicoEnterprise).Controller(fakeController{}) + s.Variant(operatorv1.CalicoEnterprise).Controller(extensions.InstallationController, fakeController{}) rc, _, err := s.ExtendContext(enterpriseContext()) Expect(err).NotTo(HaveOccurred()) Expect(rc.ClusterDomain).To(Equal("from-fake")) }) It("ignores an extension registered for a different variant", func() { - s.Variant(operatorv1.CalicoEnterprise).Controller(fakeController{}) + s.Variant(operatorv1.CalicoEnterprise).Controller(extensions.InstallationController, fakeController{}) rc, _, err := s.ExtendContext(extensions.ControllerContext{ RenderContext: extensions.RenderContext{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.Calico}, ClusterDomain: "real"}, + Controller: extensions.InstallationController, }) Expect(err).NotTo(HaveOccurred()) Expect(rc.ClusterDomain).To(Equal("real")) }) It("surfaces the extension error", func() { - s.Variant(operatorv1.CalicoEnterprise).Controller(fakeController{err: errors.New("boom")}) + s.Variant(operatorv1.CalicoEnterprise).Controller(extensions.InstallationController, fakeController{err: errors.New("boom")}) _, _, err := s.ExtendContext(enterpriseContext()) Expect(err).To(MatchError("boom")) }) It("runs the extension's validation", func() { - s.Variant(operatorv1.CalicoEnterprise).Controller(fakeController{validateErr: errors.New("invalid")}) + s.Variant(operatorv1.CalicoEnterprise).Controller(extensions.InstallationController, fakeController{validateErr: errors.New("invalid")}) Expect(s.Validate(enterpriseContext())).To(MatchError("invalid")) }) @@ -83,6 +85,7 @@ var _ = Describe("controller extension", func() { func enterpriseContext() extensions.ControllerContext { return extensions.ControllerContext{ RenderContext: extensions.RenderContext{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.CalicoEnterprise}}, + Controller: extensions.InstallationController, } } diff --git a/pkg/extensions/set.go b/pkg/extensions/set.go index ac97f821d5..146d62db6c 100644 --- a/pkg/extensions/set.go +++ b/pkg/extensions/set.go @@ -50,9 +50,10 @@ func NewSet() *Set { func (s *Set) Variant(v operatorv1.ProductVariant) *Variant { if s.variants[v] == nil { s.variants[v] = &Variant{ - variant: v, - modifiers: map[string]decorator{}, - images: s.images, + variant: v, + controllers: map[ControllerName]ControllerExtension{}, + modifiers: map[string]decorator{}, + images: s.images, } } return s.variants[v] @@ -79,7 +80,7 @@ func (s *Set) Decorate(component render.Component, ctx RenderContext) render.Com return s.variant(ctx.Installation.Variant).decorate(component, ctx) } -// Validate runs the controller extension's validation for the installation's +// Validate runs the cc.Controller extension's validation for the installation's // variant, or returns nil when no extension is registered. Nil-safe. func (s *Set) Validate(cc ControllerContext) error { if cc.Installation == nil { @@ -88,9 +89,9 @@ func (s *Set) Validate(cc ControllerContext) error { return s.variant(cc.Installation.Variant).validate(cc) } -// ExtendContext runs the controller extension for the installation's variant and -// returns the resulting RenderContext plus any keypairs the extension wants the -// controller to manage, or the base render context and no keypairs when no +// ExtendContext runs the cc.Controller extension for the installation's variant +// and returns the resulting RenderContext plus any keypairs the extension wants +// the controller to manage, or the base render context and no keypairs when no // extension is registered. Nil-safe. func (s *Set) ExtendContext(cc ControllerContext) (RenderContext, []certificatemanagement.KeyPairInterface, error) { if cc.Installation == nil { diff --git a/pkg/extensions/variant.go b/pkg/extensions/variant.go index 08c920bbc5..ca1c553c4c 100644 --- a/pkg/extensions/variant.go +++ b/pkg/extensions/variant.go @@ -31,20 +31,20 @@ import ( // variant, so within a Variant there is at most one extension per component and // nothing here is itself keyed by variant. type Variant struct { - variant operatorv1.ProductVariant - controller ControllerExtension - modifiers map[string]decorator - images *imageoverride.Overrides // shared with the owning Set + variant operatorv1.ProductVariant + controllers map[ControllerName]ControllerExtension + modifiers map[string]decorator + images *imageoverride.Overrides // shared with the owning Set } // decorator wraps a base component, returning one whose Objects() are augmented // by a registered modifier. type decorator func(base render.Component, rc RenderContext) render.Component -// Controller registers the variant's controller-side extension. A variant has -// at most one; registering replaces any prior one. -func (v *Variant) Controller(c ControllerExtension) { - v.controller = c +// Controller registers the variant's controller-side extension for the named +// controller. A controller has at most one; registering replaces any prior one. +func (v *Variant) Controller(name ControllerName, c ControllerExtension) { + v.controllers[name] = c } // Image registers an image override for the named component. @@ -108,22 +108,22 @@ func (v *Variant) decorate(component render.Component, rc RenderContext) render. return build(component, rc) } -// validate runs the controller extension's validation, or nil when the variant -// has none. Nil-safe. +// validate runs the cc.Controller extension's validation, or nil when the +// variant has none for it. Nil-safe. func (v *Variant) validate(cc ControllerContext) error { - if v == nil || v.controller == nil { + if v == nil || v.controllers[cc.Controller] == nil { return nil } - return v.controller.Validate(cc) + return v.controllers[cc.Controller].Validate(cc) } -// extendContext runs the controller extension, or returns the base render -// context and no managed keypairs when the variant has none. Nil-safe. +// extendContext runs the cc.Controller extension, or returns the base render +// context and no managed keypairs when the variant has none for it. Nil-safe. func (v *Variant) extendContext(cc ControllerContext) (RenderContext, []certificatemanagement.KeyPairInterface, error) { - if v == nil || v.controller == nil { + if v == nil || v.controllers[cc.Controller] == nil { return cc.RenderContext, nil, nil } - return v.controller.ExtendContext(cc) + return v.controllers[cc.Controller].ExtendContext(cc) } // decoratedComponent is the render.Component produced by decorate: it renders diff --git a/pkg/render/node_enterprise_test.go b/pkg/render/node_enterprise_test.go index 9ea5db8d2e..a513ba72d0 100644 --- a/pkg/render/node_enterprise_test.go +++ b/pkg/render/node_enterprise_test.go @@ -102,6 +102,7 @@ var _ = Describe("node enterprise modifier integration", func() { TrustedBundle: typhaNodeTLS.TrustedBundle, ClusterDomain: dns.DefaultClusterDomain, }, + Controller: extensions.InstallationController, Ctx: context.Background(), Client: cli, CertificateManager: certManager, diff --git a/pkg/render/render_test.go b/pkg/render/render_test.go index 242c821bb2..22d7fa63a4 100644 --- a/pkg/render/render_test.go +++ b/pkg/render/render_test.go @@ -66,7 +66,6 @@ func allCalicoComponents( nodeAppArmorProfile string, clusterDomain string, kubeControllersMetricsPort int, - nodeReporterMetricsPort int, bgpLayout *corev1.ConfigMap, logCollector *operatorv1.LogCollector, ) ([]render.Component, error) { @@ -110,13 +109,12 @@ func allCalicoComponents( } winCfg := &render.WindowsConfiguration{ - K8sServiceEp: k8sServiceEp, - K8sDNSServers: []string{}, - Installation: cr, - ClusterDomain: clusterDomain, - TLS: typhaNodeTLS, - NodeReporterMetricsPort: nodeReporterMetricsPort, - VXLANVNI: 4096, + K8sServiceEp: k8sServiceEp, + K8sDNSServers: []string{}, + Installation: cr, + ClusterDomain: clusterDomain, + TLS: typhaNodeTLS, + VXLANVNI: 4096, } nodeCertComponent := rcertificatemanagement.CertificateManagement(&rcertificatemanagement.Config{ @@ -218,7 +216,7 @@ var _ = Describe("Rendering tests", func() { // - 6 kube-controllers resources (ServiceAccount, ClusterRole, Binding, Deployment, Service, Secret,RoleBinding) // - 1 namespace // - 2 Windows node resources (ConfigMap, DaemonSet) - c, err := allCalicoComponents(k8sServiceEp, instance, nil, nil, nil, typhaNodeTLS, nil, nil, false, "", dns.DefaultClusterDomain, 9094, 0, nil, nil) + c, err := allCalicoComponents(k8sServiceEp, instance, nil, nil, nil, typhaNodeTLS, nil, nil, false, "", dns.DefaultClusterDomain, 9094, nil, nil) Expect(err).To(BeNil(), "Expected Calico to create successfully %s", err) Expect(componentCount(c)).To(Equal(5 + 3 + 4 + 1 + 6 + 6 + 1 + 2)) }) @@ -231,7 +229,7 @@ var _ = Describe("Rendering tests", func() { var nodeMetricsPort int32 = 9081 instance.Variant = operatorv1.CalicoEnterprise instance.NodeMetricsPort = &nodeMetricsPort - c, err := allCalicoComponents(k8sServiceEp, instance, nil, nil, nil, typhaNodeTLS, nil, nil, false, "", dns.DefaultClusterDomain, 9094, 0, nil, nil) + c, err := allCalicoComponents(k8sServiceEp, instance, nil, nil, nil, typhaNodeTLS, nil, nil, false, "", dns.DefaultClusterDomain, 9094, nil, nil) Expect(err).To(BeNil(), "Expected Calico to create successfully %s", err) Expect(componentCount(c)).To(Equal(5 + 3 + 4 + 1 + 6 + 6 + 1 + 2)) }) @@ -244,7 +242,7 @@ var _ = Describe("Rendering tests", func() { instance.Variant = operatorv1.CalicoEnterprise instance.NodeMetricsPort = &nodeMetricsPort - c, err := allCalicoComponents(k8sServiceEp, instance, &operatorv1.ManagementCluster{}, nil, nil, typhaNodeTLS, internalManagerKeyPair, nil, false, "", dns.DefaultClusterDomain, 9094, 0, nil, nil) + c, err := allCalicoComponents(k8sServiceEp, instance, &operatorv1.ManagementCluster{}, nil, nil, typhaNodeTLS, internalManagerKeyPair, nil, false, "", dns.DefaultClusterDomain, 9094, nil, nil) Expect(err).To(BeNil(), "Expected Calico to create successfully %s", err) expectedResources := []client.Object{ @@ -305,7 +303,7 @@ var _ = Describe("Rendering tests", func() { It("should render calico with a apparmor profile if annotation is present in installation", func() { apparmorProf := "foobar" - comps, err := allCalicoComponents(k8sServiceEp, instance, nil, nil, nil, typhaNodeTLS, nil, nil, false, apparmorProf, dns.DefaultClusterDomain, 0, 0, nil, nil) + comps, err := allCalicoComponents(k8sServiceEp, instance, nil, nil, nil, typhaNodeTLS, nil, nil, false, apparmorProf, dns.DefaultClusterDomain, 0, nil, nil) Expect(err).To(BeNil(), "Expected Calico to create successfully %s", err) var cn *appsv1.DaemonSet for _, comp := range comps { @@ -329,7 +327,7 @@ var _ = Describe("Rendering tests", func() { } bgpLayout.Name = "bgp-layout" bgpLayout.Namespace = common.OperatorNamespace() - comps, err := allCalicoComponents(k8sServiceEp, instance, nil, nil, nil, typhaNodeTLS, nil, nil, false, "", dns.DefaultClusterDomain, 0, 0, bgpLayout, nil) + comps, err := allCalicoComponents(k8sServiceEp, instance, nil, nil, nil, typhaNodeTLS, nil, nil, false, "", dns.DefaultClusterDomain, 0, bgpLayout, nil) Expect(err).To(BeNil(), "Expected Calico to create successfully %s", err) var cm *corev1.ConfigMap var ds *appsv1.DaemonSet @@ -354,7 +352,7 @@ var _ = Describe("Rendering tests", func() { testNode := func(processPath operatorv1.CollectProcessPathOption, expectedHostPID bool) { var logCollector operatorv1.LogCollector logCollector.Spec.CollectProcessPath = &processPath - comps, err := allCalicoComponents(k8sServiceEp, instance, nil, nil, nil, typhaNodeTLS, nil, nil, false, "", dns.DefaultClusterDomain, 0, 0, nil, &logCollector) + comps, err := allCalicoComponents(k8sServiceEp, instance, nil, nil, nil, typhaNodeTLS, nil, nil, false, "", dns.DefaultClusterDomain, 0, nil, &logCollector) Expect(err).To(BeNil(), "Expected Calico to create successfully %s", err) var ds *appsv1.DaemonSet for _, comp := range comps { @@ -385,7 +383,7 @@ var _ = Describe("Rendering tests", func() { }) It("should set node priority class to system-node-critical", func() { - comps, err := allCalicoComponents(k8sServiceEp, instance, nil, nil, nil, typhaNodeTLS, nil, nil, false, "", dns.DefaultClusterDomain, 0, 0, nil, nil) + comps, err := allCalicoComponents(k8sServiceEp, instance, nil, nil, nil, typhaNodeTLS, nil, nil, false, "", dns.DefaultClusterDomain, 0, nil, nil) Expect(err).To(BeNil(), "Expected Calico to create successfully %s", err) var cn *appsv1.DaemonSet for _, comp := range comps { @@ -401,7 +399,7 @@ var _ = Describe("Rendering tests", func() { }) It("should set typha priority class to system-cluster-critical", func() { - comps, err := allCalicoComponents(k8sServiceEp, instance, nil, nil, nil, typhaNodeTLS, nil, nil, false, "", dns.DefaultClusterDomain, 0, 0, nil, nil) + comps, err := allCalicoComponents(k8sServiceEp, instance, nil, nil, nil, typhaNodeTLS, nil, nil, false, "", dns.DefaultClusterDomain, 0, nil, nil) Expect(err).To(BeNil(), "Expected Calico to create successfully %s", err) var cn *appsv1.Deployment for _, comp := range comps { @@ -417,7 +415,7 @@ var _ = Describe("Rendering tests", func() { }) It("should set kube controllers priority class to system-cluster-critical", func() { - comps, err := allCalicoComponents(k8sServiceEp, instance, nil, nil, nil, typhaNodeTLS, nil, nil, false, "", dns.DefaultClusterDomain, 0, 0, nil, nil) + comps, err := allCalicoComponents(k8sServiceEp, instance, nil, nil, nil, typhaNodeTLS, nil, nil, false, "", dns.DefaultClusterDomain, 0, nil, nil) Expect(err).To(BeNil(), "Expected Calico to create successfully %s", err) var cn *appsv1.Deployment for _, comp := range comps { diff --git a/pkg/render/windows.go b/pkg/render/windows.go index 7392094458..dd9692c77f 100644 --- a/pkg/render/windows.go +++ b/pkg/render/windows.go @@ -34,7 +34,6 @@ import ( rcomp "github.com/tigera/operator/pkg/render/common/components" rmeta "github.com/tigera/operator/pkg/render/common/meta" "github.com/tigera/operator/pkg/render/common/securitycontext" - "github.com/tigera/operator/pkg/tls/certificatemanagement" ) const ( @@ -49,14 +48,12 @@ func Windows( } type WindowsConfiguration struct { - K8sServiceEp k8sapi.ServiceEndpoint - K8sDNSServers []string - Installation *operatorv1.InstallationSpec - ClusterDomain string - TLS *TyphaNodeTLS - PrometheusServerTLS certificatemanagement.KeyPairInterface - NodeReporterMetricsPort int - VXLANVNI int + K8sServiceEp k8sapi.ServiceEndpoint + K8sDNSServers []string + Installation *operatorv1.InstallationSpec + ClusterDomain string + TLS *TyphaNodeTLS + VXLANVNI int // ImageOverrides lets a variant swap the windows node and CNI images. The // controller wires in the operator's image overrides; nil resolves to the @@ -99,25 +96,6 @@ func (c *windowsComponent) SupportedOSType() rmeta.OSType { func (c *windowsComponent) ModifierKey() string { return ComponentNameWindows } -// WindowsExtensionContext is the per-component context the windows modifier -// reads (via RenderContext.Component). It carries the enterprise inputs the -// windows controller has but a modifier can't derive from the installation: the -// reporter metrics port, the node prometheus keypair, and the trusted bundle -// the cert env vars reference. -type WindowsExtensionContext struct { - NodeReporterMetricsPort int - PrometheusServerTLS certificatemanagement.KeyPairInterface - TrustedBundle certificatemanagement.TrustedBundleRO -} - -func (c *windowsComponent) ExtensionContext() any { - return WindowsExtensionContext{ - NodeReporterMetricsPort: c.cfg.NodeReporterMetricsPort, - PrometheusServerTLS: c.cfg.PrometheusServerTLS, - TrustedBundle: c.cfg.TLS.TrustedBundle, - } -} - func (c *windowsComponent) Objects() ([]client.Object, []client.Object) { // Clean up old windows upgrader daemonset if present objsToDelete := []client.Object{ diff --git a/pkg/render/windows_test.go b/pkg/render/windows_test.go index 903fb3349b..1d92427a6b 100644 --- a/pkg/render/windows_test.go +++ b/pkg/render/windows_test.go @@ -51,11 +51,7 @@ func renderWindows(cfg *render.WindowsConfiguration) []client.Object { ExpectWithOffset(1, comp.ResolveImages(nil)).To(BeNil()) objs, _ := comp.Objects() rc := extensions.RenderContext{Installation: cfg.Installation} - var extCtx any - if p, ok := comp.(render.ExtensionContextProvider); ok { - extCtx = p.ExtensionContext() - } - out, _ := applyExtensionsWithContext(ext, render.ComponentNameWindows, rc, extCtx, objs, nil) + out, _ := applyExtensions(ext, render.ComponentNameWindows, rc, objs, nil) return out } @@ -718,7 +714,6 @@ var _ = Describe("Windows rendering tests", func() { {name: "calico-node-metrics-windows", ns: "calico-system", group: "", version: "v1", kind: "Service"}, } defaultInstance.Variant = operatorv1.CalicoEnterprise - cfg.NodeReporterMetricsPort = 9081 resources := renderWindows(&cfg) Expect(len(resources)).To(Equal(len(expectedResources))) @@ -1710,7 +1705,6 @@ var _ = Describe("Windows rendering tests", func() { defaultInstance.Variant = operatorv1.CalicoEnterprise defaultInstance.KubernetesProvider = operatorv1.ProviderOpenShift - cfg.NodeReporterMetricsPort = 9081 resources := renderWindows(&cfg) Expect(len(resources)).To(Equal(len(expectedResources))) @@ -1864,7 +1858,6 @@ var _ = Describe("Windows rendering tests", func() { defaultInstance.Variant = operatorv1.CalicoEnterprise defaultInstance.KubernetesProvider = operatorv1.ProviderRKE2 - cfg.NodeReporterMetricsPort = 9081 resources := renderWindows(&cfg) Expect(len(resources)).To(Equal(len(expectedResources)), fmt.Sprintf("Actual resources: %#v", resources)) @@ -2149,7 +2142,6 @@ var _ = Describe("Windows rendering tests", func() { It("should not enable prometheus metrics if NodeMetricsPort is nil", func() { defaultInstance.Variant = operatorv1.CalicoEnterprise defaultInstance.NodeMetricsPort = nil - cfg.NodeReporterMetricsPort = 9081 resources := renderWindows(&cfg) Expect(len(resources)).To(Equal(defaultNumExpectedResources + 1)) From 9a053384379ccad7a7b84b07f5d28a641e332c31 Mon Sep 17 00:00:00 2001 From: Casey Davenport Date: Thu, 18 Jun 2026 12:51:15 -0700 Subject: [PATCH 33/54] Move the enterprise manager-internal cert fetch into the controller hook The installation hook already fetches the prometheus and esgw certs into the trusted bundle; fold the manager-internal cert in too and drop the IsEnterprise branch from the core controller. --- pkg/controller/installation/core_controller.go | 12 ------------ pkg/enterprise/installation.go | 10 ++++++++++ 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/pkg/controller/installation/core_controller.go b/pkg/controller/installation/core_controller.go index 29fafcccf6..09a55030ea 100644 --- a/pkg/controller/installation/core_controller.go +++ b/pkg/controller/installation/core_controller.go @@ -1086,18 +1086,6 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile return reconcile.Result{}, err } - if instance.Spec.Variant.IsEnterprise() { - managerInternalTLSSecret, err := certificateManager.GetCertificate(r.client, render.ManagerInternalTLSSecretName, common.OperatorNamespace()) - if err != nil { - r.status.SetDegraded(operatorv1.ResourceReadError, fmt.Sprintf("Error fetching TLS secret %s in namespace %s", render.ManagerInternalTLSSecretName, common.OperatorNamespace()), err, reqLogger) - return reconcile.Result{}, nil - } else if managerInternalTLSSecret != nil { - // It may seem odd to add the manager internal TLS secret to the trusted bundle for Typha / calico-node, but this bundle is also used - // for other components in this namespace such as es-kube-controllers, who communicates with Voltron and thus needs to trust this certificate. - typhaNodeTLS.TrustedBundle.AddCertificates(managerInternalTLSSecret) - } - } - birdTemplates, err := getBirdTemplates(r.client) if err != nil { r.status.SetDegraded(operatorv1.ResourceReadError, "Error retrieving confd templates", err, reqLogger) diff --git a/pkg/enterprise/installation.go b/pkg/enterprise/installation.go index 6b3658f62a..934c78d0e7 100644 --- a/pkg/enterprise/installation.go +++ b/pkg/enterprise/installation.go @@ -86,6 +86,16 @@ func (coreControllerExtension) ExtendContext(cc extensions.ControllerContext) (e cc.TrustedBundle.AddCertificates(esgwCertificate) } + // es-kube-controllers talks to Voltron, so the shared bundle must trust the + // manager internal cert. + managerInternalTLS, err := cc.CertificateManager.GetCertificate(cc.Client, render.ManagerInternalTLSSecretName, common.OperatorNamespace()) + if err != nil { + return rc, nil, fmt.Errorf("failed to retrieve %s: %w", render.ManagerInternalTLSSecretName, err) + } + if managerInternalTLS != nil { + cc.TrustedBundle.AddCertificates(managerInternalTLS) + } + var managed []certificatemanagement.KeyPairInterface if nodePrometheusTLS != nil { managed = append(managed, nodePrometheusTLS) From 7f472881942ee128ae3a79fd56480132aec909b2 Mon Sep 17 00:00:00 2001 From: Casey Davenport Date: Thu, 18 Jun 2026 13:12:20 -0700 Subject: [PATCH 34/54] Let extensions declare their own watches Add a Watcher companion interface to ControllerExtension; each controller's Add() calls Set.SetupWatches, which runs the watch hook of every variant's extension for that controller. The enterprise installation and windows extensions register the enterprise CR and secret watches they need, so core no longer names ManagementCluster, ManagementClusterConnection, LogCollector, or the enterprise prometheus secrets. Still gated by EnterpriseCRDExists for now. --- .../installation/core_controller.go | 23 ++----------------- .../installation/windows_controller.go | 10 ++------ pkg/enterprise/installation.go | 22 ++++++++++++++++++ pkg/enterprise/windows.go | 16 +++++++++++++ pkg/extensions/controllerextension.go | 9 ++++++++ pkg/extensions/controllerextension_test.go | 20 ++++++++++++++++ pkg/extensions/set.go | 21 +++++++++++++++++ 7 files changed, 92 insertions(+), 29 deletions(-) diff --git a/pkg/controller/installation/core_controller.go b/pkg/controller/installation/core_controller.go index 09a55030ea..49121f472f 100644 --- a/pkg/controller/installation/core_controller.go +++ b/pkg/controller/installation/core_controller.go @@ -233,27 +233,8 @@ func Add(mgr manager.Manager, opts options.ControllerOptions) error { } if opts.EnterpriseCRDExists { - // Watch for changes to primary resource ManagementCluster - err = c.WatchObject(&operatorv1.ManagementCluster{}, &handler.EnqueueRequestForObject{}) - if err != nil { - return fmt.Errorf("tigera-installation-controller failed to watch primary resource: %v", err) - } - - // Watch for changes to primary resource ManagementClusterConnection - err = c.WatchObject(&operatorv1.ManagementClusterConnection{}, &handler.EnqueueRequestForObject{}) - if err != nil { - return fmt.Errorf("tigera-installation-controller failed to watch primary resource: %v", err) - } - - // watch for change to primary resource LogCollector - err = c.WatchObject(&operatorv1.LogCollector{}, &handler.EnqueueRequestForObject{}) - if err != nil { - return fmt.Errorf("tigera-installation-controller failed to watch primary resource: %v", err) - } - - // Watch the internal manager TLS secret in the operator namespace, which included in the bundle for es-kube-controllers. - if err = utils.AddSecretsWatch(c, render.ManagerInternalTLSSecretName, common.OperatorNamespace()); err != nil { - return fmt.Errorf("tigera-installation-controller failed to watch secret: %v", err) + if err = opts.Extensions.SetupWatches(extensions.InstallationController, c); err != nil { + return fmt.Errorf("tigera-installation-controller failed to set up extension watches: %w", err) } if opts.ManageCRDs { diff --git a/pkg/controller/installation/windows_controller.go b/pkg/controller/installation/windows_controller.go index 0e8152d85b..1c05e72227 100644 --- a/pkg/controller/installation/windows_controller.go +++ b/pkg/controller/installation/windows_controller.go @@ -51,7 +51,6 @@ import ( "github.com/tigera/operator/pkg/ctrlruntime" "github.com/tigera/operator/pkg/extensions" "github.com/tigera/operator/pkg/render" - "github.com/tigera/operator/pkg/render/monitor" ) var logw = logf.Log.WithName("controller_windows") @@ -149,13 +148,8 @@ func AddWindowsController(mgr manager.Manager, opts options.ControllerOptions) e go utils.WaitToAddResourceWatch(c, opts.K8sClientset, logw, ri.ipamConfigWatchReady, []client.Object{&v3.IPAMConfiguration{TypeMeta: metav1.TypeMeta{Kind: v3.KindIPAMConfiguration}}}) if ri.opts.EnterpriseCRDExists { - for _, ns := range []string{common.CalicoNamespace, common.OperatorNamespace()} { - if err = utils.AddSecretsWatch(c, render.NodePrometheusTLSServerSecret, ns); err != nil { - return fmt.Errorf("tigera-windows-controller failed to watch secret '%s' in '%s' namespace: %w", render.NodePrometheusTLSServerSecret, ns, err) - } - if err = utils.AddSecretsWatch(c, monitor.PrometheusClientTLSSecretName, ns); err != nil { - return fmt.Errorf("tigera-windows-controller failed to watch secret '%s' in '%s' namespace: %w", monitor.PrometheusClientTLSSecretName, ns, err) - } + if err = ri.opts.Extensions.SetupWatches(extensions.WindowsController, c); err != nil { + return fmt.Errorf("tigera-windows-controller failed to set up extension watches: %w", err) } } diff --git a/pkg/enterprise/installation.go b/pkg/enterprise/installation.go index 934c78d0e7..58b1360ac5 100644 --- a/pkg/enterprise/installation.go +++ b/pkg/enterprise/installation.go @@ -17,7 +17,13 @@ package enterprise import ( "fmt" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/handler" + + operatorv1 "github.com/tigera/operator/api/v1" "github.com/tigera/operator/pkg/common" + "github.com/tigera/operator/pkg/controller/utils" + "github.com/tigera/operator/pkg/ctrlruntime" "github.com/tigera/operator/pkg/dns" "github.com/tigera/operator/pkg/extensions" "github.com/tigera/operator/pkg/render" @@ -49,6 +55,22 @@ func (coreControllerExtension) Validate(cc extensions.ControllerContext) error { return validateReporterPort(cc.FelixConfiguration) } +// Watches registers the enterprise resources the installation controller +// reconciles on. +func (coreControllerExtension) Watches(c ctrlruntime.Controller) error { + for _, obj := range []client.Object{ + &operatorv1.ManagementCluster{}, + &operatorv1.ManagementClusterConnection{}, + &operatorv1.LogCollector{}, + } { + if err := c.WatchObject(obj, &handler.EnqueueRequestForObject{}); err != nil { + return err + } + } + // es-kube-controllers includes the manager internal TLS secret in its bundle. + return utils.AddSecretsWatch(c, render.ManagerInternalTLSSecretName, common.OperatorNamespace()) +} + // ExtendContext does the controller-side work the modifiers can't: creating and // fetching the certificates that feed the trusted bundle. It returns the render // context carrying the produced node prometheus keypair, and that keypair as one diff --git a/pkg/enterprise/windows.go b/pkg/enterprise/windows.go index bffb7ec1d9..183bfd08ae 100644 --- a/pkg/enterprise/windows.go +++ b/pkg/enterprise/windows.go @@ -26,10 +26,13 @@ import ( operatorv1 "github.com/tigera/operator/api/v1" "github.com/tigera/operator/pkg/common" "github.com/tigera/operator/pkg/components" + "github.com/tigera/operator/pkg/controller/utils" + "github.com/tigera/operator/pkg/ctrlruntime" "github.com/tigera/operator/pkg/dns" "github.com/tigera/operator/pkg/extensions" "github.com/tigera/operator/pkg/render" rmeta "github.com/tigera/operator/pkg/render/common/meta" + "github.com/tigera/operator/pkg/render/monitor" "github.com/tigera/operator/pkg/tls/certificatemanagement" ) @@ -65,6 +68,19 @@ func (windowsControllerExtension) Validate(cc extensions.ControllerContext) erro return validateReporterPort(cc.FelixConfiguration) } +// Watches registers the enterprise secrets the windows controller reconciles on. +func (windowsControllerExtension) Watches(c ctrlruntime.Controller) error { + for _, ns := range []string{common.CalicoNamespace, common.OperatorNamespace()} { + if err := utils.AddSecretsWatch(c, render.NodePrometheusTLSServerSecret, ns); err != nil { + return err + } + if err := utils.AddSecretsWatch(c, monitor.PrometheusClientTLSSecretName, ns); err != nil { + return err + } + } + return nil +} + // ExtendContext fetches the node prometheus keypair the installation controller // created and stashes it in the render context for the windows modifier. func (windowsControllerExtension) ExtendContext(cc extensions.ControllerContext) (extensions.RenderContext, []certificatemanagement.KeyPairInterface, error) { diff --git a/pkg/extensions/controllerextension.go b/pkg/extensions/controllerextension.go index ce5b713dfd..d61b4cee6f 100644 --- a/pkg/extensions/controllerextension.go +++ b/pkg/extensions/controllerextension.go @@ -20,6 +20,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "github.com/tigera/operator/pkg/controller/certificatemanager" + "github.com/tigera/operator/pkg/ctrlruntime" "github.com/tigera/operator/pkg/tls/certificatemanagement" ) @@ -50,6 +51,14 @@ type ControllerExtension interface { ExtendContext(cc ControllerContext) (RenderContext, []certificatemanagement.KeyPairInterface, error) } +// Watcher is an optional companion to ControllerExtension. A controller's Add() +// calls Set.SetupWatches, which invokes Watches on any registered extension that +// implements this, so the extension registers the watches it needs (its CRs, its +// secrets) instead of the controller naming them. +type Watcher interface { + Watches(c ctrlruntime.Controller) error +} + // ControllerContext is the controller-phase context, the corollary to the // render-phase RenderContext. It is the embedded RenderContext (the same data // the render phase sees) plus the controller-side machinery a ControllerExtension diff --git a/pkg/extensions/controllerextension_test.go b/pkg/extensions/controllerextension_test.go index f8808e860a..729e8783d6 100644 --- a/pkg/extensions/controllerextension_test.go +++ b/pkg/extensions/controllerextension_test.go @@ -21,6 +21,7 @@ import ( . "github.com/onsi/gomega" operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/ctrlruntime" "github.com/tigera/operator/pkg/extensions" "github.com/tigera/operator/pkg/tls/certificatemanagement" ) @@ -71,6 +72,13 @@ var _ = Describe("controller extension", func() { Expect(s.Validate(enterpriseContext())).To(MatchError("invalid")) }) + It("runs the watch hook of an extension that implements Watcher", func() { + called := false + s.Variant(operatorv1.CalicoEnterprise).Controller(extensions.InstallationController, watchingController{called: &called}) + Expect(s.SetupWatches(extensions.InstallationController, nil)).NotTo(HaveOccurred()) + Expect(called).To(BeTrue()) + }) + It("returns the base context and no validation error for a nil Set", func() { var nilSet *extensions.Set cc := enterpriseContext() @@ -106,3 +114,15 @@ func (f fakeController) ExtendContext(_ extensions.ControllerContext) (extension } return extensions.RenderContext{ClusterDomain: "from-fake"}, nil, nil } + +// watchingController is a fakeController that also implements the Watcher +// companion, recording that its watch hook ran. +type watchingController struct { + fakeController + called *bool +} + +func (w watchingController) Watches(ctrlruntime.Controller) error { + *w.called = true + return nil +} diff --git a/pkg/extensions/set.go b/pkg/extensions/set.go index 146d62db6c..ee95199a15 100644 --- a/pkg/extensions/set.go +++ b/pkg/extensions/set.go @@ -17,6 +17,7 @@ package extensions import ( operatorv1 "github.com/tigera/operator/api/v1" "github.com/tigera/operator/pkg/components" + "github.com/tigera/operator/pkg/ctrlruntime" "github.com/tigera/operator/pkg/imageoverride" "github.com/tigera/operator/pkg/render" "github.com/tigera/operator/pkg/tls/certificatemanagement" @@ -100,6 +101,26 @@ func (s *Set) ExtendContext(cc ControllerContext) (RenderContext, []certificatem return s.variant(cc.Installation.Variant).extendContext(cc) } +// SetupWatches registers the watches every variant's extension declares for the +// named controller. It runs at controller startup, which is variant-agnostic, so +// it registers the union across variants (in practice the one active extension +// build's). Nil-safe. +func (s *Set) SetupWatches(controller ControllerName, c ctrlruntime.Controller) error { + if s == nil { + return nil + } + for _, v := range s.variants { + w, ok := v.controllers[controller].(Watcher) + if !ok { + continue + } + if err := w.Watches(c); err != nil { + return err + } + } + return nil +} + // Images returns the shared image override table. The render package resolves a // component's image through it directly (the imageoverride leaf, so render need // not import extensions). Nil-safe, returning nil overrides that resolve to the From a6e87f94d0bd1044efcc3f08bd277cf88ec54b15 Mon Sep 17 00:00:00 2001 From: Casey Davenport Date: Thu, 18 Jun 2026 13:24:51 -0700 Subject: [PATCH 35/54] Move LogCollector process-path collection into the node modifier The OSS node render read the enterprise LogCollector CR to set HostPID and FELIX_FLOWLOGSCOLLECTPROCESSPATH - enterprise flow-log behavior (OSS flow logs go through Goldmane, not LogCollector). The installation hook now fetches the LogCollector and records whether process-path collection is on; the node modifier sets HostPID and the env from it. LogCollector is gone from NodeConfiguration and the core controller. --- .../installation/core_controller.go | 10 ---- pkg/enterprise/installation.go | 20 ++++++- pkg/enterprise/node.go | 12 ++++- pkg/render/node.go | 15 ------ pkg/render/node_enterprise_test.go | 37 +++++++++++-- pkg/render/render_test.go | 52 +++---------------- 6 files changed, 70 insertions(+), 76 deletions(-) diff --git a/pkg/controller/installation/core_controller.go b/pkg/controller/installation/core_controller.go index 49121f472f..a52e196046 100644 --- a/pkg/controller/installation/core_controller.go +++ b/pkg/controller/installation/core_controller.go @@ -1007,16 +1007,7 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile var managementCluster *operatorv1.ManagementCluster var managementClusterConnection *operatorv1.ManagementClusterConnection - var logCollector *operatorv1.LogCollector if r.opts.EnterpriseCRDExists { - logCollector, err = utils.GetLogCollector(ctx, r.client) - if logCollector != nil { - if err != nil { - r.status.SetDegraded(operatorv1.ResourceReadError, "Error reading LogCollector", err, reqLogger) - return reconcile.Result{}, err - } - } - managementCluster, err = utils.GetManagementCluster(ctx, r.client) if err != nil { r.status.SetDegraded(operatorv1.ResourceReadError, "Error reading ManagementCluster", err, reqLogger) @@ -1520,7 +1511,6 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile K8sServiceEp: k8sapi.Endpoint, Installation: &instance.Spec, IPPools: crdPoolsToOperator(currentPools.Items), - LogCollector: logCollector, BirdTemplates: birdTemplates, TLS: typhaNodeTLS, ClusterDomain: r.opts.ClusterDomain, diff --git a/pkg/enterprise/installation.go b/pkg/enterprise/installation.go index 58b1360ac5..3126e90880 100644 --- a/pkg/enterprise/installation.go +++ b/pkg/enterprise/installation.go @@ -41,6 +41,10 @@ type coreControllerExtension struct{} // modifier type-asserts it back out. type installationRenderData struct { nodePrometheusTLS certificatemanagement.KeyPairInterface + + // collectProcessPath mirrors LogCollector.Spec.CollectProcessPath being + // enabled; the node modifier uses it to set HostPID and the felix env. + collectProcessPath bool } // installationData pulls the installation extension's render data back out of the @@ -50,6 +54,12 @@ func installationData(rc extensions.RenderContext) installationRenderData { return data } +func collectProcessPathEnabled(lc *operatorv1.LogCollector) bool { + return lc != nil && + lc.Spec.CollectProcessPath != nil && + *lc.Spec.CollectProcessPath == operatorv1.CollectProcessPathEnable +} + // Validate rejects installation config Calico Enterprise does not support. func (coreControllerExtension) Validate(cc extensions.ControllerContext) error { return validateReporterPort(cc.FelixConfiguration) @@ -90,7 +100,15 @@ func (coreControllerExtension) ExtendContext(cc extensions.ControllerContext) (e if nodePrometheusTLS != nil { cc.TrustedBundle.AddCertificates(nodePrometheusTLS) } - rc.Extension = installationRenderData{nodePrometheusTLS: nodePrometheusTLS} + + logCollector, err := utils.GetLogCollector(cc.Ctx, cc.Client) + if err != nil { + return rc, nil, fmt.Errorf("error reading LogCollector: %w", err) + } + rc.Extension = installationRenderData{ + nodePrometheusTLS: nodePrometheusTLS, + collectProcessPath: collectProcessPathEnabled(logCollector), + } prometheusClientCert, err := cc.CertificateManager.GetCertificate(cc.Client, monitor.PrometheusClientTLSSecretName, common.OperatorNamespace()) if err != nil { diff --git a/pkg/enterprise/node.go b/pkg/enterprise/node.go index ad4469a8b7..d4e35dae68 100644 --- a/pkg/enterprise/node.go +++ b/pkg/enterprise/node.go @@ -120,6 +120,11 @@ func nodeEnterpriseRules() []rbacv1.PolicyRule { func modifyNodeDaemonSet(rc extensions.RenderContext, ds *appsv1.DaemonSet) { spec := &ds.Spec.Template.Spec + // Collecting process info for flow logs reads from the host's process table. + if installationData(rc).collectProcessPath { + spec.HostPID = true + } + multiInterfaceMode := multiInterfaceModeEnv(rc.Installation) for i := range spec.InitContainers { @@ -182,7 +187,7 @@ func mountNodePrometheusTLS(rc extensions.RenderContext, ds *appsv1.DaemonSet) { // nodeEnterpriseEnv is the Enterprise felix configuration added to the // calico/node container. func nodeEnterpriseEnv(rc extensions.RenderContext) []corev1.EnvVar { - tls := installationData(rc).nodePrometheusTLS + data := installationData(rc) env := []corev1.EnvVar{ {Name: "FELIX_PROMETHEUSREPORTERENABLED", Value: "true"}, {Name: "FELIX_PROMETHEUSREPORTERPORT", Value: fmt.Sprintf("%d", nodeReporterPort(rc.FelixConfiguration))}, @@ -196,10 +201,15 @@ func nodeEnterpriseEnv(rc extensions.RenderContext) []corev1.EnvVar { {Name: "FELIX_DNSLOGSFILEPERNODELIMIT", Value: "1000"}, } + if data.collectProcessPath { + env = append(env, corev1.EnvVar{Name: "FELIX_FLOWLOGSCOLLECTPROCESSPATH", Value: "true"}) + } + if mode := multiInterfaceModeEnv(rc.Installation); mode != nil { env = append(env, *mode) } + tls := data.nodePrometheusTLS if tls != nil && rc.TrustedBundle != nil { env = append(env, corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERCERTFILE", Value: tls.VolumeMountCertificateFilePath()}, diff --git a/pkg/render/node.go b/pkg/render/node.go index 4077e43db2..9a838cf5cc 100644 --- a/pkg/render/node.go +++ b/pkg/render/node.go @@ -116,7 +116,6 @@ type NodeConfiguration struct { GoldmaneIP string // Optional fields. - LogCollector *operatorv1.LogCollector MigrateNamespaces bool NodeAppArmorProfile string BirdTemplates map[string]string @@ -1025,10 +1024,6 @@ func (c *nodeComponent) nodeDaemonset(cniCfgMap *corev1.ConfigMap) *appsv1.Daemo ds.Spec.Template.Spec.InitContainers = append(ds.Spec.Template.Spec.InitContainers, c.cniContainer()) } - if c.collectProcessPathEnabled() { - ds.Spec.Template.Spec.HostPID = true - } - setNodeCriticalPod(&(ds.Spec.Template)) if c.cfg.MigrateNamespaces { migration.LimitDaemonSetToMigratedNodes(&ds) @@ -1144,12 +1139,6 @@ func (c *nodeComponent) vppDataplaneEnabled() bool { *c.cfg.Installation.CalicoNetwork.LinuxDataplane == operatorv1.LinuxDataplaneVPP } -func (c *nodeComponent) collectProcessPathEnabled() bool { - return c.cfg.LogCollector != nil && - c.cfg.LogCollector.Spec.CollectProcessPath != nil && - *c.cfg.LogCollector.Spec.CollectProcessPath == operatorv1.CollectProcessPathEnable -} - // cniContainer creates the node's init container that installs CNI. func (c *nodeComponent) cniContainer() corev1.Container { // Determine environment to pass to the CNI init container. @@ -1509,10 +1498,6 @@ func (c *nodeComponent) nodeEnvVars() []corev1.EnvVar { } } - if c.collectProcessPathEnabled() { - nodeEnv = append(nodeEnv, corev1.EnvVar{Name: "FELIX_FLOWLOGSCOLLECTPROCESSPATH", Value: "true"}) - } - // Determine MTU to use. If specified explicitly, use that. Otherwise, set defaults based on an overall // MTU of 1460. mtu := getMTU(c.cfg.Installation) diff --git a/pkg/render/node_enterprise_test.go b/pkg/render/node_enterprise_test.go index a513ba72d0..7696646454 100644 --- a/pkg/render/node_enterprise_test.go +++ b/pkg/render/node_enterprise_test.go @@ -23,6 +23,7 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -113,7 +114,7 @@ var _ = Describe("node enterprise modifier integration", func() { // renderNodeObjects renders the real node component and applies the registered // modifier, exactly as the componentHandler does. - renderNodeObjects := func() []client.Object { + renderNodeObjects := func(rc extensions.RenderContext) []client.Object { cfg := &render.NodeConfiguration{ K8sServiceEp: k8sapi.ServiceEndpoint{}, Installation: instance, @@ -125,19 +126,19 @@ var _ = Describe("node enterprise modifier integration", func() { comp := render.Node(cfg) Expect(comp.ResolveImages(nil)).NotTo(HaveOccurred()) objs, _ := comp.Objects() - out, _ := applyExtensions(ext, render.ComponentNameNode, renderCtx, objs, nil) + out, _ := applyExtensions(ext, render.ComponentNameNode, rc, objs, nil) return out } It("appends the node metrics service to the real render output", func() { - objs := renderNodeObjects() + objs := renderNodeObjects(renderCtx) svc, ok := extensions.FindObject[*corev1.Service](objs, render.CalicoNodeMetricsService) Expect(ok).To(BeTrue(), "expected the modifier to append %s", render.CalicoNodeMetricsService) Expect(svc.Namespace).To(Equal(common.CalicoNamespace)) }) It("adds the enterprise rules to the real cluster roles", func() { - objs := renderNodeObjects() + objs := renderNodeObjects(renderCtx) nodeRole, ok := extensions.FindObject[*rbacv1.ClusterRole](objs, render.CalicoNodeObjectName) Expect(ok).To(BeTrue()) @@ -149,7 +150,7 @@ var _ = Describe("node enterprise modifier integration", func() { }) It("rewrites the real node daemonset for enterprise", func() { - objs := renderNodeObjects() + objs := renderNodeObjects(renderCtx) ds, ok := extensions.FindObject[*appsv1.DaemonSet](objs, common.NodeDaemonSetName) Expect(ok).To(BeTrue()) @@ -172,6 +173,32 @@ var _ = Describe("node enterprise modifier integration", func() { Expect(c.ReadinessProbe.Exec.Command).To(ContainElement("--bgp-metrics-ready")) }) + It("enables process-path collection when the LogCollector requests it", func() { + enable := operatorv1.CollectProcessPathEnable + Expect(cli.Create(context.Background(), &operatorv1.LogCollector{ + ObjectMeta: metav1.ObjectMeta{Name: "tigera-secure"}, + Spec: operatorv1.LogCollectorSpec{CollectProcessPath: &enable}, + })).NotTo(HaveOccurred()) + + rc, _, err := ext.ExtendContext(extensions.ControllerContext{ + RenderContext: extensions.RenderContext{ + Installation: instance, + TrustedBundle: typhaNodeTLS.TrustedBundle, + ClusterDomain: dns.DefaultClusterDomain, + }, + Controller: extensions.InstallationController, + Ctx: context.Background(), + Client: cli, + CertificateManager: certManager, + }) + Expect(err).NotTo(HaveOccurred()) + + ds, ok := extensions.FindObject[*appsv1.DaemonSet](renderNodeObjects(rc), common.NodeDaemonSetName) + Expect(ok).To(BeTrue()) + Expect(ds.Spec.Template.Spec.HostPID).To(BeTrue()) + Expect(nodeContainer(ds).Env).To(ContainElement(corev1.EnvVar{Name: "FELIX_FLOWLOGSCOLLECTPROCESSPATH", Value: "true"})) + }) + It("adds the enterprise rules to the real typha cluster role", func() { comp := render.Typha(&render.TyphaConfiguration{ K8sServiceEp: k8sapi.ServiceEndpoint{}, diff --git a/pkg/render/render_test.go b/pkg/render/render_test.go index 22d7fa63a4..beef705146 100644 --- a/pkg/render/render_test.go +++ b/pkg/render/render_test.go @@ -67,7 +67,6 @@ func allCalicoComponents( clusterDomain string, kubeControllersMetricsPort int, bgpLayout *corev1.ConfigMap, - logCollector *operatorv1.LogCollector, ) ([]render.Component, error) { namespaces := render.Namespaces(&render.NamespaceConfiguration{Installation: cr, PullSecrets: pullSecrets}) @@ -84,7 +83,6 @@ func allCalicoComponents( NodeAppArmorProfile: nodeAppArmorProfile, ClusterDomain: clusterDomain, BGPLayouts: bgpLayout, - LogCollector: logCollector, BirdTemplates: bt, MigrateNamespaces: up, FelixHealthPort: 9099, @@ -216,7 +214,7 @@ var _ = Describe("Rendering tests", func() { // - 6 kube-controllers resources (ServiceAccount, ClusterRole, Binding, Deployment, Service, Secret,RoleBinding) // - 1 namespace // - 2 Windows node resources (ConfigMap, DaemonSet) - c, err := allCalicoComponents(k8sServiceEp, instance, nil, nil, nil, typhaNodeTLS, nil, nil, false, "", dns.DefaultClusterDomain, 9094, nil, nil) + c, err := allCalicoComponents(k8sServiceEp, instance, nil, nil, nil, typhaNodeTLS, nil, nil, false, "", dns.DefaultClusterDomain, 9094, nil) Expect(err).To(BeNil(), "Expected Calico to create successfully %s", err) Expect(componentCount(c)).To(Equal(5 + 3 + 4 + 1 + 6 + 6 + 1 + 2)) }) @@ -229,7 +227,7 @@ var _ = Describe("Rendering tests", func() { var nodeMetricsPort int32 = 9081 instance.Variant = operatorv1.CalicoEnterprise instance.NodeMetricsPort = &nodeMetricsPort - c, err := allCalicoComponents(k8sServiceEp, instance, nil, nil, nil, typhaNodeTLS, nil, nil, false, "", dns.DefaultClusterDomain, 9094, nil, nil) + c, err := allCalicoComponents(k8sServiceEp, instance, nil, nil, nil, typhaNodeTLS, nil, nil, false, "", dns.DefaultClusterDomain, 9094, nil) Expect(err).To(BeNil(), "Expected Calico to create successfully %s", err) Expect(componentCount(c)).To(Equal(5 + 3 + 4 + 1 + 6 + 6 + 1 + 2)) }) @@ -242,7 +240,7 @@ var _ = Describe("Rendering tests", func() { instance.Variant = operatorv1.CalicoEnterprise instance.NodeMetricsPort = &nodeMetricsPort - c, err := allCalicoComponents(k8sServiceEp, instance, &operatorv1.ManagementCluster{}, nil, nil, typhaNodeTLS, internalManagerKeyPair, nil, false, "", dns.DefaultClusterDomain, 9094, nil, nil) + c, err := allCalicoComponents(k8sServiceEp, instance, &operatorv1.ManagementCluster{}, nil, nil, typhaNodeTLS, internalManagerKeyPair, nil, false, "", dns.DefaultClusterDomain, 9094, nil) Expect(err).To(BeNil(), "Expected Calico to create successfully %s", err) expectedResources := []client.Object{ @@ -303,7 +301,7 @@ var _ = Describe("Rendering tests", func() { It("should render calico with a apparmor profile if annotation is present in installation", func() { apparmorProf := "foobar" - comps, err := allCalicoComponents(k8sServiceEp, instance, nil, nil, nil, typhaNodeTLS, nil, nil, false, apparmorProf, dns.DefaultClusterDomain, 0, nil, nil) + comps, err := allCalicoComponents(k8sServiceEp, instance, nil, nil, nil, typhaNodeTLS, nil, nil, false, apparmorProf, dns.DefaultClusterDomain, 0, nil) Expect(err).To(BeNil(), "Expected Calico to create successfully %s", err) var cn *appsv1.DaemonSet for _, comp := range comps { @@ -327,7 +325,7 @@ var _ = Describe("Rendering tests", func() { } bgpLayout.Name = "bgp-layout" bgpLayout.Namespace = common.OperatorNamespace() - comps, err := allCalicoComponents(k8sServiceEp, instance, nil, nil, nil, typhaNodeTLS, nil, nil, false, "", dns.DefaultClusterDomain, 0, bgpLayout, nil) + comps, err := allCalicoComponents(k8sServiceEp, instance, nil, nil, nil, typhaNodeTLS, nil, nil, false, "", dns.DefaultClusterDomain, 0, bgpLayout) Expect(err).To(BeNil(), "Expected Calico to create successfully %s", err) var cm *corev1.ConfigMap var ds *appsv1.DaemonSet @@ -348,42 +346,8 @@ var _ = Describe("Rendering tests", func() { Expect(ds.Spec.Template.Annotations["hash.operator.tigera.io/bgp-layout"]).NotTo(BeEmpty()) }) - It("should handle collectProcessPath in logCollector", func() { - testNode := func(processPath operatorv1.CollectProcessPathOption, expectedHostPID bool) { - var logCollector operatorv1.LogCollector - logCollector.Spec.CollectProcessPath = &processPath - comps, err := allCalicoComponents(k8sServiceEp, instance, nil, nil, nil, typhaNodeTLS, nil, nil, false, "", dns.DefaultClusterDomain, 0, nil, &logCollector) - Expect(err).To(BeNil(), "Expected Calico to create successfully %s", err) - var ds *appsv1.DaemonSet - for _, comp := range comps { - resources, _ := comp.Objects() - r := rtest.GetResource(resources, "calico-node", "calico-system", "apps", "v1", "DaemonSet") - if r != nil { - ds = r.(*appsv1.DaemonSet) - } - } - checkEnvVar := func(ds *appsv1.DaemonSet) bool { - envPresent := false - for _, env := range ds.Spec.Template.Spec.Containers[0].Env { - if env.Name == "FELIX_FLOWLOGSCOLLECTPROCESSPATH" { - envPresent = true - if env.Value == "true" { - return true - } - } - } - return !envPresent - } - Expect(ds).ToNot(BeNil()) - Expect(ds.Spec.Template.Spec.HostPID).To(Equal(expectedHostPID)) - Expect(checkEnvVar(ds)).To(Equal(true)) - } - testNode(operatorv1.CollectProcessPathEnable, true) - testNode(operatorv1.CollectProcessPathDisable, false) - }) - It("should set node priority class to system-node-critical", func() { - comps, err := allCalicoComponents(k8sServiceEp, instance, nil, nil, nil, typhaNodeTLS, nil, nil, false, "", dns.DefaultClusterDomain, 0, nil, nil) + comps, err := allCalicoComponents(k8sServiceEp, instance, nil, nil, nil, typhaNodeTLS, nil, nil, false, "", dns.DefaultClusterDomain, 0, nil) Expect(err).To(BeNil(), "Expected Calico to create successfully %s", err) var cn *appsv1.DaemonSet for _, comp := range comps { @@ -399,7 +363,7 @@ var _ = Describe("Rendering tests", func() { }) It("should set typha priority class to system-cluster-critical", func() { - comps, err := allCalicoComponents(k8sServiceEp, instance, nil, nil, nil, typhaNodeTLS, nil, nil, false, "", dns.DefaultClusterDomain, 0, nil, nil) + comps, err := allCalicoComponents(k8sServiceEp, instance, nil, nil, nil, typhaNodeTLS, nil, nil, false, "", dns.DefaultClusterDomain, 0, nil) Expect(err).To(BeNil(), "Expected Calico to create successfully %s", err) var cn *appsv1.Deployment for _, comp := range comps { @@ -415,7 +379,7 @@ var _ = Describe("Rendering tests", func() { }) It("should set kube controllers priority class to system-cluster-critical", func() { - comps, err := allCalicoComponents(k8sServiceEp, instance, nil, nil, nil, typhaNodeTLS, nil, nil, false, "", dns.DefaultClusterDomain, 0, nil, nil) + comps, err := allCalicoComponents(k8sServiceEp, instance, nil, nil, nil, typhaNodeTLS, nil, nil, false, "", dns.DefaultClusterDomain, 0, nil) Expect(err).To(BeNil(), "Expected Calico to create successfully %s", err) var cn *appsv1.Deployment for _, comp := range comps { From 632cfa2577a2274899db357deac267e12719d188 Mon Sep 17 00:00:00 2001 From: Casey Davenport Date: Thu, 18 Jun 2026 14:15:24 -0700 Subject: [PATCH 36/54] Remove variant branches from the kube-controllers render The kube-controllers component renders from a generic config (name, rules, enabled controllers, extra env, network policy) with no IsEnterprise or component-name branching. es-calico-kube-controllers assembly and its constants live in pkg/enterprise and fill that config; calico-kube-controllers still assembles in render. --- pkg/controller/logstorage/common/common.go | 18 +- .../kubecontrollers/es_kube_controllers.go | 10 +- .../es_kube_controllers_test.go | 10 +- pkg/enterprise/kubecontrollers.go | 177 +++++++++ .../kubecontrollers/kube-controllers.go | 363 +++++++----------- .../kubecontrollers/kube-controllers_test.go | 56 ++- 6 files changed, 355 insertions(+), 279 deletions(-) create mode 100644 pkg/enterprise/kubecontrollers.go diff --git a/pkg/controller/logstorage/common/common.go b/pkg/controller/logstorage/common/common.go index 2c070439d7..e15f982b49 100644 --- a/pkg/controller/logstorage/common/common.go +++ b/pkg/controller/logstorage/common/common.go @@ -26,7 +26,7 @@ import ( "github.com/tigera/operator/pkg/controller/utils" "github.com/tigera/operator/pkg/crypto" - "github.com/tigera/operator/pkg/render/kubecontrollers" + "github.com/tigera/operator/pkg/enterprise" ) const ( @@ -45,7 +45,7 @@ const ( // the gateway credentials, and a secret containing real admin level credentials is created and stored in the tigera-elasticsearch namespace to be swapped in once // ES Gateway has confirmed that the gateway credentials match. func CreateKubeControllersSecrets(ctx context.Context, esAdminUserSecret *corev1.Secret, esAdminUserName string, cli client.Client, h utils.NamespaceHelper) (*corev1.Secret, *corev1.Secret, *corev1.Secret, error) { - kubeControllersGatewaySecret, err := utils.GetSecret(ctx, cli, kubecontrollers.ElasticsearchKubeControllersUserSecret, h.TruthNamespace()) + kubeControllersGatewaySecret, err := utils.GetSecret(ctx, cli, enterprise.ElasticsearchKubeControllersUserSecret, h.TruthNamespace()) if err != nil { return nil, nil, nil, err } @@ -53,11 +53,11 @@ func CreateKubeControllersSecrets(ctx context.Context, esAdminUserSecret *corev1 password := crypto.GeneratePassword(16) kubeControllersGatewaySecret = &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ - Name: kubecontrollers.ElasticsearchKubeControllersUserSecret, + Name: enterprise.ElasticsearchKubeControllersUserSecret, Namespace: h.TruthNamespace(), }, Data: map[string][]byte{ - "username": []byte(kubecontrollers.ElasticsearchKubeControllersUserName), + "username": []byte(enterprise.ElasticsearchKubeControllersUserName), "password": []byte(password), }, } @@ -67,34 +67,34 @@ func CreateKubeControllersSecrets(ctx context.Context, esAdminUserSecret *corev1 return nil, nil, nil, err } - kubeControllersVerificationSecret, err := utils.GetSecret(ctx, cli, kubecontrollers.ElasticsearchKubeControllersVerificationUserSecret, h.InstallNamespace()) + kubeControllersVerificationSecret, err := utils.GetSecret(ctx, cli, enterprise.ElasticsearchKubeControllersVerificationUserSecret, h.InstallNamespace()) if err != nil { return nil, nil, nil, err } if kubeControllersVerificationSecret == nil { kubeControllersVerificationSecret = &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ - Name: kubecontrollers.ElasticsearchKubeControllersVerificationUserSecret, + Name: enterprise.ElasticsearchKubeControllersVerificationUserSecret, Namespace: h.InstallNamespace(), Labels: map[string]string{ ESGatewaySelectorLabel: ESGatewaySelectorLabelValue, }, }, Data: map[string][]byte{ - "username": []byte(kubecontrollers.ElasticsearchKubeControllersUserName), + "username": []byte(enterprise.ElasticsearchKubeControllersUserName), "password": hashedPassword, }, } } - kubeControllersSecureUserSecret, err := utils.GetSecret(ctx, cli, kubecontrollers.ElasticsearchKubeControllersSecureUserSecret, h.InstallNamespace()) + kubeControllersSecureUserSecret, err := utils.GetSecret(ctx, cli, enterprise.ElasticsearchKubeControllersSecureUserSecret, h.InstallNamespace()) if err != nil { return nil, nil, nil, err } if kubeControllersSecureUserSecret == nil { kubeControllersSecureUserSecret = &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ - Name: kubecontrollers.ElasticsearchKubeControllersSecureUserSecret, + Name: enterprise.ElasticsearchKubeControllersSecureUserSecret, Namespace: h.InstallNamespace(), Labels: map[string]string{ ESGatewaySelectorLabel: ESGatewaySelectorLabelValue, diff --git a/pkg/controller/logstorage/kubecontrollers/es_kube_controllers.go b/pkg/controller/logstorage/kubecontrollers/es_kube_controllers.go index c51cbc076a..b8f056159e 100644 --- a/pkg/controller/logstorage/kubecontrollers/es_kube_controllers.go +++ b/pkg/controller/logstorage/kubecontrollers/es_kube_controllers.go @@ -44,6 +44,7 @@ import ( "github.com/tigera/operator/pkg/controller/utils" "github.com/tigera/operator/pkg/controller/utils/imageset" "github.com/tigera/operator/pkg/ctrlruntime" + "github.com/tigera/operator/pkg/enterprise" "github.com/tigera/operator/pkg/render" "github.com/tigera/operator/pkg/render/common/networkpolicy" "github.com/tigera/operator/pkg/render/kubecontrollers" @@ -140,7 +141,7 @@ func Add(mgr manager.Manager, opts options.ControllerOptions) error { if err := utils.AddDeploymentWatch(c, esgateway.DeploymentName, esKubeControllersNamespace.InstallNamespace()); err != nil { return fmt.Errorf("log-storage-access-controller failed to watch the Service resource: %w", err) } - if err := utils.AddDeploymentWatch(c, kubecontrollers.EsKubeController, esKubeControllersNamespace.InstallNamespace()); err != nil { + if err := utils.AddDeploymentWatch(c, enterprise.EsKubeController, esKubeControllersNamespace.InstallNamespace()); err != nil { return fmt.Errorf("log-storage-access-controller failed to watch the Service resource: %w", err) } @@ -168,7 +169,7 @@ func Add(mgr manager.Manager, opts options.ControllerOptions) error { // Start goroutines to establish watches against projectcalico.org/v3 resources. go utils.WaitToAddTierWatch(networkpolicy.CalicoTierName, c, opts.K8sClientset, log, r.tierWatchReady) go utils.WaitToAddNetworkPolicyWatches(c, opts.K8sClientset, log, []types.NamespacedName{ - {Name: kubecontrollers.EsKubeControllerNetworkPolicyName, Namespace: esKubeControllersNamespace.InstallNamespace()}, + {Name: enterprise.EsKubeControllerNetworkPolicyName, Namespace: esKubeControllersNamespace.InstallNamespace()}, }) return nil @@ -262,7 +263,7 @@ func (r *ESKubeControllersController) Reconcile(ctx context.Context, request rec // Get secrets needed for kube-controllers to talk to elastic. This is needed for zero-tenants and single-tenants // that deploy es-kube-controllers and need to talk to es-gateway var kubeControllersUserSecret *core.Secret - kubeControllersUserSecret, err = utils.GetSecret(ctx, r.client, kubecontrollers.ElasticsearchKubeControllersUserSecret, helper.TruthNamespace()) + kubeControllersUserSecret, err = utils.GetSecret(ctx, r.client, enterprise.ElasticsearchKubeControllersUserSecret, helper.TruthNamespace()) if err != nil { r.status.SetDegraded(operatorv1.ResourceReadError, "Failed to get kube controllers gateway secret", err, reqLogger) return reconcile.Result{}, err @@ -338,13 +339,12 @@ func (r *ESKubeControllersController) Reconcile(ctx context.Context, request rec ClusterDomain: r.clusterDomain, Authentication: authentication, KubeControllersGatewaySecret: kubeControllersUserSecret, - LogStorageExists: logStorage != nil, TrustedBundle: trustedBundle, Namespace: helper.InstallNamespace(), BindingNamespaces: namespaces, Tenant: nil, } - esKubeControllerComponents := kubecontrollers.NewElasticsearchKubeControllers(&kubeControllersCfg) + esKubeControllerComponents := enterprise.NewElasticsearchKubeControllers(&kubeControllersCfg) imageSet, err := imageset.GetImageSet(ctx, r.client, variant) if err != nil { diff --git a/pkg/controller/logstorage/kubecontrollers/es_kube_controllers_test.go b/pkg/controller/logstorage/kubecontrollers/es_kube_controllers_test.go index b35d072664..c053617282 100644 --- a/pkg/controller/logstorage/kubecontrollers/es_kube_controllers_test.go +++ b/pkg/controller/logstorage/kubecontrollers/es_kube_controllers_test.go @@ -46,8 +46,8 @@ import ( "github.com/tigera/operator/pkg/controller/utils" ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" "github.com/tigera/operator/pkg/dns" + "github.com/tigera/operator/pkg/enterprise" "github.com/tigera/operator/pkg/render" - "github.com/tigera/operator/pkg/render/kubecontrollers" "github.com/tigera/operator/pkg/render/logstorage" "github.com/tigera/operator/pkg/render/logstorage/esgateway" "github.com/tigera/operator/pkg/tls/certificatemanagement" @@ -235,7 +235,7 @@ var _ = Describe("LogStorage ES kube-controllers controller", func() { dep := appsv1.Deployment{ TypeMeta: metav1.TypeMeta{Kind: "Deployment", APIVersion: "apps/v1"}, ObjectMeta: metav1.ObjectMeta{ - Name: kubecontrollers.EsKubeController, + Name: enterprise.EsKubeController, Namespace: common.CalicoNamespace, }, } @@ -275,12 +275,12 @@ var _ = Describe("LogStorage ES kube-controllers controller", func() { dep := appsv1.Deployment{ TypeMeta: metav1.TypeMeta{Kind: "Deployment", APIVersion: "apps/v1"}, ObjectMeta: metav1.ObjectMeta{ - Name: kubecontrollers.EsKubeController, + Name: enterprise.EsKubeController, Namespace: common.CalicoNamespace, }, } Expect(test.GetResource(cli, &dep)).To(BeNil()) - kc := test.GetContainer(dep.Spec.Template.Spec.Containers, kubecontrollers.EsKubeController) + kc := test.GetContainer(dep.Spec.Template.Spec.Containers, enterprise.EsKubeController) Expect(kc).ToNot(BeNil()) Expect(kc.Image).To(Equal(fmt.Sprintf("some.registry.org/%s%s@%s", components.TigeraImagePath, components.ComponentTigeraCalico.Image, "sha256:kubecontrollershash"))) }) @@ -325,7 +325,7 @@ var _ = Describe("LogStorage ES kube-controllers controller", func() { dep := appsv1.Deployment{ TypeMeta: metav1.TypeMeta{Kind: "Deployment", APIVersion: "apps/v1"}, ObjectMeta: metav1.ObjectMeta{ - Name: kubecontrollers.EsKubeController, + Name: enterprise.EsKubeController, Namespace: common.CalicoNamespace, }, } diff --git a/pkg/enterprise/kubecontrollers.go b/pkg/enterprise/kubecontrollers.go new file mode 100644 index 0000000000..388e7dac4a --- /dev/null +++ b/pkg/enterprise/kubecontrollers.go @@ -0,0 +1,177 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package enterprise + +import ( + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" + + "github.com/tigera/operator/pkg/render" + relasticsearch "github.com/tigera/operator/pkg/render/common/elasticsearch" + rmeta "github.com/tigera/operator/pkg/render/common/meta" + "github.com/tigera/operator/pkg/render/common/networkpolicy" + "github.com/tigera/operator/pkg/render/kubecontrollers" + "github.com/tigera/operator/pkg/url" +) + +const ( + EsKubeController = "es-calico-kube-controllers" + EsKubeControllerRole = "es-calico-kube-controllers" + EsKubeControllerRoleBinding = "es-calico-kube-controllers" + EsKubeControllerMetrics = "es-calico-kube-controllers-metrics" + EsKubeControllerNetworkPolicyName = networkpolicy.CalicoComponentPolicyPrefix + "es-kube-controller-access" + + ElasticsearchKubeControllersUserSecret = "tigera-ee-kube-controllers-elasticsearch-access" + ElasticsearchKubeControllersUserName = "tigera-ee-kube-controllers" + ElasticsearchKubeControllersSecureUserSecret = "tigera-ee-kube-controllers-elasticsearch-access-gateway" + ElasticsearchKubeControllersVerificationUserSecret = "tigera-ee-kube-controllers-gateway-verification-credentials" +) + +// NewElasticsearchKubeControllers fills the generic kube-controllers configuration +// for the enterprise es-calico-kube-controllers deployment and returns the rendered +// component. es-kube-controllers is a distinct deployment (talks to Elasticsearch via +// es-gateway) reconciled by the logstorage kube-controllers controller, so it's +// assembled here rather than through the render-time modifier mechanism. +func NewElasticsearchKubeControllers(cfg *kubecontrollers.KubeControllersConfiguration) render.Component { + cfg.Name = EsKubeController + cfg.ConfigName = "elasticsearch" + cfg.RoleName = EsKubeControllerRole + cfg.RoleBindingName = EsKubeControllerRoleBinding + cfg.MetricsName = EsKubeControllerMetrics + cfg.DisableConfigAPI = cfg.Tenant.MultiTenant() + + cfg.Rules = kubecontrollers.KubeControllersRoleCommonRules(cfg) + cfg.Rules = append(cfg.Rules, kubecontrollers.KubeControllersRoleEnterpriseCommonRules(cfg)...) + cfg.Rules = append(cfg.Rules, + rbacv1.PolicyRule{ + APIGroups: []string{"elasticsearch.k8s.elastic.co"}, + Resources: []string{"elasticsearches"}, + Verbs: []string{"watch", "get", "list"}, + }, + rbacv1.PolicyRule{ + APIGroups: []string{"rbac.authorization.k8s.io"}, + Resources: []string{"clusterroles", "clusterrolebindings"}, + Verbs: []string{"watch", "list", "get"}, + }, + ) + + if !cfg.Tenant.MultiTenant() { + // Zero and single tenant clusters need elasticsearch configuration. + cfg.EnabledControllers = append(cfg.EnabledControllers, "authorization", "elasticsearchconfiguration") + if cfg.ManagementCluster != nil && cfg.Tenant == nil { + // Enterprise requires the managedcluster controller to push licenses. + cfg.EnabledControllers = append(cfg.EnabledControllers, "managedcluster") + } + } + + cfg.NetworkPolicy = esKubeControllersCalicoSystemPolicy(cfg) + cfg.DeprecatedNetworkPolicyName = "es-kube-controller-access" + cfg.ExtraEnv = esKubeControllersEnv(cfg) + + return kubecontrollers.NewKubeControllers(cfg) +} + +// esKubeControllersEnv builds the enterprise env vars for es-calico-kube-controllers. +func esKubeControllersEnv(cfg *kubecontrollers.KubeControllersConfiguration) []corev1.EnvVar { + var env []corev1.EnvVar + + if cfg.Tenant != nil { + env = append(env, corev1.EnvVar{Name: "TENANT_ID", Value: cfg.Tenant.Spec.ID}) + } + + // What started as a workaround is now the default behaviour. This feature uses our backend in order to + // log into Kibana for users from external identity providers, rather than configuring an authn realm + // in the Elastic stack. + env = append(env, corev1.EnvVar{Name: "ENABLE_ELASTICSEARCH_OIDC_WORKAROUND", Value: "true"}) + if cfg.Authentication != nil { + env = append(env, + corev1.EnvVar{Name: "OIDC_AUTH_USERNAME_PREFIX", Value: cfg.Authentication.Spec.UsernamePrefix}, + corev1.EnvVar{Name: "OIDC_AUTH_GROUP_PREFIX", Value: cfg.Authentication.Spec.GroupsPrefix}, + ) + } + + if cfg.TrustedBundle != nil { + env = append(env, corev1.EnvVar{Name: "MULTI_CLUSTER_FORWARDING_CA", Value: cfg.TrustedBundle.MountPath()}) + } + if cfg.Installation.CalicoNetwork != nil && cfg.Installation.CalicoNetwork.MultiInterfaceMode != nil { + env = append(env, corev1.EnvVar{Name: "MULTI_INTERFACE_MODE", Value: cfg.Installation.CalicoNetwork.MultiInterfaceMode.Value()}) + } + + if !cfg.Tenant.MultiTenant() { + _, esHost, esPort, _ := url.ParseEndpoint(relasticsearch.GatewayEndpoint(rmeta.OSTypeLinux, cfg.ClusterDomain, render.ElasticsearchNamespace)) + env = append(env, + relasticsearch.ElasticHostEnvVar(esHost), + relasticsearch.ElasticPortEnvVar(esPort), + relasticsearch.ElasticUsernameEnvVar(ElasticsearchKubeControllersUserSecret), + relasticsearch.ElasticPasswordEnvVar(ElasticsearchKubeControllersUserSecret), + relasticsearch.ElasticCAEnvVar(rmeta.OSTypeLinux), + ) + } + + return env +} + +func esKubeControllersCalicoSystemPolicy(cfg *kubecontrollers.KubeControllersConfiguration) *v3.NetworkPolicy { + if cfg.ManagementClusterConnection != nil { + return nil + } + + egressRules := []v3.Rule{} + egressRules = networkpolicy.AppendDNSEgressRules(egressRules, cfg.Installation.KubernetesProvider.IsOpenShift()) + egressRules = append(egressRules, []v3.Rule{ + { + Action: v3.Allow, + Protocol: &networkpolicy.TCPProtocol, + Destination: v3.EntityRule{ + Ports: networkpolicy.Ports(443, 6443, 12388), + }, + }, + }...) + + egressRules = append(egressRules, []v3.Rule{ + { + Action: v3.Allow, + Protocol: &networkpolicy.TCPProtocol, + Destination: networkpolicy.DefaultHelper().ESGatewayEntityRule(), + }, + }...) + + networkpolicyHelper := networkpolicy.Helper(cfg.Tenant.MultiTenant(), cfg.Namespace) + egressRules = append(egressRules, []v3.Rule{ + { + Action: v3.Allow, + Protocol: &networkpolicy.TCPProtocol, + Destination: networkpolicyHelper.ManagerEntityRule(), + }, + }...) + + return &v3.NetworkPolicy{ + TypeMeta: metav1.TypeMeta{Kind: "NetworkPolicy", APIVersion: "projectcalico.org/v3"}, + ObjectMeta: metav1.ObjectMeta{ + Name: EsKubeControllerNetworkPolicyName, + Namespace: cfg.Namespace, + }, + Spec: v3.NetworkPolicySpec{ + Order: &networkpolicy.HighPrecedenceOrder, + Tier: networkpolicy.CalicoTierName, + Selector: networkpolicy.KubernetesAppSelector(EsKubeController), + Types: []v3.PolicyType{v3.PolicyTypeEgress}, + Egress: egressRules, + }, + } +} diff --git a/pkg/render/kubecontrollers/kube-controllers.go b/pkg/render/kubecontrollers/kube-controllers.go index 1f592f8358..b6340efb74 100644 --- a/pkg/render/kubecontrollers/kube-controllers.go +++ b/pkg/render/kubecontrollers/kube-controllers.go @@ -38,7 +38,6 @@ import ( "github.com/tigera/operator/pkg/render" "github.com/tigera/operator/pkg/render/applicationlayer" rcomp "github.com/tigera/operator/pkg/render/common/components" - relasticsearch "github.com/tigera/operator/pkg/render/common/elasticsearch" rmeta "github.com/tigera/operator/pkg/render/common/meta" "github.com/tigera/operator/pkg/render/common/networkpolicy" "github.com/tigera/operator/pkg/render/common/secret" @@ -46,7 +45,6 @@ import ( "github.com/tigera/operator/pkg/render/common/securitycontextconstraints" "github.com/tigera/operator/pkg/render/monitor" "github.com/tigera/operator/pkg/tls/certificatemanagement" - "github.com/tigera/operator/pkg/url" ) const ( @@ -72,18 +70,12 @@ const ( // bundle, provisioned by the core controller and passed in as WASMCACert. WASMCACertName = "tigera-waf-ca-bundle" - EsKubeController = "es-calico-kube-controllers" - EsKubeControllerRole = "es-calico-kube-controllers" - EsKubeControllerRoleBinding = "es-calico-kube-controllers" - EsKubeControllerMetrics = "es-calico-kube-controllers-metrics" - EsKubeControllerNetworkPolicyName = networkpolicy.CalicoComponentPolicyPrefix + "es-kube-controller-access" + // ManagedClustersWatchRoleBindingName binds kube-controllers to the managed-cluster + // watch ClusterRole. Used by both calico-kube-controllers (in a management cluster) + // and the enterprise es-calico-kube-controllers, so the binding stays generic here. ManagedClustersWatchRoleBindingName = "es-calico-kube-controllers-managed-cluster-watch" - ElasticsearchKubeControllersUserSecret = "tigera-ee-kube-controllers-elasticsearch-access" - ElasticsearchKubeControllersUserName = "tigera-ee-kube-controllers" - ElasticsearchKubeControllersSecureUserSecret = "tigera-ee-kube-controllers-elasticsearch-access-gateway" - ElasticsearchKubeControllersVerificationUserSecret = "tigera-ee-kube-controllers-gateway-verification-credentials" - KubeControllerPrometheusTLSSecret = "calico-kube-controllers-metrics-tls" + KubeControllerPrometheusTLSSecret = "calico-kube-controllers-metrics-tls" // KubeControllersHealthPort is the port the kube-controllers HealthAggregator listens on when run from the // combined calico binary. The legacy per-component image uses file-based health checks instead. @@ -99,9 +91,6 @@ type KubeControllersConfiguration struct { ManagementClusterConnection *operatorv1.ManagementClusterConnection Authentication *operatorv1.Authentication - // Whether or not the LogStorage CRD is present in the cluster. - LogStorageExists bool - ClusterDomain string MetricsPort int @@ -150,6 +139,39 @@ type KubeControllersConfiguration struct { // caBundle so the apiserver can verify the in-process webhook endpoint. // Only consulted when WAFGatewayExtensionEnabled is true. WAFWebhookCABundle []byte + + // The fields below parameterize the generic kube-controllers component. The + // variant assemblers (NewCalicoKubeControllers, the enterprise es builder) + // fill them; the component renders them without any variant or component-name + // branching. + + // Name is the deployment / pod / container name (and the value the metrics + // Service selects on). + Name string + // ConfigName is the KUBE_CONTROLLERS_CONFIG_NAME the binary reconciles. + ConfigName string + // RoleName / RoleBindingName / MetricsName name the ClusterRole, its binding, + // and the Prometheus metrics Service. + RoleName string + RoleBindingName string + MetricsName string + // EnabledControllers is the ENABLED_CONTROLLERS list. The deployment is only + // rendered when it is non-empty. + EnabledControllers []string + // Rules are the ClusterRole policy rules. + Rules []rbacv1.PolicyRule + // NetworkPolicy, when set, is rendered into the install namespace (and the + // deprecated allow-tigera policy named DeprecatedNetworkPolicyName is deleted). + NetworkPolicy *v3.NetworkPolicy + DeprecatedNetworkPolicyName string + // ExtraEnv is appended to the deployment's container env. + ExtraEnv []corev1.EnvVar + // DisableConfigAPI sets DISABLE_KUBE_CONTROLLERS_CONFIG_API. + DisableConfigAPI bool + // ManageWAFWebhook makes this component own the in-process WAF admission + // webhook surface lifecycle (rendered when WAFGatewayExtensionEnabled, deleted + // otherwise). Only the calico-kube-controllers component sets this. + ManageWAFWebhook bool } func NewCalicoKubeControllersPolicy(cfg *KubeControllersConfiguration, defaultDeny *v3.NetworkPolicy) render.Component { @@ -169,12 +191,27 @@ func NewCalicoKubeControllersPolicy(cfg *KubeControllersConfiguration, defaultDe ) } -func NewCalicoKubeControllers(cfg *KubeControllersConfiguration) *kubeControllersComponent { - kubeControllerRolePolicyRules := kubeControllersRoleCommonRules(cfg) - enabledControllers := []string{"node", "loadbalancer"} +// NewKubeControllers builds a kube-controllers component from a fully-populated +// configuration. Callers (NewCalicoKubeControllers, the enterprise es-kube-controllers +// builder) fill the generic Name/Rules/EnabledControllers/ExtraEnv/NetworkPolicy fields; +// the component renders them with no variant branching. +func NewKubeControllers(cfg *KubeControllersConfiguration) render.Component { + return &kubeControllersComponent{cfg: cfg} +} + +func NewCalicoKubeControllers(cfg *KubeControllersConfiguration) render.Component { + cfg.Name = KubeController + cfg.ConfigName = "default" + cfg.RoleName = KubeControllerRole + cfg.RoleBindingName = KubeControllerRoleBinding + cfg.MetricsName = KubeControllerMetrics + cfg.ManageWAFWebhook = true + + cfg.Rules = KubeControllersRoleCommonRules(cfg) + cfg.EnabledControllers = []string{"node", "loadbalancer"} if cfg.Installation.Variant.IsEnterprise() { - kubeControllerRolePolicyRules = append(kubeControllerRolePolicyRules, kubeControllersRoleEnterpriseCommonRules(cfg)...) - kubeControllerRolePolicyRules = append(kubeControllerRolePolicyRules, + cfg.Rules = append(cfg.Rules, KubeControllersRoleEnterpriseCommonRules(cfg)...) + cfg.Rules = append(cfg.Rules, rbacv1.PolicyRule{ APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, Resources: []string{"remoteclusterconfigurations"}, @@ -196,69 +233,31 @@ func NewCalicoKubeControllers(cfg *KubeControllersConfiguration) *kubeController Verbs: []string{"create", "update", "delete", "watch", "list", "get"}, }, ) - enabledControllers = append(enabledControllers, "service", "federatedservices", "usage") + cfg.EnabledControllers = append(cfg.EnabledControllers, "service", "federatedservices", "usage") if cfg.WAFGatewayExtensionEnabled { - enabledControllers = append(enabledControllers, "applicationlayer") + cfg.EnabledControllers = append(cfg.EnabledControllers, "applicationlayer") } + cfg.ExtraEnv = calicoEnterpriseEnv(cfg) } - return &kubeControllersComponent{ - cfg: cfg, - kubeControllerServiceAccountName: KubeControllerServiceAccount, - kubeControllerRoleName: KubeControllerRole, - kubeControllerRoleBindingName: KubeControllerRoleBinding, - kubeControllerName: KubeController, - kubeControllerConfigName: "default", - kubeControllerMetricsName: KubeControllerMetrics, - kubeControllersRules: kubeControllerRolePolicyRules, - enabledControllers: enabledControllers, - } + return NewKubeControllers(cfg) } -func NewElasticsearchKubeControllers(cfg *KubeControllersConfiguration) *kubeControllersComponent { - var kubeControllerCalicoSystemPolicy *v3.NetworkPolicy - kubeControllerRolePolicyRules := kubeControllersRoleCommonRules(cfg) - - if cfg.Installation.Variant.IsEnterprise() { - kubeControllerRolePolicyRules = append(kubeControllerRolePolicyRules, kubeControllersRoleEnterpriseCommonRules(cfg)...) - kubeControllerRolePolicyRules = append(kubeControllerRolePolicyRules, - rbacv1.PolicyRule{ - APIGroups: []string{"elasticsearch.k8s.elastic.co"}, - Resources: []string{"elasticsearches"}, - Verbs: []string{"watch", "get", "list"}, - }, - rbacv1.PolicyRule{ - APIGroups: []string{"rbac.authorization.k8s.io"}, - Resources: []string{"clusterroles", "clusterrolebindings"}, - Verbs: []string{"watch", "list", "get"}, - }, - ) - - kubeControllerCalicoSystemPolicy = esKubeControllersCalicoSystemPolicy(cfg) +// calicoEnterpriseEnv builds the enterprise-only static env vars for +// calico-kube-controllers. The dynamic WASM_* vars depend on resolved images and +// are added at deployment-render time. +func calicoEnterpriseEnv(cfg *KubeControllersConfiguration) []corev1.EnvVar { + var env []corev1.EnvVar + if cfg.Tenant != nil { + env = append(env, corev1.EnvVar{Name: "TENANT_ID", Value: cfg.Tenant.Spec.ID}) } - - var enabledControllers []string - if !cfg.Tenant.MultiTenant() { - // Zero and single tenant cluster needs elasticsearch configuration - enabledControllers = append(enabledControllers, "authorization", "elasticsearchconfiguration") - if cfg.ManagementCluster != nil && cfg.Tenant == nil { - // Enterprise will require the managedcluster controller to push licenses - enabledControllers = append(enabledControllers, "managedcluster") - } + if cfg.TrustedBundle != nil { + env = append(env, corev1.EnvVar{Name: "MULTI_CLUSTER_FORWARDING_CA", Value: cfg.TrustedBundle.MountPath()}) } - - return &kubeControllersComponent{ - cfg: cfg, - kubeControllerServiceAccountName: KubeControllerServiceAccount, - kubeControllerRoleName: EsKubeControllerRole, - kubeControllerRoleBindingName: EsKubeControllerRoleBinding, - kubeControllerName: EsKubeController, - kubeControllerConfigName: "elasticsearch", - kubeControllerMetricsName: EsKubeControllerMetrics, - kubeControllersRules: kubeControllerRolePolicyRules, - kubeControllerCalicoSystemPolicy: kubeControllerCalicoSystemPolicy, - enabledControllers: enabledControllers, + if cfg.Installation.CalicoNetwork != nil && cfg.Installation.CalicoNetwork.MultiInterfaceMode != nil { + env = append(env, corev1.EnvVar{Name: "MULTI_INTERFACE_MODE", Value: cfg.Installation.CalicoNetwork.MultiInterfaceMode.Value()}) } + return env } type kubeControllersComponent struct { @@ -268,18 +267,6 @@ type kubeControllersComponent struct { // Internal state generated by the given configuration. calicoImage string - kubeControllerServiceAccountName string - kubeControllerRoleName string - kubeControllerRoleBindingName string - kubeControllerName string - kubeControllerConfigName string - kubeControllerMetricsName string - - kubeControllersRules []rbacv1.PolicyRule - kubeControllerCalicoSystemPolicy *v3.NetworkPolicy - - enabledControllers []string - // wasmImage is the fully-resolved OCI reference for the Coraza WAF wasm // binary (Enterprise only). Surfaced to the kube-controllers binary via // the WASM_IMAGE env var; consumed by the applicationlayer reconcilers @@ -296,7 +283,7 @@ func (c *kubeControllersComponent) ResolveImages(is *operatorv1.ImageSet) error if err != nil { return err } - if c.cfg.Installation.Variant.IsEnterprise() && c.cfg.WAFGatewayExtensionEnabled { + if c.cfg.WAFGatewayExtensionEnabled { // The Coraza WAF wasm is baked into the gateway envoy-proxy image as its // final layer; Envoy Gateway extracts it from there. Point WASM_IMAGE at // that same image (no standalone coraza-wasm image needed). @@ -316,12 +303,14 @@ func (c *kubeControllersComponent) Objects() ([]client.Object, []client.Object) objectsToCreate := []client.Object{} objectsToDelete := []client.Object{} - if c.kubeControllerCalicoSystemPolicy != nil { - objectsToCreate = append(objectsToCreate, c.kubeControllerCalicoSystemPolicy) - // allow-tigera Tier was renamed to calico-system - objectsToDelete = append(objectsToDelete, - networkpolicy.DeprecatedAllowTigeraNetworkPolicyObject("es-kube-controller-access", c.cfg.Namespace), - ) + if c.cfg.NetworkPolicy != nil { + objectsToCreate = append(objectsToCreate, c.cfg.NetworkPolicy) + if c.cfg.DeprecatedNetworkPolicyName != "" { + // allow-tigera Tier was renamed to calico-system + objectsToDelete = append(objectsToDelete, + networkpolicy.DeprecatedAllowTigeraNetworkPolicyObject(c.cfg.DeprecatedNetworkPolicyName, c.cfg.Namespace), + ) + } } objectsToCreate = append(objectsToCreate, @@ -331,7 +320,7 @@ func (c *kubeControllersComponent) Objects() ([]client.Object, []client.Object) ) objectsToCreate = append(objectsToCreate, c.managedClusterRoleBindings()...) - if len(c.enabledControllers) > 0 { + if len(c.cfg.EnabledControllers) > 0 { // There's something to run, so create the deployment. objectsToCreate = append(objectsToCreate, c.controllersDeployment()) } else { @@ -358,7 +347,7 @@ func (c *kubeControllersComponent) Objects() ([]client.Object, []client.Object) // ValidatingWebhookConfiguration). Rendered here, rather than as a // passthrough in the core controller, so the objects are cleaned up when the // WAF extension is disabled or the GatewayAPI CR is removed. - if c.kubeControllerName == KubeController { + if c.cfg.ManageWAFWebhook { webhookObjs := applicationlayer.WAFAdmissionWebhookComponents(c.cfg.WAFWebhookCABundle) if c.cfg.WAFGatewayExtensionEnabled { objectsToCreate = append(objectsToCreate, webhookObjs...) @@ -385,7 +374,7 @@ func (c *kubeControllersComponent) Ready() bool { return true } -func kubeControllersRoleCommonRules(cfg *KubeControllersConfiguration) []rbacv1.PolicyRule { +func KubeControllersRoleCommonRules(cfg *KubeControllersConfiguration) []rbacv1.PolicyRule { rules := []rbacv1.PolicyRule{ { // Nodes are watched to monitor for deletions. @@ -511,7 +500,7 @@ func kubeControllersRoleCommonRules(cfg *KubeControllersConfiguration) []rbacv1. return rules } -func kubeControllersRoleEnterpriseCommonRules(cfg *KubeControllersConfiguration) []rbacv1.PolicyRule { +func KubeControllersRoleEnterpriseCommonRules(cfg *KubeControllersConfiguration) []rbacv1.PolicyRule { rules := []rbacv1.PolicyRule{ { APIGroups: []string{""}, @@ -662,7 +651,7 @@ func (c *kubeControllersComponent) controllersServiceAccount() *corev1.ServiceAc return &corev1.ServiceAccount{ TypeMeta: metav1.TypeMeta{Kind: "ServiceAccount", APIVersion: "v1"}, ObjectMeta: metav1.ObjectMeta{ - Name: c.kubeControllerServiceAccountName, + Name: KubeControllerServiceAccount, Namespace: c.cfg.Namespace, Labels: map[string]string{}, }, @@ -673,9 +662,9 @@ func (c *kubeControllersComponent) controllersClusterRole() *rbacv1.ClusterRole role := &rbacv1.ClusterRole{ TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, ObjectMeta: metav1.ObjectMeta{ - Name: c.kubeControllerRoleName, + Name: c.cfg.RoleName, }, - Rules: c.kubeControllersRules, + Rules: c.cfg.Rules, } return role @@ -698,7 +687,7 @@ func (c *kubeControllersComponent) controllersOCPFederationRoleBinding() *rbacv1 Subjects: []rbacv1.Subject{ { Kind: "ServiceAccount", - Name: KubeController, + Name: KubeControllerServiceAccount, Namespace: c.cfg.Namespace, }, }, @@ -707,71 +696,46 @@ func (c *kubeControllersComponent) controllersOCPFederationRoleBinding() *rbacv1 func (c *kubeControllersComponent) controllersDeployment() *appsv1.Deployment { env := []corev1.EnvVar{ - {Name: "KUBE_CONTROLLERS_CONFIG_NAME", Value: c.kubeControllerConfigName}, + {Name: "KUBE_CONTROLLERS_CONFIG_NAME", Value: c.cfg.ConfigName}, {Name: "DATASTORE_TYPE", Value: "kubernetes"}, - {Name: "ENABLED_CONTROLLERS", Value: strings.Join(c.enabledControllers, ",")}, - {Name: "DISABLE_KUBE_CONTROLLERS_CONFIG_API", Value: strconv.FormatBool(c.cfg.Tenant.MultiTenant() && c.kubeControllerConfigName == "elasticsearch")}, + {Name: "ENABLED_CONTROLLERS", Value: strings.Join(c.cfg.EnabledControllers, ",")}, + {Name: "DISABLE_KUBE_CONTROLLERS_CONFIG_API", Value: strconv.FormatBool(c.cfg.DisableConfigAPI)}, } env = append(env, c.cfg.K8sServiceEpPodNetwork.EnvVars()...) - - if c.cfg.Installation.Variant.IsEnterprise() { - if c.cfg.Tenant != nil { - env = append(env, corev1.EnvVar{Name: "TENANT_ID", Value: c.cfg.Tenant.Spec.ID}) - } - - if c.kubeControllerName == EsKubeController { - // What started as a workaround is now the default behaviour. This feature uses our backend in order to - // log into Kibana for users from external identity providers, rather than configuring an authn realm - // in the Elastic stack. - env = append(env, corev1.EnvVar{Name: "ENABLE_ELASTICSEARCH_OIDC_WORKAROUND", Value: "true"}) - - if c.cfg.Authentication != nil { - env = append(env, - corev1.EnvVar{Name: "OIDC_AUTH_USERNAME_PREFIX", Value: c.cfg.Authentication.Spec.UsernamePrefix}, - corev1.EnvVar{Name: "OIDC_AUTH_GROUP_PREFIX", Value: c.cfg.Authentication.Spec.GroupsPrefix}, - ) - } - } - if c.cfg.TrustedBundle != nil { - env = append(env, corev1.EnvVar{Name: "MULTI_CLUSTER_FORWARDING_CA", Value: c.cfg.TrustedBundle.MountPath()}) + env = append(env, c.cfg.ExtraEnv...) + + // Application-layer (gateway-addons / WAF v3) env vars, gated by + // GatewayAPI.spec.extensions.waf.state == Enabled. When the gate is off + // (default), none of the WASM_* env vars are rendered and the kube-controllers + // binary skips the WAF reconcilers entirely (see the applicationlayer entry in + // EnabledControllers). The WASM_IMAGE value depends on resolved images, so this + // is rendered here rather than in the static ExtraEnv. + if c.cfg.WAFGatewayExtensionEnabled { + // Application-layer (gateway-addons) reconcilers consume the Coraza WAF + // wasm OCI reference from this env var to program WAF policy attachments. + // Empty when ResolveImages was not called for the Calico variant; the + // reconciler stamps Programmed=False/WASMUnavailable in that case. + if c.wasmImage != "" { + env = append(env, corev1.EnvVar{Name: "WASM_IMAGE", Value: c.wasmImage}) } - if c.cfg.Installation.CalicoNetwork != nil && c.cfg.Installation.CalicoNetwork.MultiInterfaceMode != nil { - env = append(env, corev1.EnvVar{Name: "MULTI_INTERFACE_MODE", Value: c.cfg.Installation.CalicoNetwork.MultiInterfaceMode.Value()}) + // WASM_PULL_SECRET names the imagePullSecret the reconciler replicates + // from the kube-controllers namespace into a WAFPolicy's namespace so + // the rendered EnvoyExtensionPolicy can pull the wasm OCI artifact from + // a private Tigera registry. Source the name from the first + // Installation.ImagePullSecrets entry so multi-tenant / BYO-registry + // installs reuse whatever pull secret operator already attaches here. + if c.cfg.WASMPullSecret != nil { + env = append(env, corev1.EnvVar{Name: "WASM_PULL_SECRET", Value: c.cfg.WASMPullSecret.Name}) } - // Application-layer (gateway-addons / WAF v3) env vars, gated by - // GatewayAPI.spec.extensions.waf.state == Enabled. When the gate is - // off (default), none of the WASM_* env vars are rendered and the - // kube-controllers binary skips the WAF reconcilers entirely (see the - // applicationlayer entry in enabledControllers). - if c.cfg.WAFGatewayExtensionEnabled { - // Application-layer (gateway-addons) reconcilers consume the Coraza WAF - // wasm OCI reference from this env var to program WAF policy attachments. - // Empty when ResolveImages was not called for the Calico variant; the - // reconciler stamps Programmed=False/WASMUnavailable in that case. - if c.wasmImage != "" { - env = append(env, corev1.EnvVar{Name: "WASM_IMAGE", Value: c.wasmImage}) - } - - // WASM_PULL_SECRET names the imagePullSecret the reconciler replicates - // from the kube-controllers namespace into a WAFPolicy's namespace so - // the rendered EnvoyExtensionPolicy can pull the wasm OCI artifact from - // a private Tigera registry. Source the name from the first - // Installation.ImagePullSecrets entry so multi-tenant / BYO-registry - // installs reuse whatever pull secret operator already attaches here. - if c.cfg.WASMPullSecret != nil { - env = append(env, corev1.EnvVar{Name: "WASM_PULL_SECRET", Value: c.cfg.WASMPullSecret.Name}) - } - - // WASM_CA_CERT names the dedicated CA bundle ConfigMap (provisioned as - // WASMCACert) that the reconciler replicates alongside WASM_PULL_SECRET - // so the EnvoyExtensionPolicy wasm fetcher trusts the registry's TLS - // chain. Only set when the source ConfigMap is actually rendered. - if c.cfg.WASMCACert != nil { - env = append(env, corev1.EnvVar{Name: "WASM_CA_CERT", Value: c.cfg.WASMCACert.Name}) - } + // WASM_CA_CERT names the dedicated CA bundle ConfigMap (provisioned as + // WASMCACert) that the reconciler replicates alongside WASM_PULL_SECRET + // so the EnvoyExtensionPolicy wasm fetcher trusts the registry's TLS + // chain. Only set when the source ConfigMap is actually rendered. + if c.cfg.WASMCACert != nil { + env = append(env, corev1.EnvVar{Name: "WASM_CA_CERT", Value: c.cfg.WASMCACert.Name}) } } @@ -828,7 +792,7 @@ func (c *kubeControllersComponent) controllersDeployment() *appsv1.Deployment { } container := corev1.Container{ - Name: c.kubeControllerName, + Name: c.cfg.Name, Image: c.calicoImage, Command: containerCommand, Env: env, @@ -849,17 +813,6 @@ func (c *kubeControllersComponent) controllersDeployment() *appsv1.Deployment { }) } - if c.kubeControllerName == EsKubeController && !c.cfg.Tenant.MultiTenant() { - _, esHost, esPort, _ := url.ParseEndpoint(relasticsearch.GatewayEndpoint(c.SupportedOSType(), c.cfg.ClusterDomain, render.ElasticsearchNamespace)) - container.Env = append(container.Env, []corev1.EnvVar{ - relasticsearch.ElasticHostEnvVar(esHost), - relasticsearch.ElasticPortEnvVar(esPort), - relasticsearch.ElasticUsernameEnvVar(ElasticsearchKubeControllersUserSecret), - relasticsearch.ElasticPasswordEnvVar(ElasticsearchKubeControllersUserSecret), - relasticsearch.ElasticCAEnvVar(c.SupportedOSType()), - }...) - } - var initContainers []corev1.Container if c.cfg.MetricsServerTLS != nil && c.cfg.MetricsServerTLS.UseCertificateManagement() { initContainers = append(initContainers, c.cfg.MetricsServerTLS.InitContainer(c.cfg.Namespace, sc)) @@ -875,7 +828,7 @@ func (c *kubeControllersComponent) controllersDeployment() *appsv1.Deployment { NodeSelector: c.cfg.Installation.ControlPlaneNodeSelector, Tolerations: tolerations, ImagePullSecrets: c.cfg.Installation.ImagePullSecrets, - ServiceAccountName: c.kubeControllerServiceAccountName, + ServiceAccountName: KubeControllerServiceAccount, InitContainers: initContainers, Containers: []corev1.Container{container}, Volumes: c.kubeControllersVolumes(), @@ -886,7 +839,7 @@ func (c *kubeControllersComponent) controllersDeployment() *appsv1.Deployment { d := appsv1.Deployment{ TypeMeta: metav1.TypeMeta{Kind: "Deployment", APIVersion: "apps/v1"}, ObjectMeta: metav1.ObjectMeta{ - Name: c.kubeControllerName, + Name: c.cfg.Name, Namespace: c.cfg.Namespace, }, Spec: appsv1.DeploymentSpec{ @@ -896,7 +849,7 @@ func (c *kubeControllersComponent) controllersDeployment() *appsv1.Deployment { }, Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ - Name: c.kubeControllerName, + Name: c.cfg.Name, Namespace: c.cfg.Namespace, Annotations: c.annotations(), }, @@ -928,20 +881,20 @@ func (c *kubeControllersComponent) controllersClusterRoleBinding() *rbacv1.Clust for _, ns := range c.cfg.BindingNamespaces { subjects = append(subjects, rbacv1.Subject{ Kind: "ServiceAccount", - Name: c.kubeControllerServiceAccountName, + Name: KubeControllerServiceAccount, Namespace: ns, }) } return &rbacv1.ClusterRoleBinding{ TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, ObjectMeta: metav1.ObjectMeta{ - Name: c.kubeControllerRoleBindingName, + Name: c.cfg.RoleBindingName, Labels: map[string]string{}, }, RoleRef: rbacv1.RoleRef{ APIGroup: "rbac.authorization.k8s.io", Kind: "ClusterRole", - Name: c.kubeControllerRoleName, + Name: c.cfg.RoleName, }, Subjects: subjects, } @@ -950,7 +903,7 @@ func (c *kubeControllersComponent) controllersClusterRoleBinding() *rbacv1.Clust func (c *kubeControllersComponent) managedClusterRoleBindings() []client.Object { if c.cfg.ManagementCluster != nil { return []client.Object{ - rcomp.ClusterRoleBinding(ManagedClustersWatchRoleBindingName, render.ManagedClustersWatchClusterRoleName, c.kubeControllerServiceAccountName, []string{c.cfg.Namespace}), + rcomp.ClusterRoleBinding(ManagedClustersWatchRoleBindingName, render.ManagedClustersWatchClusterRoleName, KubeControllerServiceAccount, []string{c.cfg.Namespace}), } } return []client.Object{} @@ -962,16 +915,16 @@ func (c *kubeControllersComponent) prometheusService() *corev1.Service { return &corev1.Service{ TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}, ObjectMeta: metav1.ObjectMeta{ - Name: c.kubeControllerMetricsName, + Name: c.cfg.MetricsName, Namespace: c.cfg.Namespace, Annotations: map[string]string{ "prometheus.io/scrape": "true", "prometheus.io/port": fmt.Sprintf("%d", c.cfg.MetricsPort), }, - Labels: map[string]string{"k8s-app": c.kubeControllerName}, + Labels: map[string]string{"k8s-app": c.cfg.Name}, }, Spec: corev1.ServiceSpec{ - Selector: map[string]string{"k8s-app": c.kubeControllerName}, + Selector: map[string]string{"k8s-app": c.cfg.Name}, // "Headless" service; prevent kube-proxy from rendering any rules for this service // (which is only intended for Prometheus to scrape). ClusterIP: "None", @@ -1114,53 +1067,3 @@ func kubeControllersCalicoSystemPolicy(cfg *KubeControllersConfiguration) *v3.Ne }, } } - -func esKubeControllersCalicoSystemPolicy(cfg *KubeControllersConfiguration) *v3.NetworkPolicy { - if cfg.ManagementClusterConnection != nil { - return nil - } - - egressRules := []v3.Rule{} - egressRules = networkpolicy.AppendDNSEgressRules(egressRules, cfg.Installation.KubernetesProvider.IsOpenShift()) - egressRules = append(egressRules, []v3.Rule{ - { - Action: v3.Allow, - Protocol: &networkpolicy.TCPProtocol, - Destination: v3.EntityRule{ - Ports: networkpolicy.Ports(443, 6443, 12388), - }, - }, - }...) - - egressRules = append(egressRules, []v3.Rule{ - { - Action: v3.Allow, - Protocol: &networkpolicy.TCPProtocol, - Destination: networkpolicy.DefaultHelper().ESGatewayEntityRule(), - }, - }...) - - networkpolicyHelper := networkpolicy.Helper(cfg.Tenant.MultiTenant(), cfg.Namespace) - egressRules = append(egressRules, []v3.Rule{ - { - Action: v3.Allow, - Protocol: &networkpolicy.TCPProtocol, - Destination: networkpolicyHelper.ManagerEntityRule(), - }, - }...) - - return &v3.NetworkPolicy{ - TypeMeta: metav1.TypeMeta{Kind: "NetworkPolicy", APIVersion: "projectcalico.org/v3"}, - ObjectMeta: metav1.ObjectMeta{ - Name: EsKubeControllerNetworkPolicyName, - Namespace: cfg.Namespace, - }, - Spec: v3.NetworkPolicySpec{ - Order: &networkpolicy.HighPrecedenceOrder, - Tier: networkpolicy.CalicoTierName, - Selector: networkpolicy.KubernetesAppSelector(EsKubeController), - Types: []v3.PolicyType{v3.PolicyTypeEgress}, - Egress: egressRules, - }, - } -} diff --git a/pkg/render/kubecontrollers/kube-controllers_test.go b/pkg/render/kubecontrollers/kube-controllers_test.go index 9f30802d99..8680614827 100644 --- a/pkg/render/kubecontrollers/kube-controllers_test.go +++ b/pkg/render/kubecontrollers/kube-controllers_test.go @@ -41,6 +41,7 @@ import ( "github.com/tigera/operator/pkg/controller/k8sapi" ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" "github.com/tigera/operator/pkg/dns" + "github.com/tigera/operator/pkg/enterprise" "github.com/tigera/operator/pkg/render" "github.com/tigera/operator/pkg/render/applicationlayer" rmeta "github.com/tigera/operator/pkg/render/common/meta" @@ -436,23 +437,22 @@ var _ = Describe("kube-controllers rendering tests", func() { version string kind string }{ - {name: kubecontrollers.EsKubeControllerNetworkPolicyName, ns: common.CalicoNamespace, group: "projectcalico.org", version: "v3", kind: "NetworkPolicy"}, + {name: enterprise.EsKubeControllerNetworkPolicyName, ns: common.CalicoNamespace, group: "projectcalico.org", version: "v3", kind: "NetworkPolicy"}, {name: "calico-kube-controllers", ns: common.CalicoNamespace, group: "", version: "v1", kind: "ServiceAccount"}, - {name: kubecontrollers.EsKubeControllerRole, ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRole"}, - {name: kubecontrollers.EsKubeControllerRoleBinding, ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRoleBinding"}, - {name: kubecontrollers.EsKubeController, ns: common.CalicoNamespace, group: "apps", version: "v1", kind: "Deployment"}, - {name: kubecontrollers.ElasticsearchKubeControllersUserSecret, ns: common.CalicoNamespace, group: "", version: "v1", kind: "Secret"}, - {name: kubecontrollers.EsKubeControllerMetrics, ns: common.CalicoNamespace, group: "", version: "v1", kind: "Service"}, + {name: enterprise.EsKubeControllerRole, ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRole"}, + {name: enterprise.EsKubeControllerRoleBinding, ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRoleBinding"}, + {name: enterprise.EsKubeController, ns: common.CalicoNamespace, group: "apps", version: "v1", kind: "Deployment"}, + {name: enterprise.ElasticsearchKubeControllersUserSecret, ns: common.CalicoNamespace, group: "", version: "v1", kind: "Secret"}, + {name: enterprise.EsKubeControllerMetrics, ns: common.CalicoNamespace, group: "", version: "v1", kind: "Service"}, } instance.Variant = operatorv1.CalicoEnterprise - cfg.LogStorageExists = true cfg.KubeControllersGatewaySecret = &testutils.KubeControllersUserSecret cfg.MetricsPort = 9094 // Opt in to the WAF Gateway API add-on so the WAF env vars + RBAC are rendered. cfg.WAFGatewayExtensionEnabled = true - component := kubecontrollers.NewElasticsearchKubeControllers(&cfg) + component := enterprise.NewElasticsearchKubeControllers(&cfg) Expect(component.ResolveImages(nil)).To(BeNil()) resources, _ := component.Objects() Expect(len(resources)).To(Equal(len(expectedResources))) @@ -465,7 +465,7 @@ var _ = Describe("kube-controllers rendering tests", func() { } // The Deployment should have the correct configuration. - dp := rtest.GetResource(resources, kubecontrollers.EsKubeController, common.CalicoNamespace, "apps", "v1", "Deployment").(*appsv1.Deployment) + dp := rtest.GetResource(resources, enterprise.EsKubeController, common.CalicoNamespace, "apps", "v1", "Deployment").(*appsv1.Deployment) Expect(dp.Spec.Template.Spec.Containers[0].Image).To(Equal("test-reg/tigera/calico:" + components.ComponentTigeraCalico.Version)) envs := dp.Spec.Template.Spec.Containers[0].Env @@ -482,7 +482,7 @@ var _ = Describe("kube-controllers rendering tests", func() { Expect(dp.Spec.Template.Spec.Volumes[0].Name).To(Equal("tigera-ca-bundle")) Expect(dp.Spec.Template.Spec.Volumes[0].ConfigMap.Name).To(Equal("tigera-ca-bundle")) - clusterRole := rtest.GetResource(resources, kubecontrollers.EsKubeControllerRole, "", "rbac.authorization.k8s.io", "v1", "ClusterRole").(*rbacv1.ClusterRole) + clusterRole := rtest.GetResource(resources, enterprise.EsKubeControllerRole, "", "rbac.authorization.k8s.io", "v1", "ClusterRole").(*rbacv1.ClusterRole) Expect(clusterRole.Rules).To(HaveLen(36), "cluster role should have 36 rules") Expect(clusterRole.Rules).To(ContainElement( rbacv1.PolicyRule{ @@ -693,26 +693,25 @@ var _ = Describe("kube-controllers rendering tests", func() { version string kind string }{ - {name: kubecontrollers.EsKubeControllerNetworkPolicyName, ns: common.CalicoNamespace, group: "projectcalico.org", version: "v3", kind: "NetworkPolicy"}, + {name: enterprise.EsKubeControllerNetworkPolicyName, ns: common.CalicoNamespace, group: "projectcalico.org", version: "v3", kind: "NetworkPolicy"}, {name: "calico-kube-controllers", ns: common.CalicoNamespace, group: "", version: "v1", kind: "ServiceAccount"}, - {name: kubecontrollers.EsKubeControllerRole, ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRole"}, - {name: kubecontrollers.EsKubeControllerRoleBinding, ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRoleBinding"}, + {name: enterprise.EsKubeControllerRole, ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRole"}, + {name: enterprise.EsKubeControllerRoleBinding, ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRoleBinding"}, {name: kubecontrollers.ManagedClustersWatchRoleBindingName, ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRoleBinding"}, - {name: kubecontrollers.EsKubeController, ns: common.CalicoNamespace, group: "apps", version: "v1", kind: "Deployment"}, - {name: kubecontrollers.ElasticsearchKubeControllersUserSecret, ns: common.CalicoNamespace, group: "", version: "v1", kind: "Secret"}, - {name: kubecontrollers.EsKubeControllerMetrics, ns: common.CalicoNamespace, group: "", version: "v1", kind: "Service"}, + {name: enterprise.EsKubeController, ns: common.CalicoNamespace, group: "apps", version: "v1", kind: "Deployment"}, + {name: enterprise.ElasticsearchKubeControllersUserSecret, ns: common.CalicoNamespace, group: "", version: "v1", kind: "Secret"}, + {name: enterprise.EsKubeControllerMetrics, ns: common.CalicoNamespace, group: "", version: "v1", kind: "Service"}, } // Override configuration to match expected Enterprise config. instance.Variant = operatorv1.CalicoEnterprise - cfg.LogStorageExists = true cfg.ManagementCluster = &operatorv1.ManagementCluster{} cfg.KubeControllersGatewaySecret = &testutils.KubeControllersUserSecret cfg.MetricsPort = 9094 // Opt in to the WAF Gateway API add-on so the WAF env vars + RBAC are rendered. cfg.WAFGatewayExtensionEnabled = true - component := kubecontrollers.NewElasticsearchKubeControllers(&cfg) + component := enterprise.NewElasticsearchKubeControllers(&cfg) Expect(component.ResolveImages(nil)).To(BeNil()) resources, _ := component.Objects() Expect(len(resources)).To(Equal(len(expectedResources))) @@ -725,7 +724,7 @@ var _ = Describe("kube-controllers rendering tests", func() { } // The Deployment should have the correct configuration. - dp := rtest.GetResource(resources, kubecontrollers.EsKubeController, common.CalicoNamespace, "apps", "v1", "Deployment").(*appsv1.Deployment) + dp := rtest.GetResource(resources, enterprise.EsKubeController, common.CalicoNamespace, "apps", "v1", "Deployment").(*appsv1.Deployment) envs := dp.Spec.Template.Spec.Containers[0].Env Expect(envs).To(ContainElement(corev1.EnvVar{ @@ -743,7 +742,7 @@ var _ = Describe("kube-controllers rendering tests", func() { Expect(dp.Spec.Template.Spec.Containers[0].Image).To(Equal("test-reg/tigera/calico:" + components.ComponentTigeraCalico.Version)) - clusterRole := rtest.GetResource(resources, kubecontrollers.EsKubeControllerRole, "", "rbac.authorization.k8s.io", "v1", "ClusterRole").(*rbacv1.ClusterRole) + clusterRole := rtest.GetResource(resources, enterprise.EsKubeControllerRole, "", "rbac.authorization.k8s.io", "v1", "ClusterRole").(*rbacv1.ClusterRole) Expect(clusterRole.Rules).To(HaveLen(36), "cluster role should have 36 rules") Expect(clusterRole.Rules).To(ContainElement( rbacv1.PolicyRule{ @@ -865,7 +864,6 @@ var _ = Describe("kube-controllers rendering tests", func() { It("should add the OIDC prefix env variables", func() { instance.Variant = operatorv1.CalicoEnterprise - cfg.LogStorageExists = true cfg.ManagementCluster = &operatorv1.ManagementCluster{} cfg.KubeControllersGatewaySecret = &testutils.KubeControllersUserSecret cfg.MetricsPort = 9094 @@ -875,17 +873,17 @@ var _ = Describe("kube-controllers rendering tests", func() { Openshift: &operatorv1.AuthenticationOpenshift{IssuerURL: "https://api.example.com"}, }} - component := kubecontrollers.NewElasticsearchKubeControllers(&cfg) + component := enterprise.NewElasticsearchKubeControllers(&cfg) Expect(component.ResolveImages(nil)).To(BeNil()) resources, _ := component.Objects() - depResource := rtest.GetResource(resources, kubecontrollers.EsKubeController, common.CalicoNamespace, "apps", "v1", "Deployment") + depResource := rtest.GetResource(resources, enterprise.EsKubeController, common.CalicoNamespace, "apps", "v1", "Deployment") Expect(depResource).ToNot(BeNil()) deployment := depResource.(*appsv1.Deployment) var usernamePrefix, groupPrefix string for _, container := range deployment.Spec.Template.Spec.Containers { - if container.Name == kubecontrollers.EsKubeController { + if container.Name == enterprise.EsKubeController { for _, env := range container.Env { switch env.Name { case "OIDC_AUTH_USERNAME_PREFIX": @@ -1132,20 +1130,19 @@ var _ = Describe("kube-controllers rendering tests", func() { When("enableESOIDCWorkaround is true", func() { It("should set the ENABLE_ELASTICSEARCH_OIDC_WORKAROUND env variable to true", func() { instance.Variant = operatorv1.CalicoEnterprise - cfg.LogStorageExists = true cfg.ManagementCluster = &operatorv1.ManagementCluster{} cfg.KubeControllersGatewaySecret = &testutils.KubeControllersUserSecret cfg.MetricsPort = 9094 - component := kubecontrollers.NewElasticsearchKubeControllers(&cfg) + component := enterprise.NewElasticsearchKubeControllers(&cfg) resources, _ := component.Objects() - depResource := rtest.GetResource(resources, kubecontrollers.EsKubeController, common.CalicoNamespace, "apps", "v1", "Deployment") + depResource := rtest.GetResource(resources, enterprise.EsKubeController, common.CalicoNamespace, "apps", "v1", "Deployment") Expect(depResource).ToNot(BeNil()) deployment := depResource.(*appsv1.Deployment) var esLicenseType string for _, container := range deployment.Spec.Template.Spec.Containers { - if container.Name == kubecontrollers.EsKubeController { + if container.Name == enterprise.EsKubeController { for _, env := range container.Env { if env.Name == "ENABLE_ELASTICSEARCH_OIDC_WORKAROUND" { esLicenseType = env.Value @@ -1289,10 +1286,9 @@ var _ = Describe("kube-controllers rendering tests", func() { cfg.ManagementClusterConnection = nil } instance.Variant = operatorv1.CalicoEnterprise - cfg.LogStorageExists = true cfg.KubeControllersGatewaySecret = &testutils.KubeControllersUserSecret - component := kubecontrollers.NewElasticsearchKubeControllers(&cfg) + component := enterprise.NewElasticsearchKubeControllers(&cfg) resources, _ := component.Objects() policy := testutils.GetCalicoSystemPolicyFromResources(policyName, resources) From 698254f6190979cf2c1d29e7f197974e8758fbbe Mon Sep 17 00:00:00 2001 From: Casey Davenport Date: Thu, 18 Jun 2026 14:33:23 -0700 Subject: [PATCH 37/54] Move calico-kube-controllers metrics TLS into the enterprise extension The installation extension hook creates the kube-controllers metrics serving keypair and returns it as a managed keypair; a calico-kube-controllers modifier mounts it onto the deployment (env, volume, mount, cert-management init container, hash annotation). The kube-controllers render base no longer carries MetricsServerTLS, and the installation controller no longer creates that certificate. The component reports a config-driven modifier key so the shared es-calico-kube-controllers deployment, which leaves it empty, is never decorated. --- .../installation/core_controller.go | 28 ---- pkg/controller/utils/component.go | 2 +- pkg/enterprise/installation.go | 25 ++++ pkg/enterprise/installation_test.go | 11 +- pkg/enterprise/kubecontrollers.go | 55 ++++++++ pkg/enterprise/kubecontrollers_test.go | 124 ++++++++++++++++++ pkg/enterprise/register.go | 1 + pkg/render/component.go | 5 + .../kubecontrollers/kube-controllers.go | 35 ++--- .../kubecontrollers/kube-controllers_test.go | 60 +-------- 10 files changed, 236 insertions(+), 110 deletions(-) create mode 100644 pkg/enterprise/kubecontrollers_test.go diff --git a/pkg/controller/installation/core_controller.go b/pkg/controller/installation/core_controller.go index a52e196046..6c5686fee8 100644 --- a/pkg/controller/installation/core_controller.go +++ b/pkg/controller/installation/core_controller.go @@ -85,7 +85,6 @@ import ( "github.com/tigera/operator/pkg/render/common/resourcequota" "github.com/tigera/operator/pkg/render/goldmane" "github.com/tigera/operator/pkg/render/kubecontrollers" - "github.com/tigera/operator/pkg/render/monitor" "github.com/tigera/operator/pkg/tls/certificatemanagement" ) @@ -1191,30 +1190,6 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile return reconcile.Result{}, err } - // Secure calico kube controller metrics. - var kubeControllerTLS certificatemanagement.KeyPairInterface - if instance.Spec.Variant.IsEnterprise() { - // Create or Get TLS certificates for kube controller. - kubeControllerTLS, err = certificateManager.GetOrCreateKeyPair( - r.client, - kubecontrollers.KubeControllerPrometheusTLSSecret, - common.OperatorNamespace(), - dns.GetServiceDNSNames(kubecontrollers.KubeControllerMetrics, common.CalicoNamespace, r.opts.ClusterDomain)) - if err != nil { - r.status.SetDegraded(operatorv1.ResourceReadError, "Error finding or creating TLS certificate kube controllers metric", err, reqLogger) - return reconcile.Result{}, err - } - - // Add prometheus client certificate to Trusted bundle. - kubeControllerPrometheusTLS, err := certificateManager.GetCertificate(r.client, monitor.PrometheusClientTLSSecretName, common.OperatorNamespace()) - if err != nil { - r.status.SetDegraded(operatorv1.ResourceReadError, "Failed to get certificate for kube controllers", err, reqLogger) - return reconcile.Result{}, err - } else if kubeControllerPrometheusTLS != nil { - typhaNodeTLS.TrustedBundle.AddCertificates(kubeControllerTLS, kubeControllerPrometheusTLS) - } - } - nodeAppArmorProfile := "" a := instance.GetObjectMeta().GetAnnotations() if val, ok := a[techPreviewFeatureSeccompApparmor]; ok { @@ -1324,7 +1299,6 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile rcertificatemanagement.NewKeyPairOption(typhaNodeTLS.NodeSecret, true, true), rcertificatemanagement.NewKeyPairOption(typhaNodeTLS.TyphaSecret, true, true), rcertificatemanagement.NewKeyPairOption(typhaNodeTLS.TyphaSecretNonClusterHost, true, true), - rcertificatemanagement.NewKeyPairOption(kubeControllerTLS, true, true), // Nil when the WAF v3 surface is disabled; the certificate-management // render skips nil key pairs. rcertificatemanagement.NewKeyPairOption(wafWebhookTLS, true, true), @@ -1604,7 +1578,6 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile ClusterDomain: r.opts.ClusterDomain, MetricsPort: kubeControllersMetricsPort, Terminating: installationMarkedForDeletion, - MetricsServerTLS: kubeControllerTLS, TrustedBundle: typhaNodeTLS.TrustedBundle, Namespace: common.CalicoNamespace, BindingNamespaces: []string{common.CalicoNamespace}, @@ -1756,7 +1729,6 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile render.TyphaTLSSecretName: typhaNodeTLS.TyphaSecret, render.NodeTLSSecretName: typhaNodeTLS.NodeSecret, render.TyphaTLSSecretName + render.TyphaNonClusterHostSuffix: typhaNodeTLS.TyphaSecretNonClusterHost, - kubecontrollers.KubeControllerPrometheusTLSSecret: kubeControllerTLS, } for _, kp := range managedKeyPairs { keyPairWarnings[kp.GetName()] = kp diff --git a/pkg/controller/utils/component.go b/pkg/controller/utils/component.go index 52eb2dc374..b8f97b1fc3 100644 --- a/pkg/controller/utils/component.go +++ b/pkg/controller/utils/component.go @@ -458,7 +458,7 @@ func resetMetadataForCreate(obj client.Object) { } func (c *componentHandler) CreateOrUpdateOrDelete(ctx context.Context, component render.Component, status status.StatusManager) error { - if ext, ok := component.(render.Extensible); ok && c.extensions == nil { + if ext, ok := component.(render.Extensible); ok && ext.ModifierKey() != "" && c.extensions == nil { c.log.Info("BUG: extensible component rendered by a handler with no extension Set; extensions will not be applied", "component", ext.ModifierKey()) } component = c.extensions.Decorate(component, c.renderCtx) diff --git a/pkg/enterprise/installation.go b/pkg/enterprise/installation.go index 3126e90880..12eef67b13 100644 --- a/pkg/enterprise/installation.go +++ b/pkg/enterprise/installation.go @@ -28,6 +28,7 @@ import ( "github.com/tigera/operator/pkg/extensions" "github.com/tigera/operator/pkg/render" relasticsearch "github.com/tigera/operator/pkg/render/common/elasticsearch" + "github.com/tigera/operator/pkg/render/kubecontrollers" "github.com/tigera/operator/pkg/render/monitor" "github.com/tigera/operator/pkg/tls/certificatemanagement" ) @@ -42,6 +43,10 @@ type coreControllerExtension struct{} type installationRenderData struct { nodePrometheusTLS certificatemanagement.KeyPairInterface + // kubeControllerTLS is the calico-kube-controllers metrics serving keypair; the + // kube-controllers modifier mounts it onto the deployment. + kubeControllerTLS certificatemanagement.KeyPairInterface + // collectProcessPath mirrors LogCollector.Spec.CollectProcessPath being // enabled; the node modifier uses it to set HostPID and the felix env. collectProcessPath bool @@ -101,12 +106,29 @@ func (coreControllerExtension) ExtendContext(cc extensions.ControllerContext) (e cc.TrustedBundle.AddCertificates(nodePrometheusTLS) } + // The calico-kube-controllers metrics endpoint is served with mTLS in + // Enterprise; the keypair is created here (cluster side effect) and mounted by + // the kube-controllers modifier. + kubeControllerTLS, err := cc.CertificateManager.GetOrCreateKeyPair( + cc.Client, + kubecontrollers.KubeControllerPrometheusTLSSecret, + common.OperatorNamespace(), + dns.GetServiceDNSNames(kubecontrollers.KubeControllerMetrics, common.CalicoNamespace, cc.ClusterDomain), + ) + if err != nil { + return rc, nil, fmt.Errorf("error creating kube-controllers metrics TLS certificate: %w", err) + } + if kubeControllerTLS != nil { + cc.TrustedBundle.AddCertificates(kubeControllerTLS) + } + logCollector, err := utils.GetLogCollector(cc.Ctx, cc.Client) if err != nil { return rc, nil, fmt.Errorf("error reading LogCollector: %w", err) } rc.Extension = installationRenderData{ nodePrometheusTLS: nodePrometheusTLS, + kubeControllerTLS: kubeControllerTLS, collectProcessPath: collectProcessPathEnabled(logCollector), } @@ -140,5 +162,8 @@ func (coreControllerExtension) ExtendContext(cc extensions.ControllerContext) (e if nodePrometheusTLS != nil { managed = append(managed, nodePrometheusTLS) } + if kubeControllerTLS != nil { + managed = append(managed, kubeControllerTLS) + } return rc, managed, nil } diff --git a/pkg/enterprise/installation_test.go b/pkg/enterprise/installation_test.go index 2c0f917984..d99e18eb00 100644 --- a/pkg/enterprise/installation_test.go +++ b/pkg/enterprise/installation_test.go @@ -30,10 +30,10 @@ import ( ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" "github.com/tigera/operator/pkg/extensions" "github.com/tigera/operator/pkg/render" + "github.com/tigera/operator/pkg/render/kubecontrollers" ) var _ = Describe("installation controller extension", func() { - It("rejects a zero prometheus reporter port", func() { port := 0 cc := newControllerContext(operatorv1.CalicoEnterprise) @@ -43,11 +43,14 @@ var _ = Describe("installation controller extension", func() { Expect(ext.Validate(cc)).To(HaveOccurred()) }) - It("creates the node prometheus keypair for the enterprise variant", func() { + It("manages the node prometheus and kube-controllers metrics keypairs for the enterprise variant", func() { _, managed, err := ext.ExtendContext(newControllerContext(operatorv1.CalicoEnterprise)) Expect(err).NotTo(HaveOccurred()) - Expect(managed).To(HaveLen(1), "expected the node prometheus keypair to be managed") - Expect(managed[0].GetName()).To(Equal(render.NodePrometheusTLSServerSecret)) + names := []string{} + for _, kp := range managed { + names = append(names, kp.GetName()) + } + Expect(names).To(ConsistOf(render.NodePrometheusTLSServerSecret, kubecontrollers.KubeControllerPrometheusTLSSecret)) }) It("is a no-op for the Calico variant", func() { diff --git a/pkg/enterprise/kubecontrollers.go b/pkg/enterprise/kubecontrollers.go index 388e7dac4a..884f928750 100644 --- a/pkg/enterprise/kubecontrollers.go +++ b/pkg/enterprise/kubecontrollers.go @@ -15,20 +15,75 @@ package enterprise import ( + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" + "github.com/tigera/operator/pkg/common" + "github.com/tigera/operator/pkg/extensions" "github.com/tigera/operator/pkg/render" relasticsearch "github.com/tigera/operator/pkg/render/common/elasticsearch" rmeta "github.com/tigera/operator/pkg/render/common/meta" "github.com/tigera/operator/pkg/render/common/networkpolicy" "github.com/tigera/operator/pkg/render/kubecontrollers" + "github.com/tigera/operator/pkg/render/monitor" "github.com/tigera/operator/pkg/url" ) +// registerKubeControllers registers the calico-kube-controllers modifier. There is +// no image override: kube-controllers runs from the combined calico image, which +// resolves by variant in the base render. +func registerKubeControllers(v *extensions.Variant) { + v.Modify(render.ComponentNameKubeControllers, modifyKubeControllers) +} + +// modifyKubeControllers layers the Calico Enterprise metrics serving TLS onto the +// rendered calico-kube-controllers deployment: the env pointing at the keypair, the +// volume + mount, the cert-management init container (when in use), and the pod hash +// annotation that rolls the pod on cert rotation. The keypair has cluster side +// effects, so the installation extension creates it and hands it in via rc. In core +// (calico) it is never created, so the base deployment carries no metrics TLS. +func modifyKubeControllers(rc extensions.RenderContext, objs, del []client.Object) ([]client.Object, []client.Object) { + tls := installationData(rc).kubeControllerTLS + if tls == nil { + return objs, del + } + + dp, ok := extensions.FindObject[*appsv1.Deployment](objs, kubecontrollers.KubeController) + if !ok { + return objs, del + } + spec := &dp.Spec.Template.Spec + spec.Volumes = append(spec.Volumes, tls.Volume()) + + for i := range spec.Containers { + c := &spec.Containers[i] + if c.Name != kubecontrollers.KubeController { + continue + } + c.Env = append(c.Env, + corev1.EnvVar{Name: "TLS_KEY_PATH", Value: tls.VolumeMountKeyFilePath()}, + corev1.EnvVar{Name: "TLS_CRT_PATH", Value: tls.VolumeMountCertificateFilePath()}, + corev1.EnvVar{Name: "CLIENT_COMMON_NAME", Value: monitor.PrometheusClientTLSSecretName}, + ) + c.VolumeMounts = append(c.VolumeMounts, tls.VolumeMount(rmeta.OSTypeLinux)) + if tls.UseCertificateManagement() { + spec.InitContainers = append(spec.InitContainers, tls.InitContainer(common.CalicoNamespace, c.SecurityContext)) + } + } + + if dp.Spec.Template.Annotations == nil { + dp.Spec.Template.Annotations = map[string]string{} + } + dp.Spec.Template.Annotations[tls.HashAnnotationKey()] = tls.HashAnnotationValue() + + return objs, del +} + const ( EsKubeController = "es-calico-kube-controllers" EsKubeControllerRole = "es-calico-kube-controllers" diff --git a/pkg/enterprise/kubecontrollers_test.go b/pkg/enterprise/kubecontrollers_test.go new file mode 100644 index 0000000000..41ee40ca72 --- /dev/null +++ b/pkg/enterprise/kubecontrollers_test.go @@ -0,0 +1,124 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package enterprise_test + +import ( + "context" + "fmt" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/apis" + "github.com/tigera/operator/pkg/common" + "github.com/tigera/operator/pkg/controller/certificatemanager" + ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" + "github.com/tigera/operator/pkg/extensions" + "github.com/tigera/operator/pkg/render" + rmeta "github.com/tigera/operator/pkg/render/common/meta" + "github.com/tigera/operator/pkg/render/kubecontrollers" + "github.com/tigera/operator/pkg/render/monitor" + "github.com/tigera/operator/pkg/tls" +) + +var _ = Describe("kube-controllers enterprise modifier", func() { + // kubeControllersDeployment is a minimal stand-in for the calico-kube-controllers + // deployment the base render produces, so the modifier has something to mount onto. + kubeControllersDeployment := func() *appsv1.Deployment { + return &appsv1.Deployment{ + TypeMeta: metav1.TypeMeta{Kind: "Deployment", APIVersion: "apps/v1"}, + ObjectMeta: metav1.ObjectMeta{Name: kubecontrollers.KubeController, Namespace: common.CalicoNamespace}, + Spec: appsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{{Name: kubecontrollers.KubeController}}, + }, + }, + }, + } + } + + It("mounts the metrics serving TLS keypair onto the deployment", func() { + rc, _, err := ext.ExtendContext(newControllerContext(operatorv1.CalicoEnterprise)) + Expect(err).NotTo(HaveOccurred()) + + objs, _ := applyExtensions(ext, render.ComponentNameKubeControllers, rc, []client.Object{kubeControllersDeployment()}, nil) + dp, ok := extensions.FindObject[*appsv1.Deployment](objs, kubecontrollers.KubeController) + Expect(ok).To(BeTrue()) + + c := dp.Spec.Template.Spec.Containers[0] + Expect(c.Env).To(ContainElements( + corev1.EnvVar{Name: "TLS_KEY_PATH", Value: "/calico-kube-controllers-metrics-tls/tls.key"}, + corev1.EnvVar{Name: "TLS_CRT_PATH", Value: "/calico-kube-controllers-metrics-tls/tls.crt"}, + corev1.EnvVar{Name: "CLIENT_COMMON_NAME", Value: monitor.PrometheusClientTLSSecretName}, + )) + Expect(c.VolumeMounts).To(ContainElement(HaveField("Name", kubecontrollers.KubeControllerPrometheusTLSSecret))) + Expect(dp.Spec.Template.Spec.Volumes).To(ContainElement(HaveField("Name", kubecontrollers.KubeControllerPrometheusTLSSecret))) + Expect(dp.Spec.Template.Annotations).NotTo(BeEmpty(), "expected the cert hash annotation") + }) + + It("adds the cert-management init container when certificate management is enabled", func() { + rc, _, err := ext.ExtendContext(certManagementControllerContext()) + Expect(err).NotTo(HaveOccurred()) + + objs, _ := applyExtensions(ext, render.ComponentNameKubeControllers, rc, []client.Object{kubeControllersDeployment()}, nil) + dp, ok := extensions.FindObject[*appsv1.Deployment](objs, kubecontrollers.KubeController) + Expect(ok).To(BeTrue()) + + Expect(dp.Spec.Template.Spec.InitContainers).To(HaveLen(1)) + Expect(dp.Spec.Template.Spec.InitContainers[0].Name).To(Equal(fmt.Sprintf("%s-key-cert-provisioner", kubecontrollers.KubeControllerPrometheusTLSSecret))) + }) +}) + +// certManagementControllerContext builds a controller context whose certificate +// manager issues cert-management (CSR-based) keypairs. +func certManagementControllerContext() extensions.ControllerContext { + scheme := runtime.NewScheme() + Expect(apis.AddToScheme(scheme, false)).NotTo(HaveOccurred()) + c := ctrlrfake.DefaultFakeClientBuilder(scheme).Build() + + ca, err := tls.MakeCA(rmeta.DefaultOperatorCASignerName()) + Expect(err).NotTo(HaveOccurred()) + caCert, _, err := ca.Config.GetPEMBytes() + Expect(err).NotTo(HaveOccurred()) + + installation := &operatorv1.InstallationSpec{ + Variant: operatorv1.CalicoEnterprise, + CertificateManagement: &operatorv1.CertificateManagement{CACert: caCert}, + } + certManager, err := certificatemanager.Create(c, installation, "", common.OperatorNamespace(), certificatemanager.AllowCACreation()) + Expect(err).NotTo(HaveOccurred()) + + return extensions.ControllerContext{ + RenderContext: extensions.RenderContext{ + Installation: installation, + FelixConfiguration: &v3.FelixConfiguration{}, + TrustedBundle: certManager.CreateTrustedBundle(), + ClusterDomain: "cluster.local", + }, + Controller: extensions.InstallationController, + Ctx: context.Background(), + Client: c, + CertificateManager: certManager, + } +} diff --git a/pkg/enterprise/register.go b/pkg/enterprise/register.go index 4d533a0e33..839e56b66a 100644 --- a/pkg/enterprise/register.go +++ b/pkg/enterprise/register.go @@ -35,6 +35,7 @@ func New() *extensions.Set { registerWindows(ent) registerGuardian(ent) registerAPIServer(ent) + registerKubeControllers(ent) // When the enterprise operator manages a Calico installation, clean up the // Enterprise objects left behind by a prior Enterprise installation. diff --git a/pkg/render/component.go b/pkg/render/component.go index 778e1a3c2f..6fdf85303a 100644 --- a/pkg/render/component.go +++ b/pkg/render/component.go @@ -76,4 +76,9 @@ const ( ComponentNameWindows = "windows" ComponentNameWindowsNodeImg = "windows-node-image" ComponentNameWindowsCNIImg = "windows-cni-image" + + // ComponentNameKubeControllers keys the calico-kube-controllers modifier. The + // es-calico-kube-controllers deployment shares the component type but leaves + // its modifier key empty, so it is not decorated. + ComponentNameKubeControllers = "kube-controllers" ) diff --git a/pkg/render/kubecontrollers/kube-controllers.go b/pkg/render/kubecontrollers/kube-controllers.go index b6340efb74..46dbf2cd8a 100644 --- a/pkg/render/kubecontrollers/kube-controllers.go +++ b/pkg/render/kubecontrollers/kube-controllers.go @@ -43,7 +43,6 @@ import ( "github.com/tigera/operator/pkg/render/common/secret" "github.com/tigera/operator/pkg/render/common/securitycontext" "github.com/tigera/operator/pkg/render/common/securitycontextconstraints" - "github.com/tigera/operator/pkg/render/monitor" "github.com/tigera/operator/pkg/tls/certificatemanagement" ) @@ -105,8 +104,6 @@ type KubeControllersConfiguration struct { WASMCACert *corev1.ConfigMap TrustedBundle certificatemanagement.TrustedBundleRO - MetricsServerTLS certificatemanagement.KeyPairInterface - // Namespace to be installed into. Namespace string @@ -172,6 +169,12 @@ type KubeControllersConfiguration struct { // webhook surface lifecycle (rendered when WAFGatewayExtensionEnabled, deleted // otherwise). Only the calico-kube-controllers component sets this. ManageWAFWebhook bool + + // ModifierKey is the extension modifier key the component reports through + // render.Extensible. calico-kube-controllers sets it so the enterprise modifier + // can layer on its metrics TLS; es-calico-kube-controllers leaves it empty so it + // is never decorated. + ModifierKey string } func NewCalicoKubeControllersPolicy(cfg *KubeControllersConfiguration, defaultDeny *v3.NetworkPolicy) render.Component { @@ -206,6 +209,7 @@ func NewCalicoKubeControllers(cfg *KubeControllersConfiguration) render.Componen cfg.RoleBindingName = KubeControllerRoleBinding cfg.MetricsName = KubeControllerMetrics cfg.ManageWAFWebhook = true + cfg.ModifierKey = render.ComponentNameKubeControllers cfg.Rules = KubeControllersRoleCommonRules(cfg) cfg.EnabledControllers = []string{"node", "loadbalancer"} @@ -374,6 +378,12 @@ func (c *kubeControllersComponent) Ready() bool { return true } +// ModifierKey implements render.Extensible. It is empty for es-calico-kube-controllers +// (never decorated) and set for calico-kube-controllers. +func (c *kubeControllersComponent) ModifierKey() string { + return c.cfg.ModifierKey +} + func KubeControllersRoleCommonRules(cfg *KubeControllersConfiguration) []rbacv1.PolicyRule { rules := []rbacv1.PolicyRule{ { @@ -739,13 +749,6 @@ func (c *kubeControllersComponent) controllersDeployment() *appsv1.Deployment { } } - if c.cfg.MetricsServerTLS != nil { - env = append(env, - corev1.EnvVar{Name: "TLS_KEY_PATH", Value: c.cfg.MetricsServerTLS.VolumeMountKeyFilePath()}, - corev1.EnvVar{Name: "TLS_CRT_PATH", Value: c.cfg.MetricsServerTLS.VolumeMountCertificateFilePath()}, - corev1.EnvVar{Name: "CLIENT_COMMON_NAME", Value: monitor.PrometheusClientTLSSecretName}, - ) - } if c.cfg.TrustedBundle != nil { env = append(env, corev1.EnvVar{Name: "CA_CRT_PATH", Value: c.cfg.TrustedBundle.MountPath()}, @@ -814,9 +817,6 @@ func (c *kubeControllersComponent) controllersDeployment() *appsv1.Deployment { } var initContainers []corev1.Container - if c.cfg.MetricsServerTLS != nil && c.cfg.MetricsServerTLS.UseCertificateManagement() { - initContainers = append(initContainers, c.cfg.MetricsServerTLS.InitContainer(c.cfg.Namespace, sc)) - } if c.cfg.WAFWebhookServerTLS != nil && c.cfg.WAFWebhookServerTLS.UseCertificateManagement() { initContainers = append(initContainers, c.cfg.WAFWebhookServerTLS.InitContainer(c.cfg.Namespace, sc)) } @@ -953,9 +953,6 @@ func (c *kubeControllersComponent) annotations() map[string]string { am = make(map[string]string) } - if c.cfg.MetricsServerTLS != nil { - am[c.cfg.MetricsServerTLS.HashAnnotationKey()] = c.cfg.MetricsServerTLS.HashAnnotationValue() - } if c.cfg.KubeControllersGatewaySecret != nil { am[render.ElasticsearchUserHashAnnotation] = rmeta.AnnotationHash(c.cfg.KubeControllersGatewaySecret.Data) } @@ -967,9 +964,6 @@ func (c *kubeControllersComponent) kubeControllersVolumeMounts() []corev1.Volume if c.cfg.TrustedBundle != nil { mounts = append(mounts, c.cfg.TrustedBundle.VolumeMounts(c.SupportedOSType())...) } - if c.cfg.MetricsServerTLS != nil { - mounts = append(mounts, c.cfg.MetricsServerTLS.VolumeMount(c.SupportedOSType())) - } if c.cfg.WAFWebhookServerTLS != nil { mounts = append(mounts, c.cfg.WAFWebhookServerTLS.VolumeMount(c.SupportedOSType())) } @@ -981,9 +975,6 @@ func (c *kubeControllersComponent) kubeControllersVolumes() []corev1.Volume { if c.cfg.TrustedBundle != nil { volumes = append(volumes, c.cfg.TrustedBundle.Volume()) } - if c.cfg.MetricsServerTLS != nil { - volumes = append(volumes, c.cfg.MetricsServerTLS.Volume()) - } if c.cfg.WAFWebhookServerTLS != nil { volumes = append(volumes, c.cfg.WAFWebhookServerTLS.Volume()) } diff --git a/pkg/render/kubecontrollers/kube-controllers_test.go b/pkg/render/kubecontrollers/kube-controllers_test.go index 8680614827..801c4b2687 100644 --- a/pkg/render/kubecontrollers/kube-controllers_test.go +++ b/pkg/render/kubecontrollers/kube-controllers_test.go @@ -49,7 +49,6 @@ import ( rtest "github.com/tigera/operator/pkg/render/common/test" "github.com/tigera/operator/pkg/render/kubecontrollers" "github.com/tigera/operator/pkg/render/testutils" - "github.com/tigera/operator/pkg/tls" "github.com/tigera/operator/pkg/tls/certificatemanagement" ) @@ -550,8 +549,6 @@ var _ = Describe("kube-controllers rendering tests", func() { Expect(dp.Spec.Template.Spec.Containers[0].Image).To(Equal("test-reg/tigera/calico:" + components.ComponentTigeraCalico.Version)) }) It("should render all calico-kube-controllers resources for a default configuration using CalicoEnterprise", func() { - var defaultMode int32 = 420 - var kubeControllerTLS certificatemanagement.KeyPairInterface expectedResources := []struct { name string ns string @@ -566,15 +563,14 @@ var _ = Describe("kube-controllers rendering tests", func() { {name: kubecontrollers.KubeControllerMetrics, ns: common.CalicoNamespace, group: "", version: "v1", kind: "Service"}, } + // The metrics serving TLS (TLS_KEY_PATH/TLS_CRT_PATH/CLIENT_COMMON_NAME env, + // the keypair volume + mount) is layered on by the enterprise modifier, so + // the base render here carries only the trusted bundle. expectedEnv := []corev1.EnvVar{ - {Name: "TLS_KEY_PATH", Value: "/calico-kube-controllers-metrics-tls/tls.key"}, - {Name: "TLS_CRT_PATH", Value: "/calico-kube-controllers-metrics-tls/tls.crt"}, - {Name: "CLIENT_COMMON_NAME", Value: "calico-node-prometheus-client-tls"}, {Name: "CA_CRT_PATH", Value: "/etc/pki/tls/certs/tigera-ca-bundle.crt"}, } expectedVolumeMounts := []corev1.VolumeMount{ {Name: "tigera-ca-bundle", MountPath: "/etc/pki/tls/certs", ReadOnly: true}, - {Name: "calico-kube-controllers-metrics-tls", MountPath: "/calico-kube-controllers-metrics-tls", ReadOnly: true}, } expectedVolume := []corev1.Volume{ { @@ -585,34 +581,11 @@ var _ = Describe("kube-controllers rendering tests", func() { }, }, }, - { - Name: "calico-kube-controllers-metrics-tls", - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: "calico-kube-controllers-metrics-tls", - DefaultMode: &defaultMode, - }, - }, - }, } - scheme := runtime.NewScheme() - Expect(apis.AddToScheme(scheme, false)).NotTo(HaveOccurred()) - cli := ctrlrfake.DefaultFakeClientBuilder(scheme).Build() - - certificateManager, err := certificatemanager.Create(cli, nil, dns.DefaultClusterDomain, common.OperatorNamespace(), certificatemanager.AllowCACreation()) - Expect(err).NotTo(HaveOccurred()) - - kubeControllerTLS, err = certificateManager.GetOrCreateKeyPair(cli, - kubecontrollers.KubeControllerPrometheusTLSSecret, - common.OperatorNamespace(), - dns.GetServiceDNSNames(kubecontrollers.KubeControllerMetrics, common.CalicoNamespace, dns.DefaultClusterDomain)) - Expect(err).NotTo(HaveOccurred()) - // Override configuration to match expected Enterprise config. instance.Variant = operatorv1.CalicoEnterprise cfg.MetricsPort = 9094 - cfg.MetricsServerTLS = kubeControllerTLS component := kubecontrollers.NewCalicoKubeControllers(&cfg) Expect(component.ResolveImages(nil)).To(BeNil()) @@ -632,10 +605,10 @@ var _ = Describe("kube-controllers rendering tests", func() { envs := dp.Spec.Template.Spec.Containers[0].Env Expect(envs).To(ContainElements(expectedEnv)) - Expect(len(dp.Spec.Template.Spec.Containers[0].VolumeMounts)).To(Equal(2)) + Expect(len(dp.Spec.Template.Spec.Containers[0].VolumeMounts)).To(Equal(1)) Expect(dp.Spec.Template.Spec.Containers[0].VolumeMounts).To(ContainElements(expectedVolumeMounts)) - Expect(len(dp.Spec.Template.Spec.Volumes)).To(Equal(2)) + Expect(len(dp.Spec.Template.Spec.Volumes)).To(Equal(1)) Expect(dp.Spec.Template.Spec.Volumes).To(ContainElements(expectedVolume)) Expect(dp.Spec.Template.Spec.Containers[0].Image).To(Equal("test-reg/tigera/calico:" + components.ComponentTigeraCalico.Version)) @@ -1302,29 +1275,6 @@ var _ = Describe("kube-controllers rendering tests", func() { ) }) - It("should render init containers when certificate management is enabled", func() { - instance.Variant = operatorv1.CalicoEnterprise - cfg.MetricsPort = 9094 - ca, _ := tls.MakeCA(rmeta.DefaultOperatorCASignerName()) - cert, _, _ := ca.Config.GetPEMBytes() // create a valid pem block - cfg.Installation.CertificateManagement = &operatorv1.CertificateManagement{CACert: cert} - - certificateManager, err := certificatemanager.Create(cli, cfg.Installation, dns.DefaultClusterDomain, common.OperatorNamespace(), certificatemanager.AllowCACreation()) - Expect(err).NotTo(HaveOccurred()) - - tls, err := certificateManager.GetOrCreateKeyPair(cli, kubecontrollers.KubeControllerPrometheusTLSSecret, common.OperatorNamespace(), []string{""}) - Expect(err).NotTo(HaveOccurred()) - - cfg.MetricsServerTLS = tls - - resources, _ := kubecontrollers.NewCalicoKubeControllers(&cfg).Objects() - - dp := rtest.GetResource(resources, kubecontrollers.KubeController, common.CalicoNamespace, "apps", "v1", "Deployment").(*appsv1.Deployment) - Expect(dp.Spec.Template.Spec.InitContainers).To(HaveLen(1)) - csrInitContainer := dp.Spec.Template.Spec.InitContainers[0] - Expect(csrInitContainer.Name).To(Equal(fmt.Sprintf("%v-key-cert-provisioner", kubecontrollers.KubeControllerPrometheusTLSSecret))) - }) - It("should add egress policy with Enterprise variant and K8SServiceEndpoint defined", func() { cfg.K8sServiceEp.Host = "k8shost" cfg.K8sServiceEp.Port = "1234" From 8d2ff05ea6e6af8f29c2f98bd6f48d6bbf78ee8d Mon Sep 17 00:00:00 2001 From: Casey Davenport Date: Thu, 18 Jun 2026 16:28:45 -0700 Subject: [PATCH 38/54] Move the calico-kube-controllers enterprise surface into the extension calico-kube-controllers renders as pure OSS now: the common rules plus the node and loadbalancer controllers, no IsEnterprise. The enterprise extension layers on the rest through a modifier - the enterprise RBAC, the service/federatedservices/usage controllers, the metrics serving TLS, and the WAF v3 (Gateway API add-on) surface (the WASM env, the in-process admission webhook, and the network policy ingress rule). The installation hook produces the controller-side inputs the modifier can't (the webhook keypair, the merged wasm pull secret, the resolved wasm image, the operator CA) and hands them over through the render context; the WASM image resolves with the same GetReference the base uses, via the ImageSet the hook reads itself. The WAF/WASM symbols and the es-kube-controllers pull-secret helper move to pkg/enterprise. The base kube-controllers config no longer carries any WAF/WASM fields. --- .../installation/core_controller.go | 86 +-- pkg/enterprise/installation.go | 35 +- pkg/enterprise/kubecontrollers.go | 534 +++++++++++++++++- pkg/enterprise/kubecontrollers_test.go | 168 ++++++ .../waf_pull_secret_test.go | 20 +- pkg/render/component.go | 4 + .../kubecontrollers/kube-controllers.go | 375 +----------- .../kubecontrollers/kube-controllers_test.go | 275 +-------- pkg/render/kubecontrollers/waf_pull_secret.go | 101 ---- .../logstorage/esgateway/esgateway_test.go | 20 +- 10 files changed, 756 insertions(+), 862 deletions(-) rename pkg/{render/kubecontrollers => enterprise}/waf_pull_secret_test.go (88%) delete mode 100644 pkg/render/kubecontrollers/waf_pull_secret.go diff --git a/pkg/controller/installation/core_controller.go b/pkg/controller/installation/core_controller.go index 6c5686fee8..b01f01cfca 100644 --- a/pkg/controller/installation/core_controller.go +++ b/pkg/controller/installation/core_controller.go @@ -63,7 +63,6 @@ import ( "github.com/tigera/operator/pkg/common/discovery" "github.com/tigera/operator/pkg/components" "github.com/tigera/operator/pkg/controller/certificatemanager" - "github.com/tigera/operator/pkg/controller/gatewayapi" "github.com/tigera/operator/pkg/controller/ippool" "github.com/tigera/operator/pkg/controller/k8sapi" "github.com/tigera/operator/pkg/controller/migration" @@ -74,12 +73,10 @@ import ( "github.com/tigera/operator/pkg/controller/utils" "github.com/tigera/operator/pkg/controller/utils/imageset" "github.com/tigera/operator/pkg/ctrlruntime" - "github.com/tigera/operator/pkg/dns" "github.com/tigera/operator/pkg/extensions" "github.com/tigera/operator/pkg/imports/admission" "github.com/tigera/operator/pkg/imports/crds" "github.com/tigera/operator/pkg/render" - "github.com/tigera/operator/pkg/render/applicationlayer" rcertificatemanagement "github.com/tigera/operator/pkg/render/certificatemanagement" "github.com/tigera/operator/pkg/render/common/networkpolicy" "github.com/tigera/operator/pkg/render/common/resourcequota" @@ -207,13 +204,6 @@ func Add(mgr manager.Manager, opts options.ControllerOptions) error { } // Watch for changes to KubeControllersConfiguration. - // Watch GatewayAPI: spec.extensions.waf.state gates the WAF v3 surface on - // calico-kube-controllers. See design tigera/designs#25 (PMREQ-384) §Gating. - if err := c.WatchObject(&operatorv1.GatewayAPI{}, &handler.EnqueueRequestForObject{}); err != nil { - log.V(5).Info("Failed to create GatewayAPI watch", "err", err) - return fmt.Errorf("core-controller failed to watch operator GatewayAPI resource: %w", err) - } - err = c.WatchObject(&v3.KubeControllersConfiguration{}, &handler.EnqueueRequestForObject{}) if err != nil { return fmt.Errorf("tigera-installation-controller failed to watch KubeControllersConfiguration resource: %w", err) @@ -1261,47 +1251,10 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile } - // Read the GatewayAPI CR (if present) to decide whether to render the WAF - // v3 (Gateway API add-on) surface — env vars, RBAC, applicationlayer - // reconciler, and the in-process admission webhook — on - // calico-kube-controllers. Default-off: if no GatewayAPI CR exists or - // spec.extensions.waf.state != Enabled, the WAF surface is not rendered. - // See design tigera/designs#25 (PMREQ-384) §Gating. - wafGatewayExtensionEnabled := false - if gatewayAPI, msg, err := gatewayapi.GetGatewayAPI(ctx, r.client); err == nil { - wafGatewayExtensionEnabled = gatewayAPI.Spec.IsWAFGatewayExtensionEnabled() - } else if !apierrors.IsNotFound(err) { - // Mirrors the GatewayAPI controller's handling: a read error or a - // duplicate default/tigera-secure pair degrades rather than guessing. - r.status.SetDegraded(operatorv1.ResourceReadError, msg, err, reqLogger) - return reconcile.Result{}, err - } - - // When the WAF v3 surface is enabled, issue the serving cert for the - // in-process WAF admission webhook (hosted by calico-kube-controllers, - // fronted by the tigera-waf-webhook Service). It is materialized into - // calico-system alongside the other kube-controllers certs below and mounted - // into the Pod by the kube-controllers render. - var wafWebhookTLS certificatemanagement.KeyPairInterface - if wafGatewayExtensionEnabled { - wafWebhookTLS, err = certificateManager.GetOrCreateKeyPair( - r.client, - applicationlayer.WAFWebhookServerTLSSecretName, - common.OperatorNamespace(), - dns.GetServiceDNSNames(applicationlayer.WAFWebhookServiceName, common.CalicoNamespace, r.opts.ClusterDomain)) - if err != nil { - r.status.SetDegraded(operatorv1.ResourceCreateError, "Error creating WAF admission webhook TLS certificate", err, reqLogger) - return reconcile.Result{}, err - } - } - keyPairOptions := []rcertificatemanagement.KeyPairOption{ rcertificatemanagement.NewKeyPairOption(typhaNodeTLS.NodeSecret, true, true), rcertificatemanagement.NewKeyPairOption(typhaNodeTLS.TyphaSecret, true, true), rcertificatemanagement.NewKeyPairOption(typhaNodeTLS.TyphaSecretNonClusterHost, true, true), - // Nil when the WAF v3 surface is disabled; the certificate-management - // render skips nil key pairs. - rcertificatemanagement.NewKeyPairOption(wafWebhookTLS, true, true), } // Manage any key pairs the variant extension created controller-side. for _, kp := range managedKeyPairs { @@ -1542,33 +1495,9 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile } components = append(components, render.CSI(&csiCfg)) - // Build a configuration for rendering calico/kube-controllers. - // Provision a dedicated WAF wasm pull secret so the WAF reconciler - // replicates it into tenant namespaces without clashing with the - // operator-managed tigera-pull-secret the GatewayAPI render also copies - // there (EV-6386). The EnvoyExtensionPolicy image source takes a single - // pullSecretRef, so the registry auths of all Installation pull secrets - // are merged into it rather than picking one. - var wasmPullSecret *corev1.Secret - if wafGatewayExtensionEnabled && len(pullSecrets) > 0 { - var skipped []string - wasmPullSecret, skipped = kubecontrollers.MergeWAFPullSecret(pullSecrets) - if len(skipped) > 0 { - reqLogger.Info("Skipped unparseable imagePullSecrets when building the WAF wasm pull secret", "skipped", skipped) - } - } - // Provision the dedicated WAF wasm CA-bundle ConfigMap as a renamed copy of - // the trusted CA bundle, so the WAF reconciler replicates it into tenant - // namespaces for the Coraza wasm OCI registry TLS check without clashing with - // the operator-managed tigera-ca-bundle the GatewayAPI render also copies - // there (EV-6386). The dedicated source was previously a TODO; the full - // TrustedBundle (not the RO interface the kube-controllers render sees) is - // available here, so build it in the core controller. - var wasmCACert *corev1.ConfigMap - if wafGatewayExtensionEnabled { - wasmCACert = typhaNodeTLS.TrustedBundle.ConfigMap(common.CalicoNamespace) - wasmCACert.Name = kubecontrollers.WASMCACertName - } + // Build a configuration for rendering calico/kube-controllers. The Calico + // Enterprise surface (extra RBAC, enterprise controllers, metrics TLS, and the + // WAF v3 / Gateway API add-on) is layered on by the enterprise extension. kubeControllersCfg := kubecontrollers.KubeControllersConfiguration{ K8sServiceEp: k8sapi.Endpoint, K8sServiceEpPodNetwork: k8sapi.PodNetworkEndpoint, @@ -1581,15 +1510,6 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile TrustedBundle: typhaNodeTLS.TrustedBundle, Namespace: common.CalicoNamespace, BindingNamespaces: []string{common.CalicoNamespace}, - WAFGatewayExtensionEnabled: wafGatewayExtensionEnabled, - WAFWebhookServerTLS: wafWebhookTLS, - WASMPullSecret: wasmPullSecret, - WASMCACert: wasmCACert, - // The webhook Service + ValidatingWebhookConfiguration are rendered by - // the kube-controllers component (and deleted when the WAF extension is - // disabled); the caBundle is the operator CA that issued the serving - // cert above. - WAFWebhookCABundle: certificateManager.KeyPair().GetCertificatePEM(), } components = append(components, kubecontrollers.NewCalicoKubeControllers(&kubeControllersCfg)) diff --git a/pkg/enterprise/installation.go b/pkg/enterprise/installation.go index 12eef67b13..7df2d2bf30 100644 --- a/pkg/enterprise/installation.go +++ b/pkg/enterprise/installation.go @@ -17,6 +17,7 @@ package enterprise import ( "fmt" + rbacv1 "k8s.io/api/rbac/v1" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/handler" @@ -50,6 +51,13 @@ type installationRenderData struct { // collectProcessPath mirrors LogCollector.Spec.CollectProcessPath being // enabled; the node modifier uses it to set HostPID and the felix env. collectProcessPath bool + + // calico-kube-controllers enterprise additions the kube-controllers modifier + // applies: the enterprise cluster role rules, the enterprise enabled controllers, + // and the WAF v3 (Gateway API add-on) surface. + kubeControllerRules []rbacv1.PolicyRule + kubeControllerControllers []string + waf wafRenderData } // installationData pulls the installation extension's render data back out of the @@ -77,6 +85,8 @@ func (coreControllerExtension) Watches(c ctrlruntime.Controller) error { &operatorv1.ManagementCluster{}, &operatorv1.ManagementClusterConnection{}, &operatorv1.LogCollector{}, + // GatewayAPI.spec.extensions.waf.state gates the WAF v3 surface on calico-kube-controllers. + &operatorv1.GatewayAPI{}, } { if err := c.WatchObject(obj, &handler.EnqueueRequestForObject{}); err != nil { return err @@ -126,10 +136,26 @@ func (coreControllerExtension) ExtendContext(cc extensions.ControllerContext) (e if err != nil { return rc, nil, fmt.Errorf("error reading LogCollector: %w", err) } + + // calico-kube-controllers enterprise additions: the WAF surface, the enterprise + // cluster role rules, and the enterprise enabled controllers. A managed cluster's + // kube-controllers needs an extra license-push rule. + managementClusterConnection, err := utils.GetManagementClusterConnection(cc.Ctx, cc.Client) + if err != nil { + return rc, nil, fmt.Errorf("error reading ManagementClusterConnection: %w", err) + } + waf, wafWebhookTLS, err := buildWAFData(cc) + if err != nil { + return rc, nil, fmt.Errorf("error preparing WAF configuration: %w", err) + } + rc.Extension = installationRenderData{ - nodePrometheusTLS: nodePrometheusTLS, - kubeControllerTLS: kubeControllerTLS, - collectProcessPath: collectProcessPathEnabled(logCollector), + nodePrometheusTLS: nodePrometheusTLS, + kubeControllerTLS: kubeControllerTLS, + collectProcessPath: collectProcessPathEnabled(logCollector), + kubeControllerRules: calicoKubeControllersEnterpriseRules(waf.enabled, managementClusterConnection != nil), + kubeControllerControllers: calicoKubeControllersEnterpriseControllers(waf.enabled), + waf: waf, } prometheusClientCert, err := cc.CertificateManager.GetCertificate(cc.Client, monitor.PrometheusClientTLSSecretName, common.OperatorNamespace()) @@ -165,5 +191,8 @@ func (coreControllerExtension) ExtendContext(cc extensions.ControllerContext) (e if kubeControllerTLS != nil { managed = append(managed, kubeControllerTLS) } + if wafWebhookTLS != nil { + managed = append(managed, wafWebhookTLS) + } return rc, managed, nil } diff --git a/pkg/enterprise/kubecontrollers.go b/pkg/enterprise/kubecontrollers.go index 884f928750..2213e4c1d6 100644 --- a/pkg/enterprise/kubecontrollers.go +++ b/pkg/enterprise/kubecontrollers.go @@ -15,73 +15,202 @@ package enterprise import ( + "encoding/json" + "fmt" + "path/filepath" + "strings" + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" "github.com/tigera/operator/pkg/common" + "github.com/tigera/operator/pkg/components" + "github.com/tigera/operator/pkg/controller/gatewayapi" + "github.com/tigera/operator/pkg/controller/utils" + "github.com/tigera/operator/pkg/controller/utils/imageset" + "github.com/tigera/operator/pkg/dns" "github.com/tigera/operator/pkg/extensions" "github.com/tigera/operator/pkg/render" + "github.com/tigera/operator/pkg/render/applicationlayer" relasticsearch "github.com/tigera/operator/pkg/render/common/elasticsearch" rmeta "github.com/tigera/operator/pkg/render/common/meta" "github.com/tigera/operator/pkg/render/common/networkpolicy" + "github.com/tigera/operator/pkg/render/common/secret" "github.com/tigera/operator/pkg/render/kubecontrollers" "github.com/tigera/operator/pkg/render/monitor" + "github.com/tigera/operator/pkg/tls/certificatemanagement" "github.com/tigera/operator/pkg/url" ) -// registerKubeControllers registers the calico-kube-controllers modifier. There is +// registerKubeControllers registers the calico-kube-controllers modifiers. There is // no image override: kube-controllers runs from the combined calico image, which // resolves by variant in the base render. func registerKubeControllers(v *extensions.Variant) { v.Modify(render.ComponentNameKubeControllers, modifyKubeControllers) + v.Modify(render.ComponentNameKubeControllersPolicy, modifyKubeControllersPolicy) } -// modifyKubeControllers layers the Calico Enterprise metrics serving TLS onto the -// rendered calico-kube-controllers deployment: the env pointing at the keypair, the -// volume + mount, the cert-management init container (when in use), and the pod hash -// annotation that rolls the pod on cert rotation. The keypair has cluster side -// effects, so the installation extension creates it and hands it in via rc. In core -// (calico) it is never created, so the base deployment carries no metrics TLS. -func modifyKubeControllers(rc extensions.RenderContext, objs, del []client.Object) ([]client.Object, []client.Object) { - tls := installationData(rc).kubeControllerTLS - if tls == nil { +// modifyKubeControllersPolicy adds the WAF admission webhook ingress rule to the +// calico-kube-controllers calico-system network policy, so the kube-apiserver can +// reach the in-process webhook on :9443 (EV-6386). Without it the calico-system +// default-deny drops the apiserver->:9443 call and WAF admission times out. +func modifyKubeControllersPolicy(rc extensions.RenderContext, objs, del []client.Object) ([]client.Object, []client.Object) { + if !installationData(rc).waf.enabled { return objs, del } - - dp, ok := extensions.FindObject[*appsv1.Deployment](objs, kubecontrollers.KubeController) + policy, ok := extensions.FindObject[*v3.NetworkPolicy](objs, kubecontrollers.KubeControllerNetworkPolicyName) if !ok { return objs, del } + policy.Spec.Ingress = append(policy.Spec.Ingress, v3.Rule{ + Action: v3.Allow, + Protocol: &networkpolicy.TCPProtocol, + Destination: v3.EntityRule{ + Ports: networkpolicy.Ports(uint16(applicationlayer.WAFWebhookContainerPort)), + }, + }) + return objs, del +} + +// modifyKubeControllers layers the full Calico Enterprise surface onto the rendered +// calico-kube-controllers objects: the enterprise cluster role rules, the enterprise +// enabled controllers, the metrics serving TLS, and the WAF v3 (Gateway API add-on) +// surface. The modifier only runs for the enterprise variant, so everything it adds +// is enterprise-only by construction - the base render carries none of it. The +// controller-side inputs (keypairs, the resolved wasm image, the pull secret) are +// produced by the installation hook and handed in through rc. +func modifyKubeControllers(rc extensions.RenderContext, objs, del []client.Object) ([]client.Object, []client.Object) { + data := installationData(rc) + + if role, ok := extensions.FindObject[*rbacv1.ClusterRole](objs, kubecontrollers.KubeControllerRole); ok { + role.Rules = append(role.Rules, data.kubeControllerRules...) + } + + if dp, ok := extensions.FindObject[*appsv1.Deployment](objs, kubecontrollers.KubeController); ok { + modifyKubeControllersDeployment(rc, dp, data) + } + + // The WAF admission webhook surface (Service + ValidatingWebhookConfiguration), + // the wasm pull secret, and the wasm CA bundle. Created when WAF is enabled, + // deleted otherwise so toggling the extension off cleans them up. + webhookObjs := applicationlayer.WAFAdmissionWebhookComponents(data.waf.caBundle) + if data.waf.enabled { + objs = append(objs, webhookObjs...) + if data.waf.pullSecret != nil { + objs = append(objs, secret.ToRuntimeObjects(secret.CopyToNamespace(common.CalicoNamespace, data.waf.pullSecret)...)...) + } + if data.waf.caCert != nil { + objs = append(objs, data.waf.caCert) + } + } else { + del = append(del, webhookObjs...) + } + + return objs, del +} + +func modifyKubeControllersDeployment(rc extensions.RenderContext, dp *appsv1.Deployment, data installationRenderData) { spec := &dp.Spec.Template.Spec - spec.Volumes = append(spec.Volumes, tls.Volume()) + if dp.Spec.Template.Annotations == nil { + dp.Spec.Template.Annotations = map[string]string{} + } + + if tls := data.kubeControllerTLS; tls != nil { + spec.Volumes = append(spec.Volumes, tls.Volume()) + dp.Spec.Template.Annotations[tls.HashAnnotationKey()] = tls.HashAnnotationValue() + } + if waf := data.waf; waf.enabled && waf.webhookTLS != nil { + spec.Volumes = append(spec.Volumes, waf.webhookTLS.Volume()) + } for i := range spec.Containers { c := &spec.Containers[i] if c.Name != kubecontrollers.KubeController { continue } - c.Env = append(c.Env, - corev1.EnvVar{Name: "TLS_KEY_PATH", Value: tls.VolumeMountKeyFilePath()}, - corev1.EnvVar{Name: "TLS_CRT_PATH", Value: tls.VolumeMountCertificateFilePath()}, - corev1.EnvVar{Name: "CLIENT_COMMON_NAME", Value: monitor.PrometheusClientTLSSecretName}, - ) - c.VolumeMounts = append(c.VolumeMounts, tls.VolumeMount(rmeta.OSTypeLinux)) - if tls.UseCertificateManagement() { - spec.InitContainers = append(spec.InitContainers, tls.InitContainer(common.CalicoNamespace, c.SecurityContext)) + + appendEnabledControllers(c, data.kubeControllerControllers) + c.Env = append(c.Env, enterpriseEnv(rc)...) + + if tls := data.kubeControllerTLS; tls != nil { + c.Env = append(c.Env, + corev1.EnvVar{Name: "TLS_KEY_PATH", Value: tls.VolumeMountKeyFilePath()}, + corev1.EnvVar{Name: "TLS_CRT_PATH", Value: tls.VolumeMountCertificateFilePath()}, + corev1.EnvVar{Name: "CLIENT_COMMON_NAME", Value: monitor.PrometheusClientTLSSecretName}, + ) + c.VolumeMounts = append(c.VolumeMounts, tls.VolumeMount(rmeta.OSTypeLinux)) + if tls.UseCertificateManagement() { + spec.InitContainers = append(spec.InitContainers, tls.InitContainer(common.CalicoNamespace, c.SecurityContext)) + } + } + + if waf := data.waf; waf.enabled { + c.Env = append(c.Env, wafEnv(waf)...) + c.Ports = append(c.Ports, corev1.ContainerPort{ + Name: "waf-webhook", + ContainerPort: applicationlayer.WAFWebhookContainerPort, + Protocol: corev1.ProtocolTCP, + }) + if waf.webhookTLS != nil { + c.VolumeMounts = append(c.VolumeMounts, waf.webhookTLS.VolumeMount(rmeta.OSTypeLinux)) + if waf.webhookTLS.UseCertificateManagement() { + spec.InitContainers = append(spec.InitContainers, waf.webhookTLS.InitContainer(common.CalicoNamespace, c.SecurityContext)) + } + } } } +} - if dp.Spec.Template.Annotations == nil { - dp.Spec.Template.Annotations = map[string]string{} +// appendEnabledControllers folds the enterprise controllers into the existing +// ENABLED_CONTROLLERS env the base render set (node,loadbalancer). +func appendEnabledControllers(c *corev1.Container, extra []string) { + if len(extra) == 0 { + return + } + for i := range c.Env { + if c.Env[i].Name == "ENABLED_CONTROLLERS" { + c.Env[i].Value = c.Env[i].Value + "," + strings.Join(extra, ",") + return + } } - dp.Spec.Template.Annotations[tls.HashAnnotationKey()] = tls.HashAnnotationValue() +} - return objs, del +// enterpriseEnv is the static enterprise env for calico-kube-controllers. The +// modifier runs only for the enterprise variant, so these are never rendered for core. +func enterpriseEnv(rc extensions.RenderContext) []corev1.EnvVar { + var env []corev1.EnvVar + if rc.TrustedBundle != nil { + env = append(env, corev1.EnvVar{Name: "MULTI_CLUSTER_FORWARDING_CA", Value: rc.TrustedBundle.MountPath()}) + } + if in := rc.Installation; in != nil && in.CalicoNetwork != nil && in.CalicoNetwork.MultiInterfaceMode != nil { + env = append(env, corev1.EnvVar{Name: "MULTI_INTERFACE_MODE", Value: in.CalicoNetwork.MultiInterfaceMode.Value()}) + } + return env +} + +// wafEnv is the WAF v3 env the kube-controllers binary consumes to program WAF policy +// attachments. WASM_IMAGE is the pre-resolved reference the hook produced. +func wafEnv(waf wafRenderData) []corev1.EnvVar { + var env []corev1.EnvVar + if waf.wasmImage != "" { + env = append(env, corev1.EnvVar{Name: "WASM_IMAGE", Value: waf.wasmImage}) + } + if waf.pullSecret != nil { + env = append(env, corev1.EnvVar{Name: "WASM_PULL_SECRET", Value: waf.pullSecret.Name}) + } + if waf.caCert != nil { + env = append(env, corev1.EnvVar{Name: "WASM_CA_CERT", Value: waf.caCert.Name}) + } + if waf.webhookTLS != nil { + env = append(env, corev1.EnvVar{Name: "WAF_WEBHOOK_CERT_DIR", Value: filepath.Dir(waf.webhookTLS.VolumeMountCertificateFilePath())}) + } + return env } const ( @@ -95,6 +224,18 @@ const ( ElasticsearchKubeControllersUserName = "tigera-ee-kube-controllers" ElasticsearchKubeControllersSecureUserSecret = "tigera-ee-kube-controllers-elasticsearch-access-gateway" ElasticsearchKubeControllersVerificationUserSecret = "tigera-ee-kube-controllers-gateway-verification-credentials" + + // WASMPullSecretName is the dedicated image-pull Secret (a merged copy of the + // install pull secrets) the WAF reconciler replicates into tenant namespaces for + // the Coraza wasm OCI pull. A dedicated name avoids clashing with the + // operator-managed tigera-pull-secret the GatewayAPI render also copies there (EV-6386). + WASMPullSecretName = "tigera-waf-pull-secret" + + // WASMCACertName is the dedicated CA-bundle ConfigMap the WAF reconciler + // replicates into tenant namespaces for the Coraza wasm OCI registry TLS check - + // a dedicated name avoids clashing with the operator-managed tigera-ca-bundle the + // GatewayAPI render also copies there (EV-6386). It is a renamed copy of the trusted bundle. + WASMCACertName = "tigera-waf-ca-bundle" ) // NewElasticsearchKubeControllers fills the generic kube-controllers configuration @@ -111,7 +252,7 @@ func NewElasticsearchKubeControllers(cfg *kubecontrollers.KubeControllersConfigu cfg.DisableConfigAPI = cfg.Tenant.MultiTenant() cfg.Rules = kubecontrollers.KubeControllersRoleCommonRules(cfg) - cfg.Rules = append(cfg.Rules, kubecontrollers.KubeControllersRoleEnterpriseCommonRules(cfg)...) + cfg.Rules = append(cfg.Rules, kubeControllersEnterpriseCommonRules(false, cfg.ManagementClusterConnection != nil)...) cfg.Rules = append(cfg.Rules, rbacv1.PolicyRule{ APIGroups: []string{"elasticsearch.k8s.elastic.co"}, @@ -181,6 +322,200 @@ func esKubeControllersEnv(cfg *kubecontrollers.KubeControllersConfiguration) []c return env } +// kubeControllersEnterpriseCommonRules are the Calico Enterprise cluster role rules +// shared by calico-kube-controllers and es-calico-kube-controllers. wafEnabled adds +// the WAF v3 (Gateway API add-on) rules; managedCluster adds the license-push rule a +// managed cluster's kube-controllers needs. +func kubeControllersEnterpriseCommonRules(wafEnabled, managedCluster bool) []rbacv1.PolicyRule { + rules := []rbacv1.PolicyRule{ + { + APIGroups: []string{""}, + Resources: []string{"configmaps"}, + Verbs: []string{"watch", "list", "get", "update", "create", "delete"}, + }, + { + // The Federated Services Controller needs access to the remote kubeconfig secret + // in order to create a remote syncer. + APIGroups: []string{""}, + Resources: []string{"secrets"}, + Verbs: []string{"watch", "list", "get"}, + }, + { + // Needed to validate the license + APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, + Resources: []string{"licensekeys"}, + Verbs: []string{"get", "watch", "list"}, + }, + { + // Needed to update the status of the LicenseKey with the result of license validation. + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"licensekeys/status"}, + Verbs: []string{"update"}, + }, + { + APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, + Resources: []string{"deeppacketinspections"}, + Verbs: []string{"get", "watch", "list"}, + }, + { + APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, + Resources: []string{"deeppacketinspections/status"}, + Verbs: []string{"update"}, + }, + { + APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, + Resources: []string{"packetcaptures"}, + Verbs: []string{"get", "list", "update"}, + }, + { + APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, + Resources: []string{"packetcaptures/status"}, + Verbs: []string{"update"}, + }, + } + + if wafEnabled { + rules = append(rules, wafRules()...) + } + + if managedCluster { + rules = append(rules, + rbacv1.PolicyRule{ + APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, + Resources: []string{"licensekeys"}, + Verbs: []string{"get", "create", "update", "list", "watch"}, + }, + ) + } + + return rules +} + +// calicoKubeControllersEnterpriseRules are the enterprise cluster role rules layered +// onto calico-kube-controllers: the shared enterprise rules plus the calico-specific +// ones (federated endpoints, license usage reporting). +func calicoKubeControllersEnterpriseRules(wafEnabled, managedCluster bool) []rbacv1.PolicyRule { + rules := kubeControllersEnterpriseCommonRules(wafEnabled, managedCluster) + return append(rules, + rbacv1.PolicyRule{ + APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, + Resources: []string{"remoteclusterconfigurations"}, + Verbs: []string{"watch", "list", "get"}, + }, + rbacv1.PolicyRule{ + APIGroups: []string{""}, + Resources: []string{"endpoints"}, + Verbs: []string{"create", "update", "delete"}, + }, + rbacv1.PolicyRule{ + APIGroups: []string{""}, + Resources: []string{"namespaces"}, + Verbs: []string{"get"}, + }, + rbacv1.PolicyRule{ + APIGroups: []string{"usage.tigera.io"}, + Resources: []string{"licenseusagereports"}, + Verbs: []string{"create", "update", "delete", "watch", "list", "get"}, + }, + ) +} + +// calicoKubeControllersEnterpriseControllers are the enterprise controllers added to +// the calico-kube-controllers ENABLED_CONTROLLERS list (on top of the base +// node,loadbalancer). applicationlayer is added only when the WAF extension is on. +func calicoKubeControllersEnterpriseControllers(wafEnabled bool) []string { + controllers := []string{"service", "federatedservices", "usage"} + if wafEnabled { + controllers = append(controllers, "applicationlayer") + } + return controllers +} + +// wafRules are the WAF v3 (Gateway API add-on) cluster role rules, gated by +// GatewayAPI.spec.extensions.waf.state == Enabled. +func wafRules() []rbacv1.PolicyRule { + return []rbacv1.PolicyRule{ + // Application-layer (gateway-addons) reconcilers reconcile WAF resources + // against Gateway API targetRefs and emit events on the policy objects. + { + APIGroups: []string{"applicationlayer.projectcalico.org"}, + Resources: []string{ + "wafpolicies", "globalwafpolicies", + "wafplugins", "globalwafplugins", + "wafvalidationpolicies", "globalwafvalidationpolicies", + }, + Verbs: []string{"get", "list", "watch", "create", "update", "patch", "delete"}, + }, + { + APIGroups: []string{"applicationlayer.projectcalico.org"}, + Resources: []string{ + "wafpolicies/status", "globalwafpolicies/status", + "wafplugins/status", "globalwafplugins/status", + "wafvalidationpolicies/status", "globalwafvalidationpolicies/status", + }, + Verbs: []string{"get", "update", "patch"}, + }, + { + APIGroups: []string{"applicationlayer.projectcalico.org"}, + Resources: []string{ + "wafpolicies/finalizers", "globalwafpolicies/finalizers", + "wafplugins/finalizers", "globalwafplugins/finalizers", + "wafvalidationpolicies/finalizers", "globalwafvalidationpolicies/finalizers", + }, + Verbs: []string{"update"}, + }, + { + // Validate Gateway API targetRefs and surface attachment status. + APIGroups: []string{"gateway.networking.k8s.io"}, + Resources: []string{"gateways", "httproutes", "tcproutes", "tlsroutes", "grpcroutes"}, + Verbs: []string{"get", "list", "watch", "update", "patch"}, + }, + { + APIGroups: []string{"gateway.networking.k8s.io"}, + Resources: []string{"gateways/status", "httproutes/status", "tcproutes/status", "tlsroutes/status", "grpcroutes/status"}, + Verbs: []string{"get", "update", "patch"}, + }, + // controller-runtime Reconcilers (e.g. the applicationlayer manager) record + // events on watched objects via Recorder.Eventf; both core and events.k8s.io + // API groups are emitted depending on the kubernetes version. + { + APIGroups: []string{""}, + Resources: []string{"events"}, + Verbs: []string{"create", "patch"}, + }, + { + APIGroups: []string{"events.k8s.io"}, + Resources: []string{"events"}, + Verbs: []string{"create", "patch"}, + }, + // Application-layer reconciler replicates the WAF wasm pull Secret from + // the controller namespace (calico-system) into each WAFPolicy's + // namespace so the rendered EnvoyExtensionPolicy can reference it. Also + // replicates CA-cert ConfigMaps when WASM_CA_CERT is set. + { + APIGroups: []string{""}, + Resources: []string{"secrets", "configmaps"}, + Verbs: []string{"get", "list", "watch", "create", "update", "patch", "delete"}, + }, + // Application-layer reconciler emits one EnvoyExtensionPolicy per WAF + // targetRef to bind the Coraza wasm filter at the gateway / route. + { + APIGroups: []string{"gateway.envoyproxy.io"}, + Resources: []string{"envoyextensionpolicies"}, + Verbs: []string{"get", "list", "watch", "create", "update", "patch", "delete"}, + }, + // Application-layer reconciler stamps each namespace with its allocated WAF + // rule-id range (applicationlayer.projectcalico.org/waf-id-range annotation) + // so application operators can author in-range rules. The base role already + // grants namespaces get/list/watch; the annotation write needs patch/update. + { + APIGroups: []string{""}, + Resources: []string{"namespaces"}, + Verbs: []string{"get", "patch", "update"}, + }, + } +} + func esKubeControllersCalicoSystemPolicy(cfg *kubecontrollers.KubeControllersConfiguration) *v3.NetworkPolicy { if cfg.ManagementClusterConnection != nil { return nil @@ -230,3 +565,150 @@ func esKubeControllersCalicoSystemPolicy(cfg *kubecontrollers.KubeControllersCon }, } } + +// wafRenderData is the controller-produced WAF v3 (Gateway API add-on) state the +// installation hook hands the kube-controllers modifier through the render context. +// The zero value (enabled false) means the modifier deletes the webhook objects. +type wafRenderData struct { + enabled bool + wasmImage string + pullSecret *corev1.Secret + caCert *corev1.ConfigMap + webhookTLS certificatemanagement.KeyPairInterface + caBundle []byte +} + +// buildWAFData reads the GatewayAPI CR and, when the WAF extension is enabled, +// produces everything the modifier needs that it can't compute itself: the resolved +// wasm image, the webhook serving keypair (also returned as a managed keypair), the +// merged wasm pull secret, the wasm CA bundle ConfigMap, and the operator CA PEM. +func buildWAFData(cc extensions.ControllerContext) (wafRenderData, certificatemanagement.KeyPairInterface, error) { + gw, _, err := gatewayapi.GetGatewayAPI(cc.Ctx, cc.Client) + if err != nil && !apierrors.IsNotFound(err) { + return wafRenderData{}, nil, err + } + if gw == nil || !gw.Spec.IsWAFGatewayExtensionEnabled() { + return wafRenderData{}, nil, nil + } + + in := cc.Installation + // The wasm is baked into the gateway envoy-proxy image. Resolve it with the same + // GetReference the base render uses for every image; the hook has the ImageSet here. + imageSet, err := imageset.GetImageSet(cc.Ctx, cc.Client, in.Variant) + if err != nil { + return wafRenderData{}, nil, err + } + wasmImage, err := components.GetReference(components.ComponentGatewayAPIEnvoyProxy, in.Registry, in.ImagePath, in.ImagePrefix, imageSet) + if err != nil { + return wafRenderData{}, nil, err + } + + webhookTLS, err := cc.CertificateManager.GetOrCreateKeyPair( + cc.Client, + applicationlayer.WAFWebhookServerTLSSecretName, + common.OperatorNamespace(), + dns.GetServiceDNSNames(applicationlayer.WAFWebhookServiceName, common.CalicoNamespace, cc.ClusterDomain), + ) + if err != nil { + return wafRenderData{}, nil, err + } + + pullSecrets, err := utils.GetInstallationPullSecrets(in, cc.Client) + if err != nil { + return wafRenderData{}, nil, err + } + var pullSecret *corev1.Secret + if len(pullSecrets) > 0 { + pullSecret, _ = MergeWAFPullSecret(pullSecrets) + } + + var caCert *corev1.ConfigMap + if cc.TrustedBundle != nil { + caCert = cc.TrustedBundle.ConfigMap(common.CalicoNamespace) + caCert.Name = WASMCACertName + } + + return wafRenderData{ + enabled: true, + wasmImage: wasmImage, + pullSecret: pullSecret, + caCert: caCert, + webhookTLS: webhookTLS, + caBundle: cc.CertificateManager.KeyPair().GetCertificatePEM(), + }, webhookTLS, nil +} + +// MergeWAFPullSecret synthesizes the dedicated WAF wasm pull secret +// (tigera-waf-pull-secret) by merging the registry auths of every Installation pull +// secret. The EnvoyExtensionPolicy image source takes a single pullSecretRef, so a +// merged secret is the only way to honor multiple Installation pull secrets for the +// Coraza wasm OCI pull (e.g. the Tigera pull secret plus a private registry mirror). +// +// If the same registry appears in more than one secret, the first secret in +// Installation order wins. Secrets that cannot be parsed are skipped and their names +// returned, so the caller can log them without failing the reconcile. Returns a nil +// Secret when no registry auths could be collected. +func MergeWAFPullSecret(pullSecrets []*corev1.Secret) (*corev1.Secret, []string) { + merged := map[string]json.RawMessage{} + var skipped []string + for _, s := range pullSecrets { + auths, err := registryAuths(s) + if err != nil { + skipped = append(skipped, s.Name) + continue + } + for registry, auth := range auths { + if _, ok := merged[registry]; !ok { + merged[registry] = auth + } + } + } + if len(merged) == 0 { + return nil, skipped + } + + // Marshalling a map sorts its keys, so the rendered bytes are deterministic and + // do not churn the object on every reconcile. + data, err := json.Marshal(map[string]map[string]json.RawMessage{"auths": merged}) + if err != nil { + // Each auth entry round-trips from a successful Unmarshal above, so this + // cannot fail in practice; treat it as nothing to render. + return nil, skipped + } + + return &corev1.Secret{ + TypeMeta: metav1.TypeMeta{Kind: "Secret", APIVersion: "v1"}, + ObjectMeta: metav1.ObjectMeta{Name: WASMPullSecretName, Namespace: common.CalicoNamespace}, + Type: corev1.SecretTypeDockerConfigJson, + Data: map[string][]byte{corev1.DockerConfigJsonKey: data}, + }, skipped +} + +// registryAuths extracts the per-registry auth entries from a pull secret of either +// the dockerconfigjson type (auths nested under an "auths" key) or the legacy +// dockercfg type (a bare registry -> auth map). +func registryAuths(s *corev1.Secret) (map[string]json.RawMessage, error) { + if raw, ok := s.Data[corev1.DockerConfigJsonKey]; ok { + var cfg struct { + Auths map[string]json.RawMessage `json:"auths"` + } + if err := json.Unmarshal(raw, &cfg); err != nil { + return nil, err + } + if len(cfg.Auths) == 0 { + return nil, fmt.Errorf("secret %s has no auths entries", s.Name) + } + return cfg.Auths, nil + } + if raw, ok := s.Data[corev1.DockerConfigKey]; ok { + var auths map[string]json.RawMessage + if err := json.Unmarshal(raw, &auths); err != nil { + return nil, err + } + if len(auths) == 0 { + return nil, fmt.Errorf("secret %s has no auths entries", s.Name) + } + return auths, nil + } + return nil, fmt.Errorf("secret %s has neither a %s nor a %s key", s.Name, corev1.DockerConfigJsonKey, corev1.DockerConfigKey) +} diff --git a/pkg/enterprise/kubecontrollers_test.go b/pkg/enterprise/kubecontrollers_test.go index 41ee40ca72..3bb93c0e62 100644 --- a/pkg/enterprise/kubecontrollers_test.go +++ b/pkg/enterprise/kubecontrollers_test.go @@ -24,6 +24,7 @@ import ( v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -31,11 +32,15 @@ import ( operatorv1 "github.com/tigera/operator/api/v1" "github.com/tigera/operator/pkg/apis" "github.com/tigera/operator/pkg/common" + "github.com/tigera/operator/pkg/components" "github.com/tigera/operator/pkg/controller/certificatemanager" ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" + "github.com/tigera/operator/pkg/enterprise" "github.com/tigera/operator/pkg/extensions" "github.com/tigera/operator/pkg/render" + "github.com/tigera/operator/pkg/render/applicationlayer" rmeta "github.com/tigera/operator/pkg/render/common/meta" + "github.com/tigera/operator/pkg/render/common/networkpolicy" "github.com/tigera/operator/pkg/render/kubecontrollers" "github.com/tigera/operator/pkg/render/monitor" "github.com/tigera/operator/pkg/tls" @@ -122,3 +127,166 @@ func certManagementControllerContext() extensions.ControllerContext { CertificateManager: certManager, } } + +var _ = Describe("calico-kube-controllers enterprise surface", func() { + calicoKubeControllersCfg := func(cc extensions.ControllerContext) *kubecontrollers.KubeControllersConfiguration { + return &kubecontrollers.KubeControllersConfiguration{ + Installation: cc.Installation, + ClusterDomain: cc.ClusterDomain, + TrustedBundle: cc.TrustedBundle, + MetricsPort: 9094, + Namespace: common.CalicoNamespace, + BindingNamespaces: []string{common.CalicoNamespace}, + } + } + + // render builds the base calico-kube-controllers objects and applies the + // enterprise modifier, exactly as the component handler does. + renderKubeControllers := func(cc extensions.ControllerContext, rc extensions.RenderContext) []client.Object { + comp := kubecontrollers.NewCalicoKubeControllers(calicoKubeControllersCfg(cc)) + Expect(comp.ResolveImages(nil)).NotTo(HaveOccurred()) + create, del := comp.Objects() + out, _ := applyExtensions(ext, render.ComponentNameKubeControllers, rc, create, del) + return out + } + + kubeContainer := func(objs []client.Object) *corev1.Container { + dp, ok := extensions.FindObject[*appsv1.Deployment](objs, kubecontrollers.KubeController) + Expect(ok).To(BeTrue()) + return &dp.Spec.Template.Spec.Containers[0] + } + + It("layers the enterprise rules, controllers, and metrics TLS on (WAF off)", func() { + rc, _, err := ext.ExtendContext(newControllerContext(operatorv1.CalicoEnterprise)) + Expect(err).NotTo(HaveOccurred()) + objs := renderKubeControllers(newControllerContext(operatorv1.CalicoEnterprise), rc) + + role, ok := extensions.FindObject[*rbacv1.ClusterRole](objs, kubecontrollers.KubeControllerRole) + Expect(ok).To(BeTrue()) + Expect(role.Rules).To(ContainElement(HaveField("Resources", ContainElement("licensekeys")))) + + c := kubeContainer(objs) + Expect(c.Env).To(ContainElement(corev1.EnvVar{ + Name: "ENABLED_CONTROLLERS", Value: "node,loadbalancer,service,federatedservices,usage", + })) + // Metrics serving TLS wired from the keypair the hook created. + Expect(c.Env).To(ContainElement(HaveField("Name", "TLS_KEY_PATH"))) + // WAF is off, so no WASM env and no webhook objects. + Expect(c.Env).NotTo(ContainElement(HaveField("Name", "WASM_IMAGE"))) + _, ok = extensions.FindObject[*corev1.Service](objs, applicationlayer.WAFWebhookServiceName) + Expect(ok).To(BeFalse()) + }) + + It("layers the full WAF surface on when the GatewayAPI extension is enabled", func() { + cc := wafControllerContext() + rc, managed, err := ext.ExtendContext(cc) + Expect(err).NotTo(HaveOccurred()) + names := []string{} + for _, kp := range managed { + names = append(names, kp.GetName()) + } + Expect(names).To(ContainElement(applicationlayer.WAFWebhookServerTLSSecretName)) + + objs := renderKubeControllers(cc, rc) + + role, ok := extensions.FindObject[*rbacv1.ClusterRole](objs, kubecontrollers.KubeControllerRole) + Expect(ok).To(BeTrue()) + Expect(role.Rules).To(ContainElement(HaveField("Resources", ContainElement("wafpolicies")))) + + c := kubeContainer(objs) + Expect(c.Env).To(ContainElement(corev1.EnvVar{ + Name: "ENABLED_CONTROLLERS", Value: "node,loadbalancer,service,federatedservices,usage,applicationlayer", + })) + Expect(c.Env).To(ContainElement(corev1.EnvVar{ + Name: "WASM_IMAGE", Value: "test-reg/tigera/envoy-proxy:" + components.ComponentGatewayAPIEnvoyProxy.Version, + })) + Expect(c.Env).To(ContainElement(corev1.EnvVar{Name: "WASM_PULL_SECRET", Value: enterprise.WASMPullSecretName})) + Expect(c.Env).To(ContainElement(corev1.EnvVar{Name: "WASM_CA_CERT", Value: enterprise.WASMCACertName})) + Expect(c.Env).To(ContainElement(HaveField("Name", "WAF_WEBHOOK_CERT_DIR"))) + Expect(c.Ports).To(ContainElement(corev1.ContainerPort{Name: "waf-webhook", ContainerPort: int32(9443), Protocol: corev1.ProtocolTCP})) + + // The webhook surface, the wasm pull secret, and the wasm CA bundle are rendered. + _, ok = extensions.FindObject[*corev1.Service](objs, applicationlayer.WAFWebhookServiceName) + Expect(ok).To(BeTrue()) + _, ok = extensions.FindObject[*corev1.Secret](objs, enterprise.WASMPullSecretName) + Expect(ok).To(BeTrue()) + _, ok = extensions.FindObject[*corev1.ConfigMap](objs, enterprise.WASMCACertName) + Expect(ok).To(BeTrue()) + }) + + It("deletes the WAF webhook surface when the extension is disabled", func() { + cc := newControllerContext(operatorv1.CalicoEnterprise) + rc, _, err := ext.ExtendContext(cc) + Expect(err).NotTo(HaveOccurred()) + + comp := kubecontrollers.NewCalicoKubeControllers(calicoKubeControllersCfg(cc)) + Expect(comp.ResolveImages(nil)).NotTo(HaveOccurred()) + create, del := comp.Objects() + _, toDelete := applyExtensions(ext, render.ComponentNameKubeControllers, rc, create, del) + + _, ok := extensions.FindObject[*corev1.Service](toDelete, applicationlayer.WAFWebhookServiceName) + Expect(ok).To(BeTrue(), "the webhook Service should be queued for deletion") + }) + + It("adds the WAF webhook ingress rule to the network policy when enabled", func() { + cc := wafControllerContext() + rc, _, err := ext.ExtendContext(cc) + Expect(err).NotTo(HaveOccurred()) + + comp := kubecontrollers.NewCalicoKubeControllersPolicy(calicoKubeControllersCfg(cc), nil) + create, del := comp.Objects() + objs, _ := applyExtensions(ext, render.ComponentNameKubeControllersPolicy, rc, create, del) + + policy, ok := extensions.FindObject[*v3.NetworkPolicy](objs, kubecontrollers.KubeControllerNetworkPolicyName) + Expect(ok).To(BeTrue()) + Expect(policy.Spec.Ingress).To(ContainElement(v3.Rule{ + Action: v3.Allow, + Protocol: &networkpolicy.TCPProtocol, + Destination: v3.EntityRule{ + Ports: networkpolicy.Ports(uint16(applicationlayer.WAFWebhookContainerPort)), + }, + })) + }) +}) + +// wafControllerContext builds a controller context with a WAF-enabled GatewayAPI CR +// and an install pull secret, so the installation hook produces the full WAF data. +func wafControllerContext() extensions.ControllerContext { + scheme := runtime.NewScheme() + Expect(apis.AddToScheme(scheme, false)).NotTo(HaveOccurred()) + c := ctrlrfake.DefaultFakeClientBuilder(scheme).Build() + + Expect(c.Create(context.Background(), &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "pull", Namespace: common.OperatorNamespace()}, + Type: corev1.SecretTypeDockerConfigJson, + Data: map[string][]byte{corev1.DockerConfigJsonKey: []byte(`{"auths":{"reg.example.com":{"auth":"abc"}}}`)}, + })).NotTo(HaveOccurred()) + + enabled := operatorv1.WAFExtensionStateEnabled + Expect(c.Create(context.Background(), &operatorv1.GatewayAPI{ + ObjectMeta: metav1.ObjectMeta{Name: "default"}, + Spec: operatorv1.GatewayAPISpec{ + Extensions: &operatorv1.GatewayAPIExtensions{WAF: &operatorv1.WAFExtensionSpec{State: &enabled}}, + }, + })).NotTo(HaveOccurred()) + + certManager, err := certificatemanager.Create(c, nil, "", common.OperatorNamespace(), certificatemanager.AllowCACreation()) + Expect(err).NotTo(HaveOccurred()) + + return extensions.ControllerContext{ + RenderContext: extensions.RenderContext{ + Installation: &operatorv1.InstallationSpec{ + Variant: operatorv1.CalicoEnterprise, + Registry: "test-reg/", + ImagePullSecrets: []corev1.LocalObjectReference{{Name: "pull"}}, + }, + FelixConfiguration: &v3.FelixConfiguration{}, + TrustedBundle: certManager.CreateTrustedBundle(), + ClusterDomain: "cluster.local", + }, + Controller: extensions.InstallationController, + Ctx: context.Background(), + Client: c, + CertificateManager: certManager, + } +} diff --git a/pkg/render/kubecontrollers/waf_pull_secret_test.go b/pkg/enterprise/waf_pull_secret_test.go similarity index 88% rename from pkg/render/kubecontrollers/waf_pull_secret_test.go rename to pkg/enterprise/waf_pull_secret_test.go index 793374f169..92cab8df9d 100644 --- a/pkg/render/kubecontrollers/waf_pull_secret_test.go +++ b/pkg/enterprise/waf_pull_secret_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package kubecontrollers_test +package enterprise_test import ( "encoding/json" @@ -22,7 +22,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/tigera/operator/pkg/common" - "github.com/tigera/operator/pkg/render/kubecontrollers" + "github.com/tigera/operator/pkg/enterprise" ) func dockerConfigJSONSecret(name string, auths map[string]any) *corev1.Secret { @@ -49,7 +49,7 @@ func mergedAuths(t *testing.T, s *corev1.Secret) map[string]map[string]string { } func TestMergeWAFPullSecret_MergesDisjointRegistries(t *testing.T) { - merged, skipped := kubecontrollers.MergeWAFPullSecret([]*corev1.Secret{ + merged, skipped := enterprise.MergeWAFPullSecret([]*corev1.Secret{ dockerConfigJSONSecret("tigera-pull-secret", map[string]any{"quay.io": map[string]string{"auth": "dGlnZXJh"}}), dockerConfigJSONSecret("mirror-pull-secret", map[string]any{"registry.example.com": map[string]string{"auth": "bWlycm9y"}}), }) @@ -59,7 +59,7 @@ func TestMergeWAFPullSecret_MergesDisjointRegistries(t *testing.T) { if merged == nil { t.Fatal("expected a merged secret") } - if merged.Name != kubecontrollers.WASMPullSecretName || merged.Namespace != common.CalicoNamespace { + if merged.Name != enterprise.WASMPullSecretName || merged.Namespace != common.CalicoNamespace { t.Fatalf("unexpected name/namespace: %s/%s", merged.Namespace, merged.Name) } if merged.Type != corev1.SecretTypeDockerConfigJson { @@ -72,7 +72,7 @@ func TestMergeWAFPullSecret_MergesDisjointRegistries(t *testing.T) { } func TestMergeWAFPullSecret_FirstSecretWinsOnDuplicateRegistry(t *testing.T) { - merged, _ := kubecontrollers.MergeWAFPullSecret([]*corev1.Secret{ + merged, _ := enterprise.MergeWAFPullSecret([]*corev1.Secret{ dockerConfigJSONSecret("first", map[string]any{"quay.io": map[string]string{"auth": "Zmlyc3Q="}}), dockerConfigJSONSecret("second", map[string]any{"quay.io": map[string]string{"auth": "c2Vjb25k"}}), }) @@ -88,7 +88,7 @@ func TestMergeWAFPullSecret_SkipsUnparseableSecrets(t *testing.T) { Type: corev1.SecretTypeDockerConfigJson, Data: map[string][]byte{corev1.DockerConfigJsonKey: []byte("not-json")}, } - merged, skipped := kubecontrollers.MergeWAFPullSecret([]*corev1.Secret{ + merged, skipped := enterprise.MergeWAFPullSecret([]*corev1.Secret{ bad, dockerConfigJSONSecret("good", map[string]any{"quay.io": map[string]string{"auth": "Z29vZA=="}}), }) @@ -111,7 +111,7 @@ func TestMergeWAFPullSecret_LegacyDockercfg(t *testing.T) { Type: corev1.SecretTypeDockercfg, Data: map[string][]byte{corev1.DockerConfigKey: cfg}, } - merged, skipped := kubecontrollers.MergeWAFPullSecret([]*corev1.Secret{legacy}) + merged, skipped := enterprise.MergeWAFPullSecret([]*corev1.Secret{legacy}) if len(skipped) != 0 { t.Fatalf("expected no skipped secrets, got %v", skipped) } @@ -127,7 +127,7 @@ func TestMergeWAFPullSecret_NothingUsableReturnsNil(t *testing.T) { Type: corev1.SecretTypeDockerConfigJson, Data: map[string][]byte{corev1.DockerConfigJsonKey: []byte("not-json")}, } - merged, skipped := kubecontrollers.MergeWAFPullSecret([]*corev1.Secret{bad}) + merged, skipped := enterprise.MergeWAFPullSecret([]*corev1.Secret{bad}) if merged != nil { t.Fatalf("expected nil secret, got %v", merged) } @@ -141,8 +141,8 @@ func TestMergeWAFPullSecret_DeterministicOutput(t *testing.T) { dockerConfigJSONSecret("a", map[string]any{"z.example.com": map[string]string{"auth": "eg=="}, "a.example.com": map[string]string{"auth": "YQ=="}}), dockerConfigJSONSecret("b", map[string]any{"m.example.com": map[string]string{"auth": "bQ=="}}), } - first, _ := kubecontrollers.MergeWAFPullSecret(in) - second, _ := kubecontrollers.MergeWAFPullSecret(in) + first, _ := enterprise.MergeWAFPullSecret(in) + second, _ := enterprise.MergeWAFPullSecret(in) if string(first.Data[corev1.DockerConfigJsonKey]) != string(second.Data[corev1.DockerConfigJsonKey]) { t.Fatal("merged secret bytes must be deterministic across reconciles") } diff --git a/pkg/render/component.go b/pkg/render/component.go index 6fdf85303a..93e4824f3a 100644 --- a/pkg/render/component.go +++ b/pkg/render/component.go @@ -81,4 +81,8 @@ const ( // es-calico-kube-controllers deployment shares the component type but leaves // its modifier key empty, so it is not decorated. ComponentNameKubeControllers = "kube-controllers" + + // ComponentNameKubeControllersPolicy keys the calico-kube-controllers network + // policy modifier (the WAF admission webhook ingress rule). + ComponentNameKubeControllersPolicy = "kube-controllers-policy" ) diff --git a/pkg/render/kubecontrollers/kube-controllers.go b/pkg/render/kubecontrollers/kube-controllers.go index 46dbf2cd8a..36bc0068da 100644 --- a/pkg/render/kubecontrollers/kube-controllers.go +++ b/pkg/render/kubecontrollers/kube-controllers.go @@ -16,7 +16,6 @@ package kubecontrollers import ( "fmt" - "path/filepath" "slices" "strconv" "strings" @@ -36,7 +35,6 @@ import ( "github.com/tigera/operator/pkg/components" "github.com/tigera/operator/pkg/controller/k8sapi" "github.com/tigera/operator/pkg/render" - "github.com/tigera/operator/pkg/render/applicationlayer" rcomp "github.com/tigera/operator/pkg/render/common/components" rmeta "github.com/tigera/operator/pkg/render/common/meta" "github.com/tigera/operator/pkg/render/common/networkpolicy" @@ -54,21 +52,6 @@ const ( KubeControllerMetrics = "calico-kube-controllers-metrics" KubeControllerNetworkPolicyName = networkpolicy.CalicoComponentPolicyPrefix + "kube-controller-access" - // WASMPullSecretName is the dedicated image-pull Secret (a renamed copy of - // the install pull secret) that the WAF reconciler replicates into tenant - // namespaces for the Coraza wasm OCI pull. A dedicated name avoids clashing - // with the operator-managed tigera-pull-secret the GatewayAPI render also - // copies into those namespaces (EV-6386). - WASMPullSecretName = "tigera-waf-pull-secret" - - // WASMCACertName is the dedicated CA-bundle ConfigMap (in the controller - // namespace) the WAF reconciler replicates into tenant namespaces for the - // Coraza wasm OCI registry TLS check — a dedicated name avoids clashing with - // the operator-managed tigera-ca-bundle ConfigMap the GatewayAPI render also - // copies there (EV-6386). The source copy is a renamed copy of the trusted - // bundle, provisioned by the core controller and passed in as WASMCACert. - WASMCACertName = "tigera-waf-ca-bundle" - // ManagedClustersWatchRoleBindingName binds kube-controllers to the managed-cluster // watch ClusterRole. Used by both calico-kube-controllers (in a management cluster) // and the enterprise es-calico-kube-controllers, so the binding stays generic here. @@ -100,8 +83,6 @@ type KubeControllersConfiguration struct { // namespace to be returned by the rendered. Expected that the calling code // take care to pass the same secret on each reconcile where possible. KubeControllersGatewaySecret *corev1.Secret - WASMPullSecret *corev1.Secret - WASMCACert *corev1.ConfigMap TrustedBundle certificatemanagement.TrustedBundleRO // Namespace to be installed into. @@ -114,29 +95,6 @@ type KubeControllersConfiguration struct { // If this is nil, then we should run in zero-tenant mode. Tenant *operatorv1.Tenant - // WAFGatewayExtensionEnabled gates the WAF v3 (Gateway API add-on) surface - // on calico-kube-controllers: the applicationlayer controller enablement, - // the WAF / Gateway-API / EnvoyExtensionPolicy / event / secret-replication - // RBAC, the WASM_IMAGE / WASM_PULL_SECRET / WASM_CA_CERT env vars, and the - // gateway envoy-proxy wasm image resolution. Sourced from - // `GatewayAPI.spec.extensions.waf.state == Enabled` (default off). - // See design `tigera/designs#25` (PMREQ-384). - WAFGatewayExtensionEnabled bool - - // WAFWebhookServerTLS is the serving certificate for the in-process WAF - // SecLang validating admission webhook hosted by calico-kube-controllers. - // When set (WAF enabled), it is mounted into the Pod and the webhook server - // reads it from WAF_WEBHOOK_CERT_DIR. Issued for the tigera-waf-webhook - // Service DNS name. Nil leaves the Deployment untouched (and the in-process - // server self-disables when the cert is absent). - WAFWebhookServerTLS certificatemanagement.KeyPairInterface - - // WAFWebhookCABundle is the PEM of the CA that issued WAFWebhookServerTLS - // (the operator CA), stamped into the ValidatingWebhookConfiguration's - // caBundle so the apiserver can verify the in-process webhook endpoint. - // Only consulted when WAFGatewayExtensionEnabled is true. - WAFWebhookCABundle []byte - // The fields below parameterize the generic kube-controllers component. The // variant assemblers (NewCalicoKubeControllers, the enterprise es builder) // fill them; the component renders them without any variant or component-name @@ -165,18 +123,25 @@ type KubeControllersConfiguration struct { ExtraEnv []corev1.EnvVar // DisableConfigAPI sets DISABLE_KUBE_CONTROLLERS_CONFIG_API. DisableConfigAPI bool - // ManageWAFWebhook makes this component own the in-process WAF admission - // webhook surface lifecycle (rendered when WAFGatewayExtensionEnabled, deleted - // otherwise). Only the calico-kube-controllers component sets this. - ManageWAFWebhook bool // ModifierKey is the extension modifier key the component reports through // render.Extensible. calico-kube-controllers sets it so the enterprise modifier - // can layer on its metrics TLS; es-calico-kube-controllers leaves it empty so it - // is never decorated. + // can layer on the enterprise surface; es-calico-kube-controllers leaves it empty + // so it is never decorated. ModifierKey string } +// calicoKubeControllersPolicyComponent wraps the calico-kube-controllers network +// policy passthrough so it is render.Extensible: the enterprise modifier adds the WAF +// admission webhook ingress rule. The base policy carries no WAF. +type calicoKubeControllersPolicyComponent struct { + render.Component +} + +func (calicoKubeControllersPolicyComponent) ModifierKey() string { + return render.ComponentNameKubeControllersPolicy +} + func NewCalicoKubeControllersPolicy(cfg *KubeControllersConfiguration, defaultDeny *v3.NetworkPolicy) render.Component { toCreate := []client.Object{kubeControllersCalicoSystemPolicy(cfg)} @@ -184,14 +149,14 @@ func NewCalicoKubeControllersPolicy(cfg *KubeControllersConfiguration, defaultDe toCreate = append(toCreate, defaultDeny) } - return render.NewPassthrough( + return calicoKubeControllersPolicyComponent{render.NewPassthrough( toCreate, []client.Object{ // allow-tigera Tier was renamed to calico-system networkpolicy.DeprecatedAllowTigeraNetworkPolicyObject("kube-controller-access", cfg.Namespace), networkpolicy.DeprecatedAllowTigeraNetworkPolicyObject("default-deny", common.CalicoNamespace), }, - ) + )} } // NewKubeControllers builds a kube-controllers component from a fully-populated @@ -202,80 +167,30 @@ func NewKubeControllers(cfg *KubeControllersConfiguration) render.Component { return &kubeControllersComponent{cfg: cfg} } +// NewCalicoKubeControllers builds the calico-kube-controllers component. The base is +// pure OSS (the common rules plus the node and loadbalancer controllers); the Calico +// Enterprise additions (extra RBAC, enterprise controllers, metrics TLS, the WAF v3 +// surface) are layered on by the enterprise modifier keyed by ModifierKey. func NewCalicoKubeControllers(cfg *KubeControllersConfiguration) render.Component { cfg.Name = KubeController cfg.ConfigName = "default" cfg.RoleName = KubeControllerRole cfg.RoleBindingName = KubeControllerRoleBinding cfg.MetricsName = KubeControllerMetrics - cfg.ManageWAFWebhook = true cfg.ModifierKey = render.ComponentNameKubeControllers cfg.Rules = KubeControllersRoleCommonRules(cfg) cfg.EnabledControllers = []string{"node", "loadbalancer"} - if cfg.Installation.Variant.IsEnterprise() { - cfg.Rules = append(cfg.Rules, KubeControllersRoleEnterpriseCommonRules(cfg)...) - cfg.Rules = append(cfg.Rules, - rbacv1.PolicyRule{ - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{"remoteclusterconfigurations"}, - Verbs: []string{"watch", "list", "get"}, - }, - rbacv1.PolicyRule{ - APIGroups: []string{""}, - Resources: []string{"endpoints"}, - Verbs: []string{"create", "update", "delete"}, - }, - rbacv1.PolicyRule{ - APIGroups: []string{""}, - Resources: []string{"namespaces"}, - Verbs: []string{"get"}, - }, - rbacv1.PolicyRule{ - APIGroups: []string{"usage.tigera.io"}, - Resources: []string{"licenseusagereports"}, - Verbs: []string{"create", "update", "delete", "watch", "list", "get"}, - }, - ) - cfg.EnabledControllers = append(cfg.EnabledControllers, "service", "federatedservices", "usage") - if cfg.WAFGatewayExtensionEnabled { - cfg.EnabledControllers = append(cfg.EnabledControllers, "applicationlayer") - } - cfg.ExtraEnv = calicoEnterpriseEnv(cfg) - } return NewKubeControllers(cfg) } -// calicoEnterpriseEnv builds the enterprise-only static env vars for -// calico-kube-controllers. The dynamic WASM_* vars depend on resolved images and -// are added at deployment-render time. -func calicoEnterpriseEnv(cfg *KubeControllersConfiguration) []corev1.EnvVar { - var env []corev1.EnvVar - if cfg.Tenant != nil { - env = append(env, corev1.EnvVar{Name: "TENANT_ID", Value: cfg.Tenant.Spec.ID}) - } - if cfg.TrustedBundle != nil { - env = append(env, corev1.EnvVar{Name: "MULTI_CLUSTER_FORWARDING_CA", Value: cfg.TrustedBundle.MountPath()}) - } - if cfg.Installation.CalicoNetwork != nil && cfg.Installation.CalicoNetwork.MultiInterfaceMode != nil { - env = append(env, corev1.EnvVar{Name: "MULTI_INTERFACE_MODE", Value: cfg.Installation.CalicoNetwork.MultiInterfaceMode.Value()}) - } - return env -} - type kubeControllersComponent struct { // cfg is caller-supplied configuration for building kube-controllers Kubernetes resources. cfg *KubeControllersConfiguration // Internal state generated by the given configuration. calicoImage string - - // wasmImage is the fully-resolved OCI reference for the Coraza WAF wasm - // binary (Enterprise only). Surfaced to the kube-controllers binary via - // the WASM_IMAGE env var; consumed by the applicationlayer reconcilers - // in tigera/calico-private to program WAF policy attachments. - wasmImage string } func (c *kubeControllersComponent) ResolveImages(is *operatorv1.ImageSet) error { @@ -287,15 +202,6 @@ func (c *kubeControllersComponent) ResolveImages(is *operatorv1.ImageSet) error if err != nil { return err } - if c.cfg.WAFGatewayExtensionEnabled { - // The Coraza WAF wasm is baked into the gateway envoy-proxy image as its - // final layer; Envoy Gateway extracts it from there. Point WASM_IMAGE at - // that same image (no standalone coraza-wasm image needed). - c.wasmImage, err = components.GetReference(components.ComponentGatewayAPIEnvoyProxy, reg, path, prefix, is) - if err != nil { - return err - } - } return nil } @@ -339,26 +245,6 @@ func (c *kubeControllersComponent) Objects() ([]client.Object, []client.Object) objectsToCreate = append(objectsToCreate, secret.ToRuntimeObjects( secret.CopyToNamespace(c.cfg.Namespace, c.cfg.KubeControllersGatewaySecret)...)...) } - if c.cfg.WASMPullSecret != nil { - objectsToCreate = append(objectsToCreate, secret.ToRuntimeObjects( - secret.CopyToNamespace(c.cfg.Namespace, c.cfg.WASMPullSecret)...)...) - } - if c.cfg.WASMCACert != nil { - objectsToCreate = append(objectsToCreate, c.cfg.WASMCACert) - } - - // The in-process WAF admission webhook surface (Service fronting this Pod + - // ValidatingWebhookConfiguration). Rendered here, rather than as a - // passthrough in the core controller, so the objects are cleaned up when the - // WAF extension is disabled or the GatewayAPI CR is removed. - if c.cfg.ManageWAFWebhook { - webhookObjs := applicationlayer.WAFAdmissionWebhookComponents(c.cfg.WAFWebhookCABundle) - if c.cfg.WAFGatewayExtensionEnabled { - objectsToCreate = append(objectsToCreate, webhookObjs...) - } else { - objectsToDelete = append(objectsToDelete, webhookObjs...) - } - } if c.cfg.MetricsPort != 0 { objectsToCreate = append(objectsToCreate, c.prometheusService()) @@ -510,153 +396,6 @@ func KubeControllersRoleCommonRules(cfg *KubeControllersConfiguration) []rbacv1. return rules } -func KubeControllersRoleEnterpriseCommonRules(cfg *KubeControllersConfiguration) []rbacv1.PolicyRule { - rules := []rbacv1.PolicyRule{ - { - APIGroups: []string{""}, - Resources: []string{"configmaps"}, - Verbs: []string{"watch", "list", "get", "update", "create", "delete"}, - }, - { - // The Federated Services Controller needs access to the remote kubeconfig secret - // in order to create a remote syncer. - APIGroups: []string{""}, - Resources: []string{"secrets"}, - Verbs: []string{"watch", "list", "get"}, - }, - { - // Needed to validate the license - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{"licensekeys"}, - Verbs: []string{"get", "watch", "list"}, - }, - { - // Needed to update the status of the LicenseKey with the result of license validation. - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"licensekeys/status"}, - Verbs: []string{"update"}, - }, - { - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{"deeppacketinspections"}, - Verbs: []string{"get", "watch", "list"}, - }, - { - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{"deeppacketinspections/status"}, - Verbs: []string{"update"}, - }, - { - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{"packetcaptures"}, - Verbs: []string{"get", "list", "update"}, - }, - { - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{"packetcaptures/status"}, - Verbs: []string{"update"}, - }, - } - - if cfg.WAFGatewayExtensionEnabled { - // WAF v3 (Gateway API add-on) RBAC. Gated by - // GatewayAPI.spec.extensions.waf.state == Enabled. - rules = append(rules, - // Application-layer (gateway-addons) reconcilers reconcile WAF resources - // against Gateway API targetRefs and emit events on the policy objects. - rbacv1.PolicyRule{ - APIGroups: []string{"applicationlayer.projectcalico.org"}, - Resources: []string{ - "wafpolicies", "globalwafpolicies", - "wafplugins", "globalwafplugins", - "wafvalidationpolicies", "globalwafvalidationpolicies", - }, - Verbs: []string{"get", "list", "watch", "create", "update", "patch", "delete"}, - }, - rbacv1.PolicyRule{ - APIGroups: []string{"applicationlayer.projectcalico.org"}, - Resources: []string{ - "wafpolicies/status", "globalwafpolicies/status", - "wafplugins/status", "globalwafplugins/status", - "wafvalidationpolicies/status", "globalwafvalidationpolicies/status", - }, - Verbs: []string{"get", "update", "patch"}, - }, - rbacv1.PolicyRule{ - APIGroups: []string{"applicationlayer.projectcalico.org"}, - Resources: []string{ - "wafpolicies/finalizers", "globalwafpolicies/finalizers", - "wafplugins/finalizers", "globalwafplugins/finalizers", - "wafvalidationpolicies/finalizers", "globalwafvalidationpolicies/finalizers", - }, - Verbs: []string{"update"}, - }, - rbacv1.PolicyRule{ - // Validate Gateway API targetRefs and surface attachment status. - APIGroups: []string{"gateway.networking.k8s.io"}, - Resources: []string{"gateways", "httproutes", "tcproutes", "tlsroutes", "grpcroutes"}, - Verbs: []string{"get", "list", "watch", "update", "patch"}, - }, - rbacv1.PolicyRule{ - APIGroups: []string{"gateway.networking.k8s.io"}, - Resources: []string{"gateways/status", "httproutes/status", "tcproutes/status", "tlsroutes/status", "grpcroutes/status"}, - Verbs: []string{"get", "update", "patch"}, - }, - // controller-runtime Reconcilers (e.g. the applicationlayer manager) record - // events on watched objects via Recorder.Eventf; both core and events.k8s.io - // API groups are emitted depending on the kubernetes version. - rbacv1.PolicyRule{ - APIGroups: []string{""}, - Resources: []string{"events"}, - Verbs: []string{"create", "patch"}, - }, - rbacv1.PolicyRule{ - APIGroups: []string{"events.k8s.io"}, - Resources: []string{"events"}, - Verbs: []string{"create", "patch"}, - }, - // Application-layer reconciler replicates the WAF wasm pull Secret from - // the controller namespace (calico-system) into each WAFPolicy's - // namespace so the rendered EnvoyExtensionPolicy can reference it. Also - // replicates CA-cert ConfigMaps when WASM_CA_CERT is set. - rbacv1.PolicyRule{ - APIGroups: []string{""}, - Resources: []string{"secrets", "configmaps"}, - Verbs: []string{"get", "list", "watch", "create", "update", "patch", "delete"}, - }, - // Application-layer reconciler emits one EnvoyExtensionPolicy per WAF - // targetRef to bind the Coraza wasm filter at the gateway / route. - rbacv1.PolicyRule{ - APIGroups: []string{"gateway.envoyproxy.io"}, - Resources: []string{"envoyextensionpolicies"}, - Verbs: []string{"get", "list", "watch", "create", "update", "patch", "delete"}, - }, - // Application-layer reconciler stamps each namespace with its - // allocated WAF rule-id range (applicationlayer.projectcalico.org/waf-id-range - // annotation) so application operators can author in-range rules. The - // base role already grants namespaces get/list/watch; the annotation - // write needs patch/update, gated to the WAF path. - rbacv1.PolicyRule{ - APIGroups: []string{""}, - Resources: []string{"namespaces"}, - Verbs: []string{"get", "patch", "update"}, - }, - ) - } - - if cfg.ManagementClusterConnection != nil { - rules = append(rules, - rbacv1.PolicyRule{ - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{"licensekeys"}, - Verbs: []string{"get", "create", "update", "list", "watch"}, - }, - ) - } - - return rules -} - func (c *kubeControllersComponent) controllersServiceAccount() *corev1.ServiceAccount { return &corev1.ServiceAccount{ TypeMeta: metav1.TypeMeta{Kind: "ServiceAccount", APIVersion: "v1"}, @@ -715,54 +454,11 @@ func (c *kubeControllersComponent) controllersDeployment() *appsv1.Deployment { env = append(env, c.cfg.K8sServiceEpPodNetwork.EnvVars()...) env = append(env, c.cfg.ExtraEnv...) - // Application-layer (gateway-addons / WAF v3) env vars, gated by - // GatewayAPI.spec.extensions.waf.state == Enabled. When the gate is off - // (default), none of the WASM_* env vars are rendered and the kube-controllers - // binary skips the WAF reconcilers entirely (see the applicationlayer entry in - // EnabledControllers). The WASM_IMAGE value depends on resolved images, so this - // is rendered here rather than in the static ExtraEnv. - if c.cfg.WAFGatewayExtensionEnabled { - // Application-layer (gateway-addons) reconcilers consume the Coraza WAF - // wasm OCI reference from this env var to program WAF policy attachments. - // Empty when ResolveImages was not called for the Calico variant; the - // reconciler stamps Programmed=False/WASMUnavailable in that case. - if c.wasmImage != "" { - env = append(env, corev1.EnvVar{Name: "WASM_IMAGE", Value: c.wasmImage}) - } - - // WASM_PULL_SECRET names the imagePullSecret the reconciler replicates - // from the kube-controllers namespace into a WAFPolicy's namespace so - // the rendered EnvoyExtensionPolicy can pull the wasm OCI artifact from - // a private Tigera registry. Source the name from the first - // Installation.ImagePullSecrets entry so multi-tenant / BYO-registry - // installs reuse whatever pull secret operator already attaches here. - if c.cfg.WASMPullSecret != nil { - env = append(env, corev1.EnvVar{Name: "WASM_PULL_SECRET", Value: c.cfg.WASMPullSecret.Name}) - } - - // WASM_CA_CERT names the dedicated CA bundle ConfigMap (provisioned as - // WASMCACert) that the reconciler replicates alongside WASM_PULL_SECRET - // so the EnvoyExtensionPolicy wasm fetcher trusts the registry's TLS - // chain. Only set when the source ConfigMap is actually rendered. - if c.cfg.WASMCACert != nil { - env = append(env, corev1.EnvVar{Name: "WASM_CA_CERT", Value: c.cfg.WASMCACert.Name}) - } - } - if c.cfg.TrustedBundle != nil { env = append(env, corev1.EnvVar{Name: "CA_CRT_PATH", Value: c.cfg.TrustedBundle.MountPath()}, ) } - if c.cfg.WAFWebhookServerTLS != nil { - // The in-process WAF admission webhook server (calico-private - // applicationlayer manager) reads its serving cert (tls.crt/tls.key) - // from this directory; the controller-runtime webhook server only - // registers when the cert is present. - env = append(env, - corev1.EnvVar{Name: "WAF_WEBHOOK_CERT_DIR", Value: filepath.Dir(c.cfg.WAFWebhookServerTLS.VolumeMountCertificateFilePath())}, - ) - } // UID 999 is used in kube-controller Dockerfile. sc := securitycontext.NewNonRootContext() @@ -806,20 +502,7 @@ func (c *kubeControllersComponent) controllersDeployment() *appsv1.Deployment { VolumeMounts: c.kubeControllersVolumeMounts(), } - if c.cfg.WAFWebhookServerTLS != nil { - // Expose the in-process WAF admission-webhook port that the - // tigera-waf-webhook Service forwards to. - container.Ports = append(container.Ports, corev1.ContainerPort{ - Name: "waf-webhook", - ContainerPort: applicationlayer.WAFWebhookContainerPort, - Protocol: corev1.ProtocolTCP, - }) - } - var initContainers []corev1.Container - if c.cfg.WAFWebhookServerTLS != nil && c.cfg.WAFWebhookServerTLS.UseCertificateManagement() { - initContainers = append(initContainers, c.cfg.WAFWebhookServerTLS.InitContainer(c.cfg.Namespace, sc)) - } tolerations := appendUniqueTolerations(c.cfg.Installation.ControlPlaneTolerations, rmeta.TolerateCriticalAddonsAndControlPlane...) if c.cfg.Installation.KubernetesProvider.IsGKE() { tolerations = appendUniqueTolerations(tolerations, rmeta.TolerateGKEARM64NoSchedule) @@ -964,9 +647,6 @@ func (c *kubeControllersComponent) kubeControllersVolumeMounts() []corev1.Volume if c.cfg.TrustedBundle != nil { mounts = append(mounts, c.cfg.TrustedBundle.VolumeMounts(c.SupportedOSType())...) } - if c.cfg.WAFWebhookServerTLS != nil { - mounts = append(mounts, c.cfg.WAFWebhookServerTLS.VolumeMount(c.SupportedOSType())) - } return mounts } @@ -975,9 +655,6 @@ func (c *kubeControllersComponent) kubeControllersVolumes() []corev1.Volume { if c.cfg.TrustedBundle != nil { volumes = append(volumes, c.cfg.TrustedBundle.Volume()) } - if c.cfg.WAFWebhookServerTLS != nil { - volumes = append(volumes, c.cfg.WAFWebhookServerTLS.Volume()) - } return volumes } @@ -1020,20 +697,6 @@ func kubeControllersCalicoSystemPolicy(cfg *KubeControllersConfiguration) *v3.Ne }) } - // Allow the kube-apiserver to reach the in-process WAF admission webhook on - // :9443 (EV-6386). render-v3 wires the webhook Service/config/cert + the - // server, but without this ingress rule the calico-system default-deny drops - // the apiserver→:9443 call and every WAFPolicy/WAFPlugin admission times out. - if cfg.WAFGatewayExtensionEnabled { - ingressRules = append(ingressRules, v3.Rule{ - Action: v3.Allow, - Protocol: &networkpolicy.TCPProtocol, - Destination: v3.EntityRule{ - Ports: networkpolicy.Ports(uint16(applicationlayer.WAFWebhookContainerPort)), - }, - }) - } - if r, err := cfg.K8sServiceEp.DestinationEntityRule(); r != nil && err == nil { egressRules = append(egressRules, v3.Rule{ Action: v3.Allow, diff --git a/pkg/render/kubecontrollers/kube-controllers_test.go b/pkg/render/kubecontrollers/kube-controllers_test.go index 801c4b2687..89cfad0940 100644 --- a/pkg/render/kubecontrollers/kube-controllers_test.go +++ b/pkg/render/kubecontrollers/kube-controllers_test.go @@ -16,12 +16,10 @@ package kubecontrollers_test import ( "fmt" - "path/filepath" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - admissionregistrationv1 "k8s.io/api/admissionregistration/v1" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" @@ -43,7 +41,6 @@ import ( "github.com/tigera/operator/pkg/dns" "github.com/tigera/operator/pkg/enterprise" "github.com/tigera/operator/pkg/render" - "github.com/tigera/operator/pkg/render/applicationlayer" rmeta "github.com/tigera/operator/pkg/render/common/meta" "github.com/tigera/operator/pkg/render/common/networkpolicy" rtest "github.com/tigera/operator/pkg/render/common/test" @@ -231,175 +228,6 @@ var _ = Describe("kube-controllers rendering tests", func() { Expect(ds.Spec.Template.Spec.Tolerations).To(ConsistOf(rmeta.TolerateCriticalAddonsAndControlPlane)) }) - It("should render all calico kube-controllers resources for a default configuration (standalone) using CalicoEnterprise", func() { - expectedResources := []struct { - name string - ns string - group string - version string - kind string - }{ - {name: kubecontrollers.KubeControllerServiceAccount, ns: common.CalicoNamespace, group: "", version: "v1", kind: "ServiceAccount"}, - {name: kubecontrollers.KubeControllerRole, ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRole"}, - {name: kubecontrollers.KubeControllerRoleBinding, ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRoleBinding"}, - {name: kubecontrollers.KubeController, ns: common.CalicoNamespace, group: "apps", version: "v1", kind: "Deployment"}, - {name: kubecontrollers.WASMPullSecretName, ns: common.CalicoNamespace, group: "", version: "v1", kind: "Secret"}, - {name: kubecontrollers.WASMCACertName, ns: common.CalicoNamespace, group: "", version: "v1", kind: "ConfigMap"}, - {name: applicationlayer.WAFWebhookServiceName, ns: common.CalicoNamespace, group: "", version: "v1", kind: "Service"}, - {name: "tigera-waf.applicationlayer.projectcalico.org", ns: "", group: "admissionregistration.k8s.io", version: "v1", kind: "ValidatingWebhookConfiguration"}, - {name: kubecontrollers.KubeControllerMetrics, ns: common.CalicoNamespace, group: "", version: "v1", kind: "Service"}, - } - - instance.Variant = operatorv1.CalicoEnterprise - instance.ImagePullSecrets = []corev1.LocalObjectReference{{Name: "tigera-pull-secret"}} - cfg.MetricsPort = 9094 - // Opt in to the WAF Gateway API add-on so the WAF env vars + RBAC are rendered. - cfg.WAFGatewayExtensionEnabled = true - cfg.WAFWebhookCABundle = []byte("fake-ca-bundle") - // core_controller provisions a dedicated WAF wasm pull secret (a renamed - // copy of the install pull secret) so the reconciler can replicate it into - // WAFPolicy namespaces without clashing with the operator-managed - // tigera-pull-secret; surface it here so it renders and WASM_PULL_SECRET is set. - cfg.WASMPullSecret = &corev1.Secret{TypeMeta: metav1.TypeMeta{Kind: "Secret", APIVersion: "v1"}, ObjectMeta: metav1.ObjectMeta{Name: kubecontrollers.WASMPullSecretName, Namespace: common.CalicoNamespace}} - // Likewise core_controller provisions the dedicated WAF wasm CA-bundle - // ConfigMap (a renamed copy of the trusted bundle); surface it here so it - // renders and WASM_CA_CERT is set. - cfg.WASMCACert = &corev1.ConfigMap{TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"}, ObjectMeta: metav1.ObjectMeta{Name: kubecontrollers.WASMCACertName, Namespace: common.CalicoNamespace}} - - component := kubecontrollers.NewCalicoKubeControllers(&cfg) - Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() - Expect(len(resources)).To(Equal(len(expectedResources))) - - // Should render the correct resources. - i := 0 - for _, expectedRes := range expectedResources { - rtest.ExpectResourceTypeAndObjectMetadata(resources[i], expectedRes.name, expectedRes.ns, expectedRes.group, expectedRes.version, expectedRes.kind) - i++ - } - - // The Deployment should have the correct configuration. - dp := rtest.GetResource(resources, kubecontrollers.KubeController, common.CalicoNamespace, "apps", "v1", "Deployment").(*appsv1.Deployment) - - Expect(dp.Spec.Template.Spec.Containers[0].Image).To(Equal("test-reg/tigera/calico:" + components.ComponentTigeraCalico.Version)) - Expect(dp.Spec.Template.Spec.ImagePullSecrets).To(ContainElement(corev1.LocalObjectReference{Name: "tigera-pull-secret"})) - envs := dp.Spec.Template.Spec.Containers[0].Env - Expect(envs).To(ContainElement(corev1.EnvVar{ - Name: "ENABLED_CONTROLLERS", Value: "node,loadbalancer,service,federatedservices,usage,applicationlayer", - })) - // Application-layer reconcilers consume these env vars to program WAF - // EnvoyExtensionPolicy attachments. - Expect(envs).To(ContainElement(corev1.EnvVar{ - Name: "WASM_IMAGE", Value: "test-reg/tigera/envoy-proxy:" + components.ComponentGatewayAPIEnvoyProxy.Version, - })) - Expect(envs).To(ContainElement(corev1.EnvVar{ - Name: "WASM_PULL_SECRET", Value: kubecontrollers.WASMPullSecretName, - })) - // WASM_CA_CERT names the dedicated WAF trusted-bundle ConfigMap that the - // reconciler replicates into WAFPolicy namespaces (kept separate from the - // operator-managed tigera-ca-bundle the GatewayAPI render also copies there). - Expect(envs).To(ContainElement(corev1.EnvVar{ - Name: "WASM_CA_CERT", Value: kubecontrollers.WASMCACertName, - })) - - Expect(len(dp.Spec.Template.Spec.Containers[0].VolumeMounts)).To(Equal(1)) - Expect(len(dp.Spec.Template.Spec.Volumes)).To(Equal(1)) - - clusterRole := rtest.GetResource(resources, kubecontrollers.KubeControllerRole, "", "rbac.authorization.k8s.io", "v1", "ClusterRole").(*rbacv1.ClusterRole) - Expect(clusterRole.Rules).To(HaveLen(38), "cluster role should have 38 rules") - - // Application-layer reconciler RBAC: WAF CRDs (resources, /status, /finalizers). - Expect(clusterRole.Rules).To(ContainElement(rbacv1.PolicyRule{ - APIGroups: []string{"applicationlayer.projectcalico.org"}, - Resources: []string{ - "wafpolicies", "globalwafpolicies", - "wafplugins", "globalwafplugins", - "wafvalidationpolicies", "globalwafvalidationpolicies", - }, - Verbs: []string{"get", "list", "watch", "create", "update", "patch", "delete"}, - })) - Expect(clusterRole.Rules).To(ContainElement(rbacv1.PolicyRule{ - APIGroups: []string{"applicationlayer.projectcalico.org"}, - Resources: []string{ - "wafpolicies/status", "globalwafpolicies/status", - "wafplugins/status", "globalwafplugins/status", - "wafvalidationpolicies/status", "globalwafvalidationpolicies/status", - }, - Verbs: []string{"get", "update", "patch"}, - })) - Expect(clusterRole.Rules).To(ContainElement(rbacv1.PolicyRule{ - APIGroups: []string{"applicationlayer.projectcalico.org"}, - Resources: []string{ - "wafpolicies/finalizers", "globalwafpolicies/finalizers", - "wafplugins/finalizers", "globalwafplugins/finalizers", - "wafvalidationpolicies/finalizers", "globalwafvalidationpolicies/finalizers", - }, - Verbs: []string{"update"}, - })) - // Gateway API targetRef validation + status patching. - Expect(clusterRole.Rules).To(ContainElement(rbacv1.PolicyRule{ - APIGroups: []string{"gateway.networking.k8s.io"}, - Resources: []string{"gateways", "httproutes", "tcproutes", "tlsroutes", "grpcroutes"}, - Verbs: []string{"get", "list", "watch", "update", "patch"}, - })) - Expect(clusterRole.Rules).To(ContainElement(rbacv1.PolicyRule{ - APIGroups: []string{"gateway.networking.k8s.io"}, - Resources: []string{"gateways/status", "httproutes/status", "tcproutes/status", "tlsroutes/status", "grpcroutes/status"}, - Verbs: []string{"get", "update", "patch"}, - })) - // Recorder.Eventf emits to both core/events and events.k8s.io/events. - Expect(clusterRole.Rules).To(ContainElement(rbacv1.PolicyRule{ - APIGroups: []string{""}, - Resources: []string{"events"}, - Verbs: []string{"create", "patch"}, - })) - Expect(clusterRole.Rules).To(ContainElement(rbacv1.PolicyRule{ - APIGroups: []string{"events.k8s.io"}, - Resources: []string{"events"}, - Verbs: []string{"create", "patch"}, - })) - // Cluster-wide secrets+configmaps CRUD: reconciler replicates pull - // secrets and CA bundles from the controller namespace into target - // WAFPolicy namespaces. - Expect(clusterRole.Rules).To(ContainElement(rbacv1.PolicyRule{ - APIGroups: []string{""}, - Resources: []string{"secrets", "configmaps"}, - Verbs: []string{"get", "list", "watch", "create", "update", "patch", "delete"}, - })) - // EnvoyExtensionPolicy CRUD: reconciler renders one EEP per WAF targetRef. - Expect(clusterRole.Rules).To(ContainElement(rbacv1.PolicyRule{ - APIGroups: []string{"gateway.envoyproxy.io"}, - Resources: []string{"envoyextensionpolicies"}, - Verbs: []string{"get", "list", "watch", "create", "update", "patch", "delete"}, - })) - - ms := rtest.GetResource(resources, kubecontrollers.KubeControllerMetrics, common.CalicoNamespace, "", "v1", "Service").(*corev1.Service) - Expect(ms.Spec.ClusterIP).To(Equal("None"), "metrics service should be headless") - - // The webhook surface is rendered with the operator CA stamped into the - // ValidatingWebhookConfiguration caBundle. - vwc := rtest.GetResource(resources, "tigera-waf.applicationlayer.projectcalico.org", "", "admissionregistration.k8s.io", "v1", "ValidatingWebhookConfiguration").(*admissionregistrationv1.ValidatingWebhookConfiguration) - Expect(vwc.Webhooks).To(HaveLen(1)) - Expect(vwc.Webhooks[0].ClientConfig.CABundle).To(Equal([]byte("fake-ca-bundle"))) - }) - - It("should delete the WAF admission webhook surface when the WAF Gateway API add-on is disabled", func() { - instance.Variant = operatorv1.CalicoEnterprise - cfg.WAFGatewayExtensionEnabled = false - - component := kubecontrollers.NewCalicoKubeControllers(&cfg) - Expect(component.ResolveImages(nil)).To(BeNil()) - toCreate, toDelete := component.Objects() - - // Neither webhook object is created... - Expect(rtest.GetResource(toCreate, applicationlayer.WAFWebhookServiceName, common.CalicoNamespace, "", "v1", "Service")).To(BeNil()) - Expect(rtest.GetResource(toCreate, "tigera-waf.applicationlayer.projectcalico.org", "", "admissionregistration.k8s.io", "v1", "ValidatingWebhookConfiguration")).To(BeNil()) - // ...and both are queued for deletion, so disabling the feature (or - // removing the GatewayAPI CR) cleans up an earlier enabled render. - Expect(rtest.GetResource(toDelete, applicationlayer.WAFWebhookServiceName, common.CalicoNamespace, "", "v1", "Service")).NotTo(BeNil()) - Expect(rtest.GetResource(toDelete, "tigera-waf.applicationlayer.projectcalico.org", "", "admissionregistration.k8s.io", "v1", "ValidatingWebhookConfiguration")).NotTo(BeNil()) - }) - It("should render all calico kube-controllers resources using CalicoEnterprise on Openshift", func() { expectedResources := []struct { name string @@ -448,8 +276,6 @@ var _ = Describe("kube-controllers rendering tests", func() { instance.Variant = operatorv1.CalicoEnterprise cfg.KubeControllersGatewaySecret = &testutils.KubeControllersUserSecret cfg.MetricsPort = 9094 - // Opt in to the WAF Gateway API add-on so the WAF env vars + RBAC are rendered. - cfg.WAFGatewayExtensionEnabled = true component := enterprise.NewElasticsearchKubeControllers(&cfg) Expect(component.ResolveImages(nil)).To(BeNil()) @@ -482,7 +308,7 @@ var _ = Describe("kube-controllers rendering tests", func() { Expect(dp.Spec.Template.Spec.Volumes[0].ConfigMap.Name).To(Equal("tigera-ca-bundle")) clusterRole := rtest.GetResource(resources, enterprise.EsKubeControllerRole, "", "rbac.authorization.k8s.io", "v1", "ClusterRole").(*rbacv1.ClusterRole) - Expect(clusterRole.Rules).To(HaveLen(36), "cluster role should have 36 rules") + Expect(clusterRole.Rules).To(HaveLen(26), "cluster role should have 26 rules") Expect(clusterRole.Rules).To(ContainElement( rbacv1.PolicyRule{ APIGroups: []string{""}, @@ -497,57 +323,6 @@ var _ = Describe("kube-controllers rendering tests", func() { })) }) - It("should render all calico-kube-controllers resources for a default configuration using CalicoEnterprise and ClusterType is Management", func() { - expectedResources := []struct { - name string - ns string - group string - version string - kind string - }{ - {name: kubecontrollers.KubeControllerServiceAccount, ns: common.CalicoNamespace, group: "", version: "v1", kind: "ServiceAccount"}, - {name: kubecontrollers.KubeControllerRole, ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRole"}, - {name: kubecontrollers.KubeControllerRoleBinding, ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRoleBinding"}, - {name: kubecontrollers.ManagedClustersWatchRoleBindingName, ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRoleBinding"}, - {name: kubecontrollers.KubeController, ns: common.CalicoNamespace, group: "apps", version: "v1", kind: "Deployment"}, - {name: applicationlayer.WAFWebhookServiceName, ns: common.CalicoNamespace, group: "", version: "v1", kind: "Service"}, - {name: "tigera-waf.applicationlayer.projectcalico.org", ns: "", group: "admissionregistration.k8s.io", version: "v1", kind: "ValidatingWebhookConfiguration"}, - {name: kubecontrollers.KubeControllerMetrics, ns: common.CalicoNamespace, group: "", version: "v1", kind: "Service"}, - } - - // Override configuration to match expected Enterprise config. - instance.Variant = operatorv1.CalicoEnterprise - cfg.ManagementCluster = &operatorv1.ManagementCluster{} - cfg.MetricsPort = 9094 - // Opt in to the WAF Gateway API add-on so the WAF env vars + RBAC are rendered. - cfg.WAFGatewayExtensionEnabled = true - - component := kubecontrollers.NewCalicoKubeControllers(&cfg) - Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() - Expect(len(resources)).To(Equal(len(expectedResources))) - - // Should render the correct resources. - i := 0 - for _, expectedRes := range expectedResources { - rtest.ExpectResourceTypeAndObjectMetadata(resources[i], expectedRes.name, expectedRes.ns, expectedRes.group, expectedRes.version, expectedRes.kind) - i++ - } - - // The Deployment should have the correct configuration. - dp := rtest.GetResource(resources, kubecontrollers.KubeController, common.CalicoNamespace, "apps", "v1", "Deployment").(*appsv1.Deployment) - - envs := dp.Spec.Template.Spec.Containers[0].Env - Expect(envs).To(ContainElement(corev1.EnvVar{ - Name: "ENABLED_CONTROLLERS", - Value: "node,loadbalancer,service,federatedservices,usage,applicationlayer", - })) - - Expect(len(dp.Spec.Template.Spec.Containers[0].VolumeMounts)).To(Equal(1)) - - Expect(len(dp.Spec.Template.Spec.Volumes)).To(Equal(1)) - Expect(dp.Spec.Template.Spec.Containers[0].Image).To(Equal("test-reg/tigera/calico:" + components.ComponentTigeraCalico.Version)) - }) It("should render all calico-kube-controllers resources for a default configuration using CalicoEnterprise", func() { expectedResources := []struct { name string @@ -614,50 +389,6 @@ var _ = Describe("kube-controllers rendering tests", func() { Expect(dp.Spec.Template.Spec.Containers[0].Image).To(Equal("test-reg/tigera/calico:" + components.ComponentTigeraCalico.Version)) }) - It("should mount the WAF admission webhook serving cert and expose its port when WAF is enabled", func() { - certificateManager, err := certificatemanager.Create(cli, nil, dns.DefaultClusterDomain, common.OperatorNamespace(), certificatemanager.AllowCACreation()) - Expect(err).NotTo(HaveOccurred()) - wafTLS, err := certificateManager.GetOrCreateKeyPair(cli, - applicationlayer.WAFWebhookServerTLSSecretName, - common.OperatorNamespace(), - dns.GetServiceDNSNames(applicationlayer.WAFWebhookServiceName, common.CalicoNamespace, dns.DefaultClusterDomain)) - Expect(err).NotTo(HaveOccurred()) - - instance.Variant = operatorv1.CalicoEnterprise - cfg.WAFGatewayExtensionEnabled = true - cfg.WAFWebhookServerTLS = wafTLS - - component := kubecontrollers.NewCalicoKubeControllers(&cfg) - Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() - - dp := rtest.GetResource(resources, kubecontrollers.KubeController, common.CalicoNamespace, "apps", "v1", "Deployment").(*appsv1.Deployment) - c := dp.Spec.Template.Spec.Containers[0] - - // Serving cert is mounted and advertised to the in-process webhook server. - Expect(dp.Spec.Template.Spec.Volumes).To(ContainElement(wafTLS.Volume())) - Expect(c.VolumeMounts).To(ContainElement(wafTLS.VolumeMount(rmeta.OSTypeLinux))) - Expect(c.Env).To(ContainElement(corev1.EnvVar{ - Name: "WAF_WEBHOOK_CERT_DIR", - Value: filepath.Dir(wafTLS.VolumeMountCertificateFilePath()), - })) - - // In-process webhook port exposed for the tigera-waf-webhook Service. - Expect(c.Ports).To(ContainElement(corev1.ContainerPort{ - Name: "waf-webhook", - ContainerPort: int32(9443), - Protocol: corev1.ProtocolTCP, - })) - - // namespaces patch/update RBAC for the waf-id-range annotation. - clusterRole := rtest.GetResource(resources, "calico-kube-controllers", "", "rbac.authorization.k8s.io", "v1", "ClusterRole").(*rbacv1.ClusterRole) - Expect(clusterRole.Rules).To(ContainElement(rbacv1.PolicyRule{ - APIGroups: []string{""}, - Resources: []string{"namespaces"}, - Verbs: []string{"get", "patch", "update"}, - })) - }) - It("should render all es-calico-kube-controllers resources for a default configuration using CalicoEnterprise and ClusterType is Management", func() { expectedResources := []struct { name string @@ -681,8 +412,6 @@ var _ = Describe("kube-controllers rendering tests", func() { cfg.ManagementCluster = &operatorv1.ManagementCluster{} cfg.KubeControllersGatewaySecret = &testutils.KubeControllersUserSecret cfg.MetricsPort = 9094 - // Opt in to the WAF Gateway API add-on so the WAF env vars + RBAC are rendered. - cfg.WAFGatewayExtensionEnabled = true component := enterprise.NewElasticsearchKubeControllers(&cfg) Expect(component.ResolveImages(nil)).To(BeNil()) @@ -716,7 +445,7 @@ var _ = Describe("kube-controllers rendering tests", func() { Expect(dp.Spec.Template.Spec.Containers[0].Image).To(Equal("test-reg/tigera/calico:" + components.ComponentTigeraCalico.Version)) clusterRole := rtest.GetResource(resources, enterprise.EsKubeControllerRole, "", "rbac.authorization.k8s.io", "v1", "ClusterRole").(*rbacv1.ClusterRole) - Expect(clusterRole.Rules).To(HaveLen(36), "cluster role should have 36 rules") + Expect(clusterRole.Rules).To(HaveLen(26), "cluster role should have 26 rules") Expect(clusterRole.Rules).To(ContainElement( rbacv1.PolicyRule{ APIGroups: []string{""}, diff --git a/pkg/render/kubecontrollers/waf_pull_secret.go b/pkg/render/kubecontrollers/waf_pull_secret.go deleted file mode 100644 index 02ada09f8a..0000000000 --- a/pkg/render/kubecontrollers/waf_pull_secret.go +++ /dev/null @@ -1,101 +0,0 @@ -// Copyright (c) 2026 Tigera, Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package kubecontrollers - -import ( - "encoding/json" - "fmt" - - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - "github.com/tigera/operator/pkg/common" -) - -// MergeWAFPullSecret synthesizes the dedicated WAF wasm pull secret -// (tigera-waf-pull-secret) by merging the registry auths of every Installation -// pull secret. The EnvoyExtensionPolicy image source takes a single -// pullSecretRef, so a merged secret is the only way to honor multiple -// Installation pull secrets for the Coraza wasm OCI pull (e.g. the Tigera pull -// secret plus credentials for a private registry mirror). -// -// If the same registry appears in more than one secret, the first secret in -// Installation order wins. Secrets that cannot be parsed are skipped and their -// names returned, so the caller can log them without failing the reconcile. -// Returns a nil Secret when no registry auths could be collected. -func MergeWAFPullSecret(pullSecrets []*corev1.Secret) (*corev1.Secret, []string) { - merged := map[string]json.RawMessage{} - var skipped []string - for _, s := range pullSecrets { - auths, err := registryAuths(s) - if err != nil { - skipped = append(skipped, s.Name) - continue - } - for registry, auth := range auths { - if _, ok := merged[registry]; !ok { - merged[registry] = auth - } - } - } - if len(merged) == 0 { - return nil, skipped - } - - // Marshalling a map sorts its keys, so the rendered bytes are deterministic - // and do not churn the object on every reconcile. - data, err := json.Marshal(map[string]map[string]json.RawMessage{"auths": merged}) - if err != nil { - // Each auth entry round-trips from a successful Unmarshal above, so - // this cannot fail in practice; treat it as nothing to render. - return nil, skipped - } - - return &corev1.Secret{ - TypeMeta: metav1.TypeMeta{Kind: "Secret", APIVersion: "v1"}, - ObjectMeta: metav1.ObjectMeta{Name: WASMPullSecretName, Namespace: common.CalicoNamespace}, - Type: corev1.SecretTypeDockerConfigJson, - Data: map[string][]byte{corev1.DockerConfigJsonKey: data}, - }, skipped -} - -// registryAuths extracts the per-registry auth entries from a pull secret of -// either the dockerconfigjson type (auths nested under an "auths" key) or the -// legacy dockercfg type (a bare registry -> auth map). -func registryAuths(s *corev1.Secret) (map[string]json.RawMessage, error) { - if raw, ok := s.Data[corev1.DockerConfigJsonKey]; ok { - var cfg struct { - Auths map[string]json.RawMessage `json:"auths"` - } - if err := json.Unmarshal(raw, &cfg); err != nil { - return nil, err - } - if len(cfg.Auths) == 0 { - return nil, fmt.Errorf("secret %s has no auths entries", s.Name) - } - return cfg.Auths, nil - } - if raw, ok := s.Data[corev1.DockerConfigKey]; ok { - var auths map[string]json.RawMessage - if err := json.Unmarshal(raw, &auths); err != nil { - return nil, err - } - if len(auths) == 0 { - return nil, fmt.Errorf("secret %s has no auths entries", s.Name) - } - return auths, nil - } - return nil, fmt.Errorf("secret %s has neither a %s nor a %s key", s.Name, corev1.DockerConfigJsonKey, corev1.DockerConfigKey) -} diff --git a/pkg/render/logstorage/esgateway/esgateway_test.go b/pkg/render/logstorage/esgateway/esgateway_test.go index 7f29d298d7..5a76cc2c34 100644 --- a/pkg/render/logstorage/esgateway/esgateway_test.go +++ b/pkg/render/logstorage/esgateway/esgateway_test.go @@ -37,12 +37,12 @@ import ( "github.com/tigera/operator/pkg/controller/certificatemanager" ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" "github.com/tigera/operator/pkg/dns" + "github.com/tigera/operator/pkg/enterprise" "github.com/tigera/operator/pkg/render" relasticsearch "github.com/tigera/operator/pkg/render/common/elasticsearch" rmeta "github.com/tigera/operator/pkg/render/common/meta" "github.com/tigera/operator/pkg/render/common/podaffinity" rtest "github.com/tigera/operator/pkg/render/common/test" - "github.com/tigera/operator/pkg/render/kubecontrollers" "github.com/tigera/operator/pkg/render/testutils" "github.com/tigera/operator/pkg/tls" "github.com/tigera/operator/pkg/tls/certificatemanagement" @@ -81,9 +81,9 @@ var _ = Describe("ES Gateway rendering tests", func() { ESGatewayKeyPair: kp, TrustedBundle: bundle, KubeControllersUserSecrets: []*corev1.Secret{ - {ObjectMeta: metav1.ObjectMeta{Name: kubecontrollers.ElasticsearchKubeControllersUserSecret, Namespace: common.OperatorNamespace()}}, - {ObjectMeta: metav1.ObjectMeta{Name: kubecontrollers.ElasticsearchKubeControllersVerificationUserSecret, Namespace: render.ElasticsearchNamespace}}, - {ObjectMeta: metav1.ObjectMeta{Name: kubecontrollers.ElasticsearchKubeControllersSecureUserSecret, Namespace: render.ElasticsearchNamespace}}, + {ObjectMeta: metav1.ObjectMeta{Name: enterprise.ElasticsearchKubeControllersUserSecret, Namespace: common.OperatorNamespace()}}, + {ObjectMeta: metav1.ObjectMeta{Name: enterprise.ElasticsearchKubeControllersVerificationUserSecret, Namespace: render.ElasticsearchNamespace}}, + {ObjectMeta: metav1.ObjectMeta{Name: enterprise.ElasticsearchKubeControllersSecureUserSecret, Namespace: render.ElasticsearchNamespace}}, }, ClusterDomain: clusterDomain, EsAdminUserName: "elastic", @@ -95,9 +95,9 @@ var _ = Describe("ES Gateway rendering tests", func() { It("should render an ES Gateway deployment and all supporting resources", func() { expectedResources := []client.Object{ &v3.NetworkPolicy{ObjectMeta: metav1.ObjectMeta{Name: PolicyName, Namespace: render.ElasticsearchNamespace}}, - &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: kubecontrollers.ElasticsearchKubeControllersUserSecret, Namespace: common.OperatorNamespace()}}, - &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: kubecontrollers.ElasticsearchKubeControllersVerificationUserSecret, Namespace: render.ElasticsearchNamespace}}, - &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: kubecontrollers.ElasticsearchKubeControllersSecureUserSecret, Namespace: render.ElasticsearchNamespace}}, + &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: enterprise.ElasticsearchKubeControllersUserSecret, Namespace: common.OperatorNamespace()}}, + &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: enterprise.ElasticsearchKubeControllersVerificationUserSecret, Namespace: render.ElasticsearchNamespace}}, + &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: enterprise.ElasticsearchKubeControllersSecureUserSecret, Namespace: render.ElasticsearchNamespace}}, &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: ServiceName, Namespace: render.ElasticsearchNamespace}}, &rbacv1.Role{ObjectMeta: metav1.ObjectMeta{Name: RoleName, Namespace: render.ElasticsearchNamespace}}, &rbacv1.RoleBinding{ObjectMeta: metav1.ObjectMeta{Name: RoleName, Namespace: render.ElasticsearchNamespace}}, @@ -134,9 +134,9 @@ var _ = Describe("ES Gateway rendering tests", func() { installation.CertificateManagement = &operatorv1.CertificateManagement{CACert: secret.Data[corev1.TLSCertKey]} expectedResources := []client.Object{ &v3.NetworkPolicy{ObjectMeta: metav1.ObjectMeta{Name: PolicyName, Namespace: render.ElasticsearchNamespace}}, - &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: kubecontrollers.ElasticsearchKubeControllersUserSecret, Namespace: common.OperatorNamespace()}}, - &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: kubecontrollers.ElasticsearchKubeControllersVerificationUserSecret, Namespace: render.ElasticsearchNamespace}}, - &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: kubecontrollers.ElasticsearchKubeControllersSecureUserSecret, Namespace: render.ElasticsearchNamespace}}, + &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: enterprise.ElasticsearchKubeControllersUserSecret, Namespace: common.OperatorNamespace()}}, + &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: enterprise.ElasticsearchKubeControllersVerificationUserSecret, Namespace: render.ElasticsearchNamespace}}, + &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: enterprise.ElasticsearchKubeControllersSecureUserSecret, Namespace: render.ElasticsearchNamespace}}, &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: ServiceName, Namespace: render.ElasticsearchNamespace}}, &rbacv1.Role{ObjectMeta: metav1.ObjectMeta{Name: RoleName, Namespace: render.ElasticsearchNamespace}}, &rbacv1.RoleBinding{ObjectMeta: metav1.ObjectMeta{Name: RoleName, Namespace: render.ElasticsearchNamespace}}, From ba712b832663bd090185c98872ccb826e9b16ba0 Mon Sep 17 00:00:00 2001 From: Casey Davenport Date: Mon, 22 Jun 2026 08:16:28 -0700 Subject: [PATCH 39/54] Move enterprise FelixConfiguration defaulting into the extension The installation controller's variant-specific FelixConfiguration defaulting now goes through a FelixConfigDefaulter companion interface, so the enterprise provider-specific dnsTrustedServers default (openshift-dns, rke2-coredns) lives in the enterprise extension instead of an IsEnterprise branch in setDefaultsOnFelixConfiguration. It is a companion rather than part of ExtendContext because felix defaulting persists early in reconcile, before ExtendContext runs. --- .../installation/core_controller.go | 37 ++++-------------- pkg/enterprise/installation.go | 38 +++++++++++++++++++ pkg/enterprise/installation_test.go | 27 +++++++++++++ pkg/extensions/controllerextension.go | 13 +++++++ pkg/extensions/set.go | 21 ++++++++++ 5 files changed, 106 insertions(+), 30 deletions(-) diff --git a/pkg/controller/installation/core_controller.go b/pkg/controller/installation/core_controller.go index b01f01cfca..fdb258e76d 100644 --- a/pkg/controller/installation/core_controller.go +++ b/pkg/controller/installation/core_controller.go @@ -1901,36 +1901,13 @@ func (r *ReconcileInstallation) setDefaultsOnFelixConfiguration(ctx context.Cont updated = true } - if install.Spec.Variant.IsEnterprise() { - // Some platforms need a different default setting for dnsTrustedServers, because their DNS service is not named "kube-dns". - dnsService := "" - switch install.Spec.KubernetesProvider { - case operatorv1.ProviderOpenShift: - dnsService = "k8s-service:openshift-dns/dns-default" - case operatorv1.ProviderRKE2: - dnsService = "k8s-service:kube-system/rke2-coredns-rke2-coredns" - } - if dnsService != "" { - felixDefault := "k8s-service:kube-dns" - trustedServers := []string{dnsService} - // Keep any other values that are already configured, excepting the value - // that we are setting and the kube-dns default. - existingSetting := "" - if fc.Spec.DNSTrustedServers != nil { - existingSetting = strings.Join(*(fc.Spec.DNSTrustedServers), ",") - for _, server := range *(fc.Spec.DNSTrustedServers) { - if server != felixDefault && server != dnsService { - trustedServers = append(trustedServers, server) - } - } - } - newSetting := strings.Join(trustedServers, ",") - if newSetting != existingSetting { - fc.Spec.DNSTrustedServers = &trustedServers - updated = true - } - } + // Variant-specific FelixConfiguration defaults (e.g. the Enterprise + // provider-specific dnsTrustedServers) are owned by the variant extension. + extUpdated, err := r.opts.Extensions.DefaultFelixConfiguration(extensions.InstallationController, &install.Spec, fc) + if err != nil { + return updated, err } + updated = updated || extUpdated // If BPF is enabled, but not set on FelixConfiguration, do so here. This could happen when an older // version of operator is replaced by the new one. Older versions of the operator used an @@ -1941,7 +1918,7 @@ func (r *ReconcileInstallation) setDefaultsOnFelixConfiguration(ctx context.Cont // If calico-node daemonset exists, we need to check the ENV VAR and set FelixConfiguration accordingly. // Otherwise, this is a fresh install in eBPF mode, set the felix config. ds := &appsv1.DaemonSet{} - err := r.client.Get(ctx, types.NamespacedName{Namespace: common.CalicoNamespace, Name: common.NodeDaemonSetName}, ds) + err = r.client.Get(ctx, types.NamespacedName{Namespace: common.CalicoNamespace, Name: common.NodeDaemonSetName}, ds) if err != nil { if !apierrors.IsNotFound(err) { reqLogger.Error(err, "An error occurred when getting the Daemonset resource") diff --git a/pkg/enterprise/installation.go b/pkg/enterprise/installation.go index 7df2d2bf30..ca4f77dc36 100644 --- a/pkg/enterprise/installation.go +++ b/pkg/enterprise/installation.go @@ -16,7 +16,9 @@ package enterprise import ( "fmt" + "strings" + v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" rbacv1 "k8s.io/api/rbac/v1" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/handler" @@ -78,6 +80,42 @@ func (coreControllerExtension) Validate(cc extensions.ControllerContext) error { return validateReporterPort(cc.FelixConfiguration) } +// DefaultFelixConfiguration sets the Enterprise-only FelixConfiguration defaults. +// Some platforms run a DNS service that isn't named "kube-dns", so dnsTrustedServers +// needs a provider-specific default for Enterprise DNS logging to work. Returns +// whether it changed fc. +func (coreControllerExtension) DefaultFelixConfiguration(install *operatorv1.InstallationSpec, fc *v3.FelixConfiguration) (bool, error) { + dnsService := "" + switch install.KubernetesProvider { + case operatorv1.ProviderOpenShift: + dnsService = "k8s-service:openshift-dns/dns-default" + case operatorv1.ProviderRKE2: + dnsService = "k8s-service:kube-system/rke2-coredns-rke2-coredns" + } + if dnsService == "" { + return false, nil + } + + felixDefault := "k8s-service:kube-dns" + trustedServers := []string{dnsService} + // Keep any other values that are already configured, excepting the value we are + // setting and the kube-dns default. + existingSetting := "" + if fc.Spec.DNSTrustedServers != nil { + existingSetting = strings.Join(*fc.Spec.DNSTrustedServers, ",") + for _, server := range *fc.Spec.DNSTrustedServers { + if server != felixDefault && server != dnsService { + trustedServers = append(trustedServers, server) + } + } + } + if strings.Join(trustedServers, ",") == existingSetting { + return false, nil + } + fc.Spec.DNSTrustedServers = &trustedServers + return true, nil +} + // Watches registers the enterprise resources the installation controller // reconciles on. func (coreControllerExtension) Watches(c ctrlruntime.Controller) error { diff --git a/pkg/enterprise/installation_test.go b/pkg/enterprise/installation_test.go index d99e18eb00..a66bdcbd91 100644 --- a/pkg/enterprise/installation_test.go +++ b/pkg/enterprise/installation_test.go @@ -43,6 +43,33 @@ var _ = Describe("installation controller extension", func() { Expect(ext.Validate(cc)).To(HaveOccurred()) }) + DescribeTable("defaults dnsTrustedServers for providers whose DNS service isn't kube-dns", + func(provider operatorv1.Provider, expected []string) { + fc := &v3.FelixConfiguration{} + install := &operatorv1.InstallationSpec{Variant: operatorv1.CalicoEnterprise, KubernetesProvider: provider} + updated, err := ext.DefaultFelixConfiguration(extensions.InstallationController, install, fc) + Expect(err).NotTo(HaveOccurred()) + if expected == nil { + Expect(updated).To(BeFalse()) + Expect(fc.Spec.DNSTrustedServers).To(BeNil()) + return + } + Expect(updated).To(BeTrue()) + Expect(*fc.Spec.DNSTrustedServers).To(ConsistOf(expected)) + }, + Entry("OpenShift", operatorv1.ProviderOpenShift, []string{"k8s-service:openshift-dns/dns-default"}), + Entry("RKE2", operatorv1.ProviderRKE2, []string{"k8s-service:kube-system/rke2-coredns-rke2-coredns"}), + Entry("other providers keep the felix default", operatorv1.ProviderNone, nil), + ) + + It("does no felix defaulting for the Calico variant", func() { + fc := &v3.FelixConfiguration{} + updated, err := ext.DefaultFelixConfiguration(extensions.InstallationController, &operatorv1.InstallationSpec{Variant: operatorv1.Calico, KubernetesProvider: operatorv1.ProviderOpenShift}, fc) + Expect(err).NotTo(HaveOccurred()) + Expect(updated).To(BeFalse()) + Expect(fc.Spec.DNSTrustedServers).To(BeNil()) + }) + It("manages the node prometheus and kube-controllers metrics keypairs for the enterprise variant", func() { _, managed, err := ext.ExtendContext(newControllerContext(operatorv1.CalicoEnterprise)) Expect(err).NotTo(HaveOccurred()) diff --git a/pkg/extensions/controllerextension.go b/pkg/extensions/controllerextension.go index d61b4cee6f..a70b3f700d 100644 --- a/pkg/extensions/controllerextension.go +++ b/pkg/extensions/controllerextension.go @@ -17,8 +17,10 @@ package extensions import ( "context" + v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" "sigs.k8s.io/controller-runtime/pkg/client" + operatorv1 "github.com/tigera/operator/api/v1" "github.com/tigera/operator/pkg/controller/certificatemanager" "github.com/tigera/operator/pkg/ctrlruntime" "github.com/tigera/operator/pkg/tls/certificatemanagement" @@ -59,6 +61,17 @@ type Watcher interface { Watches(c ctrlruntime.Controller) error } +// FelixConfigDefaulter is an optional companion to ControllerExtension. A +// controller's FelixConfiguration defaulting calls Set.DefaultFelixConfiguration, +// which invokes this on a registered extension that implements it, so the variant's +// FelixConfiguration defaults (e.g. the provider-specific dnsTrustedServers) live in +// the extension instead of the controller. It returns whether it changed fc. Felix +// defaulting persists early in reconcile, before ExtendContext runs, so it can't fold +// into ExtendContext. +type FelixConfigDefaulter interface { + DefaultFelixConfiguration(install *operatorv1.InstallationSpec, fc *v3.FelixConfiguration) (bool, error) +} + // ControllerContext is the controller-phase context, the corollary to the // render-phase RenderContext. It is the embedded RenderContext (the same data // the render phase sees) plus the controller-side machinery a ControllerExtension diff --git a/pkg/extensions/set.go b/pkg/extensions/set.go index ee95199a15..afbdc114f1 100644 --- a/pkg/extensions/set.go +++ b/pkg/extensions/set.go @@ -15,6 +15,8 @@ package extensions import ( + v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" + operatorv1 "github.com/tigera/operator/api/v1" "github.com/tigera/operator/pkg/components" "github.com/tigera/operator/pkg/ctrlruntime" @@ -121,6 +123,25 @@ func (s *Set) SetupWatches(controller ControllerName, c ctrlruntime.Controller) return nil } +// DefaultFelixConfiguration runs the named controller's extension FelixConfiguration +// defaulting for the installation's variant, returning whether it changed fc. The +// extension implements the optional FelixConfigDefaulter companion; when it doesn't +// (or no extension is registered) this is a no-op. Nil-safe. +func (s *Set) DefaultFelixConfiguration(controller ControllerName, install *operatorv1.InstallationSpec, fc *v3.FelixConfiguration) (bool, error) { + if install == nil { + return false, nil + } + v := s.variant(install.Variant) + if v == nil { + return false, nil + } + d, ok := v.controllers[controller].(FelixConfigDefaulter) + if !ok { + return false, nil + } + return d.DefaultFelixConfiguration(install, fc) +} + // Images returns the shared image override table. The render package resolves a // component's image through it directly (the imageoverride leaf, so render need // not import extensions). Nil-safe, returning nil overrides that resolve to the From fb295a6b394c6f8f56d03631ad8a7df123280df7 Mon Sep 17 00:00:00 2001 From: Casey Davenport Date: Mon, 22 Jun 2026 09:51:05 -0700 Subject: [PATCH 40/54] Move the enterprise API server surface into the extension Strip the enterprise-only fields off APIServerConfiguration (ManagementCluster, ManagementClusterConnection, ApplicationLayer, RequiresQueryServer, the query-server cert, and KeyValidatorConfig) so the base render is variant-blind. A new apiserver controller hook fetches the enterprise CRs, builds the trusted bundle and query-server cert, and resolves the L7 sidecar images, stashing them on the render context. The modifier reads that slot and find-or-creates the deployment skeleton when the base did not render an aggregation server. APIServerPolicy becomes extensible so the OIDC egress rule and L7 ingress port move to a policy modifier too. --- .../apiserver/apiserver_controller.go | 200 +- pkg/enterprise/apiserver.go | 535 ++++- pkg/enterprise/apiserver_test.go | 439 ++++ pkg/enterprise/register.go | 1 + pkg/extensions/controllerextension.go | 6 + pkg/render/apiserver.go | 370 +--- pkg/render/apiserver_test.go | 1970 +---------------- 7 files changed, 1134 insertions(+), 2387 deletions(-) create mode 100644 pkg/enterprise/apiserver_test.go diff --git a/pkg/controller/apiserver/apiserver_controller.go b/pkg/controller/apiserver/apiserver_controller.go index 3b8c5e4197..212a4cfe43 100644 --- a/pkg/controller/apiserver/apiserver_controller.go +++ b/pkg/controller/apiserver/apiserver_controller.go @@ -52,7 +52,6 @@ import ( "github.com/tigera/operator/pkg/extensions" "github.com/tigera/operator/pkg/render" rcertificatemanagement "github.com/tigera/operator/pkg/render/certificatemanagement" - "github.com/tigera/operator/pkg/render/common/authentication" "github.com/tigera/operator/pkg/render/common/networkpolicy" "github.com/tigera/operator/pkg/render/monitor" "github.com/tigera/operator/pkg/render/webhooks" @@ -105,42 +104,11 @@ func Add(mgr manager.Manager, opts options.ControllerOptions) error { } if opts.EnterpriseCRDExists { - // Watch for changes to ApplicationLayer - err = c.WatchObject(&operatorv1.ApplicationLayer{ObjectMeta: metav1.ObjectMeta{Name: utils.DefaultEnterpriseInstanceKey.Name}}, &handler.EnqueueRequestForObject{}) - if err != nil { - return fmt.Errorf("apiserver-controller failed to watch ApplicationLayer resource: %v", err) - } - - // Watch for changes to primary resource ManagementCluster - err = c.WatchObject(&operatorv1.ManagementCluster{}, &handler.EnqueueRequestForObject{}) - if err != nil { - return fmt.Errorf("apiserver-controller failed to watch primary resource: %v", err) - } - - // Watch for changes to primary resource ManagementClusterConnection - err = c.WatchObject(&operatorv1.ManagementClusterConnection{}, &handler.EnqueueRequestForObject{}) - if err != nil { - return fmt.Errorf("apiserver-controller failed to watch primary resource: %v", err) - } - - for _, namespace := range []string{common.OperatorNamespace(), render.APIServerNamespace} { - for _, secretName := range []string{render.VoltronTunnelSecretName, render.ManagerTLSSecretName} { - if err = utils.AddSecretsWatch(c, secretName, namespace); err != nil { - return fmt.Errorf("apiserver-controller failed to watch the Secret resource: %v", err) - } - } + // The variant extension registers the enterprise watches it needs (the management + // cluster CRs, ApplicationLayer, Authentication, and the tunnel secrets). + if err = opts.Extensions.SetupWatches(extensions.APIServerController, c); err != nil { + return fmt.Errorf("apiserver-controller failed to set up extension watches: %w", err) } - - if err = utils.AddSecretsWatch(c, render.VoltronLinseedPublicCert, common.OperatorNamespace()); err != nil { - return fmt.Errorf("apiserver-controller failed to watch the Secret resource: %v", err) - } - - // Watch for changes to authentication - err = c.WatchObject(&operatorv1.Authentication{}, &handler.EnqueueRequestForObject{}) - if err != nil { - return fmt.Errorf("apiserver-controller failed to watch resource: %w", err) - } - } // Watch for the namespace(s) managed by this controller. @@ -319,16 +287,6 @@ func (r *ReconcileAPIServer) Reconcile(ctx context.Context, request reconcile.Re return reconcile.Result{}, err } - // Since apiserver and queryserver may have different UID:GID at run-time, we need to produce this secret in separate volumes and with different permissions. - var queryServerTLSSecretCertificateManagementOnly certificatemanagement.KeyPairInterface - if installationSpec.CertificateManagement != nil { - queryServerTLSSecretCertificateManagementOnly, err = certificateManager.GetOrCreateKeyPair(r.client, "query-server-tls", common.OperatorNamespace(), dns.GetServiceDNSNames(render.APIServerServiceName, render.APIServerNamespace, r.opts.ClusterDomain)) - if err != nil { - r.status.SetDegraded(operatorv1.ResourceCreateError, "Unable to get or create tls key pair", err, reqLogger) - return reconcile.Result{}, err - } - } - certificateManager.AddToStatusManager(r.status, render.APIServerNamespace) pullSecrets, err := utils.GetInstallationPullSecrets(installationSpec, r.client) @@ -337,114 +295,45 @@ func (r *ReconcileAPIServer) Reconcile(ctx context.Context, request reconcile.Re return reconcile.Result{}, err } - // Query enterprise-only data. - var trustedBundle certificatemanagement.TrustedBundle - var applicationLayer *operatorv1.ApplicationLayer - var managementCluster *operatorv1.ManagementCluster - var managementClusterConnection *operatorv1.ManagementClusterConnection - var keyValidatorConfig authentication.KeyValidatorConfig - includeV3NetworkPolicy := false - - if installationSpec.Variant.IsEnterprise() { - trustedBundle, err = certificateManager.CreateNamedTrustedBundleFromSecrets(render.APIServerResourceName, r.client, - common.OperatorNamespace(), false) - if err != nil { - r.status.SetDegraded(operatorv1.ResourceCreateError, "Unable to create the trusted bundle", err, reqLogger) - return reconcile.Result{}, err - } - - applicationLayer, err = utils.GetApplicationLayer(ctx, r.client) - if err != nil { - r.status.SetDegraded(operatorv1.ResourceReadError, "Error reading ApplicationLayer", err, reqLogger) - return reconcile.Result{}, err - } + // Run the variant extension: it validates the configuration and produces the + // enterprise render data (the trusted bundle, the query server certificate, the + // management cluster CRs, the OIDC config, and the resolved L7 sidecar images). For + // the core operator this is a no-op and the render context carries no extension data. + cc := extensions.ControllerContext{ + RenderContext: extensions.RenderContext{ + Installation: installationSpec, + ClusterDomain: r.opts.ClusterDomain, + }, + Controller: extensions.APIServerController, + Ctx: ctx, + Client: r.client, + CertificateManager: certificateManager, + MultiTenant: r.opts.MultiTenant, + } + if err := r.opts.Extensions.Validate(cc); err != nil { + r.status.SetDegraded(operatorv1.ResourceValidationError, "Invalid API server configuration", err, reqLogger) + return reconcile.Result{}, err + } + renderCtx, managedKeyPairs, err := r.opts.Extensions.ExtendContext(cc) + if err != nil { + r.status.SetDegraded(operatorv1.ResourceCreateError, "Error preparing the API server extension", err, reqLogger) + return reconcile.Result{}, err + } + trustedBundle := renderCtx.TrustedBundle + // The webhooks component (v3-CRD mode) needs the ManagementCluster to register the + // managed-cluster webhook. Reading it requires the enterprise CRDs. + var managementCluster *operatorv1.ManagementCluster + if r.opts.EnterpriseCRDExists { managementCluster, err = utils.GetManagementCluster(ctx, r.client) if err != nil { r.status.SetDegraded(operatorv1.ResourceReadError, "Error reading ManagementCluster", err, reqLogger) return reconcile.Result{}, err } - - managementClusterConnection, err = utils.GetManagementClusterConnection(ctx, r.client) - if err != nil { - r.status.SetDegraded(operatorv1.ResourceReadError, "Error reading ManagementClusterConnection", err, reqLogger) - return reconcile.Result{}, err - } - - if managementClusterConnection != nil && managementCluster != nil { - err = fmt.Errorf("having both a ManagementCluster and a ManagementClusterConnection is not supported") - r.status.SetDegraded(operatorv1.ResourceValidationError, "", err, reqLogger) - return reconcile.Result{}, err - } - - // Management cluster only: check if the tunnel CA secret has been created. The apiserver mounts this secret so - // it can sign certificates for managed clusters. If the managementCluster has not been defaulted then we should - // not degrade. This is because the manager_controller exits the reconcile loop if the apiserver is not available. - if managementCluster != nil && managementCluster.Spec.TLS != nil && !r.opts.MultiTenant { - tunnelSecretName := managementCluster.Spec.TLS.SecretName - // The manager_controller should have written this secret. We know this since spec.TLS has been defaulted. - // If the secret does not exist, we degrade this controller. - _, err := utils.GetSecret(ctx, r.client, tunnelSecretName, common.OperatorNamespace()) - if err != nil { - r.status.SetDegraded(operatorv1.ResourceReadError, "Unable to fetch the tunnel secret", err, reqLogger) - return reconcile.Result{}, err - } - } - - prometheusCertificate, err := certificateManager.GetCertificate(r.client, monitor.PrometheusClientTLSSecretName, common.OperatorNamespace()) - if err != nil { - r.status.SetDegraded(operatorv1.ResourceReadError, "Failed to get certificate", err, reqLogger) - return reconcile.Result{}, err - } else if prometheusCertificate != nil { - trustedBundle.AddCertificates(prometheusCertificate) - } - - if managementClusterConnection != nil { - voltronLinseedCert, err := certificateManager.GetCertificate(r.client, render.VoltronLinseedPublicCert, common.OperatorNamespace()) - if err != nil { - r.status.SetDegraded(operatorv1.ResourceReadError, fmt.Sprintf("Failed to retrieve %s", render.VoltronLinseedPublicCert), err, reqLogger) - return reconcile.Result{}, err - } - if voltronLinseedCert != nil { - trustedBundle.AddCertificates(voltronLinseedCert) - } - } - - var authenticationCR *operatorv1.Authentication - // Fetch the Authentication spec. If present, we use it to configure user authentication. - authenticationCR, err = utils.GetAuthentication(ctx, r.client) - if err != nil && !errors.IsNotFound(err) { - r.status.SetDegraded(operatorv1.ResourceReadError, "Error while fetching Authentication", err, reqLogger) - return reconcile.Result{}, err - } - - if authenticationCR != nil && authenticationCR.Status.State == operatorv1.TigeraStatusReady { - if utils.DexEnabled(authenticationCR) { - // Do not include DEX TLS Secret Name if authentication CR does not have type Dex - secret := render.DexTLSSecretName - certificate, err := certificateManager.GetCertificate(r.client, secret, common.OperatorNamespace()) - if err != nil { - r.status.SetDegraded(operatorv1.CertificateError, fmt.Sprintf("Failed to retrieve %s", secret), - err, reqLogger) - return reconcile.Result{}, err - } else if certificate == nil { - reqLogger.Info(fmt.Sprintf("Waiting for secret '%s' to become available", secret)) - r.status.SetDegraded(operatorv1.ResourceNotReady, - fmt.Sprintf("Waiting for secret '%s' to become available", secret), - nil, reqLogger) - return reconcile.Result{}, nil - } - trustedBundle.AddCertificates(certificate) - } - - keyValidatorConfig, err = utils.GetKeyValidatorConfig(ctx, r.client, authenticationCR, r.opts.ClusterDomain) - if err != nil { - r.status.SetDegraded(operatorv1.ResourceReadError, "Failed to get KeyValidator Config", err, reqLogger) - return reconcile.Result{}, err - } - } } + includeV3NetworkPolicy := false + // Ensure the calico-system tier exists, before rendering any network policies within it. // // The creation of the Tier depends on this controller to reconcile it's non-NetworkPolicy resources so that @@ -480,7 +369,7 @@ func (r *ReconcileAPIServer) Reconcile(ctx context.Context, request reconcile.Re r.client, r.scheme, instance, - utils.WithRenderContext(extensions.RenderContext{Installation: installationSpec}), + utils.WithRenderContext(renderCtx), utils.WithExtensions(r.opts.Extensions), ) @@ -493,20 +382,14 @@ func (r *ReconcileAPIServer) Reconcile(ctx context.Context, request reconcile.Re Installation: installationSpec, APIServer: &instance.Spec, ForceHostNetwork: false, - ApplicationLayer: applicationLayer, - ManagementCluster: managementCluster, - ManagementClusterConnection: managementClusterConnection, TLSKeyPair: tlsSecret, PullSecrets: pullSecrets, OpenShift: r.opts.DetectedProvider.IsOpenShift(), TrustedBundle: trustedBundle, MultiTenant: r.opts.MultiTenant, - KeyValidatorConfig: keyValidatorConfig, KubernetesVersion: r.opts.KubernetesVersion, ClusterDomain: r.opts.ClusterDomain, RequiresAggregationServer: !r.opts.UseV3CRDs, - RequiresQueryServer: installationSpec.Variant.IsEnterprise(), - QueryServerTLSKeyPairCertificateManagementOnly: queryServerTLSSecretCertificateManagementOnly, } var components []render.Component @@ -515,6 +398,9 @@ func (r *ReconcileAPIServer) Reconcile(ctx context.Context, request reconcile.Re certKeyPairOptions := []rcertificatemanagement.KeyPairOption{ rcertificatemanagement.NewKeyPairOption(tlsSecret, true, true), } + for _, kp := range managedKeyPairs { + certKeyPairOptions = append(certKeyPairOptions, rcertificatemanagement.NewKeyPairOption(kp, true, true)) + } if r.opts.UseV3CRDs { // If using v3 CRDs, we render the webhooks component that handles various RBAC and validation // responsibilities. The ordering of resources here is important to avoid a deadlock: @@ -594,10 +480,14 @@ func (r *ReconcileAPIServer) Reconcile(ctx context.Context, request reconcile.Re } // Check BYO certificate expiry warnings. - certificatemanagement.CheckKeyPairWarnings(map[string]certificatemanagement.KeyPairInterface{ + keyPairWarnings := map[string]certificatemanagement.KeyPairInterface{ render.CalicoAPIServerTLSSecretName: tlsSecret, webhooks.WebhooksTLSSecretName: webhooksTLS, - }, r.status) + } + for _, kp := range managedKeyPairs { + keyPairWarnings[kp.GetName()] = kp + } + certificatemanagement.CheckKeyPairWarnings(keyPairWarnings, r.status) // Clear the degraded bit if we've reached this far. r.status.ClearDegraded() diff --git a/pkg/enterprise/apiserver.go b/pkg/enterprise/apiserver.go index b20f3579e5..0ad765e362 100644 --- a/pkg/enterprise/apiserver.go +++ b/pkg/enterprise/apiserver.go @@ -16,25 +16,40 @@ package enterprise import ( "fmt" + "net/url" "slices" "strings" + admregv1 "k8s.io/api/admissionregistration/v1" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/handler" + v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" + "github.com/tigera/api/pkg/lib/numorstring" + + operatorv1 "github.com/tigera/operator/api/v1" "github.com/tigera/operator/pkg/common" "github.com/tigera/operator/pkg/components" + "github.com/tigera/operator/pkg/controller/utils" + "github.com/tigera/operator/pkg/controller/utils/imageset" + "github.com/tigera/operator/pkg/ctrlruntime" + "github.com/tigera/operator/pkg/dns" "github.com/tigera/operator/pkg/extensions" "github.com/tigera/operator/pkg/render" + "github.com/tigera/operator/pkg/render/common/authentication" rcomp "github.com/tigera/operator/pkg/render/common/components" relasticsearch "github.com/tigera/operator/pkg/render/common/elasticsearch" rmeta "github.com/tigera/operator/pkg/render/common/meta" + "github.com/tigera/operator/pkg/render/common/networkpolicy" "github.com/tigera/operator/pkg/render/common/securitycontext" + "github.com/tigera/operator/pkg/render/monitor" "github.com/tigera/operator/pkg/tls/certificatemanagement" ) @@ -43,16 +58,218 @@ const ( auditPolicyVolumeName = "calico-audit-policy" ) -// apiServer carries the rendered API server configuration and resolved image so the -// enterprise builders (moved verbatim from the render package) can construct the +// apiServerRenderData is the controller-produced data the API server hook hands to its +// modifiers through RenderContext.Extension. It carries the enterprise inputs the base +// render no longer knows about: the management cluster / managed cluster CRs, the +// ApplicationLayer (which drives the L7 sidecar), the cert-management-only query server +// keypair, the OIDC key validator config, and the resolved L7 sidecar images. +type apiServerRenderData struct { + managementCluster *operatorv1.ManagementCluster + managementClusterConnection *operatorv1.ManagementClusterConnection + applicationLayer *operatorv1.ApplicationLayer + queryServerTLS certificatemanagement.KeyPairInterface + keyValidatorConfig authentication.KeyValidatorConfig + l7EnvoyImage string + dikastesImage string +} + +// apiServerData pulls the API server hook's render data back out of the render context, +// returning the zero value when none is set. +func apiServerData(rc extensions.RenderContext) apiServerRenderData { + d, _ := rc.Extension.(apiServerRenderData) + return d +} + +// apiServer carries the rendered API server configuration, the resolved image, and the +// controller-produced render data so the enterprise builders can construct the // Enterprise-only objects and deployment additions. type apiServer struct { cfg *render.APIServerConfiguration calicoImage string + data apiServerRenderData } func registerAPIServer(v *extensions.Variant) { extensions.RegisterModifier(v, render.ComponentNameAPIServer, modifyAPIServer) + extensions.RegisterModifier(v, render.ComponentNameAPIServerPolicy, modifyAPIServerPolicy) +} + +func (c *apiServer) isSidecarInjectionEnabled() bool { + al := c.data.applicationLayer + return al != nil && + al.Spec.SidecarInjection != nil && + *al.Spec.SidecarInjection == operatorv1.SidecarEnabled +} + +// apiServerControllerExtension is the Calico Enterprise controller-side hook for the API +// server controller. It does the enterprise reconcile work the render phase can't: +// fetching the enterprise CRs, creating the trusted bundle and the query server cert, +// and resolving the L7 sidecar images. +type apiServerControllerExtension struct{} + +// Validate rejects an API server configuration Calico Enterprise does not support: a +// cluster cannot be both a management cluster and a managed cluster. +func (apiServerControllerExtension) Validate(cc extensions.ControllerContext) error { + managementCluster, err := utils.GetManagementCluster(cc.Ctx, cc.Client) + if err != nil { + return fmt.Errorf("error reading ManagementCluster: %w", err) + } + managementClusterConnection, err := utils.GetManagementClusterConnection(cc.Ctx, cc.Client) + if err != nil { + return fmt.Errorf("error reading ManagementClusterConnection: %w", err) + } + if managementCluster != nil && managementClusterConnection != nil { + return fmt.Errorf("having both a ManagementCluster and a ManagementClusterConnection is not supported") + } + return nil +} + +// Watches registers the enterprise resources the API server controller reconciles on. +func (apiServerControllerExtension) Watches(c ctrlruntime.Controller) error { + for _, obj := range []client.Object{ + &operatorv1.ApplicationLayer{ObjectMeta: metav1.ObjectMeta{Name: utils.DefaultEnterpriseInstanceKey.Name}}, + &operatorv1.ManagementCluster{}, + &operatorv1.ManagementClusterConnection{}, + &operatorv1.Authentication{}, + } { + if err := c.WatchObject(obj, &handler.EnqueueRequestForObject{}); err != nil { + return err + } + } + for _, namespace := range []string{common.OperatorNamespace(), render.APIServerNamespace} { + for _, secretName := range []string{render.VoltronTunnelSecretName, render.ManagerTLSSecretName} { + if err := utils.AddSecretsWatch(c, secretName, namespace); err != nil { + return err + } + } + } + return utils.AddSecretsWatch(c, render.VoltronLinseedPublicCert, common.OperatorNamespace()) +} + +// ExtendContext does the enterprise controller-side work: it builds the trusted bundle, +// fetches the enterprise CRs, creates the query server certificate, resolves the L7 +// sidecar images, and stashes them for the modifiers. The base API server render carries +// none of this. +func (apiServerControllerExtension) ExtendContext(cc extensions.ControllerContext) (extensions.RenderContext, []certificatemanagement.KeyPairInterface, error) { + rc := cc.RenderContext + in := cc.Installation + + trustedBundle, err := cc.CertificateManager.CreateNamedTrustedBundleFromSecrets(render.APIServerResourceName, cc.Client, common.OperatorNamespace(), false) + if err != nil { + return rc, nil, fmt.Errorf("unable to create the trusted bundle: %w", err) + } + + applicationLayer, err := utils.GetApplicationLayer(cc.Ctx, cc.Client) + if err != nil { + return rc, nil, fmt.Errorf("error reading ApplicationLayer: %w", err) + } + + managementCluster, err := utils.GetManagementCluster(cc.Ctx, cc.Client) + if err != nil { + return rc, nil, fmt.Errorf("error reading ManagementCluster: %w", err) + } + + managementClusterConnection, err := utils.GetManagementClusterConnection(cc.Ctx, cc.Client) + if err != nil { + return rc, nil, fmt.Errorf("error reading ManagementClusterConnection: %w", err) + } + + // Management cluster only: the apiserver mounts the tunnel CA secret so it can sign + // certificates for managed clusters. The manager controller writes it once + // ManagementCluster.Spec.TLS is defaulted; degrade until it exists. + if managementCluster != nil && managementCluster.Spec.TLS != nil && !cc.MultiTenant { + if _, err := utils.GetSecret(cc.Ctx, cc.Client, managementCluster.Spec.TLS.SecretName, common.OperatorNamespace()); err != nil { + return rc, nil, fmt.Errorf("unable to fetch the tunnel secret: %w", err) + } + } + + prometheusCertificate, err := cc.CertificateManager.GetCertificate(cc.Client, monitor.PrometheusClientTLSSecretName, common.OperatorNamespace()) + if err != nil { + return rc, nil, fmt.Errorf("failed to get certificate: %w", err) + } + if prometheusCertificate != nil { + trustedBundle.AddCertificates(prometheusCertificate) + } + + if managementClusterConnection != nil { + voltronLinseedCert, err := cc.CertificateManager.GetCertificate(cc.Client, render.VoltronLinseedPublicCert, common.OperatorNamespace()) + if err != nil { + return rc, nil, fmt.Errorf("failed to retrieve %s: %w", render.VoltronLinseedPublicCert, err) + } + if voltronLinseedCert != nil { + trustedBundle.AddCertificates(voltronLinseedCert) + } + } + + // Authentication: when a Dex-backed Authentication CR is ready, add its cert to the + // bundle and build the key validator config for the query server and the policy. + var keyValidatorConfig authentication.KeyValidatorConfig + authenticationCR, err := utils.GetAuthentication(cc.Ctx, cc.Client) + if err != nil && !apierrors.IsNotFound(err) { + return rc, nil, fmt.Errorf("error while fetching Authentication: %w", err) + } + if authenticationCR != nil && authenticationCR.Status.State == operatorv1.TigeraStatusReady { + if utils.DexEnabled(authenticationCR) { + certificate, err := cc.CertificateManager.GetCertificate(cc.Client, render.DexTLSSecretName, common.OperatorNamespace()) + if err != nil { + return rc, nil, fmt.Errorf("failed to retrieve %s: %w", render.DexTLSSecretName, err) + } else if certificate == nil { + return rc, nil, fmt.Errorf("waiting for secret %q to become available", render.DexTLSSecretName) + } + trustedBundle.AddCertificates(certificate) + } + keyValidatorConfig, err = utils.GetKeyValidatorConfig(cc.Ctx, cc.Client, authenticationCR, cc.ClusterDomain) + if err != nil { + return rc, nil, fmt.Errorf("failed to get KeyValidator config: %w", err) + } + } + + // Under certificate management, the query server needs its own keypair so it can run + // with different permissions than the apiserver. + var queryServerTLS certificatemanagement.KeyPairInterface + if in.CertificateManagement != nil { + queryServerTLS, err = cc.CertificateManager.GetOrCreateKeyPair( + cc.Client, + "query-server-tls", + common.OperatorNamespace(), + dns.GetServiceDNSNames(render.APIServerServiceName, render.APIServerNamespace, cc.ClusterDomain), + ) + if err != nil { + return rc, nil, fmt.Errorf("unable to get or create query server tls key pair: %w", err) + } + } + + // Resolve the L7 sidecar images when sidecar injection is enabled. The modifier runs + // after image resolution, so the hook resolves them here. + var l7EnvoyImage, dikastesImage string + if applicationLayer != nil && + applicationLayer.Spec.SidecarInjection != nil && + *applicationLayer.Spec.SidecarInjection == operatorv1.SidecarEnabled { + imageSet, err := imageset.GetImageSet(cc.Ctx, cc.Client, in.Variant) + if err != nil { + return rc, nil, err + } + l7EnvoyImage, err = components.GetReference(components.ComponentEnvoyProxy, in.Registry, in.ImagePath, in.ImagePrefix, imageSet) + if err != nil { + return rc, nil, err + } + dikastesImage, err = components.GetReference(components.ComponentDikastes, in.Registry, in.ImagePath, in.ImagePrefix, imageSet) + if err != nil { + return rc, nil, err + } + } + + rc.TrustedBundle = trustedBundle + rc.Extension = apiServerRenderData{ + managementCluster: managementCluster, + managementClusterConnection: managementClusterConnection, + applicationLayer: applicationLayer, + queryServerTLS: queryServerTLS, + keyValidatorConfig: keyValidatorConfig, + l7EnvoyImage: l7EnvoyImage, + dikastesImage: dikastesImage, + } + return rc, nil, nil } // registerAPIServerCleanup registers, for the Calico variant, the cleanup that @@ -66,13 +283,19 @@ func registerAPIServerCleanup(v *extensions.Variant) { // the query server container and its volumes, audit logging on the aggregation API server // container, the Enterprise RBAC objects, and the query server port on the Service. func modifyAPIServer(rc extensions.RenderContext, ec render.APIServerExtensionContext, create, del []client.Object) ([]client.Object, []client.Object) { - c := &apiServer{cfg: ec.Config, calicoImage: ec.CalicoImage} + c := &apiServer{cfg: ec.Config, calicoImage: ec.CalicoImage, data: apiServerData(rc)} + + // Ensure the deployment and its supporting objects exist. The base renders them when + // running an aggregation API server; in v3-CRD mode it queues them for deletion, but + // Enterprise always runs a query server, so render the skeleton ourselves and pull + // those objects back out of the delete list. + create, del = c.ensureDeployment(create, del) if dep, ok := extensions.FindObject[*appsv1.Deployment](create, render.APIServerName); ok { c.layerDeployment(dep) } if svc, ok := extensions.FindObject[*corev1.Service](create, render.APIServerServiceName); ok { - c.addQueryServerPort(svc) + c.addServicePorts(svc) } // Enterprise serves staged policies through the tiered-policy passthrough role. if role, ok := extensions.FindObject[*rbacv1.ClusterRole](create, "calico-tiered-policy-passthrough"); ok { @@ -83,13 +306,21 @@ func modifyAPIServer(rc extensions.RenderContext, ec render.APIServerExtensionCo } } + // The L7 sidecar mutating webhook is driven by ApplicationLayer. The base always + // queues it for deletion; when sidecar injection is on, render it and pull it back + // out of the delete list. + if c.isSidecarInjectionEnabled() { + create = append(create, c.sidecarMutatingWebhookConfig()) + del = removeByRef(del, &admregv1.MutatingWebhookConfiguration{ObjectMeta: metav1.ObjectMeta{Name: common.SidecarMutatingWebhookConfigName}}) + } + // Global Enterprise RBAC. create = append(create, c.tigeraAPIServerClusterRole(), c.tigeraAPIServerClusterRoleBinding()) if !c.cfg.MultiTenant { // These resources are only installed in zero-tenant clusters. create = append(create, c.tigeraUserClusterRole(), c.tigeraNetworkAdminClusterRole()) } - if c.cfg.ManagementCluster != nil { + if c.data.managementCluster != nil { create = append(create, c.managedClusterWatchClusterRole()) if c.cfg.MultiTenant { create = append(create, c.multiTenantSecretsRBAC()...) @@ -109,7 +340,7 @@ func modifyAPIServer(rc extensions.RenderContext, ec render.APIServerExtensionCo if c.cfg.TrustedBundle != nil { create = append(create, c.cfg.TrustedBundle.ConfigMap(render.QueryserverNamespace)) } - if c.cfg.ManagementClusterConnection != nil { + if c.data.managementClusterConnection != nil { create = append(create, c.externalLinseedRoleBinding()) } @@ -159,14 +390,22 @@ func cleanupAPIServer(rc extensions.RenderContext, ec render.APIServerExtensionC return create, del } -// layerDeployment adds the Enterprise query server container, audit logging, and the -// query server / trusted bundle volumes to the rendered API server deployment. +// layerDeployment adds the Enterprise additions to the rendered API server deployment: +// the query server container (and, under certificate management, its init container and +// volume), audit logging and the management-cluster tunnel args on the aggregation API +// server container, the L7 admission controller sidecar, and the Linseed token and +// trusted bundle volumes. func (c *apiServer) layerDeployment(d *appsv1.Deployment) { - // Audit logging is performed through the aggregation API server container, which is - // only present when the aggregation API server is running. + spec := &d.Spec.Template.Spec + if d.Spec.Template.Annotations == nil { + d.Spec.Template.Annotations = map[string]string{} + } + + // Audit logging and the management-cluster tunnel args are layered onto the + // aggregation API server container, which is only present when that server runs. if c.cfg.RequiresAggregationServer { - for i := range d.Spec.Template.Spec.Containers { - ctr := &d.Spec.Template.Spec.Containers[i] + for i := range spec.Containers { + ctr := &spec.Containers[i] if ctr.Name != string(render.APIServerContainerName) { continue } @@ -178,24 +417,258 @@ func (c *apiServer) layerDeployment(d *appsv1.Deployment) { "--audit-policy-file=/etc/tigera/audit/policy.conf", "--audit-log-path=/var/log/calico/audit/tsee-audit.log", ) + ctr.Args = append(ctr.Args, c.managementClusterArgs()...) // In case of OpenShift, apiserver needs privileged access to write audit logs to the // host path volume. Audit logs are owned by root on hosts so we need to be root. ctr.SecurityContext = securitycontext.NewRootContext(c.cfg.OpenShift) } - d.Spec.Template.Spec.Volumes = append(d.Spec.Template.Spec.Volumes, c.auditVolumes()...) + spec.Volumes = append(spec.Volumes, c.auditVolumes()...) + } + + spec.Containers = append(spec.Containers, c.queryServerContainer()) + if c.isSidecarInjectionEnabled() { + spec.Containers = append(spec.Containers, c.l7AdmissionControllerContainer()) } - d.Spec.Template.Spec.Containers = append(d.Spec.Template.Spec.Containers, c.queryServerContainer()) + // Under certificate management the query server gets its own cert init container and + // volume, since apiserver and queryserver may run with different UID:GID. + if c.data.queryServerTLS != nil { + init := c.data.queryServerTLS.InitContainer(render.APIServerNamespace, securitycontext.NewNonRootContext()) + spec.InitContainers = append(spec.InitContainers, init) + spec.Volumes = append(spec.Volumes, c.data.queryServerTLS.Volume()) + d.Spec.Template.Annotations[c.data.queryServerTLS.HashAnnotationKey()] = c.data.queryServerTLS.HashAnnotationValue() + } + + if c.data.managementClusterConnection != nil { + // Optional: the Secret is delivered over the Guardian tunnel, which can't be + // established until calico-apiserver is Ready. + spec.Volumes = append(spec.Volumes, corev1.Volume{ + Name: render.LinseedTokenVolumeName, + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: fmt.Sprintf(render.LinseedTokenSecret, "calico-apiserver"), + Items: []corev1.KeyToPath{{Key: render.LinseedTokenKey, Path: render.LinseedTokenSubPath}}, + Optional: ptr.To(true), + }, + }, + }) + } if c.cfg.TrustedBundle != nil { - d.Spec.Template.Spec.Volumes = append(d.Spec.Template.Spec.Volumes, c.cfg.TrustedBundle.Volume()) + spec.Volumes = append(spec.Volumes, c.cfg.TrustedBundle.Volume()) for k, v := range c.cfg.TrustedBundle.HashAnnotations() { d.Spec.Template.Annotations[k] = v } } } +// managementClusterArgs returns the aggregation API server tunnel args for a management +// cluster, or nil when this isn't one. +func (c *apiServer) managementClusterArgs() []string { + mc := c.data.managementCluster + if mc == nil { + return nil + } + args := []string{"--enable-managed-clusters-create-api=true"} + if mc.Spec.Address != "" { + args = append(args, fmt.Sprintf("--managementClusterAddr=%s", mc.Spec.Address)) + } + if mc.Spec.TLS != nil && mc.Spec.TLS.SecretName != "" { + if mc.Spec.TLS.SecretName == render.ManagerTLSSecretName { + args = append(args, "--managementClusterCAType=Public") + } + args = append(args, fmt.Sprintf("--tunnelSecretName=%s", mc.Spec.TLS.SecretName)) + } + return args +} + +// ensureDeployment makes sure the API server Deployment and its supporting objects are +// in the create list. The base renders them when running an aggregation API server; when +// it doesn't (v3-CRD mode), it queues them for deletion, so render the skeleton and pull +// those objects back out of the delete list. +func (c *apiServer) ensureDeployment(create, del []client.Object) ([]client.Object, []client.Object) { + if _, ok := extensions.FindObject[*appsv1.Deployment](create, render.APIServerName); ok { + return create, del + } + skeleton := render.APIServerDeploymentObjects(c.cfg, c.calicoImage) + create = append(create, skeleton...) + for _, obj := range render.APIServerDeploymentObjectMeta() { + del = removeByRef(del, obj) + } + return create, del +} + +// removeByRef returns del with any object matching ref's kind, namespace, and name +// removed. +func removeByRef(del []client.Object, ref client.Object) []client.Object { + out := del[:0:0] + for _, o := range del { + if o.GetObjectKind().GroupVersionKind().Kind == ref.GetObjectKind().GroupVersionKind().Kind && + o.GetNamespace() == ref.GetNamespace() && + o.GetName() == ref.GetName() { + continue + } + out = append(out, o) + } + return out +} + +// addServicePorts adds the query server port and, when sidecar injection is enabled, the +// L7 admission controller port to the API server Service. +func (c *apiServer) addServicePorts(s *corev1.Service) { + queryServerTargetPort := render.GetContainerPort(c.cfg, render.TigeraAPIServerQueryServerContainerName) + s.Spec.Ports = append(s.Spec.Ports, corev1.ServicePort{ + Name: render.QueryServerPortName, + Port: render.QueryServerPort, + Protocol: corev1.ProtocolTCP, + TargetPort: intstr.FromInt32(queryServerTargetPort.ContainerPort), + }) + if c.isSidecarInjectionEnabled() { + l7Port := render.GetContainerPort(c.cfg, render.L7AdmissionControllerContainerName) + s.Spec.Ports = append(s.Spec.Ports, corev1.ServicePort{ + Name: render.L7AdmissionControllerPortName, + Port: render.L7AdmissionControllerPort, + Protocol: corev1.ProtocolTCP, + TargetPort: intstr.FromInt32(l7Port.ContainerPort), + }) + } +} + +// l7AdmissionControllerContainer is the L7 admission controller sidecar, rendered when +// ApplicationLayer sidecar injection is enabled. +func (c *apiServer) l7AdmissionControllerContainer() corev1.Container { + volumeMounts := []corev1.VolumeMount{ + c.cfg.TLSKeyPair.VolumeMount(rmeta.OSTypeLinux), + } + + l7Port := render.GetContainerPort(c.cfg, render.L7AdmissionControllerContainerName).ContainerPort + + dataplane := "iptables" + if c.cfg.Installation.IsNftables() { + dataplane = "nftables" + } + + return corev1.Container{ + Name: string(render.L7AdmissionControllerContainerName), + Image: c.calicoImage, + Command: []string{components.CalicoBinaryPath, "component", "l7-admission-controller"}, + Env: []corev1.EnvVar{ + {Name: "L7ADMCTRL_TLSCERTPATH", Value: c.cfg.TLSKeyPair.VolumeMountCertificateFilePath()}, + {Name: "L7ADMCTRL_TLSKEYPATH", Value: c.cfg.TLSKeyPair.VolumeMountKeyFilePath()}, + {Name: "L7ADMCTRL_ENVOYIMAGE", Value: c.data.l7EnvoyImage}, + {Name: "L7ADMCTRL_DIKASTESIMAGE", Value: c.data.dikastesImage}, + {Name: "L7ADMCTRL_LISTENADDR", Value: fmt.Sprintf(":%d", l7Port)}, + {Name: "DATAPLANE", Value: dataplane}, + }, + VolumeMounts: volumeMounts, + LivenessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/live", + Port: intstr.FromInt32(l7Port), + Scheme: corev1.URISchemeHTTPS, + }, + }, + }, + } +} + +// sidecarMutatingWebhookConfig is the L7 sidecar injection webhook, rendered when +// ApplicationLayer sidecar injection is enabled. +func (c *apiServer) sidecarMutatingWebhookConfig() *admregv1.MutatingWebhookConfiguration { + var cacert []byte + svcPort := render.GetContainerPort(c.cfg, render.L7AdmissionControllerContainerName).ContainerPort + + svcpath := "/sidecar-webhook" + svcref := admregv1.ServiceReference{ + Name: render.QueryserverServiceName, + Namespace: render.QueryserverNamespace, + Path: &svcpath, + Port: &svcPort, + } + failpol := admregv1.Fail + labelsel := metav1.LabelSelector{ + MatchLabels: map[string]string{ + "applicationlayer.projectcalico.org/sidecar": "true", + }, + } + rules := []admregv1.RuleWithOperations{ + { + Rule: admregv1.Rule{ + APIGroups: []string{""}, + APIVersions: []string{"v1"}, + Resources: []string{"pods"}, + }, + Operations: []admregv1.OperationType{admregv1.Create}, + }, + } + sidefx := admregv1.SideEffectClassNone + if !c.cfg.TLSKeyPair.UseCertificateManagement() { + cacert = c.cfg.TLSKeyPair.GetIssuer().GetCertificatePEM() + } else { + cacert = c.cfg.Installation.CertificateManagement.CACert + } + return &admregv1.MutatingWebhookConfiguration{ + TypeMeta: metav1.TypeMeta{ + Kind: "MutatingWebhookConfiguration", + APIVersion: "admissionregistration.k8s.io/v1", + }, + ObjectMeta: metav1.ObjectMeta{Name: common.SidecarMutatingWebhookConfigName}, + Webhooks: []admregv1.MutatingWebhook{ + { + AdmissionReviewVersions: []string{"v1"}, + ClientConfig: admregv1.WebhookClientConfig{ + Service: &svcref, + CABundle: cacert, + }, + Name: "sidecar.projectcalico.org", + FailurePolicy: &failpol, + ObjectSelector: &labelsel, + Rules: rules, + SideEffects: &sidefx, + }, + }, + } +} + +// modifyAPIServerPolicy adds the enterprise additions to the API server network policy: +// the OIDC egress rule (when an OIDC key validator is configured) and the L7 admission +// controller ingress port (when sidecar injection is enabled). The base policy carries +// neither. +func modifyAPIServerPolicy(rc extensions.RenderContext, ec render.APIServerExtensionContext, create, del []client.Object) ([]client.Object, []client.Object) { + c := &apiServer{cfg: ec.Config, data: apiServerData(rc)} + + policy, ok := extensions.FindObject[*v3.NetworkPolicy](create, render.APIServerPolicyName) + if !ok { + return create, del + } + + // Insert the OIDC egress rule before the trailing Pass rule so it is evaluated. + if c.data.keyValidatorConfig != nil { + if parsedURL, err := url.Parse(c.data.keyValidatorConfig.Issuer()); err == nil { + oidc := networkpolicy.GetOIDCEgressRule(parsedURL) + egress := policy.Spec.Egress + if n := len(egress); n > 0 && egress[n-1].Action == v3.Pass { + policy.Spec.Egress = append(egress[:n-1:n-1], oidc, egress[n-1]) + } else { + policy.Spec.Egress = append(egress, oidc) + } + } + } + + // Allow the kube-apiserver to reach the L7 admission controller. + if c.isSidecarInjectionEnabled() { + l7Port := render.GetContainerPort(c.cfg, render.L7AdmissionControllerContainerName).ContainerPort + for i := range policy.Spec.Ingress { + policy.Spec.Ingress[i].Destination.Ports = append(policy.Spec.Ingress[i].Destination.Ports, + numorstring.Port{MinPort: uint16(l7Port), MaxPort: uint16(l7Port)}) + } + } + + return create, del +} + // auditVolumes are the host-path audit log and audit policy volumes used by the // aggregation API server container. func (c *apiServer) auditVolumes() []corev1.Volume { @@ -226,30 +699,20 @@ func (c *apiServer) auditVolumes() []corev1.Volume { } } -func (c *apiServer) addQueryServerPort(s *corev1.Service) { - queryServerTargetPort := render.GetContainerPort(c.cfg, render.TigeraAPIServerQueryServerContainerName) - s.Spec.Ports = append(s.Spec.Ports, corev1.ServicePort{ - Name: render.QueryServerPortName, - Port: render.QueryServerPort, - Protocol: corev1.ProtocolTCP, - TargetPort: intstr.FromInt32(queryServerTargetPort.ContainerPort), - }) -} - func (c *apiServer) multiTenantSecretsRBAC() []client.Object { - return render.TunnelSecretRBAC(render.APIServerSecretsRBACName, render.APIServerServiceAccountName, c.cfg.ManagementCluster, true) + return render.TunnelSecretRBAC(render.APIServerSecretsRBACName, render.APIServerServiceAccountName, c.data.managementCluster, true) } func (c *apiServer) secretsRBAC() []client.Object { - return render.TunnelSecretRBAC(render.APIServerSecretsRBACName, render.APIServerServiceAccountName, c.cfg.ManagementCluster, false) + return render.TunnelSecretRBAC(render.APIServerSecretsRBACName, render.APIServerServiceAccountName, c.data.managementCluster, false) } func (c *apiServer) queryServerContainer() corev1.Container { queryServerTargetPort := render.GetContainerPort(c.cfg, render.TigeraAPIServerQueryServerContainerName).ContainerPort var tlsSecret certificatemanagement.KeyPairInterface - if c.cfg.QueryServerTLSKeyPairCertificateManagementOnly != nil { - tlsSecret = c.cfg.QueryServerTLSKeyPairCertificateManagementOnly + if c.data.queryServerTLS != nil { + tlsSecret = c.data.queryServerTLS } else { tlsSecret = c.cfg.TLSKeyPair } @@ -273,17 +736,17 @@ func (c *apiServer) queryServerContainer() corev1.Container { env = append(env, corev1.EnvVar{Name: "MULTI_INTERFACE_MODE", Value: c.cfg.Installation.CalicoNetwork.MultiInterfaceMode.Value()}) } - if c.cfg.KeyValidatorConfig != nil { - env = append(env, c.cfg.KeyValidatorConfig.RequiredEnv("")...) + if c.data.keyValidatorConfig != nil { + env = append(env, c.data.keyValidatorConfig.RequiredEnv("")...) } - linseedURL := relasticsearch.LinseedEndpoint(rmeta.OSTypeLinux, c.cfg.ClusterDomain, render.ElasticsearchNamespace, c.cfg.ManagementClusterConnection != nil, false) + linseedURL := relasticsearch.LinseedEndpoint(rmeta.OSTypeLinux, c.cfg.ClusterDomain, render.ElasticsearchNamespace, c.data.managementClusterConnection != nil, false) env = append(env, corev1.EnvVar{Name: "LINSEED_URL", Value: linseedURL}, corev1.EnvVar{Name: "LINSEED_CLIENT_CERT", Value: fmt.Sprintf("/%s/tls.crt", tlsSecret.GetName())}, corev1.EnvVar{Name: "LINSEED_CLIENT_KEY", Value: fmt.Sprintf("/%s/tls.key", tlsSecret.GetName())}, ) - if c.cfg.ManagementClusterConnection != nil { + if c.data.managementClusterConnection != nil { env = append(env, corev1.EnvVar{Name: "CLUSTER_ID", Value: ""}, corev1.EnvVar{Name: "LINSEED_TOKEN", Value: render.GetLinseedTokenPath(true)}, @@ -309,7 +772,7 @@ func (c *apiServer) queryServerContainer() corev1.Container { if c.cfg.TrustedBundle != nil { volumeMounts = append(volumeMounts, c.cfg.TrustedBundle.VolumeMounts(rmeta.OSTypeLinux)...) } - if c.cfg.ManagementClusterConnection != nil { + if c.data.managementClusterConnection != nil { volumeMounts = append(volumeMounts, corev1.VolumeMount{ Name: render.LinseedTokenVolumeName, MountPath: render.LinseedVolumeMountPath, @@ -685,7 +1148,7 @@ func (c *apiServer) tigeraUserClusterRole() *rbacv1.ClusterRole { } // Privileges for lma.tigera.io have no effect on managed clusters. - if c.cfg.ManagementClusterConnection == nil { + if c.data.managementClusterConnection == nil { // Access to flow logs, audit logs, and statistics. // Access to log into Kibana for oidc users. rules = append(rules, rbacv1.PolicyRule{ @@ -904,7 +1367,7 @@ func (c *apiServer) tigeraNetworkAdminClusterRole() *rbacv1.ClusterRole { } // Privileges for lma.tigera.io have no effect on managed clusters. - if c.cfg.ManagementClusterConnection == nil { + if c.data.managementClusterConnection == nil { // Access to flow logs, audit logs, and statistics. // Elasticsearch superuser access once logged into Kibana. rules = append(rules, rbacv1.PolicyRule{ diff --git a/pkg/enterprise/apiserver_test.go b/pkg/enterprise/apiserver_test.go new file mode 100644 index 0000000000..d8fce991cc --- /dev/null +++ b/pkg/enterprise/apiserver_test.go @@ -0,0 +1,439 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package enterprise_test + +import ( + "context" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" + admregv1 "k8s.io/api/admissionregistration/v1" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/apis" + "github.com/tigera/operator/pkg/common" + "github.com/tigera/operator/pkg/controller/certificatemanager" + "github.com/tigera/operator/pkg/controller/k8sapi" + "github.com/tigera/operator/pkg/controller/utils" + ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" + "github.com/tigera/operator/pkg/dns" + "github.com/tigera/operator/pkg/extensions" + "github.com/tigera/operator/pkg/render" + "github.com/tigera/operator/pkg/tls/certificatemanagement" +) + +const apiServerClusterDomain = "cluster.local" + +// apiServerControllerContext builds a controller context for the API server controller, +// seeded with a fake client that holds objs. The returned context carries a real +// certificate manager and trusted bundle, so ExtendContext can create the query server +// cert and the bundle the modifiers consume. +func apiServerControllerContext(variant operatorv1.ProductVariant, install *operatorv1.InstallationSpec, objs ...client.Object) extensions.ControllerContext { + scheme := runtime.NewScheme() + Expect(apis.AddToScheme(scheme, false)).NotTo(HaveOccurred()) + c := ctrlrfake.DefaultFakeClientBuilder(scheme).Build() + + for _, o := range objs { + Expect(c.Create(context.Background(), o)).NotTo(HaveOccurred()) + } + + if install == nil { + install = &operatorv1.InstallationSpec{Variant: variant} + } + + certManager, err := certificatemanager.Create(c, install, apiServerClusterDomain, common.OperatorNamespace(), certificatemanager.AllowCACreation()) + Expect(err).NotTo(HaveOccurred()) + + return extensions.ControllerContext{ + RenderContext: extensions.RenderContext{ + Installation: install, + ClusterDomain: apiServerClusterDomain, + TrustedBundle: certManager.CreateTrustedBundle(), + }, + Controller: extensions.APIServerController, + Ctx: context.Background(), + Client: c, + CertificateManager: certManager, + } +} + +// apiServerKeyPair issues the API server TLS keypair from the context's certificate +// manager, the way the controller does before rendering. +func apiServerKeyPair(cc extensions.ControllerContext) certificatemanagement.KeyPairInterface { + dnsNames := dns.GetServiceDNSNames(render.APIServerServiceName, render.APIServerNamespace, cc.ClusterDomain) + kp, err := cc.CertificateManager.GetOrCreateKeyPair(cc.Client, render.CalicoAPIServerTLSSecretName, common.OperatorNamespace(), dnsNames) + Expect(err).NotTo(HaveOccurred()) + return kp +} + +var _ = Describe("API server enterprise controller extension", func() { + managementCluster := func() *operatorv1.ManagementCluster { + return &operatorv1.ManagementCluster{ + ObjectMeta: metav1.ObjectMeta{Name: utils.DefaultEnterpriseInstanceKey.Name}, + Spec: operatorv1.ManagementClusterSpec{ + Address: "example.com:1234", + TLS: &operatorv1.TLS{SecretName: render.VoltronTunnelSecretName}, + }, + } + } + + tunnelSecret := func() *corev1.Secret { + return &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: render.VoltronTunnelSecretName, Namespace: common.OperatorNamespace()}, + Data: map[string][]byte{"cert": []byte("a"), "key": []byte("b")}, + } + } + + managementClusterConnection := func() *operatorv1.ManagementClusterConnection { + return &operatorv1.ManagementClusterConnection{ + ObjectMeta: metav1.ObjectMeta{Name: utils.DefaultEnterpriseInstanceKey.Name}, + } + } + + Describe("Validate", func() { + It("accepts a cluster with neither a ManagementCluster nor a ManagementClusterConnection", func() { + cc := apiServerControllerContext(operatorv1.CalicoEnterprise, nil) + Expect(ext.Validate(cc)).NotTo(HaveOccurred()) + }) + + It("accepts a management cluster", func() { + cc := apiServerControllerContext(operatorv1.CalicoEnterprise, nil, managementCluster(), tunnelSecret()) + Expect(ext.Validate(cc)).NotTo(HaveOccurred()) + }) + + It("accepts a managed cluster", func() { + cc := apiServerControllerContext(operatorv1.CalicoEnterprise, nil, managementClusterConnection()) + Expect(ext.Validate(cc)).NotTo(HaveOccurred()) + }) + + It("rejects a cluster that is both a management cluster and a managed cluster", func() { + cc := apiServerControllerContext(operatorv1.CalicoEnterprise, nil, managementCluster(), tunnelSecret(), managementClusterConnection()) + Expect(ext.Validate(cc)).To(HaveOccurred()) + }) + }) +}) + +var _ = Describe("API server enterprise modifier", func() { + // renderAPIServer builds the base API server objects and applies the enterprise + // modifier, the way the component handler does. It returns the create and delete + // lists after the modifier ran. + renderAPIServer := func(cc extensions.ControllerContext, rc extensions.RenderContext, kp certificatemanagement.KeyPairInterface) ([]client.Object, []client.Object) { + cfg := &render.APIServerConfiguration{ + RequiresAggregationServer: true, + K8SServiceEndpoint: k8sapi.ServiceEndpoint{}, + Installation: cc.Installation, + APIServer: &operatorv1.APIServerSpec{}, + TLSKeyPair: kp, + TrustedBundle: rc.TrustedBundle, + KubernetesVersion: &common.VersionInfo{Major: 1, Minor: 31}, + } + comp, err := render.APIServer(cfg) + Expect(err).NotTo(HaveOccurred()) + Expect(comp.ResolveImages(nil)).NotTo(HaveOccurred()) + create, del := comp.Objects() + + ec := comp.(render.ExtensionContextProvider).ExtensionContext() + return applyExtensionsWithContext(ext, render.ComponentNameAPIServer, rc, ec, create, del) + } + + apiServerDeployment := func(objs []client.Object) *appsv1.Deployment { + dp, ok := extensions.FindObject[*appsv1.Deployment](objs, render.APIServerName) + Expect(ok).To(BeTrue()) + return dp + } + + container := func(dp *appsv1.Deployment, name string) *corev1.Container { + for i := range dp.Spec.Template.Spec.Containers { + if dp.Spec.Template.Spec.Containers[i].Name == name { + return &dp.Spec.Template.Spec.Containers[i] + } + } + return nil + } + + It("is a no-op for the Calico variant (no enterprise objects added)", func() { + cc := apiServerControllerContext(operatorv1.Calico, nil) + rc, _, err := ext.ExtendContext(cc) + Expect(err).NotTo(HaveOccurred()) + + objs, _ := renderAPIServer(cc, rc, apiServerKeyPair(cc)) + + // No enterprise objects. (The Calico variant cleanup is registered on the base + // render component, not exercised here.) + _, ok := extensions.FindObject[*rbacv1.ClusterRole](objs, "tigera-ui-user") + Expect(ok).To(BeFalse()) + dp := apiServerDeployment(objs) + Expect(container(dp, string(render.TigeraAPIServerQueryServerContainerName))).To(BeNil()) + }) + + It("layers the query server, enterprise RBAC, audit policy, and query server port on", func() { + cc := apiServerControllerContext(operatorv1.CalicoEnterprise, nil) + rc, _, err := ext.ExtendContext(cc) + Expect(err).NotTo(HaveOccurred()) + + objs, _ := renderAPIServer(cc, rc, apiServerKeyPair(cc)) + + // The query server container is layered onto the deployment. + dp := apiServerDeployment(objs) + Expect(container(dp, string(render.TigeraAPIServerQueryServerContainerName))).NotTo(BeNil()) + + // Enterprise RBAC. + for _, name := range []string{"calico-apiserver", "tigera-ui-user", "tigera-network-admin", "calico-uisettingsgroup-getter", "calico-uisettings-passthrough"} { + _, ok := extensions.FindObject[*rbacv1.ClusterRole](objs, name) + Expect(ok).To(BeTrue(), "expected ClusterRole %q", name) + } + + // Audit policy configmap. + _, ok := extensions.FindObject[*corev1.ConfigMap](objs, "calico-audit-policy") + Expect(ok).To(BeTrue()) + + // The query server port is added to the Service. + svc, ok := extensions.FindObject[*corev1.Service](objs, render.APIServerServiceName) + Expect(ok).To(BeTrue()) + Expect(svc.Spec.Ports).To(ContainElement(HaveField("Name", render.QueryServerPortName))) + }) + + It("queues the enterprise RBAC for deletion when not a management cluster", func() { + cc := apiServerControllerContext(operatorv1.CalicoEnterprise, nil) + rc, _, err := ext.ExtendContext(cc) + Expect(err).NotTo(HaveOccurred()) + + _, del := renderAPIServer(cc, rc, apiServerKeyPair(cc)) + _, ok := extensions.FindObject[*rbacv1.ClusterRole](del, render.ManagedClustersWatchClusterRoleName) + Expect(ok).To(BeTrue()) + }) + + Context("management cluster", func() { + It("adds the tunnel args and the managed-cluster-watch and secrets RBAC", func() { + cc := apiServerControllerContext(operatorv1.CalicoEnterprise, nil, + &operatorv1.ManagementCluster{ + ObjectMeta: metav1.ObjectMeta{Name: utils.DefaultEnterpriseInstanceKey.Name}, + Spec: operatorv1.ManagementClusterSpec{ + Address: "example.com:1234", + TLS: &operatorv1.TLS{SecretName: render.VoltronTunnelSecretName}, + }, + }, + &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: render.VoltronTunnelSecretName, Namespace: common.OperatorNamespace()}, + Data: map[string][]byte{"cert": []byte("a"), "key": []byte("b")}, + }, + ) + rc, _, err := ext.ExtendContext(cc) + Expect(err).NotTo(HaveOccurred()) + + objs, _ := renderAPIServer(cc, rc, apiServerKeyPair(cc)) + + dp := apiServerDeployment(objs) + apiCtr := container(dp, string(render.APIServerContainerName)) + Expect(apiCtr).NotTo(BeNil()) + Expect(apiCtr.Args).To(ContainElement("--enable-managed-clusters-create-api=true")) + Expect(apiCtr.Args).To(ContainElement("--managementClusterAddr=example.com:1234")) + Expect(apiCtr.Args).To(ContainElement("--tunnelSecretName=" + render.VoltronTunnelSecretName)) + + _, ok := extensions.FindObject[*rbacv1.ClusterRole](objs, render.ManagedClustersWatchClusterRoleName) + Expect(ok).To(BeTrue()) + _, ok = extensions.FindObject[*rbacv1.Role](objs, render.APIServerSecretsRBACName) + Expect(ok).To(BeTrue()) + }) + }) + + Context("managed cluster", func() { + It("adds the external Linseed rolebinding and the query server token volume", func() { + cc := apiServerControllerContext(operatorv1.CalicoEnterprise, nil, + &operatorv1.ManagementClusterConnection{ + ObjectMeta: metav1.ObjectMeta{Name: utils.DefaultEnterpriseInstanceKey.Name}, + }, + ) + rc, _, err := ext.ExtendContext(cc) + Expect(err).NotTo(HaveOccurred()) + + objs, _ := renderAPIServer(cc, rc, apiServerKeyPair(cc)) + + _, ok := extensions.FindObject[*rbacv1.RoleBinding](objs, "tigera-linseed") + Expect(ok).To(BeTrue()) + + dp := apiServerDeployment(objs) + Expect(dp.Spec.Template.Spec.Volumes).To(ContainElement(HaveField("Name", render.LinseedTokenVolumeName))) + qs := container(dp, string(render.TigeraAPIServerQueryServerContainerName)) + Expect(qs).NotTo(BeNil()) + Expect(qs.Env).To(ContainElement(HaveField("Name", "LINSEED_TOKEN"))) + }) + }) + + Context("v3-CRD mode (no aggregation server)", func() { + It("renders the deployment skeleton with the query server and pulls it out of the delete list", func() { + cc := apiServerControllerContext(operatorv1.CalicoEnterprise, nil) + rc, _, err := ext.ExtendContext(cc) + Expect(err).NotTo(HaveOccurred()) + + cfg := &render.APIServerConfiguration{ + RequiresAggregationServer: false, + K8SServiceEndpoint: k8sapi.ServiceEndpoint{}, + Installation: cc.Installation, + APIServer: &operatorv1.APIServerSpec{}, + TLSKeyPair: apiServerKeyPair(cc), + TrustedBundle: rc.TrustedBundle, + KubernetesVersion: &common.VersionInfo{Major: 1, Minor: 31}, + } + comp, err := render.APIServer(cfg) + Expect(err).NotTo(HaveOccurred()) + Expect(comp.ResolveImages(nil)).NotTo(HaveOccurred()) + create, del := comp.Objects() + + // The base queues the deployment objects for deletion in v3-CRD mode. + _, ok := extensions.FindObject[*appsv1.Deployment](del, render.APIServerName) + Expect(ok).To(BeTrue()) + + ec := comp.(render.ExtensionContextProvider).ExtensionContext() + create, del = applyExtensionsWithContext(ext, render.ComponentNameAPIServer, rc, ec, create, del) + + // After the modifier, the deployment (with the query server container) is in the + // create list and out of the delete list. + dp, ok := extensions.FindObject[*appsv1.Deployment](create, render.APIServerName) + Expect(ok).To(BeTrue()) + Expect(container(dp, string(render.TigeraAPIServerQueryServerContainerName))).NotTo(BeNil()) + _, ok = extensions.FindObject[*appsv1.Deployment](del, render.APIServerName) + Expect(ok).To(BeFalse()) + }) + }) + + Context("sidecar / L7 injection", func() { + applicationLayerSidecar := func() *operatorv1.ApplicationLayer { + enabled := operatorv1.SidecarEnabled + return &operatorv1.ApplicationLayer{ + ObjectMeta: metav1.ObjectMeta{Name: utils.DefaultEnterpriseInstanceKey.Name}, + Spec: operatorv1.ApplicationLayerSpec{SidecarInjection: &enabled}, + } + } + + It("adds the L7 admission controller container, the sidecar webhook, and the L7 service port", func() { + cc := apiServerControllerContext(operatorv1.CalicoEnterprise, nil, applicationLayerSidecar()) + rc, _, err := ext.ExtendContext(cc) + Expect(err).NotTo(HaveOccurred()) + + objs, _ := renderAPIServer(cc, rc, apiServerKeyPair(cc)) + + dp := apiServerDeployment(objs) + Expect(container(dp, string(render.L7AdmissionControllerContainerName))).NotTo(BeNil()) + + _, ok := extensions.FindObject[*admregv1.MutatingWebhookConfiguration](objs, common.SidecarMutatingWebhookConfigName) + Expect(ok).To(BeTrue()) + + svc, ok := extensions.FindObject[*corev1.Service](objs, render.APIServerServiceName) + Expect(ok).To(BeTrue()) + Expect(svc.Spec.Ports).To(ContainElement(HaveField("Name", render.L7AdmissionControllerPortName))) + }) + + It("pulls the sidecar webhook out of the delete list", func() { + cc := apiServerControllerContext(operatorv1.CalicoEnterprise, nil, applicationLayerSidecar()) + rc, _, err := ext.ExtendContext(cc) + Expect(err).NotTo(HaveOccurred()) + + _, del := renderAPIServer(cc, rc, apiServerKeyPair(cc)) + _, ok := extensions.FindObject[*admregv1.MutatingWebhookConfiguration](del, common.SidecarMutatingWebhookConfigName) + Expect(ok).To(BeFalse()) + }) + }) +}) + +var _ = Describe("API server enterprise policy modifier", func() { + applyPolicy := func(cc extensions.ControllerContext, rc extensions.RenderContext) *v3.NetworkPolicy { + cfg := &render.APIServerConfiguration{ + RequiresAggregationServer: true, + K8SServiceEndpoint: k8sapi.ServiceEndpoint{}, + Installation: cc.Installation, + APIServer: &operatorv1.APIServerSpec{}, + } + comp := render.APIServerPolicy(cfg) + create, del := comp.Objects() + ec := comp.(render.ExtensionContextProvider).ExtensionContext() + objs, _ := applyExtensionsWithContext(ext, render.ComponentNameAPIServerPolicy, rc, ec, create, del) + policy, ok := extensions.FindObject[*v3.NetworkPolicy](objs, render.APIServerPolicyName) + Expect(ok).To(BeTrue()) + return policy + } + + It("leaves the egress rules as the base when no OIDC key validator is configured", func() { + cc := apiServerControllerContext(operatorv1.CalicoEnterprise, nil) + rc, _, err := ext.ExtendContext(cc) + Expect(err).NotTo(HaveOccurred()) + + policy := applyPolicy(cc, rc) + // The trailing rule remains the Pass rule (no OIDC egress rule inserted). + n := len(policy.Spec.Egress) + Expect(n).To(BeNumerically(">", 0)) + Expect(policy.Spec.Egress[n-1].Action).To(Equal(v3.Pass)) + }) + + It("adds the L7 admission controller ingress port when sidecar injection is enabled", func() { + enabled := operatorv1.SidecarEnabled + cc := apiServerControllerContext(operatorv1.CalicoEnterprise, nil, &operatorv1.ApplicationLayer{ + ObjectMeta: metav1.ObjectMeta{Name: utils.DefaultEnterpriseInstanceKey.Name}, + Spec: operatorv1.ApplicationLayerSpec{SidecarInjection: &enabled}, + }) + rc, _, err := ext.ExtendContext(cc) + Expect(err).NotTo(HaveOccurred()) + + policy := applyPolicy(cc, rc) + // Every ingress rule should carry the L7 admission controller port. + found := false + for _, rule := range policy.Spec.Ingress { + for _, p := range rule.Destination.Ports { + if p.MinPort == uint16(render.L7AdmissionControllerPort) { + found = true + } + } + } + Expect(found).To(BeTrue(), "expected the L7 admission controller ingress port") + }) +}) + +// cleanupAPIServer behaviour for the Calico variant: the base render component carries +// the enterprise cleanup modifier, which queues the enterprise RBAC for deletion. +var _ = Describe("API server Calico-variant cleanup", func() { + It("queues the enterprise RBAC for deletion", func() { + cc := apiServerControllerContext(operatorv1.Calico, nil) + rc, _, err := ext.ExtendContext(cc) + Expect(err).NotTo(HaveOccurred()) + + cfg := &render.APIServerConfiguration{ + RequiresAggregationServer: true, + K8SServiceEndpoint: k8sapi.ServiceEndpoint{}, + Installation: cc.Installation, + APIServer: &operatorv1.APIServerSpec{}, + TLSKeyPair: apiServerKeyPair(cc), + KubernetesVersion: &common.VersionInfo{Major: 1, Minor: 31}, + } + comp, err := render.APIServer(cfg) + Expect(err).NotTo(HaveOccurred()) + Expect(comp.ResolveImages(nil)).NotTo(HaveOccurred()) + create, del := comp.Objects() + ec := comp.(render.ExtensionContextProvider).ExtensionContext() + _, del = applyExtensionsWithContext(ext, render.ComponentNameAPIServer, rc, ec, create, del) + + _, ok := extensions.FindObject[*rbacv1.ClusterRole](del, "tigera-ui-user") + Expect(ok).To(BeTrue()) + _, ok = extensions.FindObject[*rbacv1.ClusterRole](del, "calico-apiserver") + Expect(ok).To(BeTrue()) + }) +}) diff --git a/pkg/enterprise/register.go b/pkg/enterprise/register.go index 839e56b66a..d480fc02f3 100644 --- a/pkg/enterprise/register.go +++ b/pkg/enterprise/register.go @@ -30,6 +30,7 @@ func New() *extensions.Set { ent := s.Variant(operatorv1.CalicoEnterprise) ent.Controller(extensions.InstallationController, coreControllerExtension{}) ent.Controller(extensions.WindowsController, windowsControllerExtension{}) + ent.Controller(extensions.APIServerController, apiServerControllerExtension{}) registerTypha(ent) registerNode(ent) registerWindows(ent) diff --git a/pkg/extensions/controllerextension.go b/pkg/extensions/controllerextension.go index a70b3f700d..0cdce8cc19 100644 --- a/pkg/extensions/controllerextension.go +++ b/pkg/extensions/controllerextension.go @@ -34,6 +34,7 @@ type ControllerName string const ( InstallationController ControllerName = "installation" WindowsController ControllerName = "windows" + APIServerController ControllerName = "apiserver" ) // ControllerExtension extends a controller's reconcile: it validates the @@ -92,4 +93,9 @@ type ControllerContext struct { Ctx context.Context Client client.Client CertificateManager certificatemanager.CertificateManager + + // MultiTenant reports whether the operator runs in multi-tenant mode. It's a + // controller-phase operator mode some extensions gate on (e.g. the API server's + // management-cluster tunnel-secret check). + MultiTenant bool } diff --git a/pkg/render/apiserver.go b/pkg/render/apiserver.go index 9702226fd3..90a64ca0b8 100644 --- a/pkg/render/apiserver.go +++ b/pkg/render/apiserver.go @@ -16,7 +16,6 @@ package render import ( "fmt" - "net/url" "strings" admregv1 "k8s.io/api/admissionregistration/v1" @@ -28,17 +27,14 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" apiregv1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1" - "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" - "github.com/tigera/api/pkg/lib/numorstring" operatorv1 "github.com/tigera/operator/api/v1" "github.com/tigera/operator/pkg/common" "github.com/tigera/operator/pkg/components" "github.com/tigera/operator/pkg/controller/k8sapi" - "github.com/tigera/operator/pkg/render/common/authentication" rcomp "github.com/tigera/operator/pkg/render/common/components" rmeta "github.com/tigera/operator/pkg/render/common/meta" "github.com/tigera/operator/pkg/render/common/networkpolicy" @@ -59,6 +55,11 @@ const ( // ComponentNameAPIServer is the extension key under which a variant registers // its API server modifier and image override. ComponentNameAPIServer = "apiserver" + + // ComponentNameAPIServerPolicy keys the API server network policy modifier (the + // OIDC egress rule and the L7 admission controller ingress port). The base policy + // carries neither. + ComponentNameAPIServerPolicy = "apiserver-policy" ) const ( @@ -114,14 +115,33 @@ func APIServer(cfg *APIServerConfiguration) (Component, error) { }, nil } +// apiServerPolicyComponent wraps the API server network policy passthrough so it is +// render.Extensible: the variant modifier adds the OIDC egress rule and the L7 +// admission controller ingress port. The base policy carries neither. +type apiServerPolicyComponent struct { + Component + cfg *APIServerConfiguration +} + +func (apiServerPolicyComponent) ModifierKey() string { return ComponentNameAPIServerPolicy } + +// ExtensionContext hands the policy modifier the config it needs to look up container +// ports. +func (c apiServerPolicyComponent) ExtensionContext() any { + return APIServerExtensionContext{Config: c.cfg} +} + func APIServerPolicy(cfg *APIServerConfiguration) Component { - return NewPassthrough( - []client.Object{calicoSystemAPIServerPolicy(cfg)}, - []client.Object{ - // allow-tigera Tier was renamed to calico-system - networkpolicy.DeprecatedAllowTigeraNetworkPolicyObject("apiserver-access", APIServerNamespace), - }, - ) + return apiServerPolicyComponent{ + Component: NewPassthrough( + []client.Object{calicoSystemAPIServerPolicy(cfg)}, + []client.Object{ + // allow-tigera Tier was renamed to calico-system + networkpolicy.DeprecatedAllowTigeraNetworkPolicyObject("apiserver-access", APIServerNamespace), + }, + ), + cfg: cfg, + } } // APIServerConfiguration contains all the config information needed to render the component. @@ -131,38 +151,22 @@ type APIServerConfiguration struct { Installation *operatorv1.InstallationSpec APIServer *operatorv1.APIServerSpec ForceHostNetwork bool - ApplicationLayer *operatorv1.ApplicationLayer - ManagementCluster *operatorv1.ManagementCluster - ManagementClusterConnection *operatorv1.ManagementClusterConnection TLSKeyPair certificatemanagement.KeyPairInterface PullSecrets []*corev1.Secret OpenShift bool TrustedBundle certificatemanagement.TrustedBundle MultiTenant bool - KeyValidatorConfig authentication.KeyValidatorConfig KubernetesVersion *common.VersionInfo ClusterDomain string // Whether or not we should run the aggregation API server for projectcalico.org/v3 APIs // as part of this component. RequiresAggregationServer bool - - // Whether or not the API server deployment must run a query server alongside the API - // server. The deployment (and its supporting objects) are rendered when either an - // aggregation API server or a query server is required. The query server itself, and - // the rest of its supporting configuration, is layered on by the variant's modifier. - RequiresQueryServer bool - - // When certificate management is enabled, we need a separate init container to create a cert, running - // with the same permissions as query server. - QueryServerTLSKeyPairCertificateManagementOnly certificatemanagement.KeyPairInterface } type apiServerComponent struct { - cfg *APIServerConfiguration - calicoImage string - l7AdmissionControllerEnvoyImage string - dikastesImage string + cfg *APIServerConfiguration + calicoImage string } // APIServerExtensionContext carries the API server's render configuration and resolved @@ -188,29 +192,14 @@ func (c *apiServerComponent) ResolveImages(is *operatorv1.ImageSet) error { reg := c.cfg.Installation.Registry path := c.cfg.Installation.ImagePath prefix := c.cfg.Installation.ImagePrefix - var err error - errMsgs := []string{} - - if c.cfg.RequiresAggregationServer || c.cfg.RequiresQueryServer { - c.calicoImage, err = components.GetReference(components.CombinedCalicoImage(c.cfg.Installation), reg, path, prefix, is) - if err != nil { - errMsgs = append(errMsgs, err.Error()) - } - } - if c.cfg.IsSidecarInjectionEnabled() { - c.l7AdmissionControllerEnvoyImage, err = components.GetReference(components.ComponentEnvoyProxy, reg, path, prefix, is) - if err != nil { - errMsgs = append(errMsgs, err.Error()) - } - c.dikastesImage, err = components.GetReference(components.ComponentDikastes, reg, path, prefix, is) - if err != nil { - errMsgs = append(errMsgs, err.Error()) - } - } - - if len(errMsgs) != 0 { - return fmt.Errorf("%s", strings.Join(errMsgs, ",")) + // Resolve the calico image unconditionally: the base uses it for the aggregation + // API server container, and a variant modifier needs it for the query server + // container and the deployment skeleton it may render itself. + var err error + c.calicoImage, err = components.GetReference(components.CombinedCalicoImage(c.cfg.Installation), reg, path, prefix, is) + if err != nil { + return err } return nil } @@ -239,22 +228,14 @@ func (c *apiServerComponent) Objects() ([]client.Object, []client.Object) { secrets := secret.CopyToNamespace(APIServerNamespace, c.cfg.PullSecrets...) namespacedObjects = append(namespacedObjects, secret.ToRuntimeObjects(secrets...)...) - // The deployment and its supporting objects are needed when running the aggregation API server - // or when a query server runs alongside it (the query server is added by a variant modifier). - if c.cfg.RequiresAggregationServer || c.cfg.RequiresQueryServer { - namespacedObjects = append(namespacedObjects, - c.apiServerServiceAccount(), - c.apiServerDeployment(), - c.apiServerService(), - c.apiServerPodDisruptionBudget(), - ) + // The deployment and its supporting objects are needed when running the aggregation + // API server. A variant that needs the deployment without an aggregation server (e.g. + // to run only a query server) renders the skeleton itself in its modifier and pulls + // these back out of the delete list. + if c.cfg.RequiresAggregationServer { + namespacedObjects = append(namespacedObjects, c.deploymentObjects()...) } else { - objsToDelete = append(objsToDelete, - &corev1.ServiceAccount{TypeMeta: metav1.TypeMeta{Kind: "ServiceAccount", APIVersion: "v1"}, ObjectMeta: metav1.ObjectMeta{Name: APIServerServiceAccountName, Namespace: APIServerNamespace}}, - &appsv1.Deployment{TypeMeta: metav1.TypeMeta{Kind: "Deployment", APIVersion: "apps/v1"}, ObjectMeta: metav1.ObjectMeta{Name: APIServerName, Namespace: APIServerNamespace}}, - &corev1.Service{TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}, ObjectMeta: metav1.ObjectMeta{Name: APIServerServiceName, Namespace: APIServerNamespace}}, - &policyv1.PodDisruptionBudget{TypeMeta: metav1.TypeMeta{Kind: "PodDisruptionBudget", APIVersion: "policy/v1"}, ObjectMeta: metav1.ObjectMeta{Name: APIServerName, Namespace: APIServerNamespace}}, - ) + objsToDelete = append(objsToDelete, APIServerDeploymentObjectMeta()...) } // These are objects that only need to exist when we are running an aggregation API server to @@ -274,12 +255,11 @@ func (c *apiServerComponent) Objects() ([]client.Object, []client.Object) { aggregationAPIServerObjects = append(aggregationAPIServerObjects, c.apiServiceRegistration(c.cfg.Installation.CertificateManagement.CACert)) } - // The sidecar mutating webhook is driven by ApplicationLayer configuration, not by variant. - if c.cfg.IsSidecarInjectionEnabled() { - namespacedObjects = append(namespacedObjects, c.sidecarMutatingWebhookConfig()) - } else { - objsToDelete = append(objsToDelete, &admregv1.MutatingWebhookConfiguration{ObjectMeta: metav1.ObjectMeta{Name: common.SidecarMutatingWebhookConfigName}}) - } + // The L7 sidecar mutating webhook is an enterprise (ApplicationLayer) concern added + // by the variant modifier. The base always queues it for deletion so a cluster + // without sidecar injection (including any OSS install) never retains a stale one; + // the modifier pulls it back out of the delete list when sidecar injection is on. + objsToDelete = append(objsToDelete, &admregv1.MutatingWebhookConfiguration{ObjectMeta: metav1.ObjectMeta{Name: common.SidecarMutatingWebhookConfigName}}) // Clean up deprecated k8s NetworkPolicy, regardless of variant, // avoiding leftovers in the case of switching between variants. @@ -308,6 +288,40 @@ func (c *apiServerComponent) Ready() bool { return true } +// deploymentObjects returns the API server Deployment and its supporting objects. +func (c *apiServerComponent) deploymentObjects() []client.Object { + return []client.Object{ + c.apiServerServiceAccount(), + c.apiServerDeployment(), + c.apiServerService(), + c.apiServerPodDisruptionBudget(), + } +} + +// APIServerDeploymentObjects returns the API server Deployment and its supporting +// objects (ServiceAccount, Service, PodDisruptionBudget). The base renders these when +// running an aggregation API server; a variant modifier renders them itself when it +// needs the deployment but the base did not (e.g. a query-server-only deployment in +// v3-CRD mode). calicoImage is the resolved image for any base containers. +func APIServerDeploymentObjects(cfg *APIServerConfiguration, calicoImage string) []client.Object { + c := &apiServerComponent{cfg: cfg, calicoImage: calicoImage} + return c.deploymentObjects() +} + +// APIServerDeploymentObjectMeta returns empty-shell copies of the API server Deployment +// and its supporting objects, identifying them by name/kind/namespace. The base queues +// these for deletion when it isn't running an aggregation API server; a variant modifier +// matches against them to pull them back out of the delete list when it renders the +// deployment skeleton itself. +func APIServerDeploymentObjectMeta() []client.Object { + return []client.Object{ + &corev1.ServiceAccount{TypeMeta: metav1.TypeMeta{Kind: "ServiceAccount", APIVersion: "v1"}, ObjectMeta: metav1.ObjectMeta{Name: APIServerServiceAccountName, Namespace: APIServerNamespace}}, + &appsv1.Deployment{TypeMeta: metav1.TypeMeta{Kind: "Deployment", APIVersion: "apps/v1"}, ObjectMeta: metav1.ObjectMeta{Name: APIServerName, Namespace: APIServerNamespace}}, + &corev1.Service{TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}, ObjectMeta: metav1.ObjectMeta{Name: APIServerServiceName, Namespace: APIServerNamespace}}, + &policyv1.PodDisruptionBudget{TypeMeta: metav1.TypeMeta{Kind: "PodDisruptionBudget", APIVersion: "policy/v1"}, ObjectMeta: metav1.ObjectMeta{Name: APIServerName, Namespace: APIServerNamespace}}, + } +} + // For legacy reasons we use apiserver: true here instead of the k8s-app: name label, // so we need to set it explicitly rather than use the common labeling logic. func (c *apiServerComponent) deploymentSelector() *metav1.LabelSelector { @@ -452,13 +466,6 @@ func calicoSystemAPIServerPolicy(cfg *APIServerConfiguration) *v3.NetworkPolicy }, }...) - if cfg.KeyValidatorConfig != nil { - if parsedURL, err := url.Parse(cfg.KeyValidatorConfig.Issuer()); err == nil { - oidcEgressRule := networkpolicy.GetOIDCEgressRule(parsedURL) - egressRules = append(egressRules, oidcEgressRule) - } - } - if r, err := cfg.K8SServiceEndpoint.DestinationEntityRule(); r != nil && err == nil { egressRules = append(egressRules, v3.Rule{ Action: v3.Allow, @@ -475,13 +482,10 @@ func calicoSystemAPIServerPolicy(cfg *APIServerConfiguration) *v3.NetworkPolicy apiServerContainerPort := GetContainerPort(cfg, APIServerContainerName).ContainerPort queryServerContainerPort := GetContainerPort(cfg, TigeraAPIServerQueryServerContainerName).ContainerPort - l7AdmCtrlContainerPort := GetContainerPort(cfg, L7AdmissionControllerContainerName).ContainerPort - // The ports Calico Enterprise API Server and Calico Enterprise Query Server are configured to listen on. + // The ports the API server and query server listen on. The L7 admission controller + // ingress port is added by the variant modifier when sidecar injection is enabled. ingressPorts := networkpolicy.Ports(443, uint16(apiServerContainerPort), uint16(queryServerContainerPort), 10443) - if cfg.IsSidecarInjectionEnabled() { - ingressPorts = append(ingressPorts, numorstring.Port{MinPort: uint16(l7AdmCtrlContainerPort), MaxPort: uint16(l7AdmCtrlContainerPort)}) - } return &v3.NetworkPolicy{ TypeMeta: metav1.TypeMeta{Kind: "NetworkPolicy", APIVersion: "projectcalico.org/v3"}, @@ -832,7 +836,6 @@ func GetContainerPort(cfg *APIServerConfiguration, containerName ContainerName) // additional ports (e.g. the query server port). func (c *apiServerComponent) apiServerService() *corev1.Service { apiServerTargetPort := GetContainerPort(c.cfg, APIServerContainerName) - l7AdmissionControllerTargetPort := GetContainerPort(c.cfg, L7AdmissionControllerContainerName) s := &corev1.Service{ TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}, @@ -856,17 +859,6 @@ func (c *apiServerComponent) apiServerService() *corev1.Service { }, } - if c.cfg.IsSidecarInjectionEnabled() { - s.Spec.Ports = append(s.Spec.Ports, - corev1.ServicePort{ - Name: L7AdmissionControllerPortName, - Port: L7AdmissionControllerPort, - Protocol: corev1.ProtocolTCP, - TargetPort: intstr.FromInt32(l7AdmissionControllerTargetPort.ContainerPort), - }, - ) - } - return s } @@ -885,29 +877,22 @@ func (c *apiServerComponent) apiServerDeployment() *appsv1.Deployment { c.cfg.TLSKeyPair.HashAnnotationKey(): c.cfg.TLSKeyPair.HashAnnotationValue(), } + // The API server cert init container is only needed when running the aggregation API + // server under certificate management. A variant modifier adds any further init + // containers it needs (e.g. the query server cert). var initContainers []corev1.Container - if c.cfg.TLSKeyPair.UseCertificateManagement() { - if c.cfg.RequiresAggregationServer { - // Only include the API server init container if we're running the aggregation API server! - initContainerAPIServer := c.cfg.TLSKeyPair.InitContainer(APIServerNamespace, c.apiServerContainer().SecurityContext) - initContainerAPIServer.Name = fmt.Sprintf("%s-%s", CalicoAPIServerTLSSecretName, certificatemanagement.CSRInitContainerName) - initContainers = append(initContainers, initContainerAPIServer) - } - - initContainerQueryServer := c.cfg.QueryServerTLSKeyPairCertificateManagementOnly.InitContainer(APIServerNamespace, securitycontext.NewNonRootContext()) - annotations[c.cfg.QueryServerTLSKeyPairCertificateManagementOnly.HashAnnotationKey()] = c.cfg.QueryServerTLSKeyPairCertificateManagementOnly.HashAnnotationValue() - initContainers = append(initContainers, initContainerQueryServer) + if c.cfg.TLSKeyPair.UseCertificateManagement() && c.cfg.RequiresAggregationServer { + initContainerAPIServer := c.cfg.TLSKeyPair.InitContainer(APIServerNamespace, c.apiServerContainer().SecurityContext) + initContainerAPIServer.Name = fmt.Sprintf("%s-%s", CalicoAPIServerTLSSecretName, certificatemanagement.CSRInitContainerName) + initContainers = append(initContainers, initContainerAPIServer) } // Determine which containers to run. A variant modifier may add additional - // containers (e.g. the query server). + // containers (e.g. the query server and the L7 admission controller). containers := []corev1.Container{} if c.cfg.RequiresAggregationServer { containers = append(containers, c.apiServerContainer()) } - if c.cfg.IsSidecarInjectionEnabled() { - containers = append(containers, c.l7AdmissionControllerContainer()) - } d := &appsv1.Deployment{ TypeMeta: metav1.TypeMeta{Kind: "Deployment", APIVersion: "apps/v1"}, @@ -959,65 +944,6 @@ func (c *apiServerComponent) apiServerDeployment() *appsv1.Deployment { return d } -// apiServer creates a MutatingWebhookConfiguration for sidecars. -func (c *apiServerComponent) sidecarMutatingWebhookConfig() *admregv1.MutatingWebhookConfiguration { - var cacert []byte - svcPort := GetContainerPort(c.cfg, L7AdmissionControllerContainerName).ContainerPort - - svcpath := "/sidecar-webhook" - svcref := admregv1.ServiceReference{ - Name: QueryserverServiceName, - Namespace: QueryserverNamespace, - Path: &svcpath, - Port: &svcPort, - } - failpol := admregv1.Fail - labelsel := metav1.LabelSelector{ - MatchLabels: map[string]string{ - "applicationlayer.projectcalico.org/sidecar": "true", - }, - } - rules := []admregv1.RuleWithOperations{ - { - Rule: admregv1.Rule{ - APIGroups: []string{""}, - APIVersions: []string{"v1"}, - Resources: []string{"pods"}, - }, - Operations: []admregv1.OperationType{admregv1.Create}, - }, - } - sidefx := admregv1.SideEffectClassNone - if !c.cfg.TLSKeyPair.UseCertificateManagement() { - cacert = c.cfg.TLSKeyPair.GetIssuer().GetCertificatePEM() - } else { - cacert = c.cfg.Installation.CertificateManagement.CACert - } - mwc := admregv1.MutatingWebhookConfiguration{ - TypeMeta: metav1.TypeMeta{ - Kind: "MutatingWebhookConfiguration", - APIVersion: "admissionregistration.k8s.io/v1", - }, - ObjectMeta: metav1.ObjectMeta{Name: common.SidecarMutatingWebhookConfigName}, - Webhooks: []admregv1.MutatingWebhook{ - { - AdmissionReviewVersions: []string{"v1"}, - ClientConfig: admregv1.WebhookClientConfig{ - Service: &svcref, - CABundle: cacert, - }, - Name: "sidecar.projectcalico.org", - FailurePolicy: &failpol, - ObjectSelector: &labelsel, - Rules: rules, - SideEffects: &sidefx, - }, - }, - } - - return &mwc -} - func (c *apiServerComponent) hostNetwork() bool { return HostNetwork(c.cfg) } @@ -1113,18 +1039,9 @@ func (c *apiServerComponent) startUpArgs() []string { fmt.Sprintf("--tls-cert-file=%s", c.cfg.TLSKeyPair.VolumeMountCertificateFilePath()), } - if c.cfg.ManagementCluster != nil { - args = append(args, "--enable-managed-clusters-create-api=true") - if c.cfg.ManagementCluster.Spec.Address != "" { - args = append(args, fmt.Sprintf("--managementClusterAddr=%s", c.cfg.ManagementCluster.Spec.Address)) - } - if c.cfg.ManagementCluster.Spec.TLS != nil && c.cfg.ManagementCluster.Spec.TLS.SecretName != "" { - if c.cfg.ManagementCluster.Spec.TLS.SecretName == ManagerTLSSecretName { - args = append(args, "--managementClusterCAType=Public") - } - args = append(args, fmt.Sprintf("--tunnelSecretName=%s", c.cfg.ManagementCluster.Spec.TLS.SecretName)) - } - } + // The management-cluster tunnel args (--enable-managed-clusters-create-api, + // --managementClusterAddr, --tunnelSecretName, --managementClusterCAType) are an + // enterprise concern, appended to this container by the variant modifier. if c.cfg.KubernetesVersion != nil && c.cfg.KubernetesVersion.Major < 2 && c.cfg.KubernetesVersion.Minor < 30 { // Disable this API as it is not available by default. If we don't, the server fails to start, due to trying to // establish watches for unavailable APIs. @@ -1133,31 +1050,13 @@ func (c *apiServerComponent) startUpArgs() []string { return args } -// apiServerVolumes creates the volumes used by the API server deployment. +// apiServerVolumes creates the volumes used by the API server deployment. A variant +// modifier adds any further volumes it needs (the query server cert and the Linseed +// token). func (c *apiServerComponent) apiServerVolumes() []corev1.Volume { - volumes := []corev1.Volume{ + return []corev1.Volume{ c.cfg.TLSKeyPair.Volume(), } - if c.cfg.QueryServerTLSKeyPairCertificateManagementOnly != nil { - volumes = append(volumes, c.cfg.QueryServerTLSKeyPairCertificateManagementOnly.Volume()) - } - - if c.cfg.ManagementClusterConnection != nil { - // Optional: the Secret is delivered over the Guardian tunnel, which can't be - // established until calico-apiserver is Ready. - volumes = append(volumes, corev1.Volume{ - Name: LinseedTokenVolumeName, - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: fmt.Sprintf(LinseedTokenSecret, "calico-apiserver"), - Items: []corev1.KeyToPath{{Key: LinseedTokenKey, Path: LinseedTokenSubPath}}, - Optional: ptr.To(true), - }, - }, - }) - } - - return volumes } // tolerations creates the tolerations used by the API server deployment. @@ -1317,66 +1216,3 @@ func (c *apiServerComponent) getDeprecatedResources() []client.Object { return renamedRscList } - -func (cfg *APIServerConfiguration) IsSidecarInjectionEnabled() bool { - return cfg.ApplicationLayer != nil && - cfg.ApplicationLayer.Spec.SidecarInjection != nil && - *cfg.ApplicationLayer.Spec.SidecarInjection == operatorv1.SidecarEnabled -} - -func (c *apiServerComponent) l7AdmissionControllerContainer() corev1.Container { - volumeMounts := []corev1.VolumeMount{ - c.cfg.TLSKeyPair.VolumeMount(c.SupportedOSType()), - } - - l7AdmissionControllerTargetPort := GetContainerPort(c.cfg, L7AdmissionControllerContainerName).ContainerPort - - dataplane := "iptables" - if c.cfg.Installation.IsNftables() { - dataplane = "nftables" - } - - l7AdmssCtrl := corev1.Container{ - Name: string(L7AdmissionControllerContainerName), - Image: c.calicoImage, - Command: []string{components.CalicoBinaryPath, "component", "l7-admission-controller"}, - Env: []corev1.EnvVar{ - { - Name: "L7ADMCTRL_TLSCERTPATH", - Value: c.cfg.TLSKeyPair.VolumeMountCertificateFilePath(), - }, - { - Name: "L7ADMCTRL_TLSKEYPATH", - Value: c.cfg.TLSKeyPair.VolumeMountKeyFilePath(), - }, - { - Name: "L7ADMCTRL_ENVOYIMAGE", - Value: c.l7AdmissionControllerEnvoyImage, - }, - { - Name: "L7ADMCTRL_DIKASTESIMAGE", - Value: c.dikastesImage, - }, - { - Name: "L7ADMCTRL_LISTENADDR", - Value: fmt.Sprintf(":%d", l7AdmissionControllerTargetPort), - }, - { - Name: "DATAPLANE", - Value: dataplane, - }, - }, - VolumeMounts: volumeMounts, - LivenessProbe: &corev1.Probe{ - ProbeHandler: corev1.ProbeHandler{ - HTTPGet: &corev1.HTTPGetAction{ - Path: "/live", - Port: intstr.FromInt32(l7AdmissionControllerTargetPort), - Scheme: corev1.URISchemeHTTPS, - }, - }, - }, - } - - return l7AdmssCtrl -} diff --git a/pkg/render/apiserver_test.go b/pkg/render/apiserver_test.go index 57eeb83e66..683f14b520 100644 --- a/pkg/render/apiserver_test.go +++ b/pkg/render/apiserver_test.go @@ -15,1505 +15,53 @@ package render_test import ( - "crypto/tls" - "crypto/x509" - "encoding/pem" "fmt" - "strings" - "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "github.com/onsi/gomega/gstruct" - "github.com/openshift/library-go/pkg/crypto" - - calicov3 "github.com/tigera/api/pkg/apis/projectcalico/v3" - - operatorv1 "github.com/tigera/operator/api/v1" - "github.com/tigera/operator/pkg/apis" - "github.com/tigera/operator/pkg/common" - "github.com/tigera/operator/pkg/components" - "github.com/tigera/operator/pkg/controller/certificatemanager" - "github.com/tigera/operator/pkg/controller/k8sapi" - ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" - "github.com/tigera/operator/pkg/dns" - "github.com/tigera/operator/pkg/extensions" - "github.com/tigera/operator/pkg/render" - rmeta "github.com/tigera/operator/pkg/render/common/meta" - "github.com/tigera/operator/pkg/render/common/networkpolicy" - "github.com/tigera/operator/pkg/render/common/podaffinity" - rtest "github.com/tigera/operator/pkg/render/common/test" - "github.com/tigera/operator/pkg/render/testutils" - "github.com/tigera/operator/pkg/tls/certificatemanagement" - "github.com/tigera/operator/test" - - appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" - policyv1 "k8s.io/api/policy/v1" - rbacv1 "k8s.io/api/rbac/v1" - "k8s.io/apimachinery/pkg/api/resource" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/types" - apiregv1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1" - "k8s.io/utils/ptr" - "sigs.k8s.io/controller-runtime/pkg/client" -) - -// apiServerObjects renders the API server component and applies the registered variant -// modifier the way the componentHandler does, so the Enterprise objects (query server, -// audit logging, Enterprise RBAC) and the Calico cleanup deletes are reflected in the -// returned create and delete lists. -func apiServerObjects(c render.Component) ([]client.Object, []client.Object) { - create, del := c.Objects() - rc := extensions.RenderContext{} - var extCtx any - if p, ok := c.(render.ExtensionContextProvider); ok { - ec := p.ExtensionContext().(render.APIServerExtensionContext) - rc.Installation = ec.Config.Installation - extCtx = ec - } - return applyExtensionsWithContext(ext, render.ComponentNameAPIServer, rc, extCtx, create, del) -} - -var _ = Describe("API server rendering tests (Calico Enterprise)", func() { - apiServerPolicy := testutils.GetExpectedPolicyFromFile("./testutils/expected_policies/apiserver.json") - apiServerPolicyForOCP := testutils.GetExpectedPolicyFromFile("./testutils/expected_policies/apiserver_ocp.json") - var ( - instance *operatorv1.InstallationSpec - apiserver *operatorv1.APIServerSpec - managementCluster = &operatorv1.ManagementCluster{Spec: operatorv1.ManagementClusterSpec{Address: "example.com:1234"}} - cfg *render.APIServerConfiguration - trustedBundle certificatemanagement.TrustedBundle - dnsNames []string - cli client.Client - certificateManager certificatemanager.CertificateManager - err error - ) - - BeforeEach(func() { - instance = &operatorv1.InstallationSpec{ - ControlPlaneReplicas: ptr.To(int32(2)), - Registry: "testregistry.com/", - Variant: operatorv1.CalicoEnterprise, - } - apiserver = &operatorv1.APIServerSpec{} - dnsNames = dns.GetServiceDNSNames(render.APIServerServiceName, render.APIServerNamespace, clusterDomain) - scheme := runtime.NewScheme() - Expect(apis.AddToScheme(scheme, false)).NotTo(HaveOccurred()) - - cli = ctrlrfake.DefaultFakeClientBuilder(scheme).Build() - certificateManager, err = certificatemanager.Create(cli, nil, clusterDomain, common.OperatorNamespace(), certificatemanager.AllowCACreation()) - Expect(err).NotTo(HaveOccurred()) - - kp, err := certificateManager.GetOrCreateKeyPair(cli, render.CalicoAPIServerTLSSecretName, common.OperatorNamespace(), dnsNames) - Expect(err).NotTo(HaveOccurred()) - - trustedBundle = certificatemanagement.CreateTrustedBundle(nil) - - cfg = &render.APIServerConfiguration{ - RequiresAggregationServer: true, - RequiresQueryServer: true, - K8SServiceEndpoint: k8sapi.ServiceEndpoint{}, - Installation: instance, - APIServer: apiserver, - OpenShift: true, - TLSKeyPair: kp, - TrustedBundle: trustedBundle, - KubernetesVersion: &common.VersionInfo{ - Major: 1, - Minor: 31, - }, - } - }) - - DescribeTable("should render an API server with default configuration", func(clusterDomain string) { - expectedResources := []client.Object{ - &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "calico-audit-policy", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"}}, - &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "tigera-ca-bundle", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"}}, - &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "ServiceAccount", APIVersion: "v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-crds"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver-access-calico-crds"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-tier-getter"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-tier-getter"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-tiered-policy-passthrough"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-tiered-policy-passthrough"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-uisettings-passthrough"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-uisettings-passthrough"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-extension-apiserver-auth-access"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-extension-apiserver-auth-access"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver-delegate-auth"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.RoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver-auth-reader", Namespace: "kube-system"}, TypeMeta: metav1.TypeMeta{Kind: "RoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &apiregv1.APIService{ObjectMeta: metav1.ObjectMeta{Name: "v3.projectcalico.org"}, TypeMeta: metav1.TypeMeta{Kind: "APIService", APIVersion: "apiregistration.k8s.io/v1"}}, - &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "Deployment", APIVersion: "apps/v1"}}, - &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: "calico-api", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}}, - &policyv1.PodDisruptionBudget{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "PodDisruptionBudget", APIVersion: "policy/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-uisettingsgroup-getter"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-uisettingsgroup-getter"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "tigera-ui-user"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "tigera-network-admin"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-webhook-reader"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver-webhook-reader"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - } - - dnsNames := dns.GetServiceDNSNames(render.APIServerServiceName, render.APIServerNamespace, clusterDomain) - kp, err := certificateManager.GetOrCreateKeyPair(cli, render.CalicoAPIServerTLSSecretName, common.OperatorNamespace(), dnsNames) - Expect(err).NotTo(HaveOccurred()) - cfg.TLSKeyPair = kp - component, err := render.APIServer(cfg) - Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - Expect(component.ResolveImages(nil)).To(BeNil()) - - resources, _ := apiServerObjects(component) - - // Should render the correct resources. - // - 1 namespace - // - 1 ConfigMap audit Policy - // - 1 ConfigMap Tigera CA bundle - // - 1 Service account - // - 2 ServiceAccount ClusterRole and binding for calico CRDs - // - 2 ServiceAccount ClusterRole and binding for tigera CRDs - // - 2 ClusterRole and binding for auth configmap - // - 2 calico policy passthru ClusterRole and binding - // - 2 tiered policy passthru ClusterRole and binding - // - 1 Role binding for tigera-operator to manage secrets - // - 1 delegate auth binding - // - 1 auth reader binding - // - 2 webhook reader ClusterRole and binding - // - 2 cert secrets - // - 1 api server - // - 1 service registration - // - 1 Server service - rtest.ExpectResources(resources, expectedResources) - - apiService, ok := rtest.GetResource(resources, "v3.projectcalico.org", "", "apiregistration.k8s.io", "v1", "APIService").(*apiregv1.APIService) - Expect(ok).To(BeTrue(), "Expected v1.APIService") - verifyAPIService(apiService, true, clusterDomain) - - d := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment").(*appsv1.Deployment) - - Expect(d.Name).To(Equal("calico-apiserver")) - Expect(len(d.Labels)).To(Equal(1)) - Expect(d.Labels).To(HaveKeyWithValue("apiserver", "true")) - - Expect(*d.Spec.Replicas).To(BeEquivalentTo(2)) - Expect(d.Spec.Strategy.Type).To(Equal(appsv1.RollingUpdateDeploymentStrategyType)) - Expect(len(d.Spec.Selector.MatchLabels)).To(Equal(1)) - Expect(d.Spec.Selector.MatchLabels).To(HaveKeyWithValue("apiserver", "true")) - - Expect(d.Spec.Template.Name).To(Equal("calico-apiserver")) - Expect(d.Spec.Template.Namespace).To(Equal("calico-system")) - Expect(len(d.Spec.Template.Labels)).To(Equal(1)) - Expect(d.Spec.Template.Labels).To(HaveKeyWithValue("apiserver", "true")) - - Expect(d.Spec.Template.Spec.ServiceAccountName).To(Equal("calico-apiserver")) - - Expect(d.Spec.Template.Spec.Tolerations).To(ConsistOf(rmeta.TolerateControlPlane)) - - Expect(d.Spec.Template.Spec.ImagePullSecrets).To(BeEmpty()) - Expect(d.Spec.Template.Spec.Containers).To(HaveLen(2)) - Expect(d.Spec.Template.Spec.Containers[0].Name).To(Equal("calico-apiserver")) - Expect(d.Spec.Template.Spec.Containers[0].Image).To(Equal( - fmt.Sprintf("testregistry.com/%s%s:%s", components.TigeraImagePath, components.ComponentTigeraCalico.Image, components.ComponentTigeraCalico.Version), - )) - - expectedArgs := []string{ - "--secure-port=5443", - "--tls-private-key-file=/calico-apiserver-certs/tls.key", - "--tls-cert-file=/calico-apiserver-certs/tls.crt", - "--audit-policy-file=/etc/tigera/audit/policy.conf", - "--audit-log-path=/var/log/calico/audit/tsee-audit.log", - } - Expect(d.Spec.Template.Spec.Containers[0].Args).To(ConsistOf(expectedArgs)) - Expect(len(d.Spec.Template.Spec.Containers[0].Env)).To(Equal(2)) - Expect(d.Spec.Template.Spec.Containers[0].Env[0].Name).To(Equal("DATASTORE_TYPE")) - Expect(d.Spec.Template.Spec.Containers[0].Env[0].Value).To(Equal("kubernetes")) - Expect(d.Spec.Template.Spec.Containers[0].Env[0].ValueFrom).To(BeNil()) - Expect(d.Spec.Template.Spec.Containers[0].Env[1].Name).To(Equal("LOG_LEVEL")) - Expect(d.Spec.Template.Spec.Containers[0].Env[1].Value).To(Equal("info")) - Expect(d.Spec.Template.Spec.Containers[0].Env[1].ValueFrom).To(BeNil()) - - Expect(len(d.Spec.Template.Spec.Containers[0].VolumeMounts)).To(Equal(3)) - Expect(d.Spec.Template.Spec.Containers[0].VolumeMounts[0].Name).To(Equal("calico-apiserver-certs")) - Expect(d.Spec.Template.Spec.Containers[0].VolumeMounts[1].MountPath).To(Equal("/var/log/calico/audit")) - Expect(d.Spec.Template.Spec.Containers[0].VolumeMounts[1].Name).To(Equal("calico-audit-logs")) - - Expect(d.Spec.Template.Spec.Containers[0].ReadinessProbe.HTTPGet.Path).To(Equal("/readyz")) - Expect(d.Spec.Template.Spec.Containers[0].ReadinessProbe.HTTPGet.Port.String()).To(BeEquivalentTo("5443")) - Expect(d.Spec.Template.Spec.Containers[0].ReadinessProbe.HTTPGet.Scheme).To(BeEquivalentTo("HTTPS")) - Expect(d.Spec.Template.Spec.Containers[0].ReadinessProbe.PeriodSeconds).To(BeEquivalentTo(60)) - - Expect(*d.Spec.Template.Spec.Containers[0].SecurityContext.AllowPrivilegeEscalation).To(BeTrue()) - Expect(*d.Spec.Template.Spec.Containers[0].SecurityContext.Privileged).To(BeTrue()) - Expect(*d.Spec.Template.Spec.Containers[0].SecurityContext.RunAsGroup).To(BeEquivalentTo(0)) - Expect(*d.Spec.Template.Spec.Containers[0].SecurityContext.RunAsNonRoot).To(BeFalse()) - Expect(*d.Spec.Template.Spec.Containers[0].SecurityContext.RunAsUser).To(BeEquivalentTo(0)) - Expect(d.Spec.Template.Spec.Containers[0].SecurityContext.Capabilities).To(Equal( - &corev1.Capabilities{ - Drop: []corev1.Capability{"ALL"}, - }, - )) - Expect(d.Spec.Template.Spec.Containers[0].SecurityContext.SeccompProfile).To(Equal( - &corev1.SeccompProfile{ - Type: corev1.SeccompProfileTypeRuntimeDefault, - })) - - Expect(d.Spec.Template.Spec.Containers[1].Name).To(Equal("tigera-queryserver")) - Expect(d.Spec.Template.Spec.Containers[1].Image).To(Equal( - fmt.Sprintf("testregistry.com/%s%s:%s", components.TigeraImagePath, components.ComponentTigeraCalico.Image, components.ComponentTigeraCalico.Version), - )) - Expect(d.Spec.Template.Spec.Containers[1].Args).To(BeEmpty()) - - Expect(d.Spec.Template.Spec.Containers[1].Env).To(HaveLen(10)) - - Expect(d.Spec.Template.Spec.Containers[1].Env[0].Name).To(Equal("DATASTORE_TYPE")) - Expect(d.Spec.Template.Spec.Containers[1].Env[0].Value).To(Equal("kubernetes")) - Expect(d.Spec.Template.Spec.Containers[1].Env[0].ValueFrom).To(BeNil()) - Expect(d.Spec.Template.Spec.Containers[1].Env[1].Name).To(Equal("LISTEN_ADDR")) - Expect(d.Spec.Template.Spec.Containers[1].Env[1].Value).To(Equal(":8080")) - Expect(d.Spec.Template.Spec.Containers[1].Env[1].ValueFrom).To(BeNil()) - Expect(d.Spec.Template.Spec.Containers[1].Env[2].Name).To(Equal("TLS_CERT")) - Expect(d.Spec.Template.Spec.Containers[1].Env[2].Value).To(Equal("/calico-apiserver-certs/tls.crt")) - Expect(d.Spec.Template.Spec.Containers[1].Env[2].ValueFrom).To(BeNil()) - Expect(d.Spec.Template.Spec.Containers[1].Env[3].Name).To(Equal("TLS_KEY")) - Expect(d.Spec.Template.Spec.Containers[1].Env[3].Value).To(Equal("/calico-apiserver-certs/tls.key")) - Expect(d.Spec.Template.Spec.Containers[1].Env[3].ValueFrom).To(BeNil()) - Expect(d.Spec.Template.Spec.Containers[1].Env[4].Name).To(Equal("TRUSTED_BUNDLE_PATH")) - Expect(d.Spec.Template.Spec.Containers[1].Env[4].Value).To(Equal("/etc/pki/tls/certs/tigera-ca-bundle.crt")) - Expect(d.Spec.Template.Spec.Containers[1].Env[5].Name).To(Equal("LINSEED_URL")) - Expect(d.Spec.Template.Spec.Containers[1].Env[5].Value).To(Equal("https://tigera-linseed.tigera-elasticsearch.svc")) - Expect(d.Spec.Template.Spec.Containers[1].Env[6].Name).To(Equal("LINSEED_CLIENT_CERT")) - Expect(d.Spec.Template.Spec.Containers[1].Env[6].Value).To(Equal("/calico-apiserver-certs/tls.crt")) - Expect(d.Spec.Template.Spec.Containers[1].Env[7].Name).To(Equal("LINSEED_CLIENT_KEY")) - Expect(d.Spec.Template.Spec.Containers[1].Env[7].Value).To(Equal("/calico-apiserver-certs/tls.key")) - Expect(d.Spec.Template.Spec.Containers[1].Env[8].Name).To(Equal("LINSEED_CA")) - Expect(d.Spec.Template.Spec.Containers[1].Env[8].Value).To(Equal("/etc/pki/tls/certs/tigera-ca-bundle.crt")) - Expect(d.Spec.Template.Spec.Containers[1].Env[9].Name).To(Equal("LOGLEVEL")) - Expect(d.Spec.Template.Spec.Containers[1].Env[9].Value).To(Equal("info")) - Expect(d.Spec.Template.Spec.Containers[1].Env[9].ValueFrom).To(BeNil()) - - // Expect the SECURITY_GROUP env variables to not be set - Expect(d.Spec.Template.Spec.Containers[1].Env).NotTo(ContainElement(gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{"Name": Equal("TIGERA_DEFAULT_SECURITY_GROUPS")}))) - Expect(d.Spec.Template.Spec.Containers[1].Env).NotTo(ContainElement(gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{"Name": Equal("TIGERA_POD_SECURITY_GROUP")}))) - - Expect(d.Spec.Template.Spec.Containers[1].VolumeMounts).To(HaveLen(2)) - Expect(d.Spec.Template.Spec.Containers[1].VolumeMounts[0].Name).To(Equal("calico-apiserver-certs")) - Expect(d.Spec.Template.Spec.Containers[1].VolumeMounts[0].MountPath).To(Equal("/calico-apiserver-certs")) - Expect(d.Spec.Template.Spec.Containers[1].VolumeMounts[0].ReadOnly).To(BeTrue()) - Expect(d.Spec.Template.Spec.Containers[1].VolumeMounts[0].SubPath).To(Equal("")) - Expect(d.Spec.Template.Spec.Containers[1].VolumeMounts[0].MountPropagation).To(BeNil()) - Expect(d.Spec.Template.Spec.Containers[1].VolumeMounts[0].SubPathExpr).To(Equal("")) - Expect(d.Spec.Template.Spec.Containers[1].VolumeMounts[1].Name).To(Equal("tigera-ca-bundle")) - Expect(d.Spec.Template.Spec.Containers[1].VolumeMounts[1].MountPath).To(Equal("/etc/pki/tls/certs")) - Expect(d.Spec.Template.Spec.Containers[1].VolumeMounts[1].ReadOnly).To(BeTrue()) - Expect(d.Spec.Template.Spec.Containers[1].VolumeMounts[1].SubPath).To(Equal("")) - Expect(d.Spec.Template.Spec.Containers[1].VolumeMounts[1].MountPropagation).To(BeNil()) - Expect(d.Spec.Template.Spec.Containers[1].VolumeMounts[1].SubPathExpr).To(Equal("")) - - Expect(d.Spec.Template.Spec.Containers[1].LivenessProbe.HTTPGet.Path).To(Equal("/version")) - Expect(d.Spec.Template.Spec.Containers[1].LivenessProbe.HTTPGet.Port.String()).To(BeEquivalentTo("8080")) - Expect(d.Spec.Template.Spec.Containers[1].LivenessProbe.HTTPGet.Scheme).To(BeEquivalentTo("HTTPS")) - Expect(d.Spec.Template.Spec.Containers[1].LivenessProbe.InitialDelaySeconds).To(BeEquivalentTo(90)) - - Expect(*d.Spec.Template.Spec.Containers[1].SecurityContext.AllowPrivilegeEscalation).To(BeFalse()) - Expect(*d.Spec.Template.Spec.Containers[1].SecurityContext.Privileged).To(BeFalse()) - Expect(*d.Spec.Template.Spec.Containers[1].SecurityContext.RunAsGroup).To(BeEquivalentTo(10001)) - Expect(*d.Spec.Template.Spec.Containers[1].SecurityContext.RunAsNonRoot).To(BeTrue()) - Expect(*d.Spec.Template.Spec.Containers[1].SecurityContext.RunAsUser).To(BeEquivalentTo(10001)) - Expect(d.Spec.Template.Spec.Containers[1].SecurityContext.Capabilities).To(Equal( - &corev1.Capabilities{ - Drop: []corev1.Capability{"ALL"}, - }, - )) - Expect(d.Spec.Template.Spec.Containers[1].SecurityContext.SeccompProfile).To(Equal( - &corev1.SeccompProfile{ - Type: corev1.SeccompProfileTypeRuntimeDefault, - })) - - Expect(d.Spec.Template.Spec.Volumes).To(HaveLen(4)) - Expect(d.Spec.Template.Spec.Volumes[0].Name).To(Equal("calico-apiserver-certs")) - Expect(d.Spec.Template.Spec.Volumes[0].Secret.SecretName).To(Equal("calico-apiserver-certs")) - Expect(d.Spec.Template.Spec.Volumes[1].Name).To(Equal("calico-audit-logs")) - Expect(d.Spec.Template.Spec.Volumes[1].HostPath.Path).To(Equal("/var/log/calico/audit")) - Expect(*d.Spec.Template.Spec.Volumes[1].HostPath.Type).To(BeEquivalentTo("DirectoryOrCreate")) - Expect(d.Spec.Template.Spec.Volumes[2].Name).To(Equal("calico-audit-policy")) - Expect(d.Spec.Template.Spec.Volumes[2].ConfigMap.Name).To(Equal("calico-audit-policy")) - Expect(d.Spec.Template.Spec.Volumes[2].ConfigMap.Items).To(HaveLen(1)) - Expect(d.Spec.Template.Spec.Volumes[2].ConfigMap.Items[0].Key).To(Equal("config")) - Expect(d.Spec.Template.Spec.Volumes[2].ConfigMap.Items[0].Path).To(Equal("policy.conf")) - Expect(d.Spec.Template.Spec.Volumes[3].Name).To(Equal("tigera-ca-bundle")) - Expect(d.Spec.Template.Spec.Volumes[3].ConfigMap.Name).To(Equal("tigera-ca-bundle")) - - clusterRole := rtest.GetResource(resources, "tigera-network-admin", "", "rbac.authorization.k8s.io", "v1", "ClusterRole").(*rbacv1.ClusterRole) - Expect(clusterRole.Rules).To(ConsistOf(networkAdminPolicyRules)) - - clusterRole = rtest.GetResource(resources, "tigera-ui-user", "", "rbac.authorization.k8s.io", "v1", "ClusterRole").(*rbacv1.ClusterRole) - Expect(clusterRole.Rules).To(ConsistOf(uiUserPolicyRules)) - - clusterRoleBinding := rtest.GetResource(resources, "calico-extension-apiserver-auth-access", "", "rbac.authorization.k8s.io", "v1", "ClusterRoleBinding").(*rbacv1.ClusterRoleBinding) - Expect(clusterRoleBinding.RoleRef.Name).To(Equal("calico-extension-apiserver-auth-access")) - - svc := rtest.GetResource(resources, "calico-api", "calico-system", "", "v1", "Service").(*corev1.Service) - Expect(svc.GetObjectMeta().GetLabels()).To(HaveLen(1)) - Expect(svc.GetObjectMeta().GetLabels()).To(HaveKeyWithValue("k8s-app", "calico-api")) - - Expect(svc.Spec.Ports).To(HaveLen(2)) - serviceFound := 0 - for _, p := range svc.Spec.Ports { - switch p.Name { - case render.APIServerPortName: - Expect(p.Port).To(Equal(int32(443))) - Expect(p.TargetPort.IntValue()).To(Equal(5443)) - serviceFound++ - case render.QueryServerPortName: - Expect(p.Port).To(Equal(int32(8080))) - Expect(p.TargetPort.IntValue()).To(Equal(8080)) - serviceFound++ - } - } - Expect(serviceFound).To(Equal(2)) - - cr := rtest.GetResource(resources, "calico-tiered-policy-passthrough", "", "rbac.authorization.k8s.io", "v1", "ClusterRole").(*rbacv1.ClusterRole) - var tieredPolicyRules []string - for _, rule := range cr.Rules { - tieredPolicyRules = append(tieredPolicyRules, rule.Resources...) - } - Expect(tieredPolicyRules).To(ContainElements("networkpolicies", "globalnetworkpolicies", "stagednetworkpolicies", "stagedglobalnetworkpolicies")) - - apiserverClusterRole := rtest.GetResource(resources, - "calico-crds", "", rbacv1.GroupName, "v1", "ClusterRole").(*rbacv1.ClusterRole) - Expect(apiserverClusterRole.Rules).To(ContainElement(rbacv1.PolicyRule{ - APIGroups: []string{"admissionregistration.k8s.io"}, - Resources: []string{ - "validatingadmissionpolicies", - "validatingadmissionpolicybindings", - }, - Verbs: []string{ - "get", - "list", - "watch", - }, - })) - }, - Entry("default cluster domain", dns.DefaultClusterDomain), - Entry("custom cluster domain", "custom-domain.internal"), - ) - - It("should render resources without an aggregation server", func() { - cfg.RequiresAggregationServer = false - - component, err := render.APIServer(cfg) - Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - resources, _ := apiServerObjects(component) - - expectedResources := []client.Object{ - &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "tigera-ca-bundle", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"}}, - &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "ServiceAccount", APIVersion: "v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-crds"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver-access-calico-crds"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-tier-getter"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-tier-getter"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver-delegate-auth"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "Deployment", APIVersion: "apps/v1"}}, - &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: "calico-api", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}}, - &policyv1.PodDisruptionBudget{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "PodDisruptionBudget", APIVersion: "policy/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "tigera-ui-user"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "tigera-network-admin"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-webhook-reader"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver-webhook-reader"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - } - - rtest.ExpectResources(resources, expectedResources) - - // In CRD/webhooks mode the calico-uisettings-passthrough ClusterRole is not deployed, so - // tigera-network-admin needs to grant write access to uisettings itself for the - // calico-webhooks UISettings handler to do the narrowing instead of being short-circuited - // by kube-apiserver RBAC. - networkAdmin := rtest.GetResource(resources, "tigera-network-admin", "", "rbac.authorization.k8s.io", "v1", "ClusterRole").(*rbacv1.ClusterRole) - Expect(networkAdmin.Rules).To(ContainElement(rbacv1.PolicyRule{ - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"uisettings"}, - Verbs: []string{"create", "update", "delete", "patch"}, - })) - }) - - It("should grant the calico-apiserver SA write access to globalreports/status", func() { - component, err := render.APIServer(cfg) - Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - resources, _ := apiServerObjects(component) - - cr := rtest.GetResource(resources, "calico-apiserver", "", "rbac.authorization.k8s.io", "v1", "ClusterRole").(*rbacv1.ClusterRole) - - // Find the backing-storage rule and assert it covers globalreports/status. - // Without this, compliance-controller can't update status.lastScheduledReportJob - // and no compliance jobs ever run. - var found bool - for _, rule := range cr.Rules { - hasGlobalReports := false - hasGlobalReportsStatus := false - for _, r := range rule.Resources { - if r == "globalreports" { - hasGlobalReports = true - } - if r == "globalreports/status" { - hasGlobalReportsStatus = true - } - } - if hasGlobalReports { - found = true - Expect(hasGlobalReportsStatus).To(BeTrue(), "calico-apiserver ClusterRole rule covering globalreports must also cover globalreports/status") - Expect(rule.Verbs).To(ContainElement("update")) - } - } - Expect(found).To(BeTrue(), "calico-apiserver ClusterRole should have a rule covering globalreports") - }) - - It("should render L7 Admission Controller with default config when SidecarInjection is Enabled", func() { - sidecarEnabled := operatorv1.SidecarEnabled - cfg.ApplicationLayer = &operatorv1.ApplicationLayer{ - Spec: operatorv1.ApplicationLayerSpec{ - SidecarInjection: &sidecarEnabled, - }, - } - - component, err := render.APIServer(cfg) - Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - resources, _ := apiServerObjects(component) - - d, ok := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment").(*appsv1.Deployment) - Expect(ok).To(BeTrue()) - var container corev1.Container - for _, c := range d.Spec.Template.Spec.Containers { - if c.Name == "calico-l7-admission-controller" { - container = c - } - } - Expect(container.Env[4].Name).To(Equal("L7ADMCTRL_LISTENADDR")) - Expect(container.Env[4].Value).To(Equal(":6443")) - - // Check the Service configuration - svc := rtest.GetResource(resources, "calico-api", "calico-system", "", "v1", "Service").(*corev1.Service) - var servicePort corev1.ServicePort - for _, p := range svc.Spec.Ports { - if p.Name == render.L7AdmissionControllerPortName { - servicePort = p - } - } - Expect(servicePort.Port).To(Equal(int32(6443))) - Expect(servicePort.TargetPort.IntValue()).To(Equal(6443)) - }) - - It("should render log severity when provided", func() { - errorLog := operatorv1.LogSeverityError - debugLog := operatorv1.LogSeverityDebug - cfg.APIServer.Logging = &operatorv1.APIServerPodLogging{ - APIServerLogging: &operatorv1.APIServerLogging{ - LogSeverity: &errorLog, - }, - QueryServerLogging: &operatorv1.QueryServerLogging{ - LogSeverity: &debugLog, - }, - } - component, err := render.APIServer(cfg) - Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - resources, _ := apiServerObjects(component) - - deploy, ok := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment").(*appsv1.Deployment) - Expect(ok).To(BeTrue()) - - containers := deploy.Spec.Template.Spec.Containers - for _, container := range containers { - envs := container.Env - if strings.Contains(container.Name, "apiserver") { - for _, env := range envs { - if env.Name == "LOG_LEVEL" { - Expect(env.Value).To(Equal("error")) - } - } - } else if strings.Contains(container.Name, "queryserver") { - for _, env := range envs { - if env.Name == "LOGLEVEL" { - Expect(env.Value).To(Equal("debug")) - } - } - } - } - Expect(deploy.Spec.Template.Spec.Containers).NotTo(BeNil()) - Expect(deploy.Spec.Template.Spec.Affinity).To(Equal(podaffinity.NewPodAntiAffinity("calico-apiserver", []string{"calico-system", "tigera-system", "calico-apiserver"}))) - }) - - It("should render SecurityContextConstrains properly when provider is OpenShift", func() { - cfg.Installation.KubernetesProvider = operatorv1.ProviderOpenShift - cfg.Installation.Variant = operatorv1.CalicoEnterprise - component, err := render.APIServer(cfg) - Expect(err).NotTo(HaveOccurred()) - Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := apiServerObjects(component) - - role := rtest.GetResource(resources, "calico-extension-apiserver-auth-access", "", "rbac.authorization.k8s.io", "v1", "ClusterRole").(*rbacv1.ClusterRole) - Expect(role.Rules).To(ContainElement(rbacv1.PolicyRule{ - APIGroups: []string{"security.openshift.io"}, - Resources: []string{"securitycontextconstraints"}, - Verbs: []string{"use"}, - ResourceNames: []string{"privileged"}, - })) - Expect(role.Rules).To(ContainElement(rbacv1.PolicyRule{ - APIGroups: []string{"config.openshift.io"}, - Resources: []string{"infrastructures"}, - Verbs: []string{"get", "list", "watch"}, - })) - }) - - It("should render an API server with custom configuration", func() { - expectedResources := []client.Object{ - &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "calico-audit-policy", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"}}, - &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "tigera-ca-bundle", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"}}, - &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "ServiceAccount", APIVersion: "v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-crds"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver-access-calico-crds"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-tier-getter"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-tier-getter"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-tiered-policy-passthrough"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-tiered-policy-passthrough"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-uisettings-passthrough"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-uisettings-passthrough"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-extension-apiserver-auth-access"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-extension-apiserver-auth-access"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver-delegate-auth"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.RoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver-auth-reader", Namespace: "kube-system"}, TypeMeta: metav1.TypeMeta{Kind: "RoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &apiregv1.APIService{ObjectMeta: metav1.ObjectMeta{Name: "v3.projectcalico.org"}, TypeMeta: metav1.TypeMeta{Kind: "APIService", APIVersion: "apiregistration.k8s.io/v1"}}, - &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "Deployment", APIVersion: "apps/v1"}}, - &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: "calico-api", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}}, - &policyv1.PodDisruptionBudget{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "PodDisruptionBudget", APIVersion: "policy/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-uisettingsgroup-getter"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-uisettingsgroup-getter"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "tigera-ui-user"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "tigera-network-admin"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-webhook-reader"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver-webhook-reader"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - } - - component, err := render.APIServer(cfg) - Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := apiServerObjects(component) - - rtest.ExpectResources(resources, expectedResources) - - dep := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment") - rtest.ExpectResourceTypeAndObjectMetadata(dep, "calico-apiserver", "calico-system", "apps", "v1", "Deployment") - d := dep.(*appsv1.Deployment) - - Expect(d.Spec.Template.Spec.Volumes).To(HaveLen(4)) - }) - - It("should render needed resources for k8s kube-controller", func() { - expectedResources := []client.Object{ - &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "calico-audit-policy", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"}}, - &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "tigera-ca-bundle", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"}}, - &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "ServiceAccount", APIVersion: "v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-crds"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver-access-calico-crds"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-tier-getter"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-tier-getter"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-tiered-policy-passthrough"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-tiered-policy-passthrough"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-uisettings-passthrough"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-uisettings-passthrough"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-extension-apiserver-auth-access"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-extension-apiserver-auth-access"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver-delegate-auth"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.RoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver-auth-reader", Namespace: "kube-system"}, TypeMeta: metav1.TypeMeta{Kind: "RoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &apiregv1.APIService{ObjectMeta: metav1.ObjectMeta{Name: "v3.projectcalico.org"}, TypeMeta: metav1.TypeMeta{Kind: "APIService", APIVersion: "apiregistration.k8s.io/v1"}}, - &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "Deployment", APIVersion: "apps/v1"}}, - &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: "calico-api", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}}, - &policyv1.PodDisruptionBudget{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "PodDisruptionBudget", APIVersion: "policy/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-uisettingsgroup-getter"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-uisettingsgroup-getter"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "tigera-ui-user"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "tigera-network-admin"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-webhook-reader"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver-webhook-reader"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - } - - component, err := render.APIServer(cfg) - Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := apiServerObjects(component) - - rtest.ExpectResources(resources, expectedResources) - - // Should render the correct resources. - cr := rtest.GetResource(resources, "calico-tier-getter", "", "rbac.authorization.k8s.io", "v1", "ClusterRole").(*rbacv1.ClusterRole) - Expect(len(cr.Rules)).To(Equal(1)) - Expect(len(cr.Rules[0].Resources)).To(Equal(1)) - Expect(cr.Rules[0].Resources[0]).To(Equal("tiers")) - Expect(len(cr.Rules[0].Verbs)).To(Equal(1)) - Expect(cr.Rules[0].Verbs[0]).To(Equal("get")) - - crb := rtest.GetResource(resources, "calico-tier-getter", "", "rbac.authorization.k8s.io", "v1", "ClusterRoleBinding").(*rbacv1.ClusterRoleBinding) - Expect(crb.RoleRef.Kind).To(Equal("ClusterRole")) - Expect(crb.RoleRef.Name).To(Equal("calico-tier-getter")) - Expect(len(crb.Subjects)).To(Equal(1)) - Expect(crb.Subjects[0].Kind).To(Equal("User")) - Expect(crb.Subjects[0].Name).To(Equal("system:kube-controller-manager")) - - cr = rtest.GetResource(resources, "calico-uisettingsgroup-getter", "", "rbac.authorization.k8s.io", "v1", "ClusterRole").(*rbacv1.ClusterRole) - Expect(len(cr.Rules)).To(Equal(1)) - Expect(len(cr.Rules[0].Resources)).To(Equal(1)) - Expect(cr.Rules[0].Resources[0]).To(Equal("uisettingsgroups")) - Expect(len(cr.Rules[0].Verbs)).To(Equal(1)) - Expect(cr.Rules[0].Verbs[0]).To(Equal("get")) - - crb = rtest.GetResource(resources, "calico-uisettingsgroup-getter", "", "rbac.authorization.k8s.io", "v1", "ClusterRoleBinding").(*rbacv1.ClusterRoleBinding) - Expect(crb.RoleRef.Kind).To(Equal("ClusterRole")) - Expect(crb.RoleRef.Name).To(Equal("calico-uisettingsgroup-getter")) - Expect(len(crb.Subjects)).To(Equal(1)) - Expect(crb.Subjects[0].Kind).To(Equal("User")) - Expect(crb.Subjects[0].Name).To(Equal("system:kube-controller-manager")) - }) - - It("should include a ControlPlaneNodeSelector when specified", func() { - expectedResources := []client.Object{ - &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "calico-audit-policy", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"}}, - &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "tigera-ca-bundle", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"}}, - &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "ServiceAccount", APIVersion: "v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-crds"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver-access-calico-crds"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-tier-getter"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-tier-getter"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-tiered-policy-passthrough"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-tiered-policy-passthrough"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-uisettings-passthrough"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-uisettings-passthrough"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-extension-apiserver-auth-access"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-extension-apiserver-auth-access"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver-delegate-auth"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.RoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver-auth-reader", Namespace: "kube-system"}, TypeMeta: metav1.TypeMeta{Kind: "RoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &apiregv1.APIService{ObjectMeta: metav1.ObjectMeta{Name: "v3.projectcalico.org"}, TypeMeta: metav1.TypeMeta{Kind: "APIService", APIVersion: "apiregistration.k8s.io/v1"}}, - &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "Deployment", APIVersion: "apps/v1"}}, - &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: "calico-api", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}}, - &policyv1.PodDisruptionBudget{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "PodDisruptionBudget", APIVersion: "policy/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-uisettingsgroup-getter"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-uisettingsgroup-getter"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "tigera-ui-user"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "tigera-network-admin"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-webhook-reader"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver-webhook-reader"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - } - - cfg.Installation.ControlPlaneNodeSelector = map[string]string{"nodeName": "control01"} - component, err := render.APIServer(cfg) - Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := apiServerObjects(component) - - rtest.ExpectResources(resources, expectedResources) - - d := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment").(*appsv1.Deployment) - - Expect(d.Spec.Template.Spec.NodeSelector).To(HaveLen(1)) - Expect(d.Spec.Template.Spec.NodeSelector).To(HaveKeyWithValue("nodeName", "control01")) - }) - - It("should include a ControlPlaneToleration when specified", func() { - tol := corev1.Toleration{ - Key: "foo", - Operator: corev1.TolerationOpEqual, - Value: "bar", - Effect: corev1.TaintEffectNoExecute, - } - cfg.Installation.ControlPlaneTolerations = []corev1.Toleration{tol} - - component, err := render.APIServer(cfg) - Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - resources, _ := apiServerObjects(component) - d := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment").(*appsv1.Deployment) - Expect(d.Spec.Template.Spec.Tolerations).To(ContainElements(append(rmeta.TolerateControlPlane, tol))) - }) - - It("should include a ClusterRole and ClusterRoleBindings for reading webhook configuration", func() { - expectedResources := []client.Object{ - &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "calico-audit-policy", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"}}, - &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "tigera-ca-bundle", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"}}, - &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "ServiceAccount", APIVersion: "v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-crds"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver-access-calico-crds"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-tier-getter"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-tier-getter"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-tiered-policy-passthrough"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-tiered-policy-passthrough"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-uisettings-passthrough"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-uisettings-passthrough"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-extension-apiserver-auth-access"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-extension-apiserver-auth-access"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver-delegate-auth"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.RoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver-auth-reader", Namespace: "kube-system"}, TypeMeta: metav1.TypeMeta{Kind: "RoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &apiregv1.APIService{ObjectMeta: metav1.ObjectMeta{Name: "v3.projectcalico.org"}, TypeMeta: metav1.TypeMeta{Kind: "APIService", APIVersion: "apiregistration.k8s.io/v1"}}, - &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "Deployment", APIVersion: "apps/v1"}}, - &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: "calico-api", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}}, - &policyv1.PodDisruptionBudget{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "PodDisruptionBudget", APIVersion: "policy/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-uisettingsgroup-getter"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-uisettingsgroup-getter"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "tigera-ui-user"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "tigera-network-admin"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-webhook-reader"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver-webhook-reader"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - } - - component, err := render.APIServer(cfg) - Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := apiServerObjects(component) - - rtest.ExpectResources(resources, expectedResources) - - // Should render the correct resources. - cr := rtest.GetResource(resources, "calico-webhook-reader", "", "rbac.authorization.k8s.io", "v1", "ClusterRole").(*rbacv1.ClusterRole) - Expect(len(cr.Rules)).To(Equal(1)) - Expect(len(cr.Rules[0].Resources)).To(Equal(2)) - Expect(cr.Rules[0].Resources[0]).To(Equal("mutatingwebhookconfigurations")) - Expect(cr.Rules[0].Resources[1]).To(Equal("validatingwebhookconfigurations")) - Expect(len(cr.Rules[0].Verbs)).To(Equal(3)) - Expect(cr.Rules[0].Verbs[0]).To(Equal("get")) - Expect(cr.Rules[0].Verbs[1]).To(Equal("list")) - Expect(cr.Rules[0].Verbs[2]).To(Equal("watch")) - - crb := rtest.GetResource(resources, "calico-apiserver-webhook-reader", "", "rbac.authorization.k8s.io", "v1", "ClusterRoleBinding").(*rbacv1.ClusterRoleBinding) - Expect(crb.RoleRef.Kind).To(Equal("ClusterRole")) - Expect(crb.RoleRef.Name).To(Equal("calico-webhook-reader")) - Expect(len(crb.Subjects)).To(Equal(1)) - Expect(crb.Subjects[0].Kind).To(Equal("ServiceAccount")) - Expect(crb.Subjects[0].Name).To(Equal("calico-apiserver")) - Expect(crb.Subjects[0].Namespace).To(Equal("calico-system")) - }) - - It("should set KUBERENETES_SERVICE_... variables if host networked", func() { - cfg.K8SServiceEndpoint.Host = "k8shost" - cfg.K8SServiceEndpoint.Port = "1234" - cfg.ForceHostNetwork = true - component, err := render.APIServer(cfg) - Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := apiServerObjects(component) - - deploymentResource := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment") - Expect(deploymentResource).ToNot(BeNil()) - - deployment := deploymentResource.(*appsv1.Deployment) - rtest.ExpectK8sServiceEpEnvVars(deployment.Spec.Template.Spec, "k8shost", "1234") - }) - - It("should set RecreateDeploymentStrategyType if host networked", func() { - cfg.ForceHostNetwork = true - component, err := render.APIServer(cfg) - Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - resources, _ := apiServerObjects(component) - d := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment").(*appsv1.Deployment) - Expect(d.Spec.Strategy.Type).To(Equal(appsv1.RecreateDeploymentStrategyType)) - }) - - It("should add egress policy with Enterprise variant and K8SServiceEndpoint defined", func() { - cfg.K8SServiceEndpoint.Host = "k8shost" - cfg.K8SServiceEndpoint.Port = "1234" - cfg.ForceHostNetwork = true - - component := render.APIServerPolicy(cfg) - resources, _ := apiServerObjects(component) - policyName := types.NamespacedName{Name: "calico-system.apiserver-access", Namespace: "calico-system"} - policy := testutils.GetCalicoSystemPolicyFromResources(policyName, resources) - Expect(policy).ToNot(BeNil()) - Expect(policy.Spec).ToNot(BeNil()) - Expect(policy.Spec.Egress).ToNot(BeNil()) - Expect(policy.Spec.Egress).To(ContainElement(calicov3.Rule{ - Action: calicov3.Allow, - Protocol: &networkpolicy.TCPProtocol, - Destination: calicov3.EntityRule{ - Ports: networkpolicy.Ports(1234), - Domains: []string{"k8shost"}, - }, - })) - }) - - It("should add egress policy with Enterprise variant and K8SServiceEndpoint as IP defined", func() { - cfg.K8SServiceEndpoint.Host = "169.169.169.169" - cfg.K8SServiceEndpoint.Port = "4321" - cfg.ForceHostNetwork = false - - component := render.APIServerPolicy(cfg) - resources, _ := apiServerObjects(component) - policyName := types.NamespacedName{Name: "calico-system.apiserver-access", Namespace: "calico-system"} - policy := testutils.GetCalicoSystemPolicyFromResources(policyName, resources) - Expect(policy).ToNot(BeNil()) - Expect(policy.Spec).ToNot(BeNil()) - Expect(policy.Spec.Egress).ToNot(BeNil()) - Expect(policy.Spec.Egress).To(ContainElement(calicov3.Rule{ - Action: calicov3.Allow, - Protocol: &networkpolicy.TCPProtocol, - Destination: calicov3.EntityRule{ - Ports: networkpolicy.Ports(4321), - Nets: []string{"169.169.169.169/32"}, - }, - })) - }) - - It("should not set KUBERENETES_SERVICE_... variables if not host networked on Docker EE with proxy.local", func() { - cfg.K8SServiceEndpoint.Host = "proxy.local" - cfg.K8SServiceEndpoint.Port = "1234" - cfg.Installation.KubernetesProvider = operatorv1.ProviderDockerEE - - component, err := render.APIServer(cfg) - Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := apiServerObjects(component) - - deploymentResource := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment") - Expect(deploymentResource).ToNot(BeNil()) - - deployment := deploymentResource.(*appsv1.Deployment) - rtest.ExpectNoK8sServiceEpEnvVars(deployment.Spec.Template.Spec) - }) - - It("should set KUBERENETES_SERVICE_... variables if not host networked on Docker EE with non-proxy address", func() { - cfg.K8SServiceEndpointPodNetwork.Host = "k8shost" - cfg.K8SServiceEndpointPodNetwork.Port = "1234" - cfg.Installation.KubernetesProvider = operatorv1.ProviderDockerEE - - component, err := render.APIServer(cfg) - Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := apiServerObjects(component) - - deploymentResource := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment") - Expect(deploymentResource).ToNot(BeNil()) - - deployment := deploymentResource.(*appsv1.Deployment) - rtest.ExpectK8sServiceEpEnvVars(deployment.Spec.Template.Spec, "k8shost", "1234") - }) - - It("should render an API server with custom configuration with MCM enabled at startup", func() { - cfg.ManagementCluster = managementCluster - component, err := render.APIServer(cfg) - Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - Expect(component.ResolveImages(nil)).To(BeNil()) - - resources, _ := apiServerObjects(component) - - expectedResources := []client.Object{ - &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "calico-audit-policy", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"}}, - &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "tigera-ca-bundle", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"}}, - &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "ServiceAccount", APIVersion: "v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-crds"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver-access-calico-crds"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-tier-getter"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-tier-getter"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-tiered-policy-passthrough"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-tiered-policy-passthrough"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-uisettings-passthrough"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-uisettings-passthrough"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-extension-apiserver-auth-access"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-extension-apiserver-auth-access"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver-delegate-auth"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.RoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver-auth-reader", Namespace: "kube-system"}, TypeMeta: metav1.TypeMeta{Kind: "RoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &apiregv1.APIService{ObjectMeta: metav1.ObjectMeta{Name: "v3.projectcalico.org"}, TypeMeta: metav1.TypeMeta{Kind: "APIService", APIVersion: "apiregistration.k8s.io/v1"}}, - &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "Deployment", APIVersion: "apps/v1"}}, - &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: "calico-api", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}}, - &policyv1.PodDisruptionBudget{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "PodDisruptionBudget", APIVersion: "policy/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-uisettingsgroup-getter"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-uisettingsgroup-getter"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "tigera-ui-user"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "tigera-network-admin"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: render.ManagedClustersWatchClusterRoleName}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-webhook-reader"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver-webhook-reader"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.Role{ObjectMeta: metav1.ObjectMeta{Name: render.APIServerSecretsRBACName, Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "Role", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.RoleBinding{ObjectMeta: metav1.ObjectMeta{Name: render.APIServerSecretsRBACName, Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "RoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - } - - rtest.ExpectResources(resources, expectedResources) - - By("Validating the newly created tunnel secret") - tunnelSecret, err := certificatemanagement.CreateSelfSignedSecret(render.VoltronTunnelSecretName, common.OperatorNamespace(), "tigera-voltron", []string{"voltron"}) - Expect(err).ToNot(HaveOccurred()) - - // Use the x509 package to validate that the cert was signed with the privatekey - validateTunnelSecret(tunnelSecret) - - dep := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment") - Expect(dep).ToNot(BeNil()) - - By("Validating startup args") - expectedArgs := []string{ - "--secure-port=5443", - "--tls-private-key-file=/calico-apiserver-certs/tls.key", - "--tls-cert-file=/calico-apiserver-certs/tls.crt", - "--audit-policy-file=/etc/tigera/audit/policy.conf", - "--audit-log-path=/var/log/calico/audit/tsee-audit.log", - "--enable-managed-clusters-create-api=true", - "--managementClusterAddr=example.com:1234", - } - Expect((dep.(*appsv1.Deployment)).Spec.Template.Spec.Containers[0].Args).To(ConsistOf(expectedArgs)) - }) - - It("should render an API server with custom configuration with MCM enabled at restart", func() { - cfg.ManagementCluster = managementCluster - component, err := render.APIServer(cfg) - Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - Expect(component.ResolveImages(nil)).To(BeNil()) - - resources, _ := apiServerObjects(component) - - expected := []client.Object{ - &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "calico-audit-policy", Namespace: "calico-system"}}, - &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "tigera-ca-bundle", Namespace: "calico-system"}}, - &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver", Namespace: "calico-system"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-crds"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver-access-calico-crds"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-tier-getter"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-tier-getter"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-tiered-policy-passthrough"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-tiered-policy-passthrough"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-uisettings-passthrough"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-uisettings-passthrough"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-extension-apiserver-auth-access"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-extension-apiserver-auth-access"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver-delegate-auth"}}, - &rbacv1.RoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver-auth-reader", Namespace: "kube-system"}}, - &apiregv1.APIService{ObjectMeta: metav1.ObjectMeta{Name: "v3.projectcalico.org"}}, - &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver", Namespace: "calico-system"}}, - &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: "calico-api", Namespace: "calico-system"}}, - &policyv1.PodDisruptionBudget{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver", Namespace: "calico-system"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-uisettingsgroup-getter"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-uisettingsgroup-getter"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "tigera-ui-user"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "tigera-network-admin"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: render.ManagedClustersWatchClusterRoleName}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-webhook-reader"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver-webhook-reader"}}, - &rbacv1.Role{ObjectMeta: metav1.ObjectMeta{Name: render.APIServerSecretsRBACName, Namespace: "calico-system"}}, - &rbacv1.RoleBinding{ObjectMeta: metav1.ObjectMeta{Name: render.APIServerSecretsRBACName, Namespace: "calico-system"}}, - } - rtest.ExpectResources(resources, expected) - - dep := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment") - Expect(dep).ToNot(BeNil()) - - By("Validating startup args") - expectedArgs := []string{ - "--secure-port=5443", - "--tls-private-key-file=/calico-apiserver-certs/tls.key", - "--tls-cert-file=/calico-apiserver-certs/tls.crt", - "--audit-policy-file=/etc/tigera/audit/policy.conf", - "--audit-log-path=/var/log/calico/audit/tsee-audit.log", - "--enable-managed-clusters-create-api=true", - "--managementClusterAddr=example.com:1234", - } - Expect((dep.(*appsv1.Deployment)).Spec.Template.Spec.Containers[0].Args).To(ConsistOf(expectedArgs)) - }) - - It("should render an API server with signed ca bundles enabled", func() { - cfg.ManagementCluster = managementCluster - cfg.ManagementCluster.Spec.TLS = &operatorv1.TLS{ - SecretName: render.ManagerTLSSecretName, - } - component, err := render.APIServer(cfg) - Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - - resources, _ := apiServerObjects(component) - - dep := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment") - Expect(dep).ToNot(BeNil()) - - Expect((dep.(*appsv1.Deployment)).Spec.Template.Spec.Containers[0].Args).To(ContainElement("--managementClusterCAType=Public")) - Expect((dep.(*appsv1.Deployment)).Spec.Template.Spec.Containers[0].Args).To(ContainElement(fmt.Sprintf("--tunnelSecretName=%s", render.ManagerTLSSecretName))) - }) - - It("should pass tunnelSecretName when TLS secret is not manager-tls", func() { - cfg.ManagementCluster = managementCluster - cfg.ManagementCluster.Spec.TLS = &operatorv1.TLS{ - SecretName: render.VoltronTunnelSecretName, - } - component, err := render.APIServer(cfg) - Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - - resources, _ := apiServerObjects(component) - - dep := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment") - Expect(dep).ToNot(BeNil()) - - args := (dep.(*appsv1.Deployment)).Spec.Template.Spec.Containers[0].Args - Expect(args).To(ContainElement(fmt.Sprintf("--tunnelSecretName=%s", render.VoltronTunnelSecretName))) - Expect(args).ToNot(ContainElement("--managementClusterCAType=Public")) - }) - - It("should add an init container if certificate management is enabled", func() { - cfg.Installation.CertificateManagement = &operatorv1.CertificateManagement{SignerName: "a.b/c", CACert: cfg.TLSKeyPair.GetCertificatePEM()} - certificateManager, err := certificatemanager.Create(cli, cfg.Installation, clusterDomain, common.OperatorNamespace(), certificatemanager.AllowCACreation()) - Expect(err).NotTo(HaveOccurred()) - kp, err := certificateManager.GetOrCreateKeyPair(cli, render.CalicoAPIServerTLSSecretName, common.OperatorNamespace(), dnsNames) - Expect(err).NotTo(HaveOccurred()) - qskp, err := certificateManager.GetOrCreateKeyPair(cli, render.CalicoAPIServerTLSSecretName, common.OperatorNamespace(), dnsNames) - cfg.TLSKeyPair = kp - cfg.QueryServerTLSKeyPairCertificateManagementOnly = qskp - Expect(err).NotTo(HaveOccurred()) - component, err := render.APIServer(cfg) - Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - resources, _ := apiServerObjects(component) - expectedResources := []client.Object{ - &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "calico-audit-policy", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"}}, - &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "tigera-ca-bundle", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"}}, - &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "ServiceAccount", APIVersion: "v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-crds"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver-access-calico-crds"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-tier-getter"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-tier-getter"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-tiered-policy-passthrough"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-tiered-policy-passthrough"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-uisettings-passthrough"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-uisettings-passthrough"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-extension-apiserver-auth-access"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-extension-apiserver-auth-access"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver-delegate-auth"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.RoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver-auth-reader", Namespace: "kube-system"}, TypeMeta: metav1.TypeMeta{Kind: "RoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &apiregv1.APIService{ObjectMeta: metav1.ObjectMeta{Name: "v3.projectcalico.org"}, TypeMeta: metav1.TypeMeta{Kind: "APIService", APIVersion: "apiregistration.k8s.io/v1"}}, - &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "Deployment", APIVersion: "apps/v1"}}, - &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: "calico-api", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}}, - &policyv1.PodDisruptionBudget{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "PodDisruptionBudget", APIVersion: "policy/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-uisettingsgroup-getter"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-uisettingsgroup-getter"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "tigera-ui-user"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "tigera-network-admin"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-webhook-reader"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver-webhook-reader"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - } - rtest.ExpectResources(resources, expectedResources) - - dep := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment") - Expect(dep).ToNot(BeNil()) - deploy, ok := dep.(*appsv1.Deployment) - Expect(ok).To(BeTrue()) - Expect(deploy.Spec.Template.Spec.InitContainers).To(HaveLen(2)) - Expect(deploy.Spec.Template.Spec.InitContainers[0].Name).To(Equal("calico-apiserver-certs-key-cert-provisioner")) - rtest.ExpectEnv(deploy.Spec.Template.Spec.InitContainers[0].Env, "SIGNER", "a.b/c") - }) - - It("should not render PodAffinity when ControlPlaneReplicas is 1", func() { - cfg.Installation.ControlPlaneReplicas = ptr.To(int32(1)) - component, err := render.APIServer(cfg) - Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - resources, _ := apiServerObjects(component) - - deploy, ok := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment").(*appsv1.Deployment) - Expect(ok).To(BeTrue()) - Expect(deploy.Spec.Template.Spec.Affinity).To(BeNil()) - }) - - It("should render PodAffinity when ControlPlaneReplicas is greater than 1", func() { - cfg.Installation.ControlPlaneReplicas = ptr.To(int32(2)) - component, err := render.APIServer(cfg) - Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - resources, _ := apiServerObjects(component) - - deploy, ok := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment").(*appsv1.Deployment) - Expect(ok).To(BeTrue()) - Expect(deploy.Spec.Template.Spec.Affinity).NotTo(BeNil()) - Expect(deploy.Spec.Template.Spec.Affinity).To(Equal(podaffinity.NewPodAntiAffinity("calico-apiserver", []string{"calico-system", "tigera-system", "calico-apiserver"}))) - }) - - It("should render Linseed routing for the queryserver when ManagementClusterConnection is set", func() { - cfg.ManagementClusterConnection = &operatorv1.ManagementClusterConnection{} - cfg.ClusterDomain = "cluster.local" - - component, err := render.APIServer(cfg) - Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - resources, _ := apiServerObjects(component) - - rb, ok := rtest.GetResource(resources, "tigera-linseed", "calico-system", "rbac.authorization.k8s.io", "v1", "RoleBinding").(*rbacv1.RoleBinding) - Expect(ok).To(BeTrue(), "expected tigera-linseed RoleBinding in calico-system") - Expect(rb.RoleRef.Name).To(Equal("tigera-linseed-secrets")) - Expect(rb.Subjects).To(ConsistOf(rbacv1.Subject{ - Kind: "ServiceAccount", - Name: render.GuardianServiceAccountName, - Namespace: render.GuardianNamespace, - })) - - deploy, ok := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment").(*appsv1.Deployment) - Expect(ok).To(BeTrue()) - var qs *corev1.Container - for i := range deploy.Spec.Template.Spec.Containers { - if deploy.Spec.Template.Spec.Containers[i].Name == "tigera-queryserver" { - qs = &deploy.Spec.Template.Spec.Containers[i] - } - } - Expect(qs).NotTo(BeNil()) - Expect(qs.Env).To(ContainElement(corev1.EnvVar{Name: "LINSEED_URL", Value: "https://guardian.calico-system.svc"})) - Expect(qs.Env).To(ContainElement(corev1.EnvVar{Name: "CLUSTER_ID", Value: ""})) - Expect(qs.Env).To(ContainElement(corev1.EnvVar{Name: "LINSEED_TOKEN", Value: "/var/run/secrets/tigera.io/linseed/token"})) - Expect(qs.VolumeMounts).To(ContainElement(corev1.VolumeMount{ - Name: render.LinseedTokenVolumeName, - MountPath: render.LinseedVolumeMountPath, - })) - - var tokenVol *corev1.Volume - for i := range deploy.Spec.Template.Spec.Volumes { - if deploy.Spec.Template.Spec.Volumes[i].Name == render.LinseedTokenVolumeName { - tokenVol = &deploy.Spec.Template.Spec.Volumes[i] - } - } - Expect(tokenVol).NotTo(BeNil()) - Expect(tokenVol.Secret).NotTo(BeNil()) - Expect(tokenVol.Secret.SecretName).To(Equal("calico-apiserver-tigera-linseed-token")) - }) - - Context("calico-system rendering", func() { - policyName := types.NamespacedName{Name: "calico-system.apiserver-access", Namespace: "calico-system"} - - DescribeTable("should render calico-system policy", - func(scenario testutils.CalicoSystemScenario) { - cfg.OpenShift = scenario.OpenShift - if scenario.ManagedCluster { - cfg.ManagementClusterConnection = &operatorv1.ManagementClusterConnection{} - } else { - cfg.ManagementClusterConnection = nil - } - - component := render.APIServerPolicy(cfg) - resources, _ := apiServerObjects(component) - - policy := testutils.GetCalicoSystemPolicyFromResources(policyName, resources) - expectedPolicy := testutils.SelectPolicyByProvider(scenario, apiServerPolicy, apiServerPolicyForOCP) - Expect(policy).To(Equal(expectedPolicy)) - }, - Entry("for management/standalone, kube-dns", testutils.CalicoSystemScenario{ManagedCluster: false, OpenShift: false}), - Entry("for management/standalone, openshift-dns", testutils.CalicoSystemScenario{ManagedCluster: false, OpenShift: true}), - Entry("for managed, kube-dns", testutils.CalicoSystemScenario{ManagedCluster: true, OpenShift: false}), - Entry("for managed, openshift-dns", testutils.CalicoSystemScenario{ManagedCluster: true, OpenShift: true}), - ) - }) - - Context("With APIServer Deployment overrides", func() { - rr1 := corev1.ResourceRequirements{ - Limits: corev1.ResourceList{ - "cpu": resource.MustParse("2"), - "memory": resource.MustParse("300Mi"), - "storage": resource.MustParse("20Gi"), - }, - Requests: corev1.ResourceList{ - "cpu": resource.MustParse("1"), - "memory": resource.MustParse("150Mi"), - "storage": resource.MustParse("10Gi"), - }, - } - - rr2 := corev1.ResourceRequirements{ - Requests: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("250m"), - corev1.ResourceMemory: resource.MustParse("64Mi"), - }, - Limits: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("500m"), - corev1.ResourceMemory: resource.MustParse("500Mi"), - }, - } - - It("should handle APIServerDeployment overrides", func() { - var minReadySeconds int32 = 20 - - affinity := &corev1.Affinity{ - NodeAffinity: &corev1.NodeAffinity{ - RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{ - NodeSelectorTerms: []corev1.NodeSelectorTerm{{ - MatchExpressions: []corev1.NodeSelectorRequirement{{ - Key: "custom-affinity-key", - Operator: corev1.NodeSelectorOpExists, - }}, - }}, - }, - }, - } - toleration := corev1.Toleration{ - Key: "foo", - Operator: corev1.TolerationOpEqual, - Value: "bar", - } - - apiServerPort := operatorv1.APIServerDeploymentContainerPort{ - Name: render.APIServerPortName, - ContainerPort: 1111, - } - queryServerPort := operatorv1.APIServerDeploymentContainerPort{ - Name: render.QueryServerPortName, - ContainerPort: 2222, - } - l7AdmCtrlPort := operatorv1.APIServerDeploymentContainerPort{ - Name: render.L7AdmissionControllerPortName, - ContainerPort: 3333, - } - - sidecarEnabled := operatorv1.SidecarEnabled - cfg.ApplicationLayer = &operatorv1.ApplicationLayer{ - Spec: operatorv1.ApplicationLayerSpec{ - SidecarInjection: &sidecarEnabled, - }, - } - - cfg.APIServer.APIServerDeployment = &operatorv1.APIServerDeployment{ - Metadata: &operatorv1.Metadata{ - Labels: map[string]string{"top-level": "label1"}, - Annotations: map[string]string{"top-level": "annot1"}, - }, - Spec: &operatorv1.APIServerDeploymentSpec{ - MinReadySeconds: &minReadySeconds, - Template: &operatorv1.APIServerDeploymentPodTemplateSpec{ - Metadata: &operatorv1.Metadata{ - Labels: map[string]string{"template-level": "label2"}, - Annotations: map[string]string{"template-level": "annot2"}, - }, - Spec: &operatorv1.APIServerDeploymentPodSpec{ - Containers: []operatorv1.APIServerDeploymentContainer{ - { - Name: "calico-apiserver", - Resources: &rr1, - Ports: []operatorv1.APIServerDeploymentContainerPort{apiServerPort}, - }, - { - Name: "tigera-queryserver", - Resources: &rr2, - Ports: []operatorv1.APIServerDeploymentContainerPort{queryServerPort}, - }, - { - Name: "calico-l7-admission-controller", - Resources: &rr2, - Ports: []operatorv1.APIServerDeploymentContainerPort{l7AdmCtrlPort}, - }, - }, - InitContainers: []operatorv1.APIServerDeploymentInitContainer{ - { - Name: "calico-apiserver-certs-key-cert-provisioner", - Resources: &rr2, - }, - }, - NodeSelector: map[string]string{ - "custom-node-selector": "value", - }, - TopologySpreadConstraints: []corev1.TopologySpreadConstraint{ - { - MaxSkew: 1, - }, - }, - Affinity: affinity, - Tolerations: []corev1.Toleration{toleration}, - }, - }, - }, - } - // Enable certificate management. - cfg.Installation.CertificateManagement = &operatorv1.CertificateManagement{SignerName: "a.b/c", CACert: cfg.TLSKeyPair.GetCertificatePEM()} - certificateManager, err := certificatemanager.Create(cli, cfg.Installation, clusterDomain, common.OperatorNamespace(), certificatemanager.AllowCACreation()) - Expect(err).NotTo(HaveOccurred()) - - // Create and add the TLS keypair so the initContainer is rendered. - dnsNames := dns.GetServiceDNSNames(render.APIServerServiceName, render.APIServerNamespace, clusterDomain) - kp, err := certificateManager.GetOrCreateKeyPair(cli, render.CalicoAPIServerTLSSecretName, common.OperatorNamespace(), dnsNames) - Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - cfg.TLSKeyPair = kp - - qsKP, err := certificateManager.GetOrCreateKeyPair(cli, render.CalicoAPIServerTLSSecretName, common.OperatorNamespace(), dnsNames) - Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - cfg.QueryServerTLSKeyPairCertificateManagementOnly = qsKP - - component, err := render.APIServer(cfg) - Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - resources, _ := apiServerObjects(component) - - d, ok := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment").(*appsv1.Deployment) - Expect(ok).To(BeTrue()) - - // API server has apiserver: true label - Expect(d.Labels).To(HaveLen(2)) - Expect(d.Labels["apiserver"]).To(Equal("true")) - Expect(d.Labels["top-level"]).To(Equal("label1")) - Expect(d.Annotations).To(HaveLen(1)) - Expect(d.Annotations["top-level"]).To(Equal("annot1")) - - Expect(d.Spec.MinReadySeconds).To(Equal(minReadySeconds)) - - // At runtime, the operator will also add some standard labels to the - // deployment such as "k8s-app=calico-apiserver". But the APIServer - // deployment object produced by the render will have no labels so we expect just the one - // provided. - Expect(d.Spec.Template.Labels).To(HaveLen(2)) - Expect(d.Spec.Template.Labels["apiserver"]).To(Equal("true")) - Expect(d.Spec.Template.Labels["template-level"]).To(Equal("label2")) - - // With the default instance we expect 2 template-level annotations - // - 1 added by the operator by default - // - 1 added by the calicoNodeDaemonSet override - Expect(d.Spec.Template.Annotations).To(HaveLen(2)) - Expect(d.Spec.Template.Annotations).To(HaveKey("tigera-operator.hash.operator.tigera.io/calico-apiserver-certs")) - Expect(d.Spec.Template.Annotations["template-level"]).To(Equal("annot2")) - - Expect(d.Spec.Template.Spec.Containers).To(HaveLen(3)) - containersFound := 0 - for _, c := range d.Spec.Template.Spec.Containers { - switch c.Name { - case "calico-apiserver": - Expect(c.Resources).To(Equal(rr1)) - Expect(c.Ports[0].Name).To(Equal(apiServerPort.Name)) - Expect(c.Ports[0].ContainerPort).To(Equal(apiServerPort.ContainerPort)) - - Expect(c.Args[0]).To(ContainSubstring(fmt.Sprintf("--secure-port=%d", apiServerPort.ContainerPort))) - containersFound++ - case "tigera-queryserver": - Expect(c.Resources).To(Equal(rr2)) - Expect(c.Ports[0].Name).To(Equal(queryServerPort.Name)) - Expect(c.Ports[0].ContainerPort).To(Equal(queryServerPort.ContainerPort)) - - Expect(c.Env[1].Name).To(Equal("LISTEN_ADDR")) - Expect(c.Env[1].Value).To(Equal(fmt.Sprintf(":%d", queryServerPort.ContainerPort))) - containersFound++ - case "calico-l7-admission-controller": - Expect(c.Resources).To(Equal(rr2)) - Expect(c.Ports[0].Name).To(Equal(l7AdmCtrlPort.Name)) - Expect(c.Ports[0].ContainerPort).To(Equal(l7AdmCtrlPort.ContainerPort)) - - Expect(c.Env[4].Name).To(Equal("L7ADMCTRL_LISTENADDR")) - Expect(c.Env[4].Value).To(Equal(fmt.Sprintf(":%d", l7AdmCtrlPort.ContainerPort))) - containersFound++ - } - } - Expect(containersFound).To(Equal(3)) - - Expect(d.Spec.Template.Spec.InitContainers).To(HaveLen(2)) - Expect(d.Spec.Template.Spec.InitContainers[0].Name).To(Equal("calico-apiserver-certs-key-cert-provisioner")) - Expect(d.Spec.Template.Spec.InitContainers[0].Resources).To(Equal(rr2)) - - Expect(d.Spec.Template.Spec.NodeSelector).To(HaveLen(1)) - Expect(d.Spec.Template.Spec.NodeSelector).To(HaveKeyWithValue("custom-node-selector", "value")) - - Expect(d.Spec.Template.Spec.TopologySpreadConstraints).To(HaveLen(1)) - Expect(d.Spec.Template.Spec.TopologySpreadConstraints[0].MaxSkew).To(Equal(int32(1))) - - Expect(d.Spec.Template.Spec.Tolerations).To(HaveLen(1)) - Expect(d.Spec.Template.Spec.Tolerations[0]).To(Equal(toleration)) - - // Check the Service configuration - svc := rtest.GetResource(resources, "calico-api", "calico-system", "", "v1", "Service").(*corev1.Service) - Expect(svc.Spec.Ports).To(HaveLen(3)) - servicesFound := 0 - for _, p := range svc.Spec.Ports { - switch p.Name { - case render.APIServerPortName: - Expect(p.Port).To(Equal(int32(443))) - Expect(p.TargetPort.IntVal).To(Equal(apiServerPort.ContainerPort)) - servicesFound++ - case render.QueryServerPortName: - Expect(p.Port).To(Equal(int32(8080))) - Expect(p.TargetPort.IntVal).To(Equal(queryServerPort.ContainerPort)) - servicesFound++ - case render.L7AdmissionControllerPortName: - Expect(p.Port).To(Equal(int32(6443))) - Expect(p.TargetPort.IntVal).To(Equal(l7AdmCtrlPort.ContainerPort)) - servicesFound++ - } - } - Expect(servicesFound).To(Equal(3)) - }) - - It("should override a ControlPlaneNodeSelector when specified", func() { - cfg.Installation.ControlPlaneNodeSelector = map[string]string{"nodeName": "control01"} - - cfg.APIServer.APIServerDeployment = &operatorv1.APIServerDeployment{ - Spec: &operatorv1.APIServerDeploymentSpec{ - Template: &operatorv1.APIServerDeploymentPodTemplateSpec{ - Spec: &operatorv1.APIServerDeploymentPodSpec{ - NodeSelector: map[string]string{ - "custom-node-selector": "value", - }, - }, - }, - }, - } - component, err := render.APIServer(cfg) - Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := apiServerObjects(component) - d, ok := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment").(*appsv1.Deployment) - Expect(ok).To(BeTrue()) - // nodeSelectors are merged - Expect(d.Spec.Template.Spec.NodeSelector).To(HaveLen(2)) - Expect(d.Spec.Template.Spec.NodeSelector).To(HaveKeyWithValue("nodeName", "control01")) - Expect(d.Spec.Template.Spec.NodeSelector).To(HaveKeyWithValue("custom-node-selector", "value")) - }) - - It("should override ControlPlaneTolerations when specified", func() { - cfg.Installation.ControlPlaneTolerations = rmeta.TolerateControlPlane - - tol := corev1.Toleration{ - Key: "foo", - Operator: corev1.TolerationOpEqual, - Value: "bar", - Effect: corev1.TaintEffectNoExecute, - } + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/apis" + "github.com/tigera/operator/pkg/common" + "github.com/tigera/operator/pkg/components" + "github.com/tigera/operator/pkg/controller/certificatemanager" + "github.com/tigera/operator/pkg/controller/k8sapi" + ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" + "github.com/tigera/operator/pkg/dns" + "github.com/tigera/operator/pkg/extensions" + "github.com/tigera/operator/pkg/render" + rmeta "github.com/tigera/operator/pkg/render/common/meta" + "github.com/tigera/operator/pkg/render/common/podaffinity" + rtest "github.com/tigera/operator/pkg/render/common/test" + "github.com/tigera/operator/test" - cfg.APIServer.APIServerDeployment = &operatorv1.APIServerDeployment{ - Spec: &operatorv1.APIServerDeploymentSpec{ - Template: &operatorv1.APIServerDeploymentPodTemplateSpec{ - Spec: &operatorv1.APIServerDeploymentPodSpec{ - Tolerations: []corev1.Toleration{tol}, - }, - }, - }, - } - component, err := render.APIServer(cfg) - Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := apiServerObjects(component) - d, ok := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment").(*appsv1.Deployment) - Expect(ok).To(BeTrue()) - Expect(d.Spec.Template.Spec.Tolerations).To(HaveLen(1)) - Expect(d.Spec.Template.Spec.Tolerations).To(ConsistOf(tol)) - }) + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + policyv1 "k8s.io/api/policy/v1" + rbacv1 "k8s.io/api/rbac/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + apiregv1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" +) - It("should disable ValidatingAdmissionPolicy on older k8s versions", func() { - cfg.KubernetesVersion = &common.VersionInfo{ - Major: 1, - Minor: 28, - } - component, err := render.APIServer(cfg) - Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - resources, _ := apiServerObjects(component) - d := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment").(*appsv1.Deployment) - Expect(d.Spec.Template.Spec.Containers[0].Args).To(ConsistOf([]string{ - "--secure-port=5443", - "--tls-private-key-file=/calico-apiserver-certs/tls.key", - "--tls-cert-file=/calico-apiserver-certs/tls.crt", - "--audit-policy-file=/etc/tigera/audit/policy.conf", - "--audit-log-path=/var/log/calico/audit/tsee-audit.log", - "--enable-validating-admission-policy=false", - })) - }) - }) -}) +// apiServerObjects renders the API server component and applies the registered variant +// modifier the way the componentHandler does, so the Enterprise objects (query server, +// audit logging, Enterprise RBAC) and the Calico cleanup deletes are reflected in the +// returned create and delete lists. +func apiServerObjects(c render.Component) ([]client.Object, []client.Object) { + create, del := c.Objects() + rc := extensions.RenderContext{} + var extCtx any + if p, ok := c.(render.ExtensionContextProvider); ok { + ec := p.ExtensionContext().(render.APIServerExtensionContext) + rc.Installation = ec.Config.Installation + extCtx = ec + } + return applyExtensionsWithContext(ext, render.ComponentNameAPIServer, rc, extCtx, create, del) +} func verifyAPIService(service *apiregv1.APIService, enterprise bool, clusterDomain string) { Expect(service.Name).To(Equal("v3.projectcalico.org")) @@ -1535,381 +83,6 @@ func verifyAPIService(service *apiregv1.APIService, enterprise bool, clusterDoma test.VerifyCertSANs(ca, expectedDNSNames...) } -func validateTunnelSecret(voltronSecret *corev1.Secret) { - var newCert *x509.Certificate - - cert := voltronSecret.Data[corev1.TLSCertKey] - key := voltronSecret.Data[corev1.TLSPrivateKeyKey] - _, err := tls.X509KeyPair(cert, key) - Expect(err).ShouldNot(HaveOccurred()) - - roots := x509.NewCertPool() - ok := roots.AppendCertsFromPEM([]byte(cert)) - Expect(ok).To(BeTrue()) - - block, _ := pem.Decode([]byte(cert)) - Expect(err).ShouldNot(HaveOccurred()) - Expect(block).To(Not(BeNil())) - - newCert, err = x509.ParseCertificate(block.Bytes) - Expect(err).ShouldNot(HaveOccurred()) - - opts := x509.VerifyOptions{ - DNSName: "voltron", - Roots: roots, - } - - _, err = newCert.Verify(opts) - Expect(err).ShouldNot(HaveOccurred()) - - opts = x509.VerifyOptions{ - DNSName: "voltron", - Roots: x509.NewCertPool(), - CurrentTime: time.Now().Add(crypto.DefaultCACertificateLifetimeDuration + 24*time.Hour), - } - _, err = newCert.Verify(opts) - Expect(err).Should(HaveOccurred()) -} - -var ( - uiUserPolicyRules = []rbacv1.PolicyRule{ - { - APIGroups: []string{ - "projectcalico.org", - "networking.k8s.io", - "extensions", - "", - }, - Resources: []string{ - "tiers", - "networkpolicies", - "tier.networkpolicies", - "globalnetworkpolicies", - "tier.globalnetworkpolicies", - "namespaces", - "globalnetworksets", - "networksets", - "managedclusters", - "stagedglobalnetworkpolicies", - "tier.stagedglobalnetworkpolicies", - "stagednetworkpolicies", - "tier.stagednetworkpolicies", - "stagedkubernetesnetworkpolicies", - "policyrecommendationscopes", - }, - Verbs: []string{"watch", "list"}, - }, - { - APIGroups: []string{"policy.networking.k8s.io"}, - Resources: []string{ - "clusternetworkpolicies", - "adminnetworkpolicies", - "baselineadminnetworkpolicies", - }, - Verbs: []string{"watch", "list"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"packetcaptures/files"}, - Verbs: []string{"get"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"packetcaptures"}, - Verbs: []string{"get", "list", "watch"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"networks"}, - Verbs: []string{"get", "list", "watch"}, - }, - { - APIGroups: []string{""}, - Resources: []string{"pods"}, - Verbs: []string{"list"}, - }, - { - APIGroups: []string{""}, - Resources: []string{"serviceaccounts"}, - Verbs: []string{"list"}, - }, - { - APIGroups: []string{""}, - Resources: []string{"configmaps"}, - ResourceNames: []string{"coreruleset-default"}, - Verbs: []string{"get"}, - }, - { - APIGroups: []string{""}, - Resources: []string{"services/proxy"}, - ResourceNames: []string{ - "https:calico-api:8080", "calico-node-prometheus:9090", - }, - Verbs: []string{"get", "create"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"tiers"}, - Verbs: []string{"get"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"globalreports"}, - Verbs: []string{"get", "list"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"globalreporttypes"}, - Verbs: []string{"get"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"clusterinformations"}, - Verbs: []string{"get", "list"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"hostendpoints"}, - Verbs: []string{"get", "list"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{ - "alertexceptions", - "globalalerts", - "globalalerts/status", - "globalalerttemplates", - "globalthreatfeeds", - "globalthreatfeeds/status", - "securityeventwebhooks", - }, - Verbs: []string{"get", "watch", "list"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"uisettingsgroups"}, - Verbs: []string{"get"}, - ResourceNames: []string{"cluster-settings", "user-settings"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"uisettingsgroups/data"}, - Verbs: []string{"get", "list", "watch"}, - ResourceNames: []string{"cluster-settings"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"uisettingsgroups/data"}, - Verbs: []string{"*"}, - ResourceNames: []string{"user-settings"}, - }, - { - APIGroups: []string{"lma.tigera.io"}, - Resources: []string{"*"}, - ResourceNames: []string{ - "flows", "audit*", "l7", "events", "dns", "waf", "kibana_login", "recommendations", - }, - Verbs: []string{"get"}, - }, - { - APIGroups: []string{"operator.tigera.io"}, - Resources: []string{"applicationlayers", "packetcaptureapis", "compliances", "intrusiondetections"}, - Verbs: []string{"get"}, - }, - { - APIGroups: []string{"apps"}, - Resources: []string{"deployments"}, - Verbs: []string{"get", "list", "watch"}, - }, - { - APIGroups: []string{""}, - Resources: []string{"services"}, - Verbs: []string{"get", "list", "watch"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"felixconfigurations"}, - Verbs: []string{"get", "list"}, - }, - { - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{"securityeventwebhooks"}, - Verbs: []string{"get", "list"}, - }, - } - networkAdminPolicyRules = []rbacv1.PolicyRule{ - { - APIGroups: []string{ - "projectcalico.org", - "networking.k8s.io", - "extensions", - }, - Resources: []string{ - "tiers", - "networkpolicies", - "tier.networkpolicies", - "globalnetworkpolicies", - "tier.globalnetworkpolicies", - "stagedglobalnetworkpolicies", - "tier.stagedglobalnetworkpolicies", - "stagednetworkpolicies", - "tier.stagednetworkpolicies", - "stagedkubernetesnetworkpolicies", - "globalnetworksets", - "networksets", - "managedclusters", - "packetcaptures", - "policyrecommendationscopes", - }, - Verbs: []string{"create", "update", "delete", "patch", "get", "watch", "list"}, - }, - { - APIGroups: []string{ - "policy.networking.k8s.io", - }, - Resources: []string{ - "clusternetworkpolicies", - "adminnetworkpolicies", - "baselineadminnetworkpolicies", - }, - Verbs: []string{"create", "update", "delete", "patch", "get", "watch", "list"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"packetcaptures/files"}, - Verbs: []string{"get", "delete"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"networks"}, - Verbs: []string{"create", "update", "delete", "patch", "get", "watch", "list"}, - }, - { - APIGroups: []string{""}, - Resources: []string{"namespaces"}, - Verbs: []string{"watch", "list"}, - }, - { - APIGroups: []string{""}, - Resources: []string{"pods"}, - Verbs: []string{"list"}, - }, - { - APIGroups: []string{""}, - Resources: []string{"serviceaccounts"}, - Verbs: []string{"list"}, - }, - { - APIGroups: []string{""}, - Resources: []string{"configmaps"}, - ResourceNames: []string{"coreruleset-default"}, - Verbs: []string{"get"}, - }, - { - APIGroups: []string{""}, - Resources: []string{"services/proxy"}, - ResourceNames: []string{ - "https:calico-api:8080", "calico-node-prometheus:9090", - }, - Verbs: []string{"get", "create"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"globalreports"}, - Verbs: []string{"*"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"globalreports/status"}, - Verbs: []string{"get", "list", "watch"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"globalreporttypes"}, - Verbs: []string{"get"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"clusterinformations"}, - Verbs: []string{"get", "list"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"hostendpoints"}, - Verbs: []string{"get", "list"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{ - "alertexceptions", - "globalalerts", - "globalalerts/status", - "globalalerttemplates", - "globalthreatfeeds", - "globalthreatfeeds/status", - "securityeventwebhooks", - }, - Verbs: []string{"create", "update", "delete", "patch", "get", "watch", "list"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"uisettingsgroups"}, - Verbs: []string{"get", "patch", "update"}, - ResourceNames: []string{"cluster-settings", "user-settings"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"uisettingsgroups/data"}, - Verbs: []string{"*"}, - ResourceNames: []string{"cluster-settings", "user-settings"}, - }, - { - APIGroups: []string{"lma.tigera.io"}, - Resources: []string{"*"}, - ResourceNames: []string{ - "flows", "audit*", "l7", "events", "dns", "waf", "kibana_login", "elasticsearch_superuser", "recommendations", - }, - Verbs: []string{"get"}, - }, - { - APIGroups: []string{"operator.tigera.io"}, - Resources: []string{"applicationlayers", "packetcaptureapis", "compliances", "intrusiondetections"}, - Verbs: []string{"get", "update", "patch", "create", "delete"}, - }, - { - APIGroups: []string{"apps"}, - Resources: []string{"deployments"}, - Verbs: []string{"get", "list", "watch", "patch"}, - }, - { - APIGroups: []string{""}, - Resources: []string{"services"}, - Verbs: []string{"get", "list", "watch", "patch"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"felixconfigurations"}, - Verbs: []string{"get", "list"}, - }, - { - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{"securityeventwebhooks"}, - Verbs: []string{"get", "list", "update", "patch", "create", "delete"}, - }, - { - APIGroups: []string{""}, - Resources: []string{"secrets"}, - Verbs: []string{"create"}, - }, - { - APIGroups: []string{""}, - Resources: []string{"secrets"}, - ResourceNames: []string{"webhooks-secret"}, - Verbs: []string{"patch"}, - }, - } -) - var _ = Describe("API server rendering tests (Calico)", func() { var instance *operatorv1.InstallationSpec var apiserver *operatorv1.APIServerSpec @@ -2381,9 +554,6 @@ var _ = Describe("API server rendering tests (Calico)", func() { kp, err := certificateManager.GetOrCreateKeyPair(cli, render.CalicoAPIServerTLSSecretName, common.OperatorNamespace(), dnsNames) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) cfg.TLSKeyPair = kp - qskp, err := certificateManager.GetOrCreateKeyPair(cli, render.CalicoAPIServerTLSSecretName, common.OperatorNamespace(), dnsNames) - Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - cfg.QueryServerTLSKeyPairCertificateManagementOnly = qskp component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) @@ -2423,7 +593,7 @@ var _ = Describe("API server rendering tests (Calico)", func() { Expect(d.Spec.Template.Spec.Containers[0].Ports[0].ContainerPort).To(Equal(apiServerPort.ContainerPort)) Expect(d.Spec.Template.Spec.Containers[0].Args[0]).To(ContainSubstring(fmt.Sprintf("--secure-port=%d", apiServerPort.ContainerPort))) - Expect(d.Spec.Template.Spec.InitContainers).To(HaveLen(2)) + Expect(d.Spec.Template.Spec.InitContainers).To(HaveLen(1)) Expect(d.Spec.Template.Spec.InitContainers[0].Name).To(Equal("calico-apiserver-certs-key-cert-provisioner")) Expect(d.Spec.Template.Spec.InitContainers[0].Resources).To(Equal(rr2)) @@ -2514,62 +684,4 @@ var _ = Describe("API server rendering tests (Calico)", func() { })) }) }) - - Context("multi-tenant", func() { - BeforeEach(func() { - cfg.MultiTenant = true - cfg.ManagementCluster = &operatorv1.ManagementCluster{Spec: operatorv1.ManagementClusterSpec{Address: "example.com:1234"}} - cfg.Installation = &operatorv1.InstallationSpec{ - ControlPlaneReplicas: ptr.To(int32(2)), - Registry: "testregistry.com/", - Variant: operatorv1.CalicoEnterprise, - } - }) - - It("should not install tigera-network-admin and tigera-ui-user", func() { - component, err := render.APIServer(cfg) - Expect(err).NotTo(HaveOccurred()) - - // Expect no UISettings / UISettingsGroups to be installed. - resources, _ := apiServerObjects(component) - obj := rtest.GetResource(resources, "tigera-network-admin", "", "rbac.authorization.k8s.io", "v1", "ClusterRole") - Expect(obj).To(BeNil()) - obj = rtest.GetResource(resources, "tigera-ui-user", "", "rbac.authorization.k8s.io", "v1", "ClusterRole") - Expect(obj).To(BeNil()) - }) - - It("should create a cluster role that get managed clusters", func() { - component, err := render.APIServer(cfg) - Expect(err).NotTo(HaveOccurred()) - - resources, _ := apiServerObjects(component) - managedClusterAccessRole := rtest.GetResource(resources, - render.MultiTenantManagedClustersAccessClusterRoleName, "", rbacv1.GroupName, "v1", "ClusterRole").(*rbacv1.ClusterRole) - expectedManagedClusterAccessRules := []rbacv1.PolicyRule{ - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"managedclusters"}, - Verbs: []string{"get"}, - }, - } - Expect(managedClusterAccessRole.Rules).To(ContainElements(expectedManagedClusterAccessRules)) - }) - - It("should create a cluster role for watching managed clusters", func() { - component, err := render.APIServer(cfg) - Expect(err).NotTo(HaveOccurred()) - - resources, _ := apiServerObjects(component) - managedClusterAccessRole := rtest.GetResource(resources, - render.ManagedClustersWatchClusterRoleName, "", rbacv1.GroupName, "v1", "ClusterRole").(*rbacv1.ClusterRole) - expectedManagedClusterAccessRules := []rbacv1.PolicyRule{ - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"managedclusters"}, - Verbs: []string{"get", "list", "watch"}, - }, - } - Expect(managedClusterAccessRole.Rules).To(ContainElements(expectedManagedClusterAccessRules)) - }) - }) }) From a2f4fcb1f115cb7eef688c12e26941d47e7090e9 Mon Sep 17 00:00:00 2001 From: Casey Davenport Date: Wed, 24 Jun 2026 11:59:30 -0700 Subject: [PATCH 41/54] Move RenderContext into render and ControllerContext into a controllers package RenderContext is a render-phase input (what a modifier consumes), and ControllerContext is a controller-phase concept (the data and machinery a controller gathers). Neither is part of the extension mechanism itself, so having them in pkg/extensions was backwards. Move RenderContext to pkg/render and ControllerContext (plus ControllerName) to a new pkg/controller/contexts. extensions already imported render and render never imports extensions, so the direction is clean: extensions and contexts both depend on render, and extensions depends on contexts. No cycle. --- .../apiserver/apiserver_controller.go | 10 +-- .../clusterconnection_controller.go | 3 +- pkg/controller/contexts/context.go | 67 +++++++++++++++++++ .../installation/core_controller.go | 12 ++-- .../installation/windows_controller.go | 10 +-- pkg/controller/utils/component.go | 6 +- .../utils/component_enterprise_test.go | 3 +- pkg/controller/utils/component_test.go | 4 +- pkg/enterprise/apiserver.go | 13 ++-- pkg/enterprise/apiserver_test.go | 15 +++-- pkg/enterprise/decorate_helpers_test.go | 5 +- pkg/enterprise/guardian.go | 4 +- pkg/enterprise/guardian_test.go | 4 +- pkg/enterprise/installation.go | 8 +-- pkg/enterprise/installation_test.go | 14 ++-- pkg/enterprise/kubecontrollers.go | 11 +-- pkg/enterprise/kubecontrollers_test.go | 21 +++--- pkg/enterprise/node.go | 10 +-- pkg/enterprise/node_test.go | 8 +-- pkg/enterprise/register.go | 7 +- pkg/enterprise/typha.go | 2 +- pkg/enterprise/typha_test.go | 7 +- pkg/enterprise/windows.go | 17 ++--- pkg/enterprise/windows_test.go | 13 ++-- pkg/extensions/controllerextension.go | 48 ++----------- pkg/extensions/controllerextension_test.go | 42 ++++++------ pkg/extensions/decorate_helpers_test.go | 5 +- pkg/extensions/extension.go | 4 +- pkg/extensions/extension_test.go | 15 +++-- pkg/extensions/set.go | 15 +++-- pkg/extensions/variant.go | 23 ++++--- pkg/render/apiserver_test.go | 3 +- pkg/render/component.go | 8 +-- pkg/render/decorate_helpers_test.go | 5 +- pkg/render/guardian_test.go | 7 +- pkg/render/node_enterprise_test.go | 17 ++--- pkg/{extensions => render}/rendercontext.go | 10 ++- pkg/render/windows_test.go | 3 +- 38 files changed, 262 insertions(+), 217 deletions(-) create mode 100644 pkg/controller/contexts/context.go rename pkg/{extensions => render}/rendercontext.go (83%) diff --git a/pkg/controller/apiserver/apiserver_controller.go b/pkg/controller/apiserver/apiserver_controller.go index 212a4cfe43..32b9a795d7 100644 --- a/pkg/controller/apiserver/apiserver_controller.go +++ b/pkg/controller/apiserver/apiserver_controller.go @@ -41,6 +41,7 @@ import ( apiserver "github.com/tigera/operator/pkg/common/validation/apiserver" webhooksvalidation "github.com/tigera/operator/pkg/common/validation/webhooks" "github.com/tigera/operator/pkg/controller/certificatemanager" + "github.com/tigera/operator/pkg/controller/contexts" "github.com/tigera/operator/pkg/controller/k8sapi" "github.com/tigera/operator/pkg/controller/migration/datastoremigration" "github.com/tigera/operator/pkg/controller/options" @@ -49,7 +50,6 @@ import ( "github.com/tigera/operator/pkg/controller/utils/imageset" "github.com/tigera/operator/pkg/ctrlruntime" "github.com/tigera/operator/pkg/dns" - "github.com/tigera/operator/pkg/extensions" "github.com/tigera/operator/pkg/render" rcertificatemanagement "github.com/tigera/operator/pkg/render/certificatemanagement" "github.com/tigera/operator/pkg/render/common/networkpolicy" @@ -106,7 +106,7 @@ func Add(mgr manager.Manager, opts options.ControllerOptions) error { if opts.EnterpriseCRDExists { // The variant extension registers the enterprise watches it needs (the management // cluster CRs, ApplicationLayer, Authentication, and the tunnel secrets). - if err = opts.Extensions.SetupWatches(extensions.APIServerController, c); err != nil { + if err = opts.Extensions.SetupWatches(contexts.APIServerController, c); err != nil { return fmt.Errorf("apiserver-controller failed to set up extension watches: %w", err) } } @@ -299,12 +299,12 @@ func (r *ReconcileAPIServer) Reconcile(ctx context.Context, request reconcile.Re // enterprise render data (the trusted bundle, the query server certificate, the // management cluster CRs, the OIDC config, and the resolved L7 sidecar images). For // the core operator this is a no-op and the render context carries no extension data. - cc := extensions.ControllerContext{ - RenderContext: extensions.RenderContext{ + cc := contexts.ControllerContext{ + RenderContext: render.RenderContext{ Installation: installationSpec, ClusterDomain: r.opts.ClusterDomain, }, - Controller: extensions.APIServerController, + Controller: contexts.APIServerController, Ctx: ctx, Client: r.client, CertificateManager: certificateManager, diff --git a/pkg/controller/clusterconnection/clusterconnection_controller.go b/pkg/controller/clusterconnection/clusterconnection_controller.go index c51bc19238..34cba8d367 100644 --- a/pkg/controller/clusterconnection/clusterconnection_controller.go +++ b/pkg/controller/clusterconnection/clusterconnection_controller.go @@ -49,7 +49,6 @@ import ( "github.com/tigera/operator/pkg/controller/utils/imageset" "github.com/tigera/operator/pkg/ctrlruntime" "github.com/tigera/operator/pkg/dns" - "github.com/tigera/operator/pkg/extensions" "github.com/tigera/operator/pkg/render" "github.com/tigera/operator/pkg/render/common/networkpolicy" "github.com/tigera/operator/pkg/render/goldmane" @@ -449,7 +448,7 @@ func (r *ReconcileConnection) Reconcile(ctx context.Context, request reconcile.R r.cli, r.scheme, managementClusterConnection, - utils.WithRenderContext(extensions.RenderContext{Installation: installationSpec}), + utils.WithRenderContext(render.RenderContext{Installation: installationSpec}), utils.WithExtensions(r.opts.Extensions), ) guardianCfg := &render.GuardianConfiguration{ diff --git a/pkg/controller/contexts/context.go b/pkg/controller/contexts/context.go new file mode 100644 index 0000000000..eb5baf4eab --- /dev/null +++ b/pkg/controller/contexts/context.go @@ -0,0 +1,67 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package contexts holds the controller-phase context types passed between a +// controller's reconcile and a variant extension. They live here, not in the +// extensions package, because they are controller concepts (the data and +// machinery a controller gathers), not part of the extension mechanism itself - +// the extensions package consumes them. +package contexts + +import ( + "context" + + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/tigera/operator/pkg/controller/certificatemanager" + "github.com/tigera/operator/pkg/render" +) + +// ControllerName identifies the controller a ControllerExtension extends, so a +// variant can register a different hook per controller. Use the constants below +// rather than bare strings so registration and lookup stay in sync. +type ControllerName string + +const ( + InstallationController ControllerName = "installation" + WindowsController ControllerName = "windows" + APIServerController ControllerName = "apiserver" +) + +// ControllerContext is the controller-phase context, the corollary to the +// render-phase render.RenderContext. It is the embedded RenderContext (the same +// data the render phase sees) plus the controller-side machinery a +// ControllerExtension needs to produce artifacts: a client, a certificate +// manager, a context. Those deps live here, not on RenderContext, so the +// modifiers that read RenderContext can't do I/O - they only transform objects. +// +// Controller names which controller is reconciling, selecting that controller's +// extension hook. The controller fills the embedded RenderContext's data fields, +// the deps, and Controller; ExtendContext does its work, sets the produced +// artifacts on the embedded context, and returns it. +type ControllerContext struct { + render.RenderContext + + // Controller identifies the reconciling controller, selecting its hook. + Controller ControllerName + + Ctx context.Context + Client client.Client + CertificateManager certificatemanager.CertificateManager + + // MultiTenant reports whether the operator runs in multi-tenant mode. It's a + // controller-phase operator mode some extensions gate on (e.g. the API server's + // management-cluster tunnel-secret check). + MultiTenant bool +} diff --git a/pkg/controller/installation/core_controller.go b/pkg/controller/installation/core_controller.go index fdb258e76d..3ef8ac48b7 100644 --- a/pkg/controller/installation/core_controller.go +++ b/pkg/controller/installation/core_controller.go @@ -63,6 +63,7 @@ import ( "github.com/tigera/operator/pkg/common/discovery" "github.com/tigera/operator/pkg/components" "github.com/tigera/operator/pkg/controller/certificatemanager" + "github.com/tigera/operator/pkg/controller/contexts" "github.com/tigera/operator/pkg/controller/ippool" "github.com/tigera/operator/pkg/controller/k8sapi" "github.com/tigera/operator/pkg/controller/migration" @@ -73,7 +74,6 @@ import ( "github.com/tigera/operator/pkg/controller/utils" "github.com/tigera/operator/pkg/controller/utils/imageset" "github.com/tigera/operator/pkg/ctrlruntime" - "github.com/tigera/operator/pkg/extensions" "github.com/tigera/operator/pkg/imports/admission" "github.com/tigera/operator/pkg/imports/crds" "github.com/tigera/operator/pkg/render" @@ -222,7 +222,7 @@ func Add(mgr manager.Manager, opts options.ControllerOptions) error { } if opts.EnterpriseCRDExists { - if err = opts.Extensions.SetupWatches(extensions.InstallationController, c); err != nil { + if err = opts.Extensions.SetupWatches(contexts.InstallationController, c); err != nil { return fmt.Errorf("tigera-installation-controller failed to set up extension watches: %w", err) } @@ -1152,14 +1152,14 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile calicoVersion = components.EnterpriseRelease } - cc := extensions.ControllerContext{ - RenderContext: extensions.RenderContext{ + cc := contexts.ControllerContext{ + RenderContext: render.RenderContext{ Installation: &instance.Spec, FelixConfiguration: felixConfiguration, ClusterDomain: r.opts.ClusterDomain, TrustedBundle: typhaNodeTLS.TrustedBundle, }, - Controller: extensions.InstallationController, + Controller: contexts.InstallationController, Ctx: ctx, Client: r.client, CertificateManager: certificateManager, @@ -1903,7 +1903,7 @@ func (r *ReconcileInstallation) setDefaultsOnFelixConfiguration(ctx context.Cont // Variant-specific FelixConfiguration defaults (e.g. the Enterprise // provider-specific dnsTrustedServers) are owned by the variant extension. - extUpdated, err := r.opts.Extensions.DefaultFelixConfiguration(extensions.InstallationController, &install.Spec, fc) + extUpdated, err := r.opts.Extensions.DefaultFelixConfiguration(contexts.InstallationController, &install.Spec, fc) if err != nil { return updated, err } diff --git a/pkg/controller/installation/windows_controller.go b/pkg/controller/installation/windows_controller.go index 1c05e72227..87750f134d 100644 --- a/pkg/controller/installation/windows_controller.go +++ b/pkg/controller/installation/windows_controller.go @@ -43,13 +43,13 @@ import ( "github.com/tigera/operator/pkg/active" "github.com/tigera/operator/pkg/common" "github.com/tigera/operator/pkg/controller/certificatemanager" + "github.com/tigera/operator/pkg/controller/contexts" "github.com/tigera/operator/pkg/controller/k8sapi" "github.com/tigera/operator/pkg/controller/options" "github.com/tigera/operator/pkg/controller/status" "github.com/tigera/operator/pkg/controller/utils" "github.com/tigera/operator/pkg/controller/utils/imageset" "github.com/tigera/operator/pkg/ctrlruntime" - "github.com/tigera/operator/pkg/extensions" "github.com/tigera/operator/pkg/render" ) @@ -148,7 +148,7 @@ func AddWindowsController(mgr manager.Manager, opts options.ControllerOptions) e go utils.WaitToAddResourceWatch(c, opts.K8sClientset, logw, ri.ipamConfigWatchReady, []client.Object{&v3.IPAMConfiguration{TypeMeta: metav1.TypeMeta{Kind: v3.KindIPAMConfiguration}}}) if ri.opts.EnterpriseCRDExists { - if err = ri.opts.Extensions.SetupWatches(extensions.WindowsController, c); err != nil { + if err = ri.opts.Extensions.SetupWatches(contexts.WindowsController, c); err != nil { return fmt.Errorf("tigera-windows-controller failed to set up extension watches: %w", err) } } @@ -340,14 +340,14 @@ func (r *ReconcileWindows) Reconcile(ctx context.Context, request reconcile.Requ // Run the variant's windows controller extension to build the render context // (creating no enterprise artifacts in core). - cc := extensions.ControllerContext{ - RenderContext: extensions.RenderContext{ + cc := contexts.ControllerContext{ + RenderContext: render.RenderContext{ Installation: &instance.Spec, FelixConfiguration: felixConfiguration, ClusterDomain: r.opts.ClusterDomain, TrustedBundle: typhaNodeTLS.TrustedBundle, }, - Controller: extensions.WindowsController, + Controller: contexts.WindowsController, Ctx: ctx, Client: r.client, CertificateManager: certificateManager, diff --git a/pkg/controller/utils/component.go b/pkg/controller/utils/component.go index b8f97b1fc3..9bfcc7c8ed 100644 --- a/pkg/controller/utils/component.go +++ b/pkg/controller/utils/component.go @@ -78,9 +78,9 @@ type ComponentHandler interface { // ComponentHandlerOption configures a componentHandler. type ComponentHandlerOption func(*componentHandler) -// WithRenderContext supplies the extensions.RenderContext passed to registered +// WithRenderContext supplies the render.RenderContext passed to registered // render modifiers. -func WithRenderContext(ctx extensions.RenderContext) ComponentHandlerOption { +func WithRenderContext(ctx render.RenderContext) ComponentHandlerOption { return func(c *componentHandler) { c.renderCtx = ctx } } @@ -114,7 +114,7 @@ type componentHandler struct { log logr.Logger createOnly bool apiGroupEnvs []v1.EnvVar - renderCtx extensions.RenderContext + renderCtx render.RenderContext extensions *extensions.Set } diff --git a/pkg/controller/utils/component_enterprise_test.go b/pkg/controller/utils/component_enterprise_test.go index 12d63fa44c..cc749be8c2 100644 --- a/pkg/controller/utils/component_enterprise_test.go +++ b/pkg/controller/utils/component_enterprise_test.go @@ -34,7 +34,6 @@ import ( ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" "github.com/tigera/operator/pkg/dns" "github.com/tigera/operator/pkg/enterprise" - "github.com/tigera/operator/pkg/extensions" "github.com/tigera/operator/pkg/render" ) @@ -73,7 +72,7 @@ var _ = Describe("componentHandler enterprise modifier integration", func() { }, }) - renderCtx := extensions.RenderContext{Installation: instance} + renderCtx := render.RenderContext{Installation: instance} handler := utils.NewComponentHandler(logf.Log, cli, scheme, nil, utils.WithRenderContext(renderCtx), utils.WithExtensions(enterprise.New())) Expect(handler.CreateOrUpdateOrDelete(context.Background(), comp, nil)).NotTo(HaveOccurred()) diff --git a/pkg/controller/utils/component_test.go b/pkg/controller/utils/component_test.go index 776ef6b4a5..8bf522013f 100644 --- a/pkg/controller/utils/component_test.go +++ b/pkg/controller/utils/component_test.go @@ -2580,7 +2580,7 @@ func (mc *mockClient) SubResource(subResource string) client.SubResourceClient { var _ = Describe("componentHandler modifier application", func() { It("applies registered modifiers to a named component before create", func() { ext := extensions.NewSet() - ext.Variant(operatorv1.CalicoEnterprise).Modify("fake", func(ctx extensions.RenderContext, objs, del []client.Object) ([]client.Object, []client.Object) { + ext.Variant(operatorv1.CalicoEnterprise).Modify("fake", func(ctx render.RenderContext, objs, del []client.Object) ([]client.Object, []client.Object) { cm := objs[0].(*corev1.ConfigMap) cm.Data = map[string]string{"patched": "yes"} return objs, del @@ -2591,7 +2591,7 @@ var _ = Describe("componentHandler modifier application", func() { Expect(corev1.SchemeBuilder.AddToScheme(s)).NotTo(HaveOccurred()) c := ctrlrfake.DefaultFakeClientBuilder(s).Build() - renderCtx := extensions.RenderContext{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.CalicoEnterprise}} + renderCtx := render.RenderContext{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.CalicoEnterprise}} handler := NewComponentHandler(logf.Log, c, s, nil, WithRenderContext(renderCtx), WithExtensions(ext)) comp := &namedFakeComponent{name: "fake", obj: &corev1.ConfigMap{ TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"}, diff --git a/pkg/enterprise/apiserver.go b/pkg/enterprise/apiserver.go index 0ad765e362..34b2978e1f 100644 --- a/pkg/enterprise/apiserver.go +++ b/pkg/enterprise/apiserver.go @@ -37,6 +37,7 @@ import ( operatorv1 "github.com/tigera/operator/api/v1" "github.com/tigera/operator/pkg/common" "github.com/tigera/operator/pkg/components" + "github.com/tigera/operator/pkg/controller/contexts" "github.com/tigera/operator/pkg/controller/utils" "github.com/tigera/operator/pkg/controller/utils/imageset" "github.com/tigera/operator/pkg/ctrlruntime" @@ -75,7 +76,7 @@ type apiServerRenderData struct { // apiServerData pulls the API server hook's render data back out of the render context, // returning the zero value when none is set. -func apiServerData(rc extensions.RenderContext) apiServerRenderData { +func apiServerData(rc render.RenderContext) apiServerRenderData { d, _ := rc.Extension.(apiServerRenderData) return d } @@ -109,7 +110,7 @@ type apiServerControllerExtension struct{} // Validate rejects an API server configuration Calico Enterprise does not support: a // cluster cannot be both a management cluster and a managed cluster. -func (apiServerControllerExtension) Validate(cc extensions.ControllerContext) error { +func (apiServerControllerExtension) Validate(cc contexts.ControllerContext) error { managementCluster, err := utils.GetManagementCluster(cc.Ctx, cc.Client) if err != nil { return fmt.Errorf("error reading ManagementCluster: %w", err) @@ -150,7 +151,7 @@ func (apiServerControllerExtension) Watches(c ctrlruntime.Controller) error { // fetches the enterprise CRs, creates the query server certificate, resolves the L7 // sidecar images, and stashes them for the modifiers. The base API server render carries // none of this. -func (apiServerControllerExtension) ExtendContext(cc extensions.ControllerContext) (extensions.RenderContext, []certificatemanagement.KeyPairInterface, error) { +func (apiServerControllerExtension) ExtendContext(cc contexts.ControllerContext) (render.RenderContext, []certificatemanagement.KeyPairInterface, error) { rc := cc.RenderContext in := cc.Installation @@ -282,7 +283,7 @@ func registerAPIServerCleanup(v *extensions.Variant) { // modifyAPIServer layers Calico Enterprise behavior onto the rendered API server objects: // the query server container and its volumes, audit logging on the aggregation API server // container, the Enterprise RBAC objects, and the query server port on the Service. -func modifyAPIServer(rc extensions.RenderContext, ec render.APIServerExtensionContext, create, del []client.Object) ([]client.Object, []client.Object) { +func modifyAPIServer(rc render.RenderContext, ec render.APIServerExtensionContext, create, del []client.Object) ([]client.Object, []client.Object) { c := &apiServer{cfg: ec.Config, calicoImage: ec.CalicoImage, data: apiServerData(rc)} // Ensure the deployment and its supporting objects exist. The base renders them when @@ -375,7 +376,7 @@ func modifyAPIServer(rc extensions.RenderContext, ec render.APIServerExtensionCo // cleanupAPIServer deletes the Enterprise API server objects when running Calico, so a // cluster switched from Enterprise to Calico does not leave them behind. -func cleanupAPIServer(rc extensions.RenderContext, ec render.APIServerExtensionContext, create, del []client.Object) ([]client.Object, []client.Object) { +func cleanupAPIServer(rc render.RenderContext, ec render.APIServerExtensionContext, create, del []client.Object) ([]client.Object, []client.Object) { c := &apiServer{cfg: ec.Config} del = append(del, c.tigeraAPIServerClusterRole(), c.tigeraAPIServerClusterRoleBinding()) @@ -636,7 +637,7 @@ func (c *apiServer) sidecarMutatingWebhookConfig() *admregv1.MutatingWebhookConf // the OIDC egress rule (when an OIDC key validator is configured) and the L7 admission // controller ingress port (when sidecar injection is enabled). The base policy carries // neither. -func modifyAPIServerPolicy(rc extensions.RenderContext, ec render.APIServerExtensionContext, create, del []client.Object) ([]client.Object, []client.Object) { +func modifyAPIServerPolicy(rc render.RenderContext, ec render.APIServerExtensionContext, create, del []client.Object) ([]client.Object, []client.Object) { c := &apiServer{cfg: ec.Config, data: apiServerData(rc)} policy, ok := extensions.FindObject[*v3.NetworkPolicy](create, render.APIServerPolicyName) diff --git a/pkg/enterprise/apiserver_test.go b/pkg/enterprise/apiserver_test.go index d8fce991cc..2c769b1ed8 100644 --- a/pkg/enterprise/apiserver_test.go +++ b/pkg/enterprise/apiserver_test.go @@ -33,6 +33,7 @@ import ( "github.com/tigera/operator/pkg/apis" "github.com/tigera/operator/pkg/common" "github.com/tigera/operator/pkg/controller/certificatemanager" + "github.com/tigera/operator/pkg/controller/contexts" "github.com/tigera/operator/pkg/controller/k8sapi" "github.com/tigera/operator/pkg/controller/utils" ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" @@ -48,7 +49,7 @@ const apiServerClusterDomain = "cluster.local" // seeded with a fake client that holds objs. The returned context carries a real // certificate manager and trusted bundle, so ExtendContext can create the query server // cert and the bundle the modifiers consume. -func apiServerControllerContext(variant operatorv1.ProductVariant, install *operatorv1.InstallationSpec, objs ...client.Object) extensions.ControllerContext { +func apiServerControllerContext(variant operatorv1.ProductVariant, install *operatorv1.InstallationSpec, objs ...client.Object) contexts.ControllerContext { scheme := runtime.NewScheme() Expect(apis.AddToScheme(scheme, false)).NotTo(HaveOccurred()) c := ctrlrfake.DefaultFakeClientBuilder(scheme).Build() @@ -64,13 +65,13 @@ func apiServerControllerContext(variant operatorv1.ProductVariant, install *oper certManager, err := certificatemanager.Create(c, install, apiServerClusterDomain, common.OperatorNamespace(), certificatemanager.AllowCACreation()) Expect(err).NotTo(HaveOccurred()) - return extensions.ControllerContext{ - RenderContext: extensions.RenderContext{ + return contexts.ControllerContext{ + RenderContext: render.RenderContext{ Installation: install, ClusterDomain: apiServerClusterDomain, TrustedBundle: certManager.CreateTrustedBundle(), }, - Controller: extensions.APIServerController, + Controller: contexts.APIServerController, Ctx: context.Background(), Client: c, CertificateManager: certManager, @@ -79,7 +80,7 @@ func apiServerControllerContext(variant operatorv1.ProductVariant, install *oper // apiServerKeyPair issues the API server TLS keypair from the context's certificate // manager, the way the controller does before rendering. -func apiServerKeyPair(cc extensions.ControllerContext) certificatemanagement.KeyPairInterface { +func apiServerKeyPair(cc contexts.ControllerContext) certificatemanagement.KeyPairInterface { dnsNames := dns.GetServiceDNSNames(render.APIServerServiceName, render.APIServerNamespace, cc.ClusterDomain) kp, err := cc.CertificateManager.GetOrCreateKeyPair(cc.Client, render.CalicoAPIServerTLSSecretName, common.OperatorNamespace(), dnsNames) Expect(err).NotTo(HaveOccurred()) @@ -137,7 +138,7 @@ var _ = Describe("API server enterprise modifier", func() { // renderAPIServer builds the base API server objects and applies the enterprise // modifier, the way the component handler does. It returns the create and delete // lists after the modifier ran. - renderAPIServer := func(cc extensions.ControllerContext, rc extensions.RenderContext, kp certificatemanagement.KeyPairInterface) ([]client.Object, []client.Object) { + renderAPIServer := func(cc contexts.ControllerContext, rc render.RenderContext, kp certificatemanagement.KeyPairInterface) ([]client.Object, []client.Object) { cfg := &render.APIServerConfiguration{ RequiresAggregationServer: true, K8SServiceEndpoint: k8sapi.ServiceEndpoint{}, @@ -357,7 +358,7 @@ var _ = Describe("API server enterprise modifier", func() { }) var _ = Describe("API server enterprise policy modifier", func() { - applyPolicy := func(cc extensions.ControllerContext, rc extensions.RenderContext) *v3.NetworkPolicy { + applyPolicy := func(cc contexts.ControllerContext, rc render.RenderContext) *v3.NetworkPolicy { cfg := &render.APIServerConfiguration{ RequiresAggregationServer: true, K8SServiceEndpoint: k8sapi.ServiceEndpoint{}, diff --git a/pkg/enterprise/decorate_helpers_test.go b/pkg/enterprise/decorate_helpers_test.go index bf9c6a72d7..042715d5c7 100644 --- a/pkg/enterprise/decorate_helpers_test.go +++ b/pkg/enterprise/decorate_helpers_test.go @@ -19,6 +19,7 @@ import ( operatorv1 "github.com/tigera/operator/api/v1" "github.com/tigera/operator/pkg/extensions" + "github.com/tigera/operator/pkg/render" rmeta "github.com/tigera/operator/pkg/render/common/meta" ) @@ -59,13 +60,13 @@ func (s stubExtComponent) ExtensionContext() any { // applyExtensions decorates a stub component holding the given objects with the // extension registered under key, then renders it. For a modifier that needs the // component's typed config, use applyExtensionsWithContext. -func applyExtensions(s *extensions.Set, key string, rc extensions.RenderContext, create, del []client.Object) ([]client.Object, []client.Object) { +func applyExtensions(s *extensions.Set, key string, rc render.RenderContext, create, del []client.Object) ([]client.Object, []client.Object) { return applyExtensionsWithContext(s, key, rc, nil, create, del) } // applyExtensionsWithContext is applyExtensions for a modifier that reads the // component's typed config: extCtx is delivered as the stub's ExtensionContext. -func applyExtensionsWithContext(s *extensions.Set, key string, rc extensions.RenderContext, extCtx any, create, del []client.Object) ([]client.Object, []client.Object) { +func applyExtensionsWithContext(s *extensions.Set, key string, rc render.RenderContext, extCtx any, create, del []client.Object) ([]client.Object, []client.Object) { stub := stubExtComponent{key: key, extCtx: extCtx, create: create, delete: del} return s.Decorate(stub, rc).Objects() } diff --git a/pkg/enterprise/guardian.go b/pkg/enterprise/guardian.go index 4886889a29..58f0b66c0f 100644 --- a/pkg/enterprise/guardian.go +++ b/pkg/enterprise/guardian.go @@ -46,7 +46,7 @@ func registerGuardian(v *extensions.Variant) { // enterprise management-cluster policy. Building the enterprise egress rules can // fail (proxy URL parsing); on failure we drop the policy entirely, matching the // core behavior of omitting it rather than installing a partial policy. -func modifyGuardianPolicy(rc extensions.RenderContext, gpc render.GuardianPolicyExtensionContext, objs, del []client.Object) ([]client.Object, []client.Object) { +func modifyGuardianPolicy(rc render.RenderContext, gpc render.GuardianPolicyExtensionContext, objs, del []client.Object) ([]client.Object, []client.Object) { policy, ok := extensions.FindObject[*v3.NetworkPolicy](objs, render.GuardianPolicyName) if !ok { return objs, del @@ -204,7 +204,7 @@ func enterpriseGuardianPolicySpec(gpc render.GuardianPolicyExtensionContext) (v3 // objects: the secrets Role/RoleBinding and default UI settings, the // elasticsearch/kibana service ports, the management-cluster-request cluster // role rules (which replace the OSS rules), and the CA bundle env vars. -func modifyGuardian(rc extensions.RenderContext, gc render.GuardianExtensionContext, objs, del []client.Object) ([]client.Object, []client.Object) { +func modifyGuardian(rc render.RenderContext, gc render.GuardianExtensionContext, objs, del []client.Object) ([]client.Object, []client.Object) { if role, ok := extensions.FindObject[*rbacv1.ClusterRole](objs, render.GuardianClusterRoleName); ok { role.Rules = guardianEnterpriseRules(gc) } diff --git a/pkg/enterprise/guardian_test.go b/pkg/enterprise/guardian_test.go index 07e6694904..6a6e965296 100644 --- a/pkg/enterprise/guardian_test.go +++ b/pkg/enterprise/guardian_test.go @@ -49,7 +49,7 @@ var _ = Describe("guardian enterprise modifier", func() { } } - entCtx := extensions.RenderContext{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.CalicoEnterprise}} + entCtx := render.RenderContext{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.CalicoEnterprise}} It("appends the secrets RBAC and UI settings", func() { out, _ := applyExtensionsWithContext(ext, render.GuardianName, entCtx, render.GuardianExtensionContext{}, newObjs(), nil) @@ -92,7 +92,7 @@ var _ = Describe("guardian enterprise modifier", func() { }) It("does nothing for the Calico variant", func() { - ctx := extensions.RenderContext{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.Calico}} + ctx := render.RenderContext{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.Calico}} out, _ := applyExtensions(ext, render.GuardianName, ctx, newObjs(), nil) Expect(out).To(HaveLen(len(newObjs()))) role, _ := extensions.FindObject[*rbacv1.ClusterRole](out, render.GuardianClusterRoleName) diff --git a/pkg/enterprise/installation.go b/pkg/enterprise/installation.go index ca4f77dc36..534dd07b98 100644 --- a/pkg/enterprise/installation.go +++ b/pkg/enterprise/installation.go @@ -25,10 +25,10 @@ import ( operatorv1 "github.com/tigera/operator/api/v1" "github.com/tigera/operator/pkg/common" + "github.com/tigera/operator/pkg/controller/contexts" "github.com/tigera/operator/pkg/controller/utils" "github.com/tigera/operator/pkg/ctrlruntime" "github.com/tigera/operator/pkg/dns" - "github.com/tigera/operator/pkg/extensions" "github.com/tigera/operator/pkg/render" relasticsearch "github.com/tigera/operator/pkg/render/common/elasticsearch" "github.com/tigera/operator/pkg/render/kubecontrollers" @@ -64,7 +64,7 @@ type installationRenderData struct { // installationData pulls the installation extension's render data back out of the // render context, returning the zero value when none is set. -func installationData(rc extensions.RenderContext) installationRenderData { +func installationData(rc render.RenderContext) installationRenderData { data, _ := rc.Extension.(installationRenderData) return data } @@ -76,7 +76,7 @@ func collectProcessPathEnabled(lc *operatorv1.LogCollector) bool { } // Validate rejects installation config Calico Enterprise does not support. -func (coreControllerExtension) Validate(cc extensions.ControllerContext) error { +func (coreControllerExtension) Validate(cc contexts.ControllerContext) error { return validateReporterPort(cc.FelixConfiguration) } @@ -138,7 +138,7 @@ func (coreControllerExtension) Watches(c ctrlruntime.Controller) error { // fetching the certificates that feed the trusted bundle. It returns the render // context carrying the produced node prometheus keypair, and that keypair as one // the controller should manage. -func (coreControllerExtension) ExtendContext(cc extensions.ControllerContext) (extensions.RenderContext, []certificatemanagement.KeyPairInterface, error) { +func (coreControllerExtension) ExtendContext(cc contexts.ControllerContext) (render.RenderContext, []certificatemanagement.KeyPairInterface, error) { rc := cc.RenderContext nodePrometheusTLS, err := cc.CertificateManager.GetOrCreateKeyPair( diff --git a/pkg/enterprise/installation_test.go b/pkg/enterprise/installation_test.go index a66bdcbd91..42f3d39d21 100644 --- a/pkg/enterprise/installation_test.go +++ b/pkg/enterprise/installation_test.go @@ -27,8 +27,8 @@ import ( "github.com/tigera/operator/pkg/apis" "github.com/tigera/operator/pkg/common" "github.com/tigera/operator/pkg/controller/certificatemanager" + "github.com/tigera/operator/pkg/controller/contexts" ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" - "github.com/tigera/operator/pkg/extensions" "github.com/tigera/operator/pkg/render" "github.com/tigera/operator/pkg/render/kubecontrollers" ) @@ -47,7 +47,7 @@ var _ = Describe("installation controller extension", func() { func(provider operatorv1.Provider, expected []string) { fc := &v3.FelixConfiguration{} install := &operatorv1.InstallationSpec{Variant: operatorv1.CalicoEnterprise, KubernetesProvider: provider} - updated, err := ext.DefaultFelixConfiguration(extensions.InstallationController, install, fc) + updated, err := ext.DefaultFelixConfiguration(contexts.InstallationController, install, fc) Expect(err).NotTo(HaveOccurred()) if expected == nil { Expect(updated).To(BeFalse()) @@ -64,7 +64,7 @@ var _ = Describe("installation controller extension", func() { It("does no felix defaulting for the Calico variant", func() { fc := &v3.FelixConfiguration{} - updated, err := ext.DefaultFelixConfiguration(extensions.InstallationController, &operatorv1.InstallationSpec{Variant: operatorv1.Calico, KubernetesProvider: operatorv1.ProviderOpenShift}, fc) + updated, err := ext.DefaultFelixConfiguration(contexts.InstallationController, &operatorv1.InstallationSpec{Variant: operatorv1.Calico, KubernetesProvider: operatorv1.ProviderOpenShift}, fc) Expect(err).NotTo(HaveOccurred()) Expect(updated).To(BeFalse()) Expect(fc.Spec.DNSTrustedServers).To(BeNil()) @@ -87,7 +87,7 @@ var _ = Describe("installation controller extension", func() { }) }) -func newControllerContext(variant operatorv1.ProductVariant) extensions.ControllerContext { +func newControllerContext(variant operatorv1.ProductVariant) contexts.ControllerContext { scheme := runtime.NewScheme() Expect(apis.AddToScheme(scheme, false)).NotTo(HaveOccurred()) c := ctrlrfake.DefaultFakeClientBuilder(scheme).Build() @@ -96,14 +96,14 @@ func newControllerContext(variant operatorv1.ProductVariant) extensions.Controll Expect(err).NotTo(HaveOccurred()) trustedBundle := certManager.CreateTrustedBundle() - return extensions.ControllerContext{ - RenderContext: extensions.RenderContext{ + return contexts.ControllerContext{ + RenderContext: render.RenderContext{ Installation: &operatorv1.InstallationSpec{Variant: variant}, FelixConfiguration: &v3.FelixConfiguration{}, TrustedBundle: trustedBundle, ClusterDomain: "cluster.local", }, - Controller: extensions.InstallationController, + Controller: contexts.InstallationController, Ctx: context.Background(), Client: c, CertificateManager: certManager, diff --git a/pkg/enterprise/kubecontrollers.go b/pkg/enterprise/kubecontrollers.go index 2213e4c1d6..7b1acbf497 100644 --- a/pkg/enterprise/kubecontrollers.go +++ b/pkg/enterprise/kubecontrollers.go @@ -31,6 +31,7 @@ import ( "github.com/tigera/operator/pkg/common" "github.com/tigera/operator/pkg/components" + "github.com/tigera/operator/pkg/controller/contexts" "github.com/tigera/operator/pkg/controller/gatewayapi" "github.com/tigera/operator/pkg/controller/utils" "github.com/tigera/operator/pkg/controller/utils/imageset" @@ -60,7 +61,7 @@ func registerKubeControllers(v *extensions.Variant) { // calico-kube-controllers calico-system network policy, so the kube-apiserver can // reach the in-process webhook on :9443 (EV-6386). Without it the calico-system // default-deny drops the apiserver->:9443 call and WAF admission times out. -func modifyKubeControllersPolicy(rc extensions.RenderContext, objs, del []client.Object) ([]client.Object, []client.Object) { +func modifyKubeControllersPolicy(rc render.RenderContext, objs, del []client.Object) ([]client.Object, []client.Object) { if !installationData(rc).waf.enabled { return objs, del } @@ -85,7 +86,7 @@ func modifyKubeControllersPolicy(rc extensions.RenderContext, objs, del []client // is enterprise-only by construction - the base render carries none of it. The // controller-side inputs (keypairs, the resolved wasm image, the pull secret) are // produced by the installation hook and handed in through rc. -func modifyKubeControllers(rc extensions.RenderContext, objs, del []client.Object) ([]client.Object, []client.Object) { +func modifyKubeControllers(rc render.RenderContext, objs, del []client.Object) ([]client.Object, []client.Object) { data := installationData(rc) if role, ok := extensions.FindObject[*rbacv1.ClusterRole](objs, kubecontrollers.KubeControllerRole); ok { @@ -115,7 +116,7 @@ func modifyKubeControllers(rc extensions.RenderContext, objs, del []client.Objec return objs, del } -func modifyKubeControllersDeployment(rc extensions.RenderContext, dp *appsv1.Deployment, data installationRenderData) { +func modifyKubeControllersDeployment(rc render.RenderContext, dp *appsv1.Deployment, data installationRenderData) { spec := &dp.Spec.Template.Spec if dp.Spec.Template.Annotations == nil { dp.Spec.Template.Annotations = map[string]string{} @@ -183,7 +184,7 @@ func appendEnabledControllers(c *corev1.Container, extra []string) { // enterpriseEnv is the static enterprise env for calico-kube-controllers. The // modifier runs only for the enterprise variant, so these are never rendered for core. -func enterpriseEnv(rc extensions.RenderContext) []corev1.EnvVar { +func enterpriseEnv(rc render.RenderContext) []corev1.EnvVar { var env []corev1.EnvVar if rc.TrustedBundle != nil { env = append(env, corev1.EnvVar{Name: "MULTI_CLUSTER_FORWARDING_CA", Value: rc.TrustedBundle.MountPath()}) @@ -582,7 +583,7 @@ type wafRenderData struct { // produces everything the modifier needs that it can't compute itself: the resolved // wasm image, the webhook serving keypair (also returned as a managed keypair), the // merged wasm pull secret, the wasm CA bundle ConfigMap, and the operator CA PEM. -func buildWAFData(cc extensions.ControllerContext) (wafRenderData, certificatemanagement.KeyPairInterface, error) { +func buildWAFData(cc contexts.ControllerContext) (wafRenderData, certificatemanagement.KeyPairInterface, error) { gw, _, err := gatewayapi.GetGatewayAPI(cc.Ctx, cc.Client) if err != nil && !apierrors.IsNotFound(err) { return wafRenderData{}, nil, err diff --git a/pkg/enterprise/kubecontrollers_test.go b/pkg/enterprise/kubecontrollers_test.go index 3bb93c0e62..4f1f967513 100644 --- a/pkg/enterprise/kubecontrollers_test.go +++ b/pkg/enterprise/kubecontrollers_test.go @@ -34,6 +34,7 @@ import ( "github.com/tigera/operator/pkg/common" "github.com/tigera/operator/pkg/components" "github.com/tigera/operator/pkg/controller/certificatemanager" + "github.com/tigera/operator/pkg/controller/contexts" ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" "github.com/tigera/operator/pkg/enterprise" "github.com/tigera/operator/pkg/extensions" @@ -97,7 +98,7 @@ var _ = Describe("kube-controllers enterprise modifier", func() { // certManagementControllerContext builds a controller context whose certificate // manager issues cert-management (CSR-based) keypairs. -func certManagementControllerContext() extensions.ControllerContext { +func certManagementControllerContext() contexts.ControllerContext { scheme := runtime.NewScheme() Expect(apis.AddToScheme(scheme, false)).NotTo(HaveOccurred()) c := ctrlrfake.DefaultFakeClientBuilder(scheme).Build() @@ -114,14 +115,14 @@ func certManagementControllerContext() extensions.ControllerContext { certManager, err := certificatemanager.Create(c, installation, "", common.OperatorNamespace(), certificatemanager.AllowCACreation()) Expect(err).NotTo(HaveOccurred()) - return extensions.ControllerContext{ - RenderContext: extensions.RenderContext{ + return contexts.ControllerContext{ + RenderContext: render.RenderContext{ Installation: installation, FelixConfiguration: &v3.FelixConfiguration{}, TrustedBundle: certManager.CreateTrustedBundle(), ClusterDomain: "cluster.local", }, - Controller: extensions.InstallationController, + Controller: contexts.InstallationController, Ctx: context.Background(), Client: c, CertificateManager: certManager, @@ -129,7 +130,7 @@ func certManagementControllerContext() extensions.ControllerContext { } var _ = Describe("calico-kube-controllers enterprise surface", func() { - calicoKubeControllersCfg := func(cc extensions.ControllerContext) *kubecontrollers.KubeControllersConfiguration { + calicoKubeControllersCfg := func(cc contexts.ControllerContext) *kubecontrollers.KubeControllersConfiguration { return &kubecontrollers.KubeControllersConfiguration{ Installation: cc.Installation, ClusterDomain: cc.ClusterDomain, @@ -142,7 +143,7 @@ var _ = Describe("calico-kube-controllers enterprise surface", func() { // render builds the base calico-kube-controllers objects and applies the // enterprise modifier, exactly as the component handler does. - renderKubeControllers := func(cc extensions.ControllerContext, rc extensions.RenderContext) []client.Object { + renderKubeControllers := func(cc contexts.ControllerContext, rc render.RenderContext) []client.Object { comp := kubecontrollers.NewCalicoKubeControllers(calicoKubeControllersCfg(cc)) Expect(comp.ResolveImages(nil)).NotTo(HaveOccurred()) create, del := comp.Objects() @@ -251,7 +252,7 @@ var _ = Describe("calico-kube-controllers enterprise surface", func() { // wafControllerContext builds a controller context with a WAF-enabled GatewayAPI CR // and an install pull secret, so the installation hook produces the full WAF data. -func wafControllerContext() extensions.ControllerContext { +func wafControllerContext() contexts.ControllerContext { scheme := runtime.NewScheme() Expect(apis.AddToScheme(scheme, false)).NotTo(HaveOccurred()) c := ctrlrfake.DefaultFakeClientBuilder(scheme).Build() @@ -273,8 +274,8 @@ func wafControllerContext() extensions.ControllerContext { certManager, err := certificatemanager.Create(c, nil, "", common.OperatorNamespace(), certificatemanager.AllowCACreation()) Expect(err).NotTo(HaveOccurred()) - return extensions.ControllerContext{ - RenderContext: extensions.RenderContext{ + return contexts.ControllerContext{ + RenderContext: render.RenderContext{ Installation: &operatorv1.InstallationSpec{ Variant: operatorv1.CalicoEnterprise, Registry: "test-reg/", @@ -284,7 +285,7 @@ func wafControllerContext() extensions.ControllerContext { TrustedBundle: certManager.CreateTrustedBundle(), ClusterDomain: "cluster.local", }, - Controller: extensions.InstallationController, + Controller: contexts.InstallationController, Ctx: context.Background(), Client: c, CertificateManager: certManager, diff --git a/pkg/enterprise/node.go b/pkg/enterprise/node.go index d4e35dae68..ee1ce308cf 100644 --- a/pkg/enterprise/node.go +++ b/pkg/enterprise/node.go @@ -61,7 +61,7 @@ func registerNode(v *extensions.Variant) { // objects: the extra RBAC rules, the node-metrics Service, and the Enterprise // daemonset configuration (flow/DNS log env, prometheus reporter, BGP metrics // readiness check, multi-interface mode, and the calico log volume). -func modifyNode(rc extensions.RenderContext, objs, del []client.Object) ([]client.Object, []client.Object) { +func modifyNode(rc render.RenderContext, objs, del []client.Object) ([]client.Object, []client.Object) { if role, ok := extensions.FindObject[*rbacv1.ClusterRole](objs, render.CalicoNodeObjectName); ok { role.Rules = append(role.Rules, nodeEnterpriseRules()...) } @@ -117,7 +117,7 @@ func nodeEnterpriseRules() []rbacv1.PolicyRule { // BGP metrics readiness check, and the prometheus reporter keypair mount. The // calico log volume is mounted by the base render for both variants, so it is // not handled here. -func modifyNodeDaemonSet(rc extensions.RenderContext, ds *appsv1.DaemonSet) { +func modifyNodeDaemonSet(rc render.RenderContext, ds *appsv1.DaemonSet) { spec := &ds.Spec.Template.Spec // Collecting process info for flow logs reads from the host's process table. @@ -158,7 +158,7 @@ func modifyNodeDaemonSet(rc extensions.RenderContext, ds *appsv1.DaemonSet) { // creates it and hands it in via rc rather than the modifier building it. In // core (calico) the keypair is never created, so the base node render carries // no prometheus mount at all. -func mountNodePrometheusTLS(rc extensions.RenderContext, ds *appsv1.DaemonSet) { +func mountNodePrometheusTLS(rc render.RenderContext, ds *appsv1.DaemonSet) { tls := installationData(rc).nodePrometheusTLS if tls == nil { return @@ -186,7 +186,7 @@ func mountNodePrometheusTLS(rc extensions.RenderContext, ds *appsv1.DaemonSet) { // nodeEnterpriseEnv is the Enterprise felix configuration added to the // calico/node container. -func nodeEnterpriseEnv(rc extensions.RenderContext) []corev1.EnvVar { +func nodeEnterpriseEnv(rc render.RenderContext) []corev1.EnvVar { data := installationData(rc) env := []corev1.EnvVar{ {Name: "FELIX_PROMETHEUSREPORTERENABLED", Value: "true"}, @@ -231,7 +231,7 @@ func multiInterfaceModeEnv(install *operatorv1.InstallationSpec) *corev1.EnvVar } // nodeMetricsService builds the enterprise-only calico-node-metrics Service. -func nodeMetricsService(rc extensions.RenderContext) *corev1.Service { +func nodeMetricsService(rc render.RenderContext) *corev1.Service { reporterPort := nodeReporterPort(rc.FelixConfiguration) felixPort := felixMetricsPort(rc.FelixConfiguration) felixEnabled := rc.FelixConfiguration != nil && utils.IsFelixPrometheusMetricsEnabled(rc.FelixConfiguration) diff --git a/pkg/enterprise/node_test.go b/pkg/enterprise/node_test.go index 4257766122..335f7153b9 100644 --- a/pkg/enterprise/node_test.go +++ b/pkg/enterprise/node_test.go @@ -76,8 +76,8 @@ var _ = Describe("node enterprise modifier", func() { return nil } - entCtx := func() extensions.RenderContext { - return extensions.RenderContext{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.CalicoEnterprise}} + entCtx := func() render.RenderContext { + return render.RenderContext{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.CalicoEnterprise}} } It("adds the enterprise cluster role rules", func() { @@ -173,7 +173,7 @@ var _ = Describe("node enterprise modifier", func() { }) It("is a no-op for the Calico variant", func() { - ctx := extensions.RenderContext{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.Calico}} + ctx := render.RenderContext{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.Calico}} out, _ := applyExtensions(ext, render.ComponentNameNode, ctx, newObjs(), nil) _, ok := extensions.FindObject[*corev1.Service](out, render.CalicoNodeMetricsService) @@ -183,7 +183,7 @@ var _ = Describe("node enterprise modifier", func() { }) It("does not panic on a zero RenderContext", func() { - out, _ := applyExtensions(ext, render.ComponentNameNode, extensions.RenderContext{}, newObjs(), nil) + out, _ := applyExtensions(ext, render.ComponentNameNode, render.RenderContext{}, newObjs(), nil) _, ok := extensions.FindObject[*corev1.Service](out, render.CalicoNodeMetricsService) Expect(ok).To(BeFalse()) }) diff --git a/pkg/enterprise/register.go b/pkg/enterprise/register.go index d480fc02f3..d8e34a4ac5 100644 --- a/pkg/enterprise/register.go +++ b/pkg/enterprise/register.go @@ -16,6 +16,7 @@ package enterprise import ( operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/controller/contexts" "github.com/tigera/operator/pkg/extensions" ) @@ -28,9 +29,9 @@ func New() *extensions.Set { s := extensions.NewSet() ent := s.Variant(operatorv1.CalicoEnterprise) - ent.Controller(extensions.InstallationController, coreControllerExtension{}) - ent.Controller(extensions.WindowsController, windowsControllerExtension{}) - ent.Controller(extensions.APIServerController, apiServerControllerExtension{}) + ent.Controller(contexts.InstallationController, coreControllerExtension{}) + ent.Controller(contexts.WindowsController, windowsControllerExtension{}) + ent.Controller(contexts.APIServerController, apiServerControllerExtension{}) registerTypha(ent) registerNode(ent) registerWindows(ent) diff --git a/pkg/enterprise/typha.go b/pkg/enterprise/typha.go index 3786e7be07..950b66415e 100644 --- a/pkg/enterprise/typha.go +++ b/pkg/enterprise/typha.go @@ -28,7 +28,7 @@ func registerTypha(v *extensions.Variant) { v.Modify(render.ComponentNameTypha, modifyTypha) } -func modifyTypha(rc extensions.RenderContext, objs, del []client.Object) ([]client.Object, []client.Object) { +func modifyTypha(rc render.RenderContext, objs, del []client.Object) ([]client.Object, []client.Object) { if role, ok := extensions.FindObject[*rbacv1.ClusterRole](objs, "calico-typha"); ok { role.Rules = append(role.Rules, rbacv1.PolicyRule{ APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, diff --git a/pkg/enterprise/typha_test.go b/pkg/enterprise/typha_test.go index cfc60b2b62..53c5ac73fa 100644 --- a/pkg/enterprise/typha_test.go +++ b/pkg/enterprise/typha_test.go @@ -24,7 +24,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" operatorv1 "github.com/tigera/operator/api/v1" - "github.com/tigera/operator/pkg/extensions" "github.com/tigera/operator/pkg/render" ) @@ -45,7 +44,7 @@ var _ = Describe("typha enterprise modifier", func() { } It("adds enterprise RBAC and MULTI_INTERFACE_MODE for the enterprise variant", func() { - ctx := extensions.RenderContext{Installation: &operatorv1.InstallationSpec{ + ctx := render.RenderContext{Installation: &operatorv1.InstallationSpec{ Variant: operatorv1.CalicoEnterprise, CalicoNetwork: &operatorv1.CalicoNetworkSpec{MultiInterfaceMode: &multiMode}, }} @@ -65,7 +64,7 @@ var _ = Describe("typha enterprise modifier", func() { }) It("is a no-op for the Calico variant", func() { - ctx := extensions.RenderContext{Installation: &operatorv1.InstallationSpec{ + ctx := render.RenderContext{Installation: &operatorv1.InstallationSpec{ Variant: operatorv1.Calico, CalicoNetwork: &operatorv1.CalicoNetworkSpec{MultiInterfaceMode: &multiMode}, }} @@ -76,7 +75,7 @@ var _ = Describe("typha enterprise modifier", func() { }) It("does not panic on a zero Context (nil Installation)", func() { - out, _ := applyExtensions(ext, render.ComponentNameTypha, extensions.RenderContext{}, newObjs(), nil) + out, _ := applyExtensions(ext, render.ComponentNameTypha, render.RenderContext{}, newObjs(), nil) Expect(out[0].(*rbacv1.ClusterRole).Rules).To(BeEmpty()) }) }) diff --git a/pkg/enterprise/windows.go b/pkg/enterprise/windows.go index 183bfd08ae..a464006d7e 100644 --- a/pkg/enterprise/windows.go +++ b/pkg/enterprise/windows.go @@ -26,6 +26,7 @@ import ( operatorv1 "github.com/tigera/operator/api/v1" "github.com/tigera/operator/pkg/common" "github.com/tigera/operator/pkg/components" + "github.com/tigera/operator/pkg/controller/contexts" "github.com/tigera/operator/pkg/controller/utils" "github.com/tigera/operator/pkg/ctrlruntime" "github.com/tigera/operator/pkg/dns" @@ -58,13 +59,13 @@ type windowsRenderData struct { // windowsData pulls the windows extension's render data back out of the render // context, returning the zero value when none is set. -func windowsData(rc extensions.RenderContext) windowsRenderData { +func windowsData(rc render.RenderContext) windowsRenderData { data, _ := rc.Extension.(windowsRenderData) return data } // Validate rejects windows installation config Calico Enterprise does not support. -func (windowsControllerExtension) Validate(cc extensions.ControllerContext) error { +func (windowsControllerExtension) Validate(cc contexts.ControllerContext) error { return validateReporterPort(cc.FelixConfiguration) } @@ -83,7 +84,7 @@ func (windowsControllerExtension) Watches(c ctrlruntime.Controller) error { // ExtendContext fetches the node prometheus keypair the installation controller // created and stashes it in the render context for the windows modifier. -func (windowsControllerExtension) ExtendContext(cc extensions.ControllerContext) (extensions.RenderContext, []certificatemanagement.KeyPairInterface, error) { +func (windowsControllerExtension) ExtendContext(cc contexts.ControllerContext) (render.RenderContext, []certificatemanagement.KeyPairInterface, error) { rc := cc.RenderContext tls, err := cc.CertificateManager.GetKeyPair( cc.Client, @@ -102,7 +103,7 @@ func (windowsControllerExtension) ExtendContext(cc extensions.ControllerContext) // calico-node-windows objects: the node-metrics Service and the Enterprise // daemonset configuration (flow/DNS log env, prometheus reporter, trusted DNS // servers, the calico log volume, and the prometheus reporter keypair mount). -func modifyWindows(rc extensions.RenderContext, objs, del []client.Object) ([]client.Object, []client.Object) { +func modifyWindows(rc render.RenderContext, objs, del []client.Object) ([]client.Object, []client.Object) { if ds, ok := extensions.FindObject[*appsv1.DaemonSet](objs, common.WindowsDaemonSetName); ok { modifyWindowsDaemonSet(rc, ds) } @@ -110,7 +111,7 @@ func modifyWindows(rc extensions.RenderContext, objs, del []client.Object) ([]cl return append(objs, windowsNodeMetricsService(rc)), del } -func modifyWindowsDaemonSet(rc extensions.RenderContext, ds *appsv1.DaemonSet) { +func modifyWindowsDaemonSet(rc render.RenderContext, ds *appsv1.DaemonSet) { dirOrCreate := corev1.HostPathDirectoryOrCreate spec := &ds.Spec.Template.Spec @@ -138,7 +139,7 @@ func modifyWindowsDaemonSet(rc extensions.RenderContext, ds *appsv1.DaemonSet) { // windowsEnterpriseEnv is the Enterprise felix configuration added to the // calico-node-windows containers. -func windowsEnterpriseEnv(rc extensions.RenderContext) []corev1.EnvVar { +func windowsEnterpriseEnv(rc render.RenderContext) []corev1.EnvVar { tls := windowsData(rc).prometheusServerTLS env := []corev1.EnvVar{ {Name: "FELIX_PROMETHEUSREPORTERENABLED", Value: "true"}, @@ -175,7 +176,7 @@ func windowsEnterpriseEnv(rc extensions.RenderContext) []corev1.EnvVar { // mountWindowsPrometheusTLS mounts the node prometheus reporter keypair onto the // windows daemonset: the volume, the volume mount on each node container, and // the pod hash annotation that rolls the pods on cert rotation. -func mountWindowsPrometheusTLS(rc extensions.RenderContext, ds *appsv1.DaemonSet) { +func mountWindowsPrometheusTLS(rc render.RenderContext, ds *appsv1.DaemonSet) { tls := windowsData(rc).prometheusServerTLS if tls == nil { return @@ -199,7 +200,7 @@ func mountWindowsPrometheusTLS(rc extensions.RenderContext, ds *appsv1.DaemonSet // windowsNodeMetricsService builds the enterprise-only calico-node-metrics-windows // Service. -func windowsNodeMetricsService(rc extensions.RenderContext) *corev1.Service { +func windowsNodeMetricsService(rc render.RenderContext) *corev1.Service { reporterPort := nodeReporterPort(rc.FelixConfiguration) return &corev1.Service{ TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}, diff --git a/pkg/enterprise/windows_test.go b/pkg/enterprise/windows_test.go index d4c7967b9b..810ec790d9 100644 --- a/pkg/enterprise/windows_test.go +++ b/pkg/enterprise/windows_test.go @@ -31,6 +31,7 @@ import ( "github.com/tigera/operator/pkg/common" "github.com/tigera/operator/pkg/components" "github.com/tigera/operator/pkg/controller/certificatemanager" + "github.com/tigera/operator/pkg/controller/contexts" ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" "github.com/tigera/operator/pkg/dns" "github.com/tigera/operator/pkg/extensions" @@ -87,8 +88,8 @@ var _ = Describe("windows enterprise modifier", func() { return nil } - ctxFor := func(provider operatorv1.Provider) extensions.RenderContext { - return extensions.RenderContext{ + ctxFor := func(provider operatorv1.Provider) render.RenderContext { + return render.RenderContext{ Installation: &operatorv1.InstallationSpec{Variant: operatorv1.CalicoEnterprise, KubernetesProvider: provider}, } } @@ -138,13 +139,13 @@ var _ = Describe("windows enterprise modifier", func() { // Build the render context the way the windows controller does: run the // windows extension, which fetches the keypair into the context. - cc := extensions.ControllerContext{ - RenderContext: extensions.RenderContext{ + cc := contexts.ControllerContext{ + RenderContext: render.RenderContext{ Installation: ctxFor(operatorv1.ProviderNone).Installation, TrustedBundle: bundle, ClusterDomain: dns.DefaultClusterDomain, }, - Controller: extensions.WindowsController, + Controller: contexts.WindowsController, Ctx: context.Background(), Client: cli, CertificateManager: cm, @@ -162,7 +163,7 @@ var _ = Describe("windows enterprise modifier", func() { }) It("does nothing for the Calico variant", func() { - ctx := extensions.RenderContext{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.Calico}} + ctx := render.RenderContext{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.Calico}} out, _ := applyExtensions(ext, render.ComponentNameWindows, ctx, newObjs(), nil) _, ok := extensions.FindObject[*corev1.Service](out, render.WindowsNodeMetricsService) Expect(ok).To(BeFalse()) diff --git a/pkg/extensions/controllerextension.go b/pkg/extensions/controllerextension.go index 0cdce8cc19..0f5025bdb7 100644 --- a/pkg/extensions/controllerextension.go +++ b/pkg/extensions/controllerextension.go @@ -15,28 +15,15 @@ package extensions import ( - "context" - v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" - "sigs.k8s.io/controller-runtime/pkg/client" operatorv1 "github.com/tigera/operator/api/v1" - "github.com/tigera/operator/pkg/controller/certificatemanager" + "github.com/tigera/operator/pkg/controller/contexts" "github.com/tigera/operator/pkg/ctrlruntime" + "github.com/tigera/operator/pkg/render" "github.com/tigera/operator/pkg/tls/certificatemanagement" ) -// ControllerName identifies the controller a ControllerExtension extends, so a -// variant can register a different hook per controller. Use the constants below -// rather than bare strings so registration and lookup stay in sync. -type ControllerName string - -const ( - InstallationController ControllerName = "installation" - WindowsController ControllerName = "windows" - APIServerController ControllerName = "apiserver" -) - // ControllerExtension extends a controller's reconcile: it validates the // configuration and builds the RenderContext the render phase consumes. The core // operator registers none and runs with the base behavior; an extension build @@ -44,14 +31,14 @@ const ( type ControllerExtension interface { // Validate rejects configuration the extension does not support, before any // rendering happens. - Validate(cc ControllerContext) error + Validate(cc contexts.ControllerContext) error // ExtendContext does the controller-side reconcile work the render phase // cannot, returning the RenderContext the render phase consumes plus any // keypairs the extension created that the controller should manage (add to // certificate management and BYO-expiry warnings), or an error that aborts the // reconcile. - ExtendContext(cc ControllerContext) (RenderContext, []certificatemanagement.KeyPairInterface, error) + ExtendContext(cc contexts.ControllerContext) (render.RenderContext, []certificatemanagement.KeyPairInterface, error) } // Watcher is an optional companion to ControllerExtension. A controller's Add() @@ -72,30 +59,3 @@ type Watcher interface { type FelixConfigDefaulter interface { DefaultFelixConfiguration(install *operatorv1.InstallationSpec, fc *v3.FelixConfiguration) (bool, error) } - -// ControllerContext is the controller-phase context, the corollary to the -// render-phase RenderContext. It is the embedded RenderContext (the same data -// the render phase sees) plus the controller-side machinery a ControllerExtension -// needs to produce artifacts: a client, a certificate manager, a context. Those -// deps live here, not on RenderContext, so the modifiers that read RenderContext -// can't do I/O - they only transform objects. -// -// Controller names which controller is reconciling, selecting that controller's -// extension hook. The controller fills the embedded RenderContext's data fields, -// the deps, and Controller; ExtendContext does its work, sets the produced -// artifacts on the embedded context, and returns it. -type ControllerContext struct { - RenderContext - - // Controller identifies the reconciling controller, selecting its hook. - Controller ControllerName - - Ctx context.Context - Client client.Client - CertificateManager certificatemanager.CertificateManager - - // MultiTenant reports whether the operator runs in multi-tenant mode. It's a - // controller-phase operator mode some extensions gate on (e.g. the API server's - // management-cluster tunnel-secret check). - MultiTenant bool -} diff --git a/pkg/extensions/controllerextension_test.go b/pkg/extensions/controllerextension_test.go index 729e8783d6..87b409fe70 100644 --- a/pkg/extensions/controllerextension_test.go +++ b/pkg/extensions/controllerextension_test.go @@ -21,8 +21,10 @@ import ( . "github.com/onsi/gomega" operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/controller/contexts" "github.com/tigera/operator/pkg/ctrlruntime" "github.com/tigera/operator/pkg/extensions" + "github.com/tigera/operator/pkg/render" "github.com/tigera/operator/pkg/tls/certificatemanagement" ) @@ -34,9 +36,9 @@ var _ = Describe("controller extension", func() { It("returns the base render context when the variant has no extension", func() { install := &operatorv1.InstallationSpec{Variant: operatorv1.Calico} - rc, _, err := s.ExtendContext(extensions.ControllerContext{ - RenderContext: extensions.RenderContext{Installation: install, ClusterDomain: "cluster.local"}, - Controller: extensions.InstallationController, + rc, _, err := s.ExtendContext(contexts.ControllerContext{ + RenderContext: render.RenderContext{Installation: install, ClusterDomain: "cluster.local"}, + Controller: contexts.InstallationController, }) Expect(err).NotTo(HaveOccurred()) Expect(rc.Installation).To(BeIdenticalTo(install)) @@ -45,37 +47,37 @@ var _ = Describe("controller extension", func() { }) It("runs the extension registered for the installation variant", func() { - s.Variant(operatorv1.CalicoEnterprise).Controller(extensions.InstallationController, fakeController{}) + s.Variant(operatorv1.CalicoEnterprise).Controller(contexts.InstallationController, fakeController{}) rc, _, err := s.ExtendContext(enterpriseContext()) Expect(err).NotTo(HaveOccurred()) Expect(rc.ClusterDomain).To(Equal("from-fake")) }) It("ignores an extension registered for a different variant", func() { - s.Variant(operatorv1.CalicoEnterprise).Controller(extensions.InstallationController, fakeController{}) - rc, _, err := s.ExtendContext(extensions.ControllerContext{ - RenderContext: extensions.RenderContext{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.Calico}, ClusterDomain: "real"}, - Controller: extensions.InstallationController, + s.Variant(operatorv1.CalicoEnterprise).Controller(contexts.InstallationController, fakeController{}) + rc, _, err := s.ExtendContext(contexts.ControllerContext{ + RenderContext: render.RenderContext{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.Calico}, ClusterDomain: "real"}, + Controller: contexts.InstallationController, }) Expect(err).NotTo(HaveOccurred()) Expect(rc.ClusterDomain).To(Equal("real")) }) It("surfaces the extension error", func() { - s.Variant(operatorv1.CalicoEnterprise).Controller(extensions.InstallationController, fakeController{err: errors.New("boom")}) + s.Variant(operatorv1.CalicoEnterprise).Controller(contexts.InstallationController, fakeController{err: errors.New("boom")}) _, _, err := s.ExtendContext(enterpriseContext()) Expect(err).To(MatchError("boom")) }) It("runs the extension's validation", func() { - s.Variant(operatorv1.CalicoEnterprise).Controller(extensions.InstallationController, fakeController{validateErr: errors.New("invalid")}) + s.Variant(operatorv1.CalicoEnterprise).Controller(contexts.InstallationController, fakeController{validateErr: errors.New("invalid")}) Expect(s.Validate(enterpriseContext())).To(MatchError("invalid")) }) It("runs the watch hook of an extension that implements Watcher", func() { called := false - s.Variant(operatorv1.CalicoEnterprise).Controller(extensions.InstallationController, watchingController{called: &called}) - Expect(s.SetupWatches(extensions.InstallationController, nil)).NotTo(HaveOccurred()) + s.Variant(operatorv1.CalicoEnterprise).Controller(contexts.InstallationController, watchingController{called: &called}) + Expect(s.SetupWatches(contexts.InstallationController, nil)).NotTo(HaveOccurred()) Expect(called).To(BeTrue()) }) @@ -90,10 +92,10 @@ var _ = Describe("controller extension", func() { }) }) -func enterpriseContext() extensions.ControllerContext { - return extensions.ControllerContext{ - RenderContext: extensions.RenderContext{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.CalicoEnterprise}}, - Controller: extensions.InstallationController, +func enterpriseContext() contexts.ControllerContext { + return contexts.ControllerContext{ + RenderContext: render.RenderContext{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.CalicoEnterprise}}, + Controller: contexts.InstallationController, } } @@ -104,15 +106,15 @@ type fakeController struct { validateErr error } -func (f fakeController) Validate(_ extensions.ControllerContext) error { +func (f fakeController) Validate(_ contexts.ControllerContext) error { return f.validateErr } -func (f fakeController) ExtendContext(_ extensions.ControllerContext) (extensions.RenderContext, []certificatemanagement.KeyPairInterface, error) { +func (f fakeController) ExtendContext(_ contexts.ControllerContext) (render.RenderContext, []certificatemanagement.KeyPairInterface, error) { if f.err != nil { - return extensions.RenderContext{}, nil, f.err + return render.RenderContext{}, nil, f.err } - return extensions.RenderContext{ClusterDomain: "from-fake"}, nil, nil + return render.RenderContext{ClusterDomain: "from-fake"}, nil, nil } // watchingController is a fakeController that also implements the Watcher diff --git a/pkg/extensions/decorate_helpers_test.go b/pkg/extensions/decorate_helpers_test.go index 82dc31b601..81b4f54800 100644 --- a/pkg/extensions/decorate_helpers_test.go +++ b/pkg/extensions/decorate_helpers_test.go @@ -19,6 +19,7 @@ import ( operatorv1 "github.com/tigera/operator/api/v1" "github.com/tigera/operator/pkg/extensions" + "github.com/tigera/operator/pkg/render" rmeta "github.com/tigera/operator/pkg/render/common/meta" ) @@ -59,13 +60,13 @@ func (s stubExtComponent) ExtensionContext() any { // applyExtensions decorates a stub component holding the given objects with the // extension registered under key, then renders it. For a modifier that needs the // component's typed config, use applyExtensionsWithContext. -func applyExtensions(s *extensions.Set, key string, rc extensions.RenderContext, create, del []client.Object) ([]client.Object, []client.Object) { +func applyExtensions(s *extensions.Set, key string, rc render.RenderContext, create, del []client.Object) ([]client.Object, []client.Object) { return applyExtensionsWithContext(s, key, rc, nil, create, del) } // applyExtensionsWithContext is applyExtensions for a modifier that reads the // component's typed config: extCtx is delivered as the stub's ExtensionContext. -func applyExtensionsWithContext(s *extensions.Set, key string, rc extensions.RenderContext, extCtx any, create, del []client.Object) ([]client.Object, []client.Object) { +func applyExtensionsWithContext(s *extensions.Set, key string, rc render.RenderContext, extCtx any, create, del []client.Object) ([]client.Object, []client.Object) { stub := stubExtComponent{key: key, extCtx: extCtx, create: create, delete: del} return s.Decorate(stub, rc).Objects() } diff --git a/pkg/extensions/extension.go b/pkg/extensions/extension.go index 106f1ee8bb..625e371afe 100644 --- a/pkg/extensions/extension.go +++ b/pkg/extensions/extension.go @@ -16,6 +16,8 @@ package extensions import ( "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/tigera/operator/pkg/render" ) // Modifier post-processes the objects a render component produced. It receives @@ -24,7 +26,7 @@ import ( // append objects to delete (e.g. to clean up resources a prior variant left // behind). It runs only for the variant it is registered under, so it need not // re-check the variant. -type Modifier func(rc RenderContext, create, delete []client.Object) (newCreate, newDelete []client.Object) +type Modifier func(rc render.RenderContext, create, delete []client.Object) (newCreate, newDelete []client.Object) // FindObject returns the first object of type T with the given name. func FindObject[T client.Object](objs []client.Object, name string) (T, bool) { diff --git a/pkg/extensions/extension_test.go b/pkg/extensions/extension_test.go index 0ea3359345..4461c7c0d9 100644 --- a/pkg/extensions/extension_test.go +++ b/pkg/extensions/extension_test.go @@ -23,6 +23,7 @@ import ( operatorv1 "github.com/tigera/operator/api/v1" "github.com/tigera/operator/pkg/extensions" + "github.com/tigera/operator/pkg/render" ) var _ = Describe("extension registry", func() { @@ -31,10 +32,10 @@ var _ = Describe("extension registry", func() { s = extensions.NewSet() }) - entCtx := extensions.RenderContext{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.CalicoEnterprise}} + entCtx := render.RenderContext{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.CalicoEnterprise}} It("applies a registered modifier to the matching component and variant", func() { - s.Variant(operatorv1.CalicoEnterprise).Modify("test", func(ctx extensions.RenderContext, objs, del []client.Object) ([]client.Object, []client.Object) { + s.Variant(operatorv1.CalicoEnterprise).Modify("test", func(ctx render.RenderContext, objs, del []client.Object) ([]client.Object, []client.Object) { cm, ok := extensions.FindObject[*corev1.ConfigMap](objs, "cm") Expect(ok).To(BeTrue()) cm.Data = map[string]string{"k": "v"} @@ -51,7 +52,7 @@ var _ = Describe("extension registry", func() { }) It("lets a modifier append to the delete list", func() { - s.Variant(operatorv1.CalicoEnterprise).Modify("test", func(_ extensions.RenderContext, objs, del []client.Object) ([]client.Object, []client.Object) { + s.Variant(operatorv1.CalicoEnterprise).Modify("test", func(_ render.RenderContext, objs, del []client.Object) ([]client.Object, []client.Object) { return objs, append(del, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "stale"}}) }) @@ -69,11 +70,11 @@ var _ = Describe("extension registry", func() { }) It("does not apply a modifier registered for a different variant", func() { - s.Variant(operatorv1.CalicoEnterprise).Modify("test", func(_ extensions.RenderContext, objs, del []client.Object) ([]client.Object, []client.Object) { + s.Variant(operatorv1.CalicoEnterprise).Modify("test", func(_ render.RenderContext, objs, del []client.Object) ([]client.Object, []client.Object) { return append(objs, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "extra"}}), del }) - calicoCtx := extensions.RenderContext{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.Calico}} + calicoCtx := render.RenderContext{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.Calico}} in := []client.Object{&corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "cm"}}} out, _ := applyExtensions(s, "test", calicoCtx, in, nil) Expect(out).To(Equal(in)) @@ -81,13 +82,13 @@ var _ = Describe("extension registry", func() { It("returns objects unchanged when no installation is set", func() { in := []client.Object{&corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "cm"}}} - out, _ := applyExtensions(s, "test", extensions.RenderContext{}, in, nil) + out, _ := applyExtensions(s, "test", render.RenderContext{}, in, nil) Expect(out).To(Equal(in)) }) It("replaces rather than stacks when a component modifier is registered twice", func() { add := func(name string) { - s.Variant(operatorv1.CalicoEnterprise).Modify("test", func(_ extensions.RenderContext, objs, del []client.Object) ([]client.Object, []client.Object) { + s.Variant(operatorv1.CalicoEnterprise).Modify("test", func(_ render.RenderContext, objs, del []client.Object) ([]client.Object, []client.Object) { return append(objs, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: name}}), del }) } diff --git a/pkg/extensions/set.go b/pkg/extensions/set.go index afbdc114f1..e7ccd405e3 100644 --- a/pkg/extensions/set.go +++ b/pkg/extensions/set.go @@ -19,6 +19,7 @@ import ( operatorv1 "github.com/tigera/operator/api/v1" "github.com/tigera/operator/pkg/components" + "github.com/tigera/operator/pkg/controller/contexts" "github.com/tigera/operator/pkg/ctrlruntime" "github.com/tigera/operator/pkg/imageoverride" "github.com/tigera/operator/pkg/render" @@ -54,7 +55,7 @@ func (s *Set) Variant(v operatorv1.ProductVariant) *Variant { if s.variants[v] == nil { s.variants[v] = &Variant{ variant: v, - controllers: map[ControllerName]ControllerExtension{}, + controllers: map[contexts.ControllerName]ControllerExtension{}, modifiers: map[string]decorator{}, images: s.images, } @@ -76,7 +77,7 @@ func (s *Set) variant(v operatorv1.ProductVariant) *Variant { // objects are post-processed by that modifier. A decorated component is itself a // render.Component, so it flows through the component handler like any other. // Returns component unchanged when no extension applies. Nil-safe. -func (s *Set) Decorate(component render.Component, ctx RenderContext) render.Component { +func (s *Set) Decorate(component render.Component, ctx render.RenderContext) render.Component { if ctx.Installation == nil { return component } @@ -85,7 +86,7 @@ func (s *Set) Decorate(component render.Component, ctx RenderContext) render.Com // Validate runs the cc.Controller extension's validation for the installation's // variant, or returns nil when no extension is registered. Nil-safe. -func (s *Set) Validate(cc ControllerContext) error { +func (s *Set) Validate(cc contexts.ControllerContext) error { if cc.Installation == nil { return nil } @@ -93,10 +94,10 @@ func (s *Set) Validate(cc ControllerContext) error { } // ExtendContext runs the cc.Controller extension for the installation's variant -// and returns the resulting RenderContext plus any keypairs the extension wants +// and returns the resulting render.RenderContext plus any keypairs the extension wants // the controller to manage, or the base render context and no keypairs when no // extension is registered. Nil-safe. -func (s *Set) ExtendContext(cc ControllerContext) (RenderContext, []certificatemanagement.KeyPairInterface, error) { +func (s *Set) ExtendContext(cc contexts.ControllerContext) (render.RenderContext, []certificatemanagement.KeyPairInterface, error) { if cc.Installation == nil { return cc.RenderContext, nil, nil } @@ -107,7 +108,7 @@ func (s *Set) ExtendContext(cc ControllerContext) (RenderContext, []certificatem // named controller. It runs at controller startup, which is variant-agnostic, so // it registers the union across variants (in practice the one active extension // build's). Nil-safe. -func (s *Set) SetupWatches(controller ControllerName, c ctrlruntime.Controller) error { +func (s *Set) SetupWatches(controller contexts.ControllerName, c ctrlruntime.Controller) error { if s == nil { return nil } @@ -127,7 +128,7 @@ func (s *Set) SetupWatches(controller ControllerName, c ctrlruntime.Controller) // defaulting for the installation's variant, returning whether it changed fc. The // extension implements the optional FelixConfigDefaulter companion; when it doesn't // (or no extension is registered) this is a no-op. Nil-safe. -func (s *Set) DefaultFelixConfiguration(controller ControllerName, install *operatorv1.InstallationSpec, fc *v3.FelixConfiguration) (bool, error) { +func (s *Set) DefaultFelixConfiguration(controller contexts.ControllerName, install *operatorv1.InstallationSpec, fc *v3.FelixConfiguration) (bool, error) { if install == nil { return false, nil } diff --git a/pkg/extensions/variant.go b/pkg/extensions/variant.go index ca1c553c4c..f714404c57 100644 --- a/pkg/extensions/variant.go +++ b/pkg/extensions/variant.go @@ -20,6 +20,7 @@ import ( operatorv1 "github.com/tigera/operator/api/v1" "github.com/tigera/operator/pkg/components" + "github.com/tigera/operator/pkg/controller/contexts" "github.com/tigera/operator/pkg/imageoverride" "github.com/tigera/operator/pkg/render" "github.com/tigera/operator/pkg/tls/certificatemanagement" @@ -32,18 +33,18 @@ import ( // nothing here is itself keyed by variant. type Variant struct { variant operatorv1.ProductVariant - controllers map[ControllerName]ControllerExtension + controllers map[contexts.ControllerName]ControllerExtension modifiers map[string]decorator images *imageoverride.Overrides // shared with the owning Set } // decorator wraps a base component, returning one whose Objects() are augmented // by a registered modifier. -type decorator func(base render.Component, rc RenderContext) render.Component +type decorator func(base render.Component, rc render.RenderContext) render.Component // Controller registers the variant's controller-side extension for the named // controller. A controller has at most one; registering replaces any prior one. -func (v *Variant) Controller(name ControllerName, c ControllerExtension) { +func (v *Variant) Controller(name contexts.ControllerName, c ControllerExtension) { v.controllers[name] = c } @@ -56,7 +57,7 @@ func (v *Variant) Image(component string, image components.Component) { // config. For components whose modifier needs the component's own typed config, // use RegisterModifier. func (v *Variant) Modify(component string, m Modifier) { - v.modifiers[component] = func(base render.Component, rc RenderContext) render.Component { + v.modifiers[component] = func(base render.Component, rc render.RenderContext) render.Component { return &decoratedComponent{Component: base, rc: rc, modify: m} } } @@ -69,9 +70,9 @@ func (v *Variant) Modify(component string, m Modifier) { func RegisterModifier[Cfg any]( v *Variant, component string, - modify func(rc RenderContext, cfg Cfg, create, delete []client.Object) ([]client.Object, []client.Object), + modify func(rc render.RenderContext, cfg Cfg, create, delete []client.Object) ([]client.Object, []client.Object), ) { - v.modifiers[component] = func(base render.Component, rc RenderContext) render.Component { + v.modifiers[component] = func(base render.Component, rc render.RenderContext) render.Component { provider, ok := base.(render.ExtensionContextProvider) if !ok { logrus.Errorf("BUG: component %q has a registered modifier but provides no extension context; leaving it unmodified", component) @@ -83,7 +84,7 @@ func RegisterModifier[Cfg any]( logrus.Errorf("BUG: component %q extension context is %T, want %T; leaving it unmodified", component, provider.ExtensionContext(), want) return base } - bound := func(rc RenderContext, create, delete []client.Object) ([]client.Object, []client.Object) { + bound := func(rc render.RenderContext, create, delete []client.Object) ([]client.Object, []client.Object) { return modify(rc, cfg, create, delete) } return &decoratedComponent{Component: base, rc: rc, modify: bound} @@ -93,7 +94,7 @@ func RegisterModifier[Cfg any]( // decorate wraps component with the modifier registered for its extension key, // or returns it unchanged when the component exposes no extension point or none // is registered. Nil-safe. -func (v *Variant) decorate(component render.Component, rc RenderContext) render.Component { +func (v *Variant) decorate(component render.Component, rc render.RenderContext) render.Component { if v == nil { return component } @@ -110,7 +111,7 @@ func (v *Variant) decorate(component render.Component, rc RenderContext) render. // validate runs the cc.Controller extension's validation, or nil when the // variant has none for it. Nil-safe. -func (v *Variant) validate(cc ControllerContext) error { +func (v *Variant) validate(cc contexts.ControllerContext) error { if v == nil || v.controllers[cc.Controller] == nil { return nil } @@ -119,7 +120,7 @@ func (v *Variant) validate(cc ControllerContext) error { // extendContext runs the cc.Controller extension, or returns the base render // context and no managed keypairs when the variant has none for it. Nil-safe. -func (v *Variant) extendContext(cc ControllerContext) (RenderContext, []certificatemanagement.KeyPairInterface, error) { +func (v *Variant) extendContext(cc contexts.ControllerContext) (render.RenderContext, []certificatemanagement.KeyPairInterface, error) { if v == nil || v.controllers[cc.Controller] == nil { return cc.RenderContext, nil, nil } @@ -132,7 +133,7 @@ func (v *Variant) extendContext(cc ControllerContext) (RenderContext, []certific // and Ready delegate to the base; only Objects is augmented. type decoratedComponent struct { render.Component - rc RenderContext + rc render.RenderContext modify Modifier } diff --git a/pkg/render/apiserver_test.go b/pkg/render/apiserver_test.go index 683f14b520..252db966ab 100644 --- a/pkg/render/apiserver_test.go +++ b/pkg/render/apiserver_test.go @@ -28,7 +28,6 @@ import ( "github.com/tigera/operator/pkg/controller/k8sapi" ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" "github.com/tigera/operator/pkg/dns" - "github.com/tigera/operator/pkg/extensions" "github.com/tigera/operator/pkg/render" rmeta "github.com/tigera/operator/pkg/render/common/meta" "github.com/tigera/operator/pkg/render/common/podaffinity" @@ -53,7 +52,7 @@ import ( // returned create and delete lists. func apiServerObjects(c render.Component) ([]client.Object, []client.Object) { create, del := c.Objects() - rc := extensions.RenderContext{} + rc := render.RenderContext{} var extCtx any if p, ok := c.(render.ExtensionContextProvider); ok { ec := p.ExtensionContext().(render.APIServerExtensionContext) diff --git a/pkg/render/component.go b/pkg/render/component.go index 93e4824f3a..ad14722304 100644 --- a/pkg/render/component.go +++ b/pkg/render/component.go @@ -51,10 +51,10 @@ type Extensible interface { // ExtensionContextProvider is an optional companion to Extensible. A component // implements it to hand its modifier component-specific context that can't be -// derived from the shared extensions.RenderContext - config only the component's -// controller has, such as a keypair the controller created. The componentHandler -// reads the returned value into RenderContext.Component before applying the -// modifier, and the modifier type-asserts it to the component's own context type. +// derived from the shared RenderContext - config only the component's controller +// has, such as a keypair the controller created. extensions.RegisterModifier +// asserts the returned value to the modifier's config type and passes it through, +// so the modifier body needs no assertion. type ExtensionContextProvider interface { ExtensionContext() any } diff --git a/pkg/render/decorate_helpers_test.go b/pkg/render/decorate_helpers_test.go index 75f0587276..8393de7e94 100644 --- a/pkg/render/decorate_helpers_test.go +++ b/pkg/render/decorate_helpers_test.go @@ -19,6 +19,7 @@ import ( operatorv1 "github.com/tigera/operator/api/v1" "github.com/tigera/operator/pkg/extensions" + "github.com/tigera/operator/pkg/render" rmeta "github.com/tigera/operator/pkg/render/common/meta" ) @@ -59,13 +60,13 @@ func (s stubExtComponent) ExtensionContext() any { // applyExtensions decorates a stub component holding the given objects with the // extension registered under key, then renders it. For a modifier that needs the // component's typed config, use applyExtensionsWithContext. -func applyExtensions(s *extensions.Set, key string, rc extensions.RenderContext, create, del []client.Object) ([]client.Object, []client.Object) { +func applyExtensions(s *extensions.Set, key string, rc render.RenderContext, create, del []client.Object) ([]client.Object, []client.Object) { return applyExtensionsWithContext(s, key, rc, nil, create, del) } // applyExtensionsWithContext is applyExtensions for a modifier that reads the // component's typed config: extCtx is delivered as the stub's ExtensionContext. -func applyExtensionsWithContext(s *extensions.Set, key string, rc extensions.RenderContext, extCtx any, create, del []client.Object) ([]client.Object, []client.Object) { +func applyExtensionsWithContext(s *extensions.Set, key string, rc render.RenderContext, extCtx any, create, del []client.Object) ([]client.Object, []client.Object) { stub := stubExtComponent{key: key, extCtx: extCtx, create: create, delete: del} return s.Decorate(stub, rc).Objects() } diff --git a/pkg/render/guardian_test.go b/pkg/render/guardian_test.go index 532b36e75f..cc8a0a9cc8 100644 --- a/pkg/render/guardian_test.go +++ b/pkg/render/guardian_test.go @@ -36,7 +36,6 @@ import ( "github.com/tigera/operator/pkg/components" "github.com/tigera/operator/pkg/controller/certificatemanager" ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" - "github.com/tigera/operator/pkg/extensions" "github.com/tigera/operator/pkg/render" rmeta "github.com/tigera/operator/pkg/render/common/meta" "github.com/tigera/operator/pkg/render/common/networkpolicy" @@ -50,7 +49,7 @@ func guardianObjects(cfg *render.GuardianConfiguration) []client.Object { g := render.Guardian(cfg) ExpectWithOffset(1, g.ResolveImages(nil)).To(BeNil()) objs, _ := g.Objects() - rc := extensions.RenderContext{Installation: cfg.Installation} + rc := render.RenderContext{Installation: cfg.Installation} var extCtx any if p, ok := g.(render.ExtensionContextProvider); ok { extCtx = p.ExtensionContext() @@ -113,7 +112,7 @@ var _ = Describe("Rendering tests", func() { resources, deleteResources = g.Objects() // Apply the registered enterprise modifier the way the componentHandler // does, so these enterprise tests exercise the integrated output. - rc := extensions.RenderContext{Installation: cfg.Installation} + rc := render.RenderContext{Installation: cfg.Installation} var extCtx any if p, ok := g.(render.ExtensionContextProvider); ok { extCtx = p.ExtensionContext() @@ -350,7 +349,7 @@ var _ = Describe("Rendering tests", func() { // Apply the registered enterprise modifier the way the componentHandler // does, so the enterprise policy is exercised. For the Calico variant the // modifier is a no-op and the OSS policy is returned. - rc := extensions.RenderContext{Installation: cfg.Installation} + rc := render.RenderContext{Installation: cfg.Installation} var extCtx any if p, ok := g.(render.ExtensionContextProvider); ok { extCtx = p.ExtensionContext() diff --git a/pkg/render/node_enterprise_test.go b/pkg/render/node_enterprise_test.go index 7696646454..9c06a22b84 100644 --- a/pkg/render/node_enterprise_test.go +++ b/pkg/render/node_enterprise_test.go @@ -31,6 +31,7 @@ import ( "github.com/tigera/operator/pkg/apis" "github.com/tigera/operator/pkg/common" "github.com/tigera/operator/pkg/controller/certificatemanager" + "github.com/tigera/operator/pkg/controller/contexts" "github.com/tigera/operator/pkg/controller/k8sapi" ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" "github.com/tigera/operator/pkg/dns" @@ -51,7 +52,7 @@ var _ = Describe("node enterprise modifier integration", func() { certManager certificatemanager.CertificateManager typhaNodeTLS *render.TyphaNodeTLS instance *operatorv1.InstallationSpec - renderCtx extensions.RenderContext + renderCtx render.RenderContext nodePrometheusTLS certificatemanagement.KeyPairInterface ) @@ -97,13 +98,13 @@ var _ = Describe("node enterprise modifier integration", func() { // Build the render context the way the controller does: run the enterprise // controller extension, which stashes the node prometheus keypair in the // context for the node modifier to read. - cc := extensions.ControllerContext{ - RenderContext: extensions.RenderContext{ + cc := contexts.ControllerContext{ + RenderContext: render.RenderContext{ Installation: instance, TrustedBundle: typhaNodeTLS.TrustedBundle, ClusterDomain: dns.DefaultClusterDomain, }, - Controller: extensions.InstallationController, + Controller: contexts.InstallationController, Ctx: context.Background(), Client: cli, CertificateManager: certManager, @@ -114,7 +115,7 @@ var _ = Describe("node enterprise modifier integration", func() { // renderNodeObjects renders the real node component and applies the registered // modifier, exactly as the componentHandler does. - renderNodeObjects := func(rc extensions.RenderContext) []client.Object { + renderNodeObjects := func(rc render.RenderContext) []client.Object { cfg := &render.NodeConfiguration{ K8sServiceEp: k8sapi.ServiceEndpoint{}, Installation: instance, @@ -180,13 +181,13 @@ var _ = Describe("node enterprise modifier integration", func() { Spec: operatorv1.LogCollectorSpec{CollectProcessPath: &enable}, })).NotTo(HaveOccurred()) - rc, _, err := ext.ExtendContext(extensions.ControllerContext{ - RenderContext: extensions.RenderContext{ + rc, _, err := ext.ExtendContext(contexts.ControllerContext{ + RenderContext: render.RenderContext{ Installation: instance, TrustedBundle: typhaNodeTLS.TrustedBundle, ClusterDomain: dns.DefaultClusterDomain, }, - Controller: extensions.InstallationController, + Controller: contexts.InstallationController, Ctx: context.Background(), Client: cli, CertificateManager: certManager, diff --git a/pkg/extensions/rendercontext.go b/pkg/render/rendercontext.go similarity index 83% rename from pkg/extensions/rendercontext.go rename to pkg/render/rendercontext.go index 3e092bf18c..b27976794a 100644 --- a/pkg/extensions/rendercontext.go +++ b/pkg/render/rendercontext.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package extensions +package render import ( v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" @@ -27,9 +27,15 @@ import ( // the shared TrustedBundle, and an opaque Extension slot for controller-produced // data specific to one extension. // +// It lives in render (not the extensions package) because it is the render-phase +// input a modifier consumes - the render-side corollary to the controller-phase +// ControllerContext. The extensions package wires modifiers to it; it is not +// itself part of the extension mechanism. +// // Per-component config a modifier needs but can't derive from these fields is // not carried here; it flows to the modifier as a typed argument (see -// RegisterModifier), supplied by the component via render.ExtensionContextProvider. +// extensions.RegisterModifier), supplied by the component via +// ExtensionContextProvider. type RenderContext struct { Installation *operatorv1.InstallationSpec FelixConfiguration *v3.FelixConfiguration diff --git a/pkg/render/windows_test.go b/pkg/render/windows_test.go index 1d92427a6b..ff1c165403 100644 --- a/pkg/render/windows_test.go +++ b/pkg/render/windows_test.go @@ -35,7 +35,6 @@ import ( "github.com/tigera/operator/pkg/controller/certificatemanager" "github.com/tigera/operator/pkg/controller/k8sapi" ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" - "github.com/tigera/operator/pkg/extensions" "github.com/tigera/operator/pkg/render" rmeta "github.com/tigera/operator/pkg/render/common/meta" rtest "github.com/tigera/operator/pkg/render/common/test" @@ -50,7 +49,7 @@ func renderWindows(cfg *render.WindowsConfiguration) []client.Object { comp := render.Windows(cfg) ExpectWithOffset(1, comp.ResolveImages(nil)).To(BeNil()) objs, _ := comp.Objects() - rc := extensions.RenderContext{Installation: cfg.Installation} + rc := render.RenderContext{Installation: cfg.Installation} out, _ := applyExtensions(ext, render.ComponentNameWindows, rc, objs, nil) return out } From c60679626b15c58de36a65c288e0455a026366a0 Mon Sep 17 00:00:00 2001 From: Casey Davenport Date: Wed, 24 Jun 2026 12:00:45 -0700 Subject: [PATCH 42/54] Add a generic ExtractExtensionData helper for the render context slot Each extension had its own one-line accessor doing the same type assertion against RenderContext.Extension. Replace that boilerplate with a single generic render.ExtractExtensionData[T] and have the per-component accessors call it. --- pkg/enterprise/apiserver.go | 3 +-- pkg/enterprise/installation.go | 3 +-- pkg/enterprise/windows.go | 3 +-- pkg/render/rendercontext.go | 9 +++++++++ 4 files changed, 12 insertions(+), 6 deletions(-) diff --git a/pkg/enterprise/apiserver.go b/pkg/enterprise/apiserver.go index 34b2978e1f..1f3f1c8395 100644 --- a/pkg/enterprise/apiserver.go +++ b/pkg/enterprise/apiserver.go @@ -77,8 +77,7 @@ type apiServerRenderData struct { // apiServerData pulls the API server hook's render data back out of the render context, // returning the zero value when none is set. func apiServerData(rc render.RenderContext) apiServerRenderData { - d, _ := rc.Extension.(apiServerRenderData) - return d + return render.ExtractExtensionData[apiServerRenderData](rc) } // apiServer carries the rendered API server configuration, the resolved image, and the diff --git a/pkg/enterprise/installation.go b/pkg/enterprise/installation.go index 534dd07b98..68f0fff7f3 100644 --- a/pkg/enterprise/installation.go +++ b/pkg/enterprise/installation.go @@ -65,8 +65,7 @@ type installationRenderData struct { // installationData pulls the installation extension's render data back out of the // render context, returning the zero value when none is set. func installationData(rc render.RenderContext) installationRenderData { - data, _ := rc.Extension.(installationRenderData) - return data + return render.ExtractExtensionData[installationRenderData](rc) } func collectProcessPathEnabled(lc *operatorv1.LogCollector) bool { diff --git a/pkg/enterprise/windows.go b/pkg/enterprise/windows.go index a464006d7e..7aba485341 100644 --- a/pkg/enterprise/windows.go +++ b/pkg/enterprise/windows.go @@ -60,8 +60,7 @@ type windowsRenderData struct { // windowsData pulls the windows extension's render data back out of the render // context, returning the zero value when none is set. func windowsData(rc render.RenderContext) windowsRenderData { - data, _ := rc.Extension.(windowsRenderData) - return data + return render.ExtractExtensionData[windowsRenderData](rc) } // Validate rejects windows installation config Calico Enterprise does not support. diff --git a/pkg/render/rendercontext.go b/pkg/render/rendercontext.go index b27976794a..d3821c168b 100644 --- a/pkg/render/rendercontext.go +++ b/pkg/render/rendercontext.go @@ -51,3 +51,12 @@ type RenderContext struct { // code never reads it. Nil when no extension is active. Extension any } + +// ExtractExtensionData returns the extension-owned data a controller extension +// stashed in the render context, asserted to T, or the zero value of T when the +// slot is empty or holds a different type. Extensions use it instead of repeating +// the type assertion in a per-component accessor. +func ExtractExtensionData[T any](rc RenderContext) T { + data, _ := rc.Extension.(T) + return data +} From e40509df1bb3e3c3033161b843d411292a6588d0 Mon Sep 17 00:00:00 2001 From: Casey Davenport Date: Wed, 24 Jun 2026 13:08:15 -0700 Subject: [PATCH 43/54] Share the extension decorate test helper instead of copying it The stub component and applyExtensions helper for exercising a modifier through Set.Decorate were copy-pasted into the extensions, render, and enterprise test packages. Pull them into pkg/extensions/extensionstest so there's one copy, and have the three test packages import it. --- pkg/enterprise/apiserver_test.go | 9 +-- pkg/enterprise/decorate_helpers_test.go | 72 --------------------- pkg/enterprise/guardian_test.go | 11 ++-- pkg/enterprise/kubecontrollers_test.go | 11 ++-- pkg/enterprise/node_test.go | 21 ++++--- pkg/enterprise/typha_test.go | 7 ++- pkg/enterprise/windows_test.go | 11 ++-- pkg/extensions/decorate_helpers_test.go | 72 --------------------- pkg/extensions/extension_test.go | 13 ++-- pkg/extensions/extensionstest/decorate.go | 77 +++++++++++++++++++++++ pkg/render/apiserver_test.go | 3 +- pkg/render/decorate_helpers_test.go | 72 --------------------- pkg/render/guardian_test.go | 7 ++- pkg/render/node_enterprise_test.go | 5 +- pkg/render/windows_test.go | 3 +- 15 files changed, 133 insertions(+), 261 deletions(-) delete mode 100644 pkg/enterprise/decorate_helpers_test.go delete mode 100644 pkg/extensions/decorate_helpers_test.go create mode 100644 pkg/extensions/extensionstest/decorate.go delete mode 100644 pkg/render/decorate_helpers_test.go diff --git a/pkg/enterprise/apiserver_test.go b/pkg/enterprise/apiserver_test.go index 2c769b1ed8..afe4292f46 100644 --- a/pkg/enterprise/apiserver_test.go +++ b/pkg/enterprise/apiserver_test.go @@ -39,6 +39,7 @@ import ( ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" "github.com/tigera/operator/pkg/dns" "github.com/tigera/operator/pkg/extensions" + "github.com/tigera/operator/pkg/extensions/extensionstest" "github.com/tigera/operator/pkg/render" "github.com/tigera/operator/pkg/tls/certificatemanagement" ) @@ -154,7 +155,7 @@ var _ = Describe("API server enterprise modifier", func() { create, del := comp.Objects() ec := comp.(render.ExtensionContextProvider).ExtensionContext() - return applyExtensionsWithContext(ext, render.ComponentNameAPIServer, rc, ec, create, del) + return extensionstest.ApplyExtensionsWithContext(ext, render.ComponentNameAPIServer, rc, ec, create, del) } apiServerDeployment := func(objs []client.Object) *appsv1.Deployment { @@ -306,7 +307,7 @@ var _ = Describe("API server enterprise modifier", func() { Expect(ok).To(BeTrue()) ec := comp.(render.ExtensionContextProvider).ExtensionContext() - create, del = applyExtensionsWithContext(ext, render.ComponentNameAPIServer, rc, ec, create, del) + create, del = extensionstest.ApplyExtensionsWithContext(ext, render.ComponentNameAPIServer, rc, ec, create, del) // After the modifier, the deployment (with the query server container) is in the // create list and out of the delete list. @@ -368,7 +369,7 @@ var _ = Describe("API server enterprise policy modifier", func() { comp := render.APIServerPolicy(cfg) create, del := comp.Objects() ec := comp.(render.ExtensionContextProvider).ExtensionContext() - objs, _ := applyExtensionsWithContext(ext, render.ComponentNameAPIServerPolicy, rc, ec, create, del) + objs, _ := extensionstest.ApplyExtensionsWithContext(ext, render.ComponentNameAPIServerPolicy, rc, ec, create, del) policy, ok := extensions.FindObject[*v3.NetworkPolicy](objs, render.APIServerPolicyName) Expect(ok).To(BeTrue()) return policy @@ -430,7 +431,7 @@ var _ = Describe("API server Calico-variant cleanup", func() { Expect(comp.ResolveImages(nil)).NotTo(HaveOccurred()) create, del := comp.Objects() ec := comp.(render.ExtensionContextProvider).ExtensionContext() - _, del = applyExtensionsWithContext(ext, render.ComponentNameAPIServer, rc, ec, create, del) + _, del = extensionstest.ApplyExtensionsWithContext(ext, render.ComponentNameAPIServer, rc, ec, create, del) _, ok := extensions.FindObject[*rbacv1.ClusterRole](del, "tigera-ui-user") Expect(ok).To(BeTrue()) diff --git a/pkg/enterprise/decorate_helpers_test.go b/pkg/enterprise/decorate_helpers_test.go deleted file mode 100644 index 042715d5c7..0000000000 --- a/pkg/enterprise/decorate_helpers_test.go +++ /dev/null @@ -1,72 +0,0 @@ -// Copyright (c) 2026 Tigera, Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package enterprise_test - -import ( - client "sigs.k8s.io/controller-runtime/pkg/client" - - operatorv1 "github.com/tigera/operator/api/v1" - "github.com/tigera/operator/pkg/extensions" - "github.com/tigera/operator/pkg/render" - rmeta "github.com/tigera/operator/pkg/render/common/meta" -) - -// stubExtComponent adapts raw object lists to a render.Component so a registered -// extension can be exercised through Set.Decorate, the same seam the component -// handler uses. key selects the extension; extCtx is delivered as the component's -// ExtensionContext (the typed config a RegisterModifier modifier reads). -type stubExtComponent struct { - key string - extCtx any - create, delete []client.Object -} - -func (s stubExtComponent) ResolveImages(*operatorv1.ImageSet) error { - return nil -} - -func (s stubExtComponent) Objects() ([]client.Object, []client.Object) { - return s.create, s.delete -} - -func (s stubExtComponent) Ready() bool { - return true -} - -func (s stubExtComponent) SupportedOSType() rmeta.OSType { - return rmeta.OSTypeAny -} - -func (s stubExtComponent) ModifierKey() string { - return s.key -} - -func (s stubExtComponent) ExtensionContext() any { - return s.extCtx -} - -// applyExtensions decorates a stub component holding the given objects with the -// extension registered under key, then renders it. For a modifier that needs the -// component's typed config, use applyExtensionsWithContext. -func applyExtensions(s *extensions.Set, key string, rc render.RenderContext, create, del []client.Object) ([]client.Object, []client.Object) { - return applyExtensionsWithContext(s, key, rc, nil, create, del) -} - -// applyExtensionsWithContext is applyExtensions for a modifier that reads the -// component's typed config: extCtx is delivered as the stub's ExtensionContext. -func applyExtensionsWithContext(s *extensions.Set, key string, rc render.RenderContext, extCtx any, create, del []client.Object) ([]client.Object, []client.Object) { - stub := stubExtComponent{key: key, extCtx: extCtx, create: create, delete: del} - return s.Decorate(stub, rc).Objects() -} diff --git a/pkg/enterprise/guardian_test.go b/pkg/enterprise/guardian_test.go index 6a6e965296..535d4f3b14 100644 --- a/pkg/enterprise/guardian_test.go +++ b/pkg/enterprise/guardian_test.go @@ -27,6 +27,7 @@ import ( v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" operatorv1 "github.com/tigera/operator/api/v1" "github.com/tigera/operator/pkg/extensions" + "github.com/tigera/operator/pkg/extensions/extensionstest" "github.com/tigera/operator/pkg/render" ) @@ -52,7 +53,7 @@ var _ = Describe("guardian enterprise modifier", func() { entCtx := render.RenderContext{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.CalicoEnterprise}} It("appends the secrets RBAC and UI settings", func() { - out, _ := applyExtensionsWithContext(ext, render.GuardianName, entCtx, render.GuardianExtensionContext{}, newObjs(), nil) + out, _ := extensionstest.ApplyExtensionsWithContext(ext, render.GuardianName, entCtx, render.GuardianExtensionContext{}, newObjs(), nil) _, ok := extensions.FindObject[*rbacv1.Role](out, render.GuardianSecretsRole) Expect(ok).To(BeTrue()) _, ok = extensions.FindObject[*rbacv1.RoleBinding](out, render.GuardianSecretsRoleBindingName) @@ -62,7 +63,7 @@ var _ = Describe("guardian enterprise modifier", func() { }) It("adds the elasticsearch and kibana service ports", func() { - out, _ := applyExtensionsWithContext(ext, render.GuardianName, entCtx, render.GuardianExtensionContext{}, newObjs(), nil) + out, _ := extensionstest.ApplyExtensionsWithContext(ext, render.GuardianName, entCtx, render.GuardianExtensionContext{}, newObjs(), nil) svc, _ := extensions.FindObject[*corev1.Service](out, render.GuardianServiceName) names := []string{} for _, p := range svc.Spec.Ports { @@ -75,7 +76,7 @@ var _ = Describe("guardian enterprise modifier", func() { gc := render.GuardianExtensionContext{ Impersonation: &operatorv1.Impersonation{Users: []string{"foo"}, Groups: []string{"bar"}}, } - out, _ := applyExtensionsWithContext(ext, render.GuardianName, entCtx, gc, newObjs(), nil) + out, _ := extensionstest.ApplyExtensionsWithContext(ext, render.GuardianName, entCtx, gc, newObjs(), nil) role, _ := extensions.FindObject[*rbacv1.ClusterRole](out, render.GuardianClusterRoleName) // The single OSS placeholder rule is gone, replaced by the enterprise set. @@ -86,14 +87,14 @@ var _ = Describe("guardian enterprise modifier", func() { It("adds the CA bundle env to the guardian container", func() { gc := render.GuardianExtensionContext{TrustedBundleMountPath: "/ca/bundle"} - out, _ := applyExtensionsWithContext(ext, render.GuardianName, entCtx, gc, newObjs(), nil) + out, _ := extensionstest.ApplyExtensionsWithContext(ext, render.GuardianName, entCtx, gc, newObjs(), nil) dep, _ := extensions.FindObject[*appsv1.Deployment](out, render.GuardianDeploymentName) Expect(dep.Spec.Template.Spec.Containers[0].Env).To(ContainElement(corev1.EnvVar{Name: "GUARDIAN_PROMETHEUS_CA_BUNDLE_PATH", Value: "/ca/bundle"})) }) It("does nothing for the Calico variant", func() { ctx := render.RenderContext{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.Calico}} - out, _ := applyExtensions(ext, render.GuardianName, ctx, newObjs(), nil) + out, _ := extensionstest.ApplyExtensions(ext, render.GuardianName, ctx, newObjs(), nil) Expect(out).To(HaveLen(len(newObjs()))) role, _ := extensions.FindObject[*rbacv1.ClusterRole](out, render.GuardianClusterRoleName) Expect(role.Rules).To(Equal([]rbacv1.PolicyRule{{Verbs: []string{"get"}}})) diff --git a/pkg/enterprise/kubecontrollers_test.go b/pkg/enterprise/kubecontrollers_test.go index 4f1f967513..0eaeda52db 100644 --- a/pkg/enterprise/kubecontrollers_test.go +++ b/pkg/enterprise/kubecontrollers_test.go @@ -38,6 +38,7 @@ import ( ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" "github.com/tigera/operator/pkg/enterprise" "github.com/tigera/operator/pkg/extensions" + "github.com/tigera/operator/pkg/extensions/extensionstest" "github.com/tigera/operator/pkg/render" "github.com/tigera/operator/pkg/render/applicationlayer" rmeta "github.com/tigera/operator/pkg/render/common/meta" @@ -68,7 +69,7 @@ var _ = Describe("kube-controllers enterprise modifier", func() { rc, _, err := ext.ExtendContext(newControllerContext(operatorv1.CalicoEnterprise)) Expect(err).NotTo(HaveOccurred()) - objs, _ := applyExtensions(ext, render.ComponentNameKubeControllers, rc, []client.Object{kubeControllersDeployment()}, nil) + objs, _ := extensionstest.ApplyExtensions(ext, render.ComponentNameKubeControllers, rc, []client.Object{kubeControllersDeployment()}, nil) dp, ok := extensions.FindObject[*appsv1.Deployment](objs, kubecontrollers.KubeController) Expect(ok).To(BeTrue()) @@ -87,7 +88,7 @@ var _ = Describe("kube-controllers enterprise modifier", func() { rc, _, err := ext.ExtendContext(certManagementControllerContext()) Expect(err).NotTo(HaveOccurred()) - objs, _ := applyExtensions(ext, render.ComponentNameKubeControllers, rc, []client.Object{kubeControllersDeployment()}, nil) + objs, _ := extensionstest.ApplyExtensions(ext, render.ComponentNameKubeControllers, rc, []client.Object{kubeControllersDeployment()}, nil) dp, ok := extensions.FindObject[*appsv1.Deployment](objs, kubecontrollers.KubeController) Expect(ok).To(BeTrue()) @@ -147,7 +148,7 @@ var _ = Describe("calico-kube-controllers enterprise surface", func() { comp := kubecontrollers.NewCalicoKubeControllers(calicoKubeControllersCfg(cc)) Expect(comp.ResolveImages(nil)).NotTo(HaveOccurred()) create, del := comp.Objects() - out, _ := applyExtensions(ext, render.ComponentNameKubeControllers, rc, create, del) + out, _ := extensionstest.ApplyExtensions(ext, render.ComponentNameKubeControllers, rc, create, del) return out } @@ -223,7 +224,7 @@ var _ = Describe("calico-kube-controllers enterprise surface", func() { comp := kubecontrollers.NewCalicoKubeControllers(calicoKubeControllersCfg(cc)) Expect(comp.ResolveImages(nil)).NotTo(HaveOccurred()) create, del := comp.Objects() - _, toDelete := applyExtensions(ext, render.ComponentNameKubeControllers, rc, create, del) + _, toDelete := extensionstest.ApplyExtensions(ext, render.ComponentNameKubeControllers, rc, create, del) _, ok := extensions.FindObject[*corev1.Service](toDelete, applicationlayer.WAFWebhookServiceName) Expect(ok).To(BeTrue(), "the webhook Service should be queued for deletion") @@ -236,7 +237,7 @@ var _ = Describe("calico-kube-controllers enterprise surface", func() { comp := kubecontrollers.NewCalicoKubeControllersPolicy(calicoKubeControllersCfg(cc), nil) create, del := comp.Objects() - objs, _ := applyExtensions(ext, render.ComponentNameKubeControllersPolicy, rc, create, del) + objs, _ := extensionstest.ApplyExtensions(ext, render.ComponentNameKubeControllersPolicy, rc, create, del) policy, ok := extensions.FindObject[*v3.NetworkPolicy](objs, kubecontrollers.KubeControllerNetworkPolicyName) Expect(ok).To(BeTrue()) diff --git a/pkg/enterprise/node_test.go b/pkg/enterprise/node_test.go index 335f7153b9..058f19d3d0 100644 --- a/pkg/enterprise/node_test.go +++ b/pkg/enterprise/node_test.go @@ -29,6 +29,7 @@ import ( "github.com/tigera/operator/pkg/common" "github.com/tigera/operator/pkg/components" "github.com/tigera/operator/pkg/extensions" + "github.com/tigera/operator/pkg/extensions/extensionstest" "github.com/tigera/operator/pkg/render" ) @@ -81,7 +82,7 @@ var _ = Describe("node enterprise modifier", func() { } It("adds the enterprise cluster role rules", func() { - out, _ := applyExtensions(ext, render.ComponentNameNode, entCtx(), newObjs(), nil) + out, _ := extensionstest.ApplyExtensions(ext, render.ComponentNameNode, entCtx(), newObjs(), nil) nodeRole, ok := extensions.FindObject[*rbacv1.ClusterRole](out, render.CalicoNodeObjectName) Expect(ok).To(BeTrue()) @@ -93,7 +94,7 @@ var _ = Describe("node enterprise modifier", func() { }) It("adds the enterprise felix env to the node container", func() { - out, _ := applyExtensions(ext, render.ComponentNameNode, entCtx(), newObjs(), nil) + out, _ := extensionstest.ApplyExtensions(ext, render.ComponentNameNode, entCtx(), newObjs(), nil) ds, _ := extensions.FindObject[*appsv1.DaemonSet](out, common.NodeDaemonSetName) c := nodeContainer(ds) @@ -110,13 +111,13 @@ var _ = Describe("node enterprise modifier", func() { ctx := entCtx() ctx.FelixConfiguration = &v3.FelixConfiguration{Spec: v3.FelixConfigurationSpec{PrometheusReporterPort: &reporter}} - out, _ := applyExtensions(ext, render.ComponentNameNode, ctx, newObjs(), nil) + out, _ := extensionstest.ApplyExtensions(ext, render.ComponentNameNode, ctx, newObjs(), nil) ds, _ := extensions.FindObject[*appsv1.DaemonSet](out, common.NodeDaemonSetName) Expect(nodeContainer(ds).Env).To(ContainElement(corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERPORT", Value: "7081"})) }) It("appends the BGP metrics readiness check when the bird check is present", func() { - out, _ := applyExtensions(ext, render.ComponentNameNode, entCtx(), newObjs(), nil) + out, _ := extensionstest.ApplyExtensions(ext, render.ComponentNameNode, entCtx(), newObjs(), nil) ds, _ := extensions.FindObject[*appsv1.DaemonSet](out, common.NodeDaemonSetName) Expect(nodeContainer(ds).ReadinessProbe.Exec.Command).To(ContainElement("--bgp-metrics-ready")) }) @@ -126,7 +127,7 @@ var _ = Describe("node enterprise modifier", func() { ds := objs[2].(*appsv1.DaemonSet) ds.Spec.Template.Spec.Containers[0].ReadinessProbe.Exec.Command = []string{"/bin/calico-node", "--felix-ready"} - out, _ := applyExtensions(ext, render.ComponentNameNode, entCtx(), objs, nil) + out, _ := extensionstest.ApplyExtensions(ext, render.ComponentNameNode, entCtx(), objs, nil) got, _ := extensions.FindObject[*appsv1.DaemonSet](out, common.NodeDaemonSetName) Expect(nodeContainer(got).ReadinessProbe.Exec.Command).NotTo(ContainElement("--bgp-metrics-ready")) }) @@ -136,7 +137,7 @@ var _ = Describe("node enterprise modifier", func() { ctx := entCtx() ctx.Installation.CalicoNetwork = &operatorv1.CalicoNetworkSpec{MultiInterfaceMode: &mode} - out, _ := applyExtensions(ext, render.ComponentNameNode, ctx, newObjs(), nil) + out, _ := extensionstest.ApplyExtensions(ext, render.ComponentNameNode, ctx, newObjs(), nil) ds, _ := extensions.FindObject[*appsv1.DaemonSet](out, common.NodeDaemonSetName) want := corev1.EnvVar{Name: "MULTI_INTERFACE_MODE", Value: mode.Value()} @@ -145,7 +146,7 @@ var _ = Describe("node enterprise modifier", func() { }) It("appends the node metrics service", func() { - out, _ := applyExtensions(ext, render.ComponentNameNode, entCtx(), newObjs(), nil) + out, _ := extensionstest.ApplyExtensions(ext, render.ComponentNameNode, entCtx(), newObjs(), nil) svc, ok := extensions.FindObject[*corev1.Service](out, render.CalicoNodeMetricsService) Expect(ok).To(BeTrue()) Expect(svc.Spec.Ports).To(HaveLen(2)) @@ -164,7 +165,7 @@ var _ = Describe("node enterprise modifier", func() { PrometheusMetricsEnabled: &enabled, }} - out, _ := applyExtensions(ext, render.ComponentNameNode, ctx, newObjs(), nil) + out, _ := extensionstest.ApplyExtensions(ext, render.ComponentNameNode, ctx, newObjs(), nil) svc, _ := extensions.FindObject[*corev1.Service](out, render.CalicoNodeMetricsService) Expect(svc.Spec.Ports).To(HaveLen(3)) Expect(svc.Spec.Ports[0].Port).To(Equal(int32(7081))) @@ -174,7 +175,7 @@ var _ = Describe("node enterprise modifier", func() { It("is a no-op for the Calico variant", func() { ctx := render.RenderContext{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.Calico}} - out, _ := applyExtensions(ext, render.ComponentNameNode, ctx, newObjs(), nil) + out, _ := extensionstest.ApplyExtensions(ext, render.ComponentNameNode, ctx, newObjs(), nil) _, ok := extensions.FindObject[*corev1.Service](out, render.CalicoNodeMetricsService) Expect(ok).To(BeFalse()) @@ -183,7 +184,7 @@ var _ = Describe("node enterprise modifier", func() { }) It("does not panic on a zero RenderContext", func() { - out, _ := applyExtensions(ext, render.ComponentNameNode, render.RenderContext{}, newObjs(), nil) + out, _ := extensionstest.ApplyExtensions(ext, render.ComponentNameNode, render.RenderContext{}, newObjs(), nil) _, ok := extensions.FindObject[*corev1.Service](out, render.CalicoNodeMetricsService) Expect(ok).To(BeFalse()) }) diff --git a/pkg/enterprise/typha_test.go b/pkg/enterprise/typha_test.go index 53c5ac73fa..8720f2091f 100644 --- a/pkg/enterprise/typha_test.go +++ b/pkg/enterprise/typha_test.go @@ -24,6 +24,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/extensions/extensionstest" "github.com/tigera/operator/pkg/render" ) @@ -48,7 +49,7 @@ var _ = Describe("typha enterprise modifier", func() { Variant: operatorv1.CalicoEnterprise, CalicoNetwork: &operatorv1.CalicoNetworkSpec{MultiInterfaceMode: &multiMode}, }} - out, _ := applyExtensions(ext, render.ComponentNameTypha, ctx, newObjs(), nil) + out, _ := extensionstest.ApplyExtensions(ext, render.ComponentNameTypha, ctx, newObjs(), nil) role := out[0].(*rbacv1.ClusterRole) Expect(role.Rules).To(ContainElement(HaveField("Resources", ContainElement("licensekeys")))) @@ -68,14 +69,14 @@ var _ = Describe("typha enterprise modifier", func() { Variant: operatorv1.Calico, CalicoNetwork: &operatorv1.CalicoNetworkSpec{MultiInterfaceMode: &multiMode}, }} - out, _ := applyExtensions(ext, render.ComponentNameTypha, ctx, newObjs(), nil) + out, _ := extensionstest.ApplyExtensions(ext, render.ComponentNameTypha, ctx, newObjs(), nil) Expect(out[0].(*rbacv1.ClusterRole).Rules).To(BeEmpty()) dep := out[1].(*appsv1.Deployment) Expect(dep.Spec.Template.Spec.Containers[0].Env).To(BeEmpty()) }) It("does not panic on a zero Context (nil Installation)", func() { - out, _ := applyExtensions(ext, render.ComponentNameTypha, render.RenderContext{}, newObjs(), nil) + out, _ := extensionstest.ApplyExtensions(ext, render.ComponentNameTypha, render.RenderContext{}, newObjs(), nil) Expect(out[0].(*rbacv1.ClusterRole).Rules).To(BeEmpty()) }) }) diff --git a/pkg/enterprise/windows_test.go b/pkg/enterprise/windows_test.go index 810ec790d9..9b68a67e32 100644 --- a/pkg/enterprise/windows_test.go +++ b/pkg/enterprise/windows_test.go @@ -35,6 +35,7 @@ import ( ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" "github.com/tigera/operator/pkg/dns" "github.com/tigera/operator/pkg/extensions" + "github.com/tigera/operator/pkg/extensions/extensionstest" "github.com/tigera/operator/pkg/render" ) @@ -95,7 +96,7 @@ var _ = Describe("windows enterprise modifier", func() { } It("appends the node-metrics service", func() { - out, _ := applyExtensions(ext, render.ComponentNameWindows, ctxFor(operatorv1.ProviderNone), newObjs(), nil) + out, _ := extensionstest.ApplyExtensions(ext, render.ComponentNameWindows, ctxFor(operatorv1.ProviderNone), newObjs(), nil) svc, ok := extensions.FindObject[*corev1.Service](out, render.WindowsNodeMetricsService) Expect(ok).To(BeTrue()) Expect(svc.Namespace).To(Equal(common.CalicoNamespace)) @@ -103,7 +104,7 @@ var _ = Describe("windows enterprise modifier", func() { }) It("swaps the cni log mount for the calico log volume and adds enterprise env", func() { - out, _ := applyExtensions(ext, render.ComponentNameWindows, ctxFor(operatorv1.ProviderNone), newObjs(), nil) + out, _ := extensionstest.ApplyExtensions(ext, render.ComponentNameWindows, ctxFor(operatorv1.ProviderNone), newObjs(), nil) d := ds(out) Expect(d.Spec.Template.Spec.Volumes).To(ContainElement(HaveField("Name", "var-log-calico"))) @@ -120,7 +121,7 @@ var _ = Describe("windows enterprise modifier", func() { }) It("sets the trusted DNS server on openshift", func() { - out, _ := applyExtensions(ext, render.ComponentNameWindows, ctxFor(operatorv1.ProviderOpenShift), newObjs(), nil) + out, _ := extensionstest.ApplyExtensions(ext, render.ComponentNameWindows, ctxFor(operatorv1.ProviderOpenShift), newObjs(), nil) Expect(container(ds(out), "node").Env).To(ContainElement(corev1.EnvVar{Name: "FELIX_DNSTRUSTEDSERVERS", Value: "k8s-service:openshift-dns/dns-default"})) }) @@ -153,7 +154,7 @@ var _ = Describe("windows enterprise modifier", func() { rc, _, err := ext.ExtendContext(cc) Expect(err).NotTo(HaveOccurred()) - out, _ := applyExtensions(ext, render.ComponentNameWindows, rc, newObjs(), nil) + out, _ := extensionstest.ApplyExtensions(ext, render.ComponentNameWindows, rc, newObjs(), nil) d := ds(out) Expect(d.Spec.Template.Spec.Volumes).To(ContainElement(tls.Volume())) @@ -164,7 +165,7 @@ var _ = Describe("windows enterprise modifier", func() { It("does nothing for the Calico variant", func() { ctx := render.RenderContext{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.Calico}} - out, _ := applyExtensions(ext, render.ComponentNameWindows, ctx, newObjs(), nil) + out, _ := extensionstest.ApplyExtensions(ext, render.ComponentNameWindows, ctx, newObjs(), nil) _, ok := extensions.FindObject[*corev1.Service](out, render.WindowsNodeMetricsService) Expect(ok).To(BeFalse()) Expect(ds(out).Spec.Template.Spec.Volumes).To(BeEmpty()) diff --git a/pkg/extensions/decorate_helpers_test.go b/pkg/extensions/decorate_helpers_test.go deleted file mode 100644 index 81b4f54800..0000000000 --- a/pkg/extensions/decorate_helpers_test.go +++ /dev/null @@ -1,72 +0,0 @@ -// Copyright (c) 2026 Tigera, Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package extensions_test - -import ( - client "sigs.k8s.io/controller-runtime/pkg/client" - - operatorv1 "github.com/tigera/operator/api/v1" - "github.com/tigera/operator/pkg/extensions" - "github.com/tigera/operator/pkg/render" - rmeta "github.com/tigera/operator/pkg/render/common/meta" -) - -// stubExtComponent adapts raw object lists to a render.Component so a registered -// extension can be exercised through Set.Decorate, the same seam the component -// handler uses. key selects the extension; extCtx is delivered as the component's -// ExtensionContext (the typed config a RegisterModifier modifier reads). -type stubExtComponent struct { - key string - extCtx any - create, delete []client.Object -} - -func (s stubExtComponent) ResolveImages(*operatorv1.ImageSet) error { - return nil -} - -func (s stubExtComponent) Objects() ([]client.Object, []client.Object) { - return s.create, s.delete -} - -func (s stubExtComponent) Ready() bool { - return true -} - -func (s stubExtComponent) SupportedOSType() rmeta.OSType { - return rmeta.OSTypeAny -} - -func (s stubExtComponent) ModifierKey() string { - return s.key -} - -func (s stubExtComponent) ExtensionContext() any { - return s.extCtx -} - -// applyExtensions decorates a stub component holding the given objects with the -// extension registered under key, then renders it. For a modifier that needs the -// component's typed config, use applyExtensionsWithContext. -func applyExtensions(s *extensions.Set, key string, rc render.RenderContext, create, del []client.Object) ([]client.Object, []client.Object) { - return applyExtensionsWithContext(s, key, rc, nil, create, del) -} - -// applyExtensionsWithContext is applyExtensions for a modifier that reads the -// component's typed config: extCtx is delivered as the stub's ExtensionContext. -func applyExtensionsWithContext(s *extensions.Set, key string, rc render.RenderContext, extCtx any, create, del []client.Object) ([]client.Object, []client.Object) { - stub := stubExtComponent{key: key, extCtx: extCtx, create: create, delete: del} - return s.Decorate(stub, rc).Objects() -} diff --git a/pkg/extensions/extension_test.go b/pkg/extensions/extension_test.go index 4461c7c0d9..69ae36aa6a 100644 --- a/pkg/extensions/extension_test.go +++ b/pkg/extensions/extension_test.go @@ -23,6 +23,7 @@ import ( operatorv1 "github.com/tigera/operator/api/v1" "github.com/tigera/operator/pkg/extensions" + "github.com/tigera/operator/pkg/extensions/extensionstest" "github.com/tigera/operator/pkg/render" ) @@ -43,7 +44,7 @@ var _ = Describe("extension registry", func() { }) in := []client.Object{&corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "cm"}}} - out, _ := applyExtensions(s, "test", entCtx, in, nil) + out, _ := extensionstest.ApplyExtensions(s, "test", entCtx, in, nil) Expect(out).To(HaveLen(2)) cm := out[0].(*corev1.ConfigMap) @@ -57,7 +58,7 @@ var _ = Describe("extension registry", func() { }) in := []client.Object{&corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "cm"}}} - out, del := applyExtensions(s, "test", entCtx, in, nil) + out, del := extensionstest.ApplyExtensions(s, "test", entCtx, in, nil) Expect(out).To(Equal(in)) Expect(del).To(HaveLen(1)) Expect(del[0].GetName()).To(Equal("stale")) @@ -65,7 +66,7 @@ var _ = Describe("extension registry", func() { It("returns objects unchanged when no modifier is registered", func() { in := []client.Object{&corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "cm"}}} - out, _ := applyExtensions(s, "unregistered", entCtx, in, nil) + out, _ := extensionstest.ApplyExtensions(s, "unregistered", entCtx, in, nil) Expect(out).To(Equal(in)) }) @@ -76,13 +77,13 @@ var _ = Describe("extension registry", func() { calicoCtx := render.RenderContext{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.Calico}} in := []client.Object{&corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "cm"}}} - out, _ := applyExtensions(s, "test", calicoCtx, in, nil) + out, _ := extensionstest.ApplyExtensions(s, "test", calicoCtx, in, nil) Expect(out).To(Equal(in)) }) It("returns objects unchanged when no installation is set", func() { in := []client.Object{&corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "cm"}}} - out, _ := applyExtensions(s, "test", render.RenderContext{}, in, nil) + out, _ := extensionstest.ApplyExtensions(s, "test", render.RenderContext{}, in, nil) Expect(out).To(Equal(in)) }) @@ -95,7 +96,7 @@ var _ = Describe("extension registry", func() { add("first") add("second") - out, _ := applyExtensions(s, "test", entCtx, nil, nil) + out, _ := extensionstest.ApplyExtensions(s, "test", entCtx, nil, nil) Expect(out).To(HaveLen(1)) Expect(out[0].GetName()).To(Equal("second")) }) diff --git a/pkg/extensions/extensionstest/decorate.go b/pkg/extensions/extensionstest/decorate.go new file mode 100644 index 0000000000..05d621e14f --- /dev/null +++ b/pkg/extensions/extensionstest/decorate.go @@ -0,0 +1,77 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package extensionstest holds shared helpers for exercising a registered +// extension through the same Set.Decorate seam the component handler uses. It is +// test support imported by the extension test suites (extensions, render, and the +// per-component enterprise packages), so the helper lives once instead of being +// copied into each test package. +package extensionstest + +import ( + client "sigs.k8s.io/controller-runtime/pkg/client" + + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/extensions" + "github.com/tigera/operator/pkg/render" + rmeta "github.com/tigera/operator/pkg/render/common/meta" +) + +// StubComponent adapts raw object lists to a render.Component so a registered +// extension can be exercised through Set.Decorate. Key selects the extension; +// ExtCtx is delivered as the component's ExtensionContext (the typed config a +// RegisterModifier modifier reads). +type StubComponent struct { + Key string + ExtCtx any + Create, Delete []client.Object +} + +func (s StubComponent) ResolveImages(*operatorv1.ImageSet) error { + return nil +} + +func (s StubComponent) Objects() ([]client.Object, []client.Object) { + return s.Create, s.Delete +} + +func (s StubComponent) Ready() bool { + return true +} + +func (s StubComponent) SupportedOSType() rmeta.OSType { + return rmeta.OSTypeAny +} + +func (s StubComponent) ModifierKey() string { + return s.Key +} + +func (s StubComponent) ExtensionContext() any { + return s.ExtCtx +} + +// ApplyExtensions decorates a stub component holding the given objects with the +// extension registered under key, then renders it. For a modifier that needs the +// component's typed config, use ApplyExtensionsWithContext. +func ApplyExtensions(s *extensions.Set, key string, rc render.RenderContext, create, del []client.Object) ([]client.Object, []client.Object) { + return ApplyExtensionsWithContext(s, key, rc, nil, create, del) +} + +// ApplyExtensionsWithContext is ApplyExtensions for a modifier that reads the +// component's typed config: extCtx is delivered as the stub's ExtensionContext. +func ApplyExtensionsWithContext(s *extensions.Set, key string, rc render.RenderContext, extCtx any, create, del []client.Object) ([]client.Object, []client.Object) { + stub := StubComponent{Key: key, ExtCtx: extCtx, Create: create, Delete: del} + return s.Decorate(stub, rc).Objects() +} diff --git a/pkg/render/apiserver_test.go b/pkg/render/apiserver_test.go index 252db966ab..955e0b5357 100644 --- a/pkg/render/apiserver_test.go +++ b/pkg/render/apiserver_test.go @@ -28,6 +28,7 @@ import ( "github.com/tigera/operator/pkg/controller/k8sapi" ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" "github.com/tigera/operator/pkg/dns" + "github.com/tigera/operator/pkg/extensions/extensionstest" "github.com/tigera/operator/pkg/render" rmeta "github.com/tigera/operator/pkg/render/common/meta" "github.com/tigera/operator/pkg/render/common/podaffinity" @@ -59,7 +60,7 @@ func apiServerObjects(c render.Component) ([]client.Object, []client.Object) { rc.Installation = ec.Config.Installation extCtx = ec } - return applyExtensionsWithContext(ext, render.ComponentNameAPIServer, rc, extCtx, create, del) + return extensionstest.ApplyExtensionsWithContext(ext, render.ComponentNameAPIServer, rc, extCtx, create, del) } func verifyAPIService(service *apiregv1.APIService, enterprise bool, clusterDomain string) { diff --git a/pkg/render/decorate_helpers_test.go b/pkg/render/decorate_helpers_test.go deleted file mode 100644 index 8393de7e94..0000000000 --- a/pkg/render/decorate_helpers_test.go +++ /dev/null @@ -1,72 +0,0 @@ -// Copyright (c) 2026 Tigera, Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package render_test - -import ( - client "sigs.k8s.io/controller-runtime/pkg/client" - - operatorv1 "github.com/tigera/operator/api/v1" - "github.com/tigera/operator/pkg/extensions" - "github.com/tigera/operator/pkg/render" - rmeta "github.com/tigera/operator/pkg/render/common/meta" -) - -// stubExtComponent adapts raw object lists to a render.Component so a registered -// extension can be exercised through Set.Decorate, the same seam the component -// handler uses. key selects the extension; extCtx is delivered as the component's -// ExtensionContext (the typed config a RegisterModifier modifier reads). -type stubExtComponent struct { - key string - extCtx any - create, delete []client.Object -} - -func (s stubExtComponent) ResolveImages(*operatorv1.ImageSet) error { - return nil -} - -func (s stubExtComponent) Objects() ([]client.Object, []client.Object) { - return s.create, s.delete -} - -func (s stubExtComponent) Ready() bool { - return true -} - -func (s stubExtComponent) SupportedOSType() rmeta.OSType { - return rmeta.OSTypeAny -} - -func (s stubExtComponent) ModifierKey() string { - return s.key -} - -func (s stubExtComponent) ExtensionContext() any { - return s.extCtx -} - -// applyExtensions decorates a stub component holding the given objects with the -// extension registered under key, then renders it. For a modifier that needs the -// component's typed config, use applyExtensionsWithContext. -func applyExtensions(s *extensions.Set, key string, rc render.RenderContext, create, del []client.Object) ([]client.Object, []client.Object) { - return applyExtensionsWithContext(s, key, rc, nil, create, del) -} - -// applyExtensionsWithContext is applyExtensions for a modifier that reads the -// component's typed config: extCtx is delivered as the stub's ExtensionContext. -func applyExtensionsWithContext(s *extensions.Set, key string, rc render.RenderContext, extCtx any, create, del []client.Object) ([]client.Object, []client.Object) { - stub := stubExtComponent{key: key, extCtx: extCtx, create: create, delete: del} - return s.Decorate(stub, rc).Objects() -} diff --git a/pkg/render/guardian_test.go b/pkg/render/guardian_test.go index cc8a0a9cc8..f54664809e 100644 --- a/pkg/render/guardian_test.go +++ b/pkg/render/guardian_test.go @@ -36,6 +36,7 @@ import ( "github.com/tigera/operator/pkg/components" "github.com/tigera/operator/pkg/controller/certificatemanager" ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" + "github.com/tigera/operator/pkg/extensions/extensionstest" "github.com/tigera/operator/pkg/render" rmeta "github.com/tigera/operator/pkg/render/common/meta" "github.com/tigera/operator/pkg/render/common/networkpolicy" @@ -54,7 +55,7 @@ func guardianObjects(cfg *render.GuardianConfiguration) []client.Object { if p, ok := g.(render.ExtensionContextProvider); ok { extCtx = p.ExtensionContext() } - out, _ := applyExtensionsWithContext(ext, render.GuardianName, rc, extCtx, objs, nil) + out, _ := extensionstest.ApplyExtensionsWithContext(ext, render.GuardianName, rc, extCtx, objs, nil) return out } @@ -117,7 +118,7 @@ var _ = Describe("Rendering tests", func() { if p, ok := g.(render.ExtensionContextProvider); ok { extCtx = p.ExtensionContext() } - resources, _ = applyExtensionsWithContext(ext, render.GuardianName, rc, extCtx, resources, nil) + resources, _ = extensionstest.ApplyExtensionsWithContext(ext, render.GuardianName, rc, extCtx, resources, nil) } BeforeEach(func() { @@ -354,7 +355,7 @@ var _ = Describe("Rendering tests", func() { if p, ok := g.(render.ExtensionContextProvider); ok { extCtx = p.ExtensionContext() } - resources, _ = applyExtensionsWithContext(ext, render.ComponentNameGuardianPolicy, rc, extCtx, objs, nil) + resources, _ = extensionstest.ApplyExtensionsWithContext(ext, render.ComponentNameGuardianPolicy, rc, extCtx, objs, nil) } Context("policy rendering based on variant and IncludeEgressNetworkPolicy", func() { diff --git a/pkg/render/node_enterprise_test.go b/pkg/render/node_enterprise_test.go index 9c06a22b84..d7afd663dd 100644 --- a/pkg/render/node_enterprise_test.go +++ b/pkg/render/node_enterprise_test.go @@ -36,6 +36,7 @@ import ( ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" "github.com/tigera/operator/pkg/dns" "github.com/tigera/operator/pkg/extensions" + "github.com/tigera/operator/pkg/extensions/extensionstest" "github.com/tigera/operator/pkg/render" rmeta "github.com/tigera/operator/pkg/render/common/meta" "github.com/tigera/operator/pkg/tls/certificatemanagement" @@ -127,7 +128,7 @@ var _ = Describe("node enterprise modifier integration", func() { comp := render.Node(cfg) Expect(comp.ResolveImages(nil)).NotTo(HaveOccurred()) objs, _ := comp.Objects() - out, _ := applyExtensions(ext, render.ComponentNameNode, rc, objs, nil) + out, _ := extensionstest.ApplyExtensions(ext, render.ComponentNameNode, rc, objs, nil) return out } @@ -210,7 +211,7 @@ var _ = Describe("node enterprise modifier integration", func() { }) Expect(comp.ResolveImages(nil)).NotTo(HaveOccurred()) objs, _ := comp.Objects() - objs, _ = applyExtensions(ext, render.ComponentNameTypha, renderCtx, objs, nil) + objs, _ = extensionstest.ApplyExtensions(ext, render.ComponentNameTypha, renderCtx, objs, nil) role, ok := extensions.FindObject[*rbacv1.ClusterRole](objs, "calico-typha") Expect(ok).To(BeTrue()) diff --git a/pkg/render/windows_test.go b/pkg/render/windows_test.go index ff1c165403..fed3de6eaf 100644 --- a/pkg/render/windows_test.go +++ b/pkg/render/windows_test.go @@ -35,6 +35,7 @@ import ( "github.com/tigera/operator/pkg/controller/certificatemanager" "github.com/tigera/operator/pkg/controller/k8sapi" ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" + "github.com/tigera/operator/pkg/extensions/extensionstest" "github.com/tigera/operator/pkg/render" rmeta "github.com/tigera/operator/pkg/render/common/meta" rtest "github.com/tigera/operator/pkg/render/common/test" @@ -50,7 +51,7 @@ func renderWindows(cfg *render.WindowsConfiguration) []client.Object { ExpectWithOffset(1, comp.ResolveImages(nil)).To(BeNil()) objs, _ := comp.Objects() rc := render.RenderContext{Installation: cfg.Installation} - out, _ := applyExtensions(ext, render.ComponentNameWindows, rc, objs, nil) + out, _ := extensionstest.ApplyExtensions(ext, render.ComponentNameWindows, rc, objs, nil) return out } From daf889777bc7b0c630c673f15dc20ff4b297343d Mon Sep 17 00:00:00 2001 From: Casey Davenport Date: Wed, 24 Jun 2026 13:38:06 -0700 Subject: [PATCH 44/54] Split pkg/enterprise into per-component subpackages The flat pkg/enterprise mixed every component's extension in one package. Split it so each subpackage maps to the render component it extends and exposes a Register func that New() composes: enterprise/typha, guardian, apiserver, windows - self-contained. enterprise/installation - the installation controller hook plus the node and calico-kube-controllers modifiers, which share installationRenderData (kept internal here). enterprise/kubecontrollers - the es-calico-kube-controllers assembly plus the enterprise kube-controllers cluster role rules it shares with the calico modifier (exported as KubeControllersEnterpriseCommonRules). The shared felix reporter-port helpers move with node (exported as NodeReporterPort/ValidateReporterPort) so the windows hook can reuse them. logstorage now imports enterprise/kubecontrollers for the es-kube-controllers symbols instead of the old flat package. Each subpackage gets its own test suite; the decorate helper is the shared extensionstest package. --- pkg/controller/logstorage/common/common.go | 18 +- .../kubecontrollers/es_kube_controllers.go | 10 +- .../es_kube_controllers_test.go | 10 +- .../{apiserver.go => apiserver/extension.go} | 10 +- .../extension_test.go} | 2 +- pkg/enterprise/apiserver/suite_test.go | 34 ++ .../{guardian.go => guardian/extension.go} | 5 +- .../extension_test.go} | 2 +- pkg/enterprise/guardian/suite_test.go | 34 ++ .../{installation.go => installation/core.go} | 13 +- .../core_test.go} | 2 +- .../{ => installation}/kubecontrollers.go | 306 +--------------- .../kubecontrollers_test.go | 12 +- pkg/enterprise/{ => installation}/node.go | 14 +- .../{ => installation}/node_test.go | 2 +- pkg/enterprise/installation/suite_test.go | 34 ++ .../waf_pull_secret_test.go | 20 +- .../kubecontrollers/kubecontrollers.go | 335 ++++++++++++++++++ pkg/enterprise/register.go | 25 +- .../{typha.go => typha/extension.go} | 5 +- .../extension_test.go} | 2 +- .../suite_test.go} | 6 +- .../{windows.go => windows/extension.go} | 13 +- .../extension_test.go} | 2 +- pkg/enterprise/windows/suite_test.go | 34 ++ .../kubecontrollers/kube-controllers_test.go | 52 +-- .../logstorage/esgateway/esgateway_test.go | 20 +- 27 files changed, 605 insertions(+), 417 deletions(-) rename pkg/enterprise/{apiserver.go => apiserver/extension.go} (99%) rename pkg/enterprise/{apiserver_test.go => apiserver/extension_test.go} (99%) create mode 100644 pkg/enterprise/apiserver/suite_test.go rename pkg/enterprise/{guardian.go => guardian/extension.go} (99%) rename pkg/enterprise/{guardian_test.go => guardian/extension_test.go} (99%) create mode 100644 pkg/enterprise/guardian/suite_test.go rename pkg/enterprise/{installation.go => installation/core.go} (95%) rename pkg/enterprise/{installation_test.go => installation/core_test.go} (99%) rename pkg/enterprise/{ => installation}/kubecontrollers.go (57%) rename pkg/enterprise/{ => installation}/kubecontrollers_test.go (97%) rename pkg/enterprise/{ => installation}/node.go (96%) rename pkg/enterprise/{ => installation}/node_test.go (99%) create mode 100644 pkg/enterprise/installation/suite_test.go rename pkg/enterprise/{ => installation}/waf_pull_secret_test.go (88%) create mode 100644 pkg/enterprise/kubecontrollers/kubecontrollers.go rename pkg/enterprise/{typha.go => typha/extension.go} (94%) rename pkg/enterprise/{typha_test.go => typha/extension_test.go} (99%) rename pkg/enterprise/{enterprise_suite_test.go => typha/suite_test.go} (91%) rename pkg/enterprise/{windows.go => windows/extension.go} (95%) rename pkg/enterprise/{windows_test.go => windows/extension_test.go} (99%) create mode 100644 pkg/enterprise/windows/suite_test.go diff --git a/pkg/controller/logstorage/common/common.go b/pkg/controller/logstorage/common/common.go index e15f982b49..92f34494a6 100644 --- a/pkg/controller/logstorage/common/common.go +++ b/pkg/controller/logstorage/common/common.go @@ -26,7 +26,7 @@ import ( "github.com/tigera/operator/pkg/controller/utils" "github.com/tigera/operator/pkg/crypto" - "github.com/tigera/operator/pkg/enterprise" + entkubecontrollers "github.com/tigera/operator/pkg/enterprise/kubecontrollers" ) const ( @@ -45,7 +45,7 @@ const ( // the gateway credentials, and a secret containing real admin level credentials is created and stored in the tigera-elasticsearch namespace to be swapped in once // ES Gateway has confirmed that the gateway credentials match. func CreateKubeControllersSecrets(ctx context.Context, esAdminUserSecret *corev1.Secret, esAdminUserName string, cli client.Client, h utils.NamespaceHelper) (*corev1.Secret, *corev1.Secret, *corev1.Secret, error) { - kubeControllersGatewaySecret, err := utils.GetSecret(ctx, cli, enterprise.ElasticsearchKubeControllersUserSecret, h.TruthNamespace()) + kubeControllersGatewaySecret, err := utils.GetSecret(ctx, cli, entkubecontrollers.ElasticsearchKubeControllersUserSecret, h.TruthNamespace()) if err != nil { return nil, nil, nil, err } @@ -53,11 +53,11 @@ func CreateKubeControllersSecrets(ctx context.Context, esAdminUserSecret *corev1 password := crypto.GeneratePassword(16) kubeControllersGatewaySecret = &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ - Name: enterprise.ElasticsearchKubeControllersUserSecret, + Name: entkubecontrollers.ElasticsearchKubeControllersUserSecret, Namespace: h.TruthNamespace(), }, Data: map[string][]byte{ - "username": []byte(enterprise.ElasticsearchKubeControllersUserName), + "username": []byte(entkubecontrollers.ElasticsearchKubeControllersUserName), "password": []byte(password), }, } @@ -67,34 +67,34 @@ func CreateKubeControllersSecrets(ctx context.Context, esAdminUserSecret *corev1 return nil, nil, nil, err } - kubeControllersVerificationSecret, err := utils.GetSecret(ctx, cli, enterprise.ElasticsearchKubeControllersVerificationUserSecret, h.InstallNamespace()) + kubeControllersVerificationSecret, err := utils.GetSecret(ctx, cli, entkubecontrollers.ElasticsearchKubeControllersVerificationUserSecret, h.InstallNamespace()) if err != nil { return nil, nil, nil, err } if kubeControllersVerificationSecret == nil { kubeControllersVerificationSecret = &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ - Name: enterprise.ElasticsearchKubeControllersVerificationUserSecret, + Name: entkubecontrollers.ElasticsearchKubeControllersVerificationUserSecret, Namespace: h.InstallNamespace(), Labels: map[string]string{ ESGatewaySelectorLabel: ESGatewaySelectorLabelValue, }, }, Data: map[string][]byte{ - "username": []byte(enterprise.ElasticsearchKubeControllersUserName), + "username": []byte(entkubecontrollers.ElasticsearchKubeControllersUserName), "password": hashedPassword, }, } } - kubeControllersSecureUserSecret, err := utils.GetSecret(ctx, cli, enterprise.ElasticsearchKubeControllersSecureUserSecret, h.InstallNamespace()) + kubeControllersSecureUserSecret, err := utils.GetSecret(ctx, cli, entkubecontrollers.ElasticsearchKubeControllersSecureUserSecret, h.InstallNamespace()) if err != nil { return nil, nil, nil, err } if kubeControllersSecureUserSecret == nil { kubeControllersSecureUserSecret = &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ - Name: enterprise.ElasticsearchKubeControllersSecureUserSecret, + Name: entkubecontrollers.ElasticsearchKubeControllersSecureUserSecret, Namespace: h.InstallNamespace(), Labels: map[string]string{ ESGatewaySelectorLabel: ESGatewaySelectorLabelValue, diff --git a/pkg/controller/logstorage/kubecontrollers/es_kube_controllers.go b/pkg/controller/logstorage/kubecontrollers/es_kube_controllers.go index b8f056159e..5e218d4978 100644 --- a/pkg/controller/logstorage/kubecontrollers/es_kube_controllers.go +++ b/pkg/controller/logstorage/kubecontrollers/es_kube_controllers.go @@ -44,7 +44,7 @@ import ( "github.com/tigera/operator/pkg/controller/utils" "github.com/tigera/operator/pkg/controller/utils/imageset" "github.com/tigera/operator/pkg/ctrlruntime" - "github.com/tigera/operator/pkg/enterprise" + entkubecontrollers "github.com/tigera/operator/pkg/enterprise/kubecontrollers" "github.com/tigera/operator/pkg/render" "github.com/tigera/operator/pkg/render/common/networkpolicy" "github.com/tigera/operator/pkg/render/kubecontrollers" @@ -141,7 +141,7 @@ func Add(mgr manager.Manager, opts options.ControllerOptions) error { if err := utils.AddDeploymentWatch(c, esgateway.DeploymentName, esKubeControllersNamespace.InstallNamespace()); err != nil { return fmt.Errorf("log-storage-access-controller failed to watch the Service resource: %w", err) } - if err := utils.AddDeploymentWatch(c, enterprise.EsKubeController, esKubeControllersNamespace.InstallNamespace()); err != nil { + if err := utils.AddDeploymentWatch(c, entkubecontrollers.EsKubeController, esKubeControllersNamespace.InstallNamespace()); err != nil { return fmt.Errorf("log-storage-access-controller failed to watch the Service resource: %w", err) } @@ -169,7 +169,7 @@ func Add(mgr manager.Manager, opts options.ControllerOptions) error { // Start goroutines to establish watches against projectcalico.org/v3 resources. go utils.WaitToAddTierWatch(networkpolicy.CalicoTierName, c, opts.K8sClientset, log, r.tierWatchReady) go utils.WaitToAddNetworkPolicyWatches(c, opts.K8sClientset, log, []types.NamespacedName{ - {Name: enterprise.EsKubeControllerNetworkPolicyName, Namespace: esKubeControllersNamespace.InstallNamespace()}, + {Name: entkubecontrollers.EsKubeControllerNetworkPolicyName, Namespace: esKubeControllersNamespace.InstallNamespace()}, }) return nil @@ -263,7 +263,7 @@ func (r *ESKubeControllersController) Reconcile(ctx context.Context, request rec // Get secrets needed for kube-controllers to talk to elastic. This is needed for zero-tenants and single-tenants // that deploy es-kube-controllers and need to talk to es-gateway var kubeControllersUserSecret *core.Secret - kubeControllersUserSecret, err = utils.GetSecret(ctx, r.client, enterprise.ElasticsearchKubeControllersUserSecret, helper.TruthNamespace()) + kubeControllersUserSecret, err = utils.GetSecret(ctx, r.client, entkubecontrollers.ElasticsearchKubeControllersUserSecret, helper.TruthNamespace()) if err != nil { r.status.SetDegraded(operatorv1.ResourceReadError, "Failed to get kube controllers gateway secret", err, reqLogger) return reconcile.Result{}, err @@ -344,7 +344,7 @@ func (r *ESKubeControllersController) Reconcile(ctx context.Context, request rec BindingNamespaces: namespaces, Tenant: nil, } - esKubeControllerComponents := enterprise.NewElasticsearchKubeControllers(&kubeControllersCfg) + esKubeControllerComponents := entkubecontrollers.NewElasticsearchKubeControllers(&kubeControllersCfg) imageSet, err := imageset.GetImageSet(ctx, r.client, variant) if err != nil { diff --git a/pkg/controller/logstorage/kubecontrollers/es_kube_controllers_test.go b/pkg/controller/logstorage/kubecontrollers/es_kube_controllers_test.go index c053617282..7fc4168dd7 100644 --- a/pkg/controller/logstorage/kubecontrollers/es_kube_controllers_test.go +++ b/pkg/controller/logstorage/kubecontrollers/es_kube_controllers_test.go @@ -46,7 +46,7 @@ import ( "github.com/tigera/operator/pkg/controller/utils" ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" "github.com/tigera/operator/pkg/dns" - "github.com/tigera/operator/pkg/enterprise" + entkubecontrollers "github.com/tigera/operator/pkg/enterprise/kubecontrollers" "github.com/tigera/operator/pkg/render" "github.com/tigera/operator/pkg/render/logstorage" "github.com/tigera/operator/pkg/render/logstorage/esgateway" @@ -235,7 +235,7 @@ var _ = Describe("LogStorage ES kube-controllers controller", func() { dep := appsv1.Deployment{ TypeMeta: metav1.TypeMeta{Kind: "Deployment", APIVersion: "apps/v1"}, ObjectMeta: metav1.ObjectMeta{ - Name: enterprise.EsKubeController, + Name: entkubecontrollers.EsKubeController, Namespace: common.CalicoNamespace, }, } @@ -275,12 +275,12 @@ var _ = Describe("LogStorage ES kube-controllers controller", func() { dep := appsv1.Deployment{ TypeMeta: metav1.TypeMeta{Kind: "Deployment", APIVersion: "apps/v1"}, ObjectMeta: metav1.ObjectMeta{ - Name: enterprise.EsKubeController, + Name: entkubecontrollers.EsKubeController, Namespace: common.CalicoNamespace, }, } Expect(test.GetResource(cli, &dep)).To(BeNil()) - kc := test.GetContainer(dep.Spec.Template.Spec.Containers, enterprise.EsKubeController) + kc := test.GetContainer(dep.Spec.Template.Spec.Containers, entkubecontrollers.EsKubeController) Expect(kc).ToNot(BeNil()) Expect(kc.Image).To(Equal(fmt.Sprintf("some.registry.org/%s%s@%s", components.TigeraImagePath, components.ComponentTigeraCalico.Image, "sha256:kubecontrollershash"))) }) @@ -325,7 +325,7 @@ var _ = Describe("LogStorage ES kube-controllers controller", func() { dep := appsv1.Deployment{ TypeMeta: metav1.TypeMeta{Kind: "Deployment", APIVersion: "apps/v1"}, ObjectMeta: metav1.ObjectMeta{ - Name: enterprise.EsKubeController, + Name: entkubecontrollers.EsKubeController, Namespace: common.CalicoNamespace, }, } diff --git a/pkg/enterprise/apiserver.go b/pkg/enterprise/apiserver/extension.go similarity index 99% rename from pkg/enterprise/apiserver.go rename to pkg/enterprise/apiserver/extension.go index 1f3f1c8395..d27cf7fc83 100644 --- a/pkg/enterprise/apiserver.go +++ b/pkg/enterprise/apiserver/extension.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package enterprise +package apiserver import ( "fmt" @@ -89,7 +89,9 @@ type apiServer struct { data apiServerRenderData } -func registerAPIServer(v *extensions.Variant) { +// Register wires the API server controller hook and modifiers into the variant. +func Register(v *extensions.Variant) { + v.Controller(contexts.APIServerController, apiServerControllerExtension{}) extensions.RegisterModifier(v, render.ComponentNameAPIServer, modifyAPIServer) extensions.RegisterModifier(v, render.ComponentNameAPIServerPolicy, modifyAPIServerPolicy) } @@ -272,10 +274,10 @@ func (apiServerControllerExtension) ExtendContext(cc contexts.ControllerContext) return rc, nil, nil } -// registerAPIServerCleanup registers, for the Calico variant, the cleanup that +// RegisterCalicoCleanup registers, for the Calico variant, the cleanup that // deletes the Enterprise API server objects left behind by a prior Enterprise // installation. -func registerAPIServerCleanup(v *extensions.Variant) { +func RegisterCalicoCleanup(v *extensions.Variant) { extensions.RegisterModifier(v, render.ComponentNameAPIServer, cleanupAPIServer) } diff --git a/pkg/enterprise/apiserver_test.go b/pkg/enterprise/apiserver/extension_test.go similarity index 99% rename from pkg/enterprise/apiserver_test.go rename to pkg/enterprise/apiserver/extension_test.go index afe4292f46..89f1e7b561 100644 --- a/pkg/enterprise/apiserver_test.go +++ b/pkg/enterprise/apiserver/extension_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package enterprise_test +package apiserver_test import ( "context" diff --git a/pkg/enterprise/apiserver/suite_test.go b/pkg/enterprise/apiserver/suite_test.go new file mode 100644 index 0000000000..eb46a99362 --- /dev/null +++ b/pkg/enterprise/apiserver/suite_test.go @@ -0,0 +1,34 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package apiserver_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/tigera/operator/pkg/enterprise" + "github.com/tigera/operator/pkg/extensions" +) + +// ext is the enterprise extension Set under test, shared across the suite. It is +// immutable once built and the specs only read it, so a single instance is safe. +var ext *extensions.Set = enterprise.New() + +func TestAPIServer(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "pkg/enterprise/apiserver Suite") +} diff --git a/pkg/enterprise/guardian.go b/pkg/enterprise/guardian/extension.go similarity index 99% rename from pkg/enterprise/guardian.go rename to pkg/enterprise/guardian/extension.go index 58f0b66c0f..6dc1a95410 100644 --- a/pkg/enterprise/guardian.go +++ b/pkg/enterprise/guardian/extension.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package enterprise +package guardian import ( "net" @@ -37,7 +37,8 @@ import ( operatorurl "github.com/tigera/operator/pkg/url" ) -func registerGuardian(v *extensions.Variant) { +// Register wires the guardian extensions into the variant. +func Register(v *extensions.Variant) { extensions.RegisterModifier(v, render.GuardianName, modifyGuardian) extensions.RegisterModifier(v, render.ComponentNameGuardianPolicy, modifyGuardianPolicy) } diff --git a/pkg/enterprise/guardian_test.go b/pkg/enterprise/guardian/extension_test.go similarity index 99% rename from pkg/enterprise/guardian_test.go rename to pkg/enterprise/guardian/extension_test.go index 535d4f3b14..e38918d924 100644 --- a/pkg/enterprise/guardian_test.go +++ b/pkg/enterprise/guardian/extension_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package enterprise_test +package guardian_test import ( . "github.com/onsi/ginkgo/v2" diff --git a/pkg/enterprise/guardian/suite_test.go b/pkg/enterprise/guardian/suite_test.go new file mode 100644 index 0000000000..b83fbf63e8 --- /dev/null +++ b/pkg/enterprise/guardian/suite_test.go @@ -0,0 +1,34 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package guardian_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/tigera/operator/pkg/enterprise" + "github.com/tigera/operator/pkg/extensions" +) + +// ext is the enterprise extension Set under test, shared across the suite. It is +// immutable once built and the specs only read it, so a single instance is safe. +var ext *extensions.Set = enterprise.New() + +func TestGuardian(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "pkg/enterprise/guardian Suite") +} diff --git a/pkg/enterprise/installation.go b/pkg/enterprise/installation/core.go similarity index 95% rename from pkg/enterprise/installation.go rename to pkg/enterprise/installation/core.go index 68f0fff7f3..6e1803bc3a 100644 --- a/pkg/enterprise/installation.go +++ b/pkg/enterprise/installation/core.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package enterprise +package installation import ( "fmt" @@ -29,6 +29,7 @@ import ( "github.com/tigera/operator/pkg/controller/utils" "github.com/tigera/operator/pkg/ctrlruntime" "github.com/tigera/operator/pkg/dns" + "github.com/tigera/operator/pkg/extensions" "github.com/tigera/operator/pkg/render" relasticsearch "github.com/tigera/operator/pkg/render/common/elasticsearch" "github.com/tigera/operator/pkg/render/kubecontrollers" @@ -36,6 +37,14 @@ import ( "github.com/tigera/operator/pkg/tls/certificatemanagement" ) +// Register wires the installation controller hook and the node and +// calico-kube-controllers modifiers into the variant. +func Register(v *extensions.Variant) { + v.Controller(contexts.InstallationController, coreControllerExtension{}) + registerNode(v) + registerKubeControllers(v) +} + // coreControllerExtension is the Calico Enterprise controller-side hook for the // installation controller. type coreControllerExtension struct{} @@ -76,7 +85,7 @@ func collectProcessPathEnabled(lc *operatorv1.LogCollector) bool { // Validate rejects installation config Calico Enterprise does not support. func (coreControllerExtension) Validate(cc contexts.ControllerContext) error { - return validateReporterPort(cc.FelixConfiguration) + return ValidateReporterPort(cc.FelixConfiguration) } // DefaultFelixConfiguration sets the Enterprise-only FelixConfiguration defaults. diff --git a/pkg/enterprise/installation_test.go b/pkg/enterprise/installation/core_test.go similarity index 99% rename from pkg/enterprise/installation_test.go rename to pkg/enterprise/installation/core_test.go index 42f3d39d21..0723e7532a 100644 --- a/pkg/enterprise/installation_test.go +++ b/pkg/enterprise/installation/core_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package enterprise_test +package installation_test import ( "context" diff --git a/pkg/enterprise/kubecontrollers.go b/pkg/enterprise/installation/kubecontrollers.go similarity index 57% rename from pkg/enterprise/kubecontrollers.go rename to pkg/enterprise/installation/kubecontrollers.go index 7b1acbf497..082e1f3fc2 100644 --- a/pkg/enterprise/kubecontrollers.go +++ b/pkg/enterprise/installation/kubecontrollers.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package enterprise +package installation import ( "encoding/json" @@ -36,17 +36,16 @@ import ( "github.com/tigera/operator/pkg/controller/utils" "github.com/tigera/operator/pkg/controller/utils/imageset" "github.com/tigera/operator/pkg/dns" + entkubecontrollers "github.com/tigera/operator/pkg/enterprise/kubecontrollers" "github.com/tigera/operator/pkg/extensions" "github.com/tigera/operator/pkg/render" "github.com/tigera/operator/pkg/render/applicationlayer" - relasticsearch "github.com/tigera/operator/pkg/render/common/elasticsearch" rmeta "github.com/tigera/operator/pkg/render/common/meta" "github.com/tigera/operator/pkg/render/common/networkpolicy" "github.com/tigera/operator/pkg/render/common/secret" "github.com/tigera/operator/pkg/render/kubecontrollers" "github.com/tigera/operator/pkg/render/monitor" "github.com/tigera/operator/pkg/tls/certificatemanagement" - "github.com/tigera/operator/pkg/url" ) // registerKubeControllers registers the calico-kube-controllers modifiers. There is @@ -215,17 +214,6 @@ func wafEnv(waf wafRenderData) []corev1.EnvVar { } const ( - EsKubeController = "es-calico-kube-controllers" - EsKubeControllerRole = "es-calico-kube-controllers" - EsKubeControllerRoleBinding = "es-calico-kube-controllers" - EsKubeControllerMetrics = "es-calico-kube-controllers-metrics" - EsKubeControllerNetworkPolicyName = networkpolicy.CalicoComponentPolicyPrefix + "es-kube-controller-access" - - ElasticsearchKubeControllersUserSecret = "tigera-ee-kube-controllers-elasticsearch-access" - ElasticsearchKubeControllersUserName = "tigera-ee-kube-controllers" - ElasticsearchKubeControllersSecureUserSecret = "tigera-ee-kube-controllers-elasticsearch-access-gateway" - ElasticsearchKubeControllersVerificationUserSecret = "tigera-ee-kube-controllers-gateway-verification-credentials" - // WASMPullSecretName is the dedicated image-pull Secret (a merged copy of the // install pull secrets) the WAF reconciler replicates into tenant namespaces for // the Coraza wasm OCI pull. A dedicated name avoids clashing with the @@ -239,164 +227,11 @@ const ( WASMCACertName = "tigera-waf-ca-bundle" ) -// NewElasticsearchKubeControllers fills the generic kube-controllers configuration -// for the enterprise es-calico-kube-controllers deployment and returns the rendered -// component. es-kube-controllers is a distinct deployment (talks to Elasticsearch via -// es-gateway) reconciled by the logstorage kube-controllers controller, so it's -// assembled here rather than through the render-time modifier mechanism. -func NewElasticsearchKubeControllers(cfg *kubecontrollers.KubeControllersConfiguration) render.Component { - cfg.Name = EsKubeController - cfg.ConfigName = "elasticsearch" - cfg.RoleName = EsKubeControllerRole - cfg.RoleBindingName = EsKubeControllerRoleBinding - cfg.MetricsName = EsKubeControllerMetrics - cfg.DisableConfigAPI = cfg.Tenant.MultiTenant() - - cfg.Rules = kubecontrollers.KubeControllersRoleCommonRules(cfg) - cfg.Rules = append(cfg.Rules, kubeControllersEnterpriseCommonRules(false, cfg.ManagementClusterConnection != nil)...) - cfg.Rules = append(cfg.Rules, - rbacv1.PolicyRule{ - APIGroups: []string{"elasticsearch.k8s.elastic.co"}, - Resources: []string{"elasticsearches"}, - Verbs: []string{"watch", "get", "list"}, - }, - rbacv1.PolicyRule{ - APIGroups: []string{"rbac.authorization.k8s.io"}, - Resources: []string{"clusterroles", "clusterrolebindings"}, - Verbs: []string{"watch", "list", "get"}, - }, - ) - - if !cfg.Tenant.MultiTenant() { - // Zero and single tenant clusters need elasticsearch configuration. - cfg.EnabledControllers = append(cfg.EnabledControllers, "authorization", "elasticsearchconfiguration") - if cfg.ManagementCluster != nil && cfg.Tenant == nil { - // Enterprise requires the managedcluster controller to push licenses. - cfg.EnabledControllers = append(cfg.EnabledControllers, "managedcluster") - } - } - - cfg.NetworkPolicy = esKubeControllersCalicoSystemPolicy(cfg) - cfg.DeprecatedNetworkPolicyName = "es-kube-controller-access" - cfg.ExtraEnv = esKubeControllersEnv(cfg) - - return kubecontrollers.NewKubeControllers(cfg) -} - -// esKubeControllersEnv builds the enterprise env vars for es-calico-kube-controllers. -func esKubeControllersEnv(cfg *kubecontrollers.KubeControllersConfiguration) []corev1.EnvVar { - var env []corev1.EnvVar - - if cfg.Tenant != nil { - env = append(env, corev1.EnvVar{Name: "TENANT_ID", Value: cfg.Tenant.Spec.ID}) - } - - // What started as a workaround is now the default behaviour. This feature uses our backend in order to - // log into Kibana for users from external identity providers, rather than configuring an authn realm - // in the Elastic stack. - env = append(env, corev1.EnvVar{Name: "ENABLE_ELASTICSEARCH_OIDC_WORKAROUND", Value: "true"}) - if cfg.Authentication != nil { - env = append(env, - corev1.EnvVar{Name: "OIDC_AUTH_USERNAME_PREFIX", Value: cfg.Authentication.Spec.UsernamePrefix}, - corev1.EnvVar{Name: "OIDC_AUTH_GROUP_PREFIX", Value: cfg.Authentication.Spec.GroupsPrefix}, - ) - } - - if cfg.TrustedBundle != nil { - env = append(env, corev1.EnvVar{Name: "MULTI_CLUSTER_FORWARDING_CA", Value: cfg.TrustedBundle.MountPath()}) - } - if cfg.Installation.CalicoNetwork != nil && cfg.Installation.CalicoNetwork.MultiInterfaceMode != nil { - env = append(env, corev1.EnvVar{Name: "MULTI_INTERFACE_MODE", Value: cfg.Installation.CalicoNetwork.MultiInterfaceMode.Value()}) - } - - if !cfg.Tenant.MultiTenant() { - _, esHost, esPort, _ := url.ParseEndpoint(relasticsearch.GatewayEndpoint(rmeta.OSTypeLinux, cfg.ClusterDomain, render.ElasticsearchNamespace)) - env = append(env, - relasticsearch.ElasticHostEnvVar(esHost), - relasticsearch.ElasticPortEnvVar(esPort), - relasticsearch.ElasticUsernameEnvVar(ElasticsearchKubeControllersUserSecret), - relasticsearch.ElasticPasswordEnvVar(ElasticsearchKubeControllersUserSecret), - relasticsearch.ElasticCAEnvVar(rmeta.OSTypeLinux), - ) - } - - return env -} - -// kubeControllersEnterpriseCommonRules are the Calico Enterprise cluster role rules -// shared by calico-kube-controllers and es-calico-kube-controllers. wafEnabled adds -// the WAF v3 (Gateway API add-on) rules; managedCluster adds the license-push rule a -// managed cluster's kube-controllers needs. -func kubeControllersEnterpriseCommonRules(wafEnabled, managedCluster bool) []rbacv1.PolicyRule { - rules := []rbacv1.PolicyRule{ - { - APIGroups: []string{""}, - Resources: []string{"configmaps"}, - Verbs: []string{"watch", "list", "get", "update", "create", "delete"}, - }, - { - // The Federated Services Controller needs access to the remote kubeconfig secret - // in order to create a remote syncer. - APIGroups: []string{""}, - Resources: []string{"secrets"}, - Verbs: []string{"watch", "list", "get"}, - }, - { - // Needed to validate the license - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{"licensekeys"}, - Verbs: []string{"get", "watch", "list"}, - }, - { - // Needed to update the status of the LicenseKey with the result of license validation. - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"licensekeys/status"}, - Verbs: []string{"update"}, - }, - { - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{"deeppacketinspections"}, - Verbs: []string{"get", "watch", "list"}, - }, - { - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{"deeppacketinspections/status"}, - Verbs: []string{"update"}, - }, - { - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{"packetcaptures"}, - Verbs: []string{"get", "list", "update"}, - }, - { - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{"packetcaptures/status"}, - Verbs: []string{"update"}, - }, - } - - if wafEnabled { - rules = append(rules, wafRules()...) - } - - if managedCluster { - rules = append(rules, - rbacv1.PolicyRule{ - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{"licensekeys"}, - Verbs: []string{"get", "create", "update", "list", "watch"}, - }, - ) - } - - return rules -} - // calicoKubeControllersEnterpriseRules are the enterprise cluster role rules layered // onto calico-kube-controllers: the shared enterprise rules plus the calico-specific // ones (federated endpoints, license usage reporting). func calicoKubeControllersEnterpriseRules(wafEnabled, managedCluster bool) []rbacv1.PolicyRule { - rules := kubeControllersEnterpriseCommonRules(wafEnabled, managedCluster) + rules := entkubecontrollers.KubeControllersEnterpriseCommonRules(wafEnabled, managedCluster) return append(rules, rbacv1.PolicyRule{ APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, @@ -432,141 +267,6 @@ func calicoKubeControllersEnterpriseControllers(wafEnabled bool) []string { return controllers } -// wafRules are the WAF v3 (Gateway API add-on) cluster role rules, gated by -// GatewayAPI.spec.extensions.waf.state == Enabled. -func wafRules() []rbacv1.PolicyRule { - return []rbacv1.PolicyRule{ - // Application-layer (gateway-addons) reconcilers reconcile WAF resources - // against Gateway API targetRefs and emit events on the policy objects. - { - APIGroups: []string{"applicationlayer.projectcalico.org"}, - Resources: []string{ - "wafpolicies", "globalwafpolicies", - "wafplugins", "globalwafplugins", - "wafvalidationpolicies", "globalwafvalidationpolicies", - }, - Verbs: []string{"get", "list", "watch", "create", "update", "patch", "delete"}, - }, - { - APIGroups: []string{"applicationlayer.projectcalico.org"}, - Resources: []string{ - "wafpolicies/status", "globalwafpolicies/status", - "wafplugins/status", "globalwafplugins/status", - "wafvalidationpolicies/status", "globalwafvalidationpolicies/status", - }, - Verbs: []string{"get", "update", "patch"}, - }, - { - APIGroups: []string{"applicationlayer.projectcalico.org"}, - Resources: []string{ - "wafpolicies/finalizers", "globalwafpolicies/finalizers", - "wafplugins/finalizers", "globalwafplugins/finalizers", - "wafvalidationpolicies/finalizers", "globalwafvalidationpolicies/finalizers", - }, - Verbs: []string{"update"}, - }, - { - // Validate Gateway API targetRefs and surface attachment status. - APIGroups: []string{"gateway.networking.k8s.io"}, - Resources: []string{"gateways", "httproutes", "tcproutes", "tlsroutes", "grpcroutes"}, - Verbs: []string{"get", "list", "watch", "update", "patch"}, - }, - { - APIGroups: []string{"gateway.networking.k8s.io"}, - Resources: []string{"gateways/status", "httproutes/status", "tcproutes/status", "tlsroutes/status", "grpcroutes/status"}, - Verbs: []string{"get", "update", "patch"}, - }, - // controller-runtime Reconcilers (e.g. the applicationlayer manager) record - // events on watched objects via Recorder.Eventf; both core and events.k8s.io - // API groups are emitted depending on the kubernetes version. - { - APIGroups: []string{""}, - Resources: []string{"events"}, - Verbs: []string{"create", "patch"}, - }, - { - APIGroups: []string{"events.k8s.io"}, - Resources: []string{"events"}, - Verbs: []string{"create", "patch"}, - }, - // Application-layer reconciler replicates the WAF wasm pull Secret from - // the controller namespace (calico-system) into each WAFPolicy's - // namespace so the rendered EnvoyExtensionPolicy can reference it. Also - // replicates CA-cert ConfigMaps when WASM_CA_CERT is set. - { - APIGroups: []string{""}, - Resources: []string{"secrets", "configmaps"}, - Verbs: []string{"get", "list", "watch", "create", "update", "patch", "delete"}, - }, - // Application-layer reconciler emits one EnvoyExtensionPolicy per WAF - // targetRef to bind the Coraza wasm filter at the gateway / route. - { - APIGroups: []string{"gateway.envoyproxy.io"}, - Resources: []string{"envoyextensionpolicies"}, - Verbs: []string{"get", "list", "watch", "create", "update", "patch", "delete"}, - }, - // Application-layer reconciler stamps each namespace with its allocated WAF - // rule-id range (applicationlayer.projectcalico.org/waf-id-range annotation) - // so application operators can author in-range rules. The base role already - // grants namespaces get/list/watch; the annotation write needs patch/update. - { - APIGroups: []string{""}, - Resources: []string{"namespaces"}, - Verbs: []string{"get", "patch", "update"}, - }, - } -} - -func esKubeControllersCalicoSystemPolicy(cfg *kubecontrollers.KubeControllersConfiguration) *v3.NetworkPolicy { - if cfg.ManagementClusterConnection != nil { - return nil - } - - egressRules := []v3.Rule{} - egressRules = networkpolicy.AppendDNSEgressRules(egressRules, cfg.Installation.KubernetesProvider.IsOpenShift()) - egressRules = append(egressRules, []v3.Rule{ - { - Action: v3.Allow, - Protocol: &networkpolicy.TCPProtocol, - Destination: v3.EntityRule{ - Ports: networkpolicy.Ports(443, 6443, 12388), - }, - }, - }...) - - egressRules = append(egressRules, []v3.Rule{ - { - Action: v3.Allow, - Protocol: &networkpolicy.TCPProtocol, - Destination: networkpolicy.DefaultHelper().ESGatewayEntityRule(), - }, - }...) - - networkpolicyHelper := networkpolicy.Helper(cfg.Tenant.MultiTenant(), cfg.Namespace) - egressRules = append(egressRules, []v3.Rule{ - { - Action: v3.Allow, - Protocol: &networkpolicy.TCPProtocol, - Destination: networkpolicyHelper.ManagerEntityRule(), - }, - }...) - - return &v3.NetworkPolicy{ - TypeMeta: metav1.TypeMeta{Kind: "NetworkPolicy", APIVersion: "projectcalico.org/v3"}, - ObjectMeta: metav1.ObjectMeta{ - Name: EsKubeControllerNetworkPolicyName, - Namespace: cfg.Namespace, - }, - Spec: v3.NetworkPolicySpec{ - Order: &networkpolicy.HighPrecedenceOrder, - Tier: networkpolicy.CalicoTierName, - Selector: networkpolicy.KubernetesAppSelector(EsKubeController), - Types: []v3.PolicyType{v3.PolicyTypeEgress}, - Egress: egressRules, - }, - } -} - // wafRenderData is the controller-produced WAF v3 (Gateway API add-on) state the // installation hook hands the kube-controllers modifier through the render context. // The zero value (enabled false) means the modifier deletes the webhook objects. diff --git a/pkg/enterprise/kubecontrollers_test.go b/pkg/enterprise/installation/kubecontrollers_test.go similarity index 97% rename from pkg/enterprise/kubecontrollers_test.go rename to pkg/enterprise/installation/kubecontrollers_test.go index 0eaeda52db..3fe2034902 100644 --- a/pkg/enterprise/kubecontrollers_test.go +++ b/pkg/enterprise/installation/kubecontrollers_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package enterprise_test +package installation_test import ( "context" @@ -36,7 +36,7 @@ import ( "github.com/tigera/operator/pkg/controller/certificatemanager" "github.com/tigera/operator/pkg/controller/contexts" ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" - "github.com/tigera/operator/pkg/enterprise" + "github.com/tigera/operator/pkg/enterprise/installation" "github.com/tigera/operator/pkg/extensions" "github.com/tigera/operator/pkg/extensions/extensionstest" "github.com/tigera/operator/pkg/render" @@ -202,17 +202,17 @@ var _ = Describe("calico-kube-controllers enterprise surface", func() { Expect(c.Env).To(ContainElement(corev1.EnvVar{ Name: "WASM_IMAGE", Value: "test-reg/tigera/envoy-proxy:" + components.ComponentGatewayAPIEnvoyProxy.Version, })) - Expect(c.Env).To(ContainElement(corev1.EnvVar{Name: "WASM_PULL_SECRET", Value: enterprise.WASMPullSecretName})) - Expect(c.Env).To(ContainElement(corev1.EnvVar{Name: "WASM_CA_CERT", Value: enterprise.WASMCACertName})) + Expect(c.Env).To(ContainElement(corev1.EnvVar{Name: "WASM_PULL_SECRET", Value: installation.WASMPullSecretName})) + Expect(c.Env).To(ContainElement(corev1.EnvVar{Name: "WASM_CA_CERT", Value: installation.WASMCACertName})) Expect(c.Env).To(ContainElement(HaveField("Name", "WAF_WEBHOOK_CERT_DIR"))) Expect(c.Ports).To(ContainElement(corev1.ContainerPort{Name: "waf-webhook", ContainerPort: int32(9443), Protocol: corev1.ProtocolTCP})) // The webhook surface, the wasm pull secret, and the wasm CA bundle are rendered. _, ok = extensions.FindObject[*corev1.Service](objs, applicationlayer.WAFWebhookServiceName) Expect(ok).To(BeTrue()) - _, ok = extensions.FindObject[*corev1.Secret](objs, enterprise.WASMPullSecretName) + _, ok = extensions.FindObject[*corev1.Secret](objs, installation.WASMPullSecretName) Expect(ok).To(BeTrue()) - _, ok = extensions.FindObject[*corev1.ConfigMap](objs, enterprise.WASMCACertName) + _, ok = extensions.FindObject[*corev1.ConfigMap](objs, installation.WASMCACertName) Expect(ok).To(BeTrue()) }) diff --git a/pkg/enterprise/node.go b/pkg/enterprise/installation/node.go similarity index 96% rename from pkg/enterprise/node.go rename to pkg/enterprise/installation/node.go index ee1ce308cf..b862fe696d 100644 --- a/pkg/enterprise/node.go +++ b/pkg/enterprise/installation/node.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package enterprise +package installation import ( "errors" @@ -190,7 +190,7 @@ func nodeEnterpriseEnv(rc render.RenderContext) []corev1.EnvVar { data := installationData(rc) env := []corev1.EnvVar{ {Name: "FELIX_PROMETHEUSREPORTERENABLED", Value: "true"}, - {Name: "FELIX_PROMETHEUSREPORTERPORT", Value: fmt.Sprintf("%d", nodeReporterPort(rc.FelixConfiguration))}, + {Name: "FELIX_PROMETHEUSREPORTERPORT", Value: fmt.Sprintf("%d", NodeReporterPort(rc.FelixConfiguration))}, {Name: "FELIX_FLOWLOGSFILEENABLED", Value: "true"}, {Name: "FELIX_FLOWLOGSFILEINCLUDELABELS", Value: "true"}, {Name: "FELIX_FLOWLOGSFILEINCLUDEPOLICIES", Value: "true"}, @@ -232,7 +232,7 @@ func multiInterfaceModeEnv(install *operatorv1.InstallationSpec) *corev1.EnvVar // nodeMetricsService builds the enterprise-only calico-node-metrics Service. func nodeMetricsService(rc render.RenderContext) *corev1.Service { - reporterPort := nodeReporterPort(rc.FelixConfiguration) + reporterPort := NodeReporterPort(rc.FelixConfiguration) felixPort := felixMetricsPort(rc.FelixConfiguration) felixEnabled := rc.FelixConfiguration != nil && utils.IsFelixPrometheusMetricsEnabled(rc.FelixConfiguration) @@ -274,19 +274,19 @@ func nodeMetricsService(rc render.RenderContext) *corev1.Service { } } -// validateReporterPort rejects the unsupported zero prometheus reporter port. +// ValidateReporterPort rejects the unsupported zero prometheus reporter port. // The node and windows controller extensions share it. -func validateReporterPort(fc *v3.FelixConfiguration) error { +func ValidateReporterPort(fc *v3.FelixConfiguration) error { if fc != nil && fc.Spec.PrometheusReporterPort != nil && *fc.Spec.PrometheusReporterPort == 0 { return errors.New("felixConfiguration prometheusReporterPort=0 not supported") } return nil } -// nodeReporterPort returns the reporter metrics port from the FelixConfiguration, +// NodeReporterPort returns the reporter metrics port from the FelixConfiguration, // falling back to the default. The node-metrics Service and the // FELIX_PROMETHEUSREPORTERPORT env var both derive from here so they can't drift. -func nodeReporterPort(fc *v3.FelixConfiguration) int { +func NodeReporterPort(fc *v3.FelixConfiguration) int { if fc != nil && fc.Spec.PrometheusReporterPort != nil { return *fc.Spec.PrometheusReporterPort } diff --git a/pkg/enterprise/node_test.go b/pkg/enterprise/installation/node_test.go similarity index 99% rename from pkg/enterprise/node_test.go rename to pkg/enterprise/installation/node_test.go index 058f19d3d0..e8f69a90ea 100644 --- a/pkg/enterprise/node_test.go +++ b/pkg/enterprise/installation/node_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package enterprise_test +package installation_test import ( . "github.com/onsi/ginkgo/v2" diff --git a/pkg/enterprise/installation/suite_test.go b/pkg/enterprise/installation/suite_test.go new file mode 100644 index 0000000000..c3aa6ebbe7 --- /dev/null +++ b/pkg/enterprise/installation/suite_test.go @@ -0,0 +1,34 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package installation_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/tigera/operator/pkg/enterprise" + "github.com/tigera/operator/pkg/extensions" +) + +// ext is the enterprise extension Set under test, shared across the suite. It is +// immutable once built and the specs only read it, so a single instance is safe. +var ext *extensions.Set = enterprise.New() + +func TestInstallation(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "pkg/enterprise/installation Suite") +} diff --git a/pkg/enterprise/waf_pull_secret_test.go b/pkg/enterprise/installation/waf_pull_secret_test.go similarity index 88% rename from pkg/enterprise/waf_pull_secret_test.go rename to pkg/enterprise/installation/waf_pull_secret_test.go index 92cab8df9d..6cb54f5f51 100644 --- a/pkg/enterprise/waf_pull_secret_test.go +++ b/pkg/enterprise/installation/waf_pull_secret_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package enterprise_test +package installation_test import ( "encoding/json" @@ -22,7 +22,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/tigera/operator/pkg/common" - "github.com/tigera/operator/pkg/enterprise" + "github.com/tigera/operator/pkg/enterprise/installation" ) func dockerConfigJSONSecret(name string, auths map[string]any) *corev1.Secret { @@ -49,7 +49,7 @@ func mergedAuths(t *testing.T, s *corev1.Secret) map[string]map[string]string { } func TestMergeWAFPullSecret_MergesDisjointRegistries(t *testing.T) { - merged, skipped := enterprise.MergeWAFPullSecret([]*corev1.Secret{ + merged, skipped := installation.MergeWAFPullSecret([]*corev1.Secret{ dockerConfigJSONSecret("tigera-pull-secret", map[string]any{"quay.io": map[string]string{"auth": "dGlnZXJh"}}), dockerConfigJSONSecret("mirror-pull-secret", map[string]any{"registry.example.com": map[string]string{"auth": "bWlycm9y"}}), }) @@ -59,7 +59,7 @@ func TestMergeWAFPullSecret_MergesDisjointRegistries(t *testing.T) { if merged == nil { t.Fatal("expected a merged secret") } - if merged.Name != enterprise.WASMPullSecretName || merged.Namespace != common.CalicoNamespace { + if merged.Name != installation.WASMPullSecretName || merged.Namespace != common.CalicoNamespace { t.Fatalf("unexpected name/namespace: %s/%s", merged.Namespace, merged.Name) } if merged.Type != corev1.SecretTypeDockerConfigJson { @@ -72,7 +72,7 @@ func TestMergeWAFPullSecret_MergesDisjointRegistries(t *testing.T) { } func TestMergeWAFPullSecret_FirstSecretWinsOnDuplicateRegistry(t *testing.T) { - merged, _ := enterprise.MergeWAFPullSecret([]*corev1.Secret{ + merged, _ := installation.MergeWAFPullSecret([]*corev1.Secret{ dockerConfigJSONSecret("first", map[string]any{"quay.io": map[string]string{"auth": "Zmlyc3Q="}}), dockerConfigJSONSecret("second", map[string]any{"quay.io": map[string]string{"auth": "c2Vjb25k"}}), }) @@ -88,7 +88,7 @@ func TestMergeWAFPullSecret_SkipsUnparseableSecrets(t *testing.T) { Type: corev1.SecretTypeDockerConfigJson, Data: map[string][]byte{corev1.DockerConfigJsonKey: []byte("not-json")}, } - merged, skipped := enterprise.MergeWAFPullSecret([]*corev1.Secret{ + merged, skipped := installation.MergeWAFPullSecret([]*corev1.Secret{ bad, dockerConfigJSONSecret("good", map[string]any{"quay.io": map[string]string{"auth": "Z29vZA=="}}), }) @@ -111,7 +111,7 @@ func TestMergeWAFPullSecret_LegacyDockercfg(t *testing.T) { Type: corev1.SecretTypeDockercfg, Data: map[string][]byte{corev1.DockerConfigKey: cfg}, } - merged, skipped := enterprise.MergeWAFPullSecret([]*corev1.Secret{legacy}) + merged, skipped := installation.MergeWAFPullSecret([]*corev1.Secret{legacy}) if len(skipped) != 0 { t.Fatalf("expected no skipped secrets, got %v", skipped) } @@ -127,7 +127,7 @@ func TestMergeWAFPullSecret_NothingUsableReturnsNil(t *testing.T) { Type: corev1.SecretTypeDockerConfigJson, Data: map[string][]byte{corev1.DockerConfigJsonKey: []byte("not-json")}, } - merged, skipped := enterprise.MergeWAFPullSecret([]*corev1.Secret{bad}) + merged, skipped := installation.MergeWAFPullSecret([]*corev1.Secret{bad}) if merged != nil { t.Fatalf("expected nil secret, got %v", merged) } @@ -141,8 +141,8 @@ func TestMergeWAFPullSecret_DeterministicOutput(t *testing.T) { dockerConfigJSONSecret("a", map[string]any{"z.example.com": map[string]string{"auth": "eg=="}, "a.example.com": map[string]string{"auth": "YQ=="}}), dockerConfigJSONSecret("b", map[string]any{"m.example.com": map[string]string{"auth": "bQ=="}}), } - first, _ := enterprise.MergeWAFPullSecret(in) - second, _ := enterprise.MergeWAFPullSecret(in) + first, _ := installation.MergeWAFPullSecret(in) + second, _ := installation.MergeWAFPullSecret(in) if string(first.Data[corev1.DockerConfigJsonKey]) != string(second.Data[corev1.DockerConfigJsonKey]) { t.Fatal("merged secret bytes must be deterministic across reconciles") } diff --git a/pkg/enterprise/kubecontrollers/kubecontrollers.go b/pkg/enterprise/kubecontrollers/kubecontrollers.go new file mode 100644 index 0000000000..6141cd6b78 --- /dev/null +++ b/pkg/enterprise/kubecontrollers/kubecontrollers.go @@ -0,0 +1,335 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package kubecontrollers holds the enterprise es-calico-kube-controllers assembly +// (a distinct deployment the logstorage controller reconciles) and the enterprise +// kube-controllers cluster role rules shared with the calico-kube-controllers +// modifier in pkg/enterprise/installation. +package kubecontrollers + +import ( + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" + + "github.com/tigera/operator/pkg/render" + relasticsearch "github.com/tigera/operator/pkg/render/common/elasticsearch" + rmeta "github.com/tigera/operator/pkg/render/common/meta" + "github.com/tigera/operator/pkg/render/common/networkpolicy" + rkc "github.com/tigera/operator/pkg/render/kubecontrollers" + "github.com/tigera/operator/pkg/url" +) + +const ( + EsKubeController = "es-calico-kube-controllers" + EsKubeControllerRole = "es-calico-kube-controllers" + EsKubeControllerRoleBinding = "es-calico-kube-controllers" + EsKubeControllerMetrics = "es-calico-kube-controllers-metrics" + EsKubeControllerNetworkPolicyName = networkpolicy.CalicoComponentPolicyPrefix + "es-kube-controller-access" + + ElasticsearchKubeControllersUserSecret = "tigera-ee-kube-controllers-elasticsearch-access" + ElasticsearchKubeControllersUserName = "tigera-ee-kube-controllers" + ElasticsearchKubeControllersSecureUserSecret = "tigera-ee-kube-controllers-elasticsearch-access-gateway" + ElasticsearchKubeControllersVerificationUserSecret = "tigera-ee-kube-controllers-gateway-verification-credentials" +) + +// NewElasticsearchKubeControllers fills the generic kube-controllers configuration +// for the enterprise es-calico-kube-controllers deployment and returns the rendered +// component. es-kube-controllers is a distinct deployment (talks to Elasticsearch via +// es-gateway) reconciled by the logstorage kube-controllers controller, so it's +// assembled here rather than through the render-time modifier mechanism. +func NewElasticsearchKubeControllers(cfg *rkc.KubeControllersConfiguration) render.Component { + cfg.Name = EsKubeController + cfg.ConfigName = "elasticsearch" + cfg.RoleName = EsKubeControllerRole + cfg.RoleBindingName = EsKubeControllerRoleBinding + cfg.MetricsName = EsKubeControllerMetrics + cfg.DisableConfigAPI = cfg.Tenant.MultiTenant() + + cfg.Rules = rkc.KubeControllersRoleCommonRules(cfg) + cfg.Rules = append(cfg.Rules, KubeControllersEnterpriseCommonRules(false, cfg.ManagementClusterConnection != nil)...) + cfg.Rules = append(cfg.Rules, + rbacv1.PolicyRule{ + APIGroups: []string{"elasticsearch.k8s.elastic.co"}, + Resources: []string{"elasticsearches"}, + Verbs: []string{"watch", "get", "list"}, + }, + rbacv1.PolicyRule{ + APIGroups: []string{"rbac.authorization.k8s.io"}, + Resources: []string{"clusterroles", "clusterrolebindings"}, + Verbs: []string{"watch", "list", "get"}, + }, + ) + + if !cfg.Tenant.MultiTenant() { + // Zero and single tenant clusters need elasticsearch configuration. + cfg.EnabledControllers = append(cfg.EnabledControllers, "authorization", "elasticsearchconfiguration") + if cfg.ManagementCluster != nil && cfg.Tenant == nil { + // Enterprise requires the managedcluster controller to push licenses. + cfg.EnabledControllers = append(cfg.EnabledControllers, "managedcluster") + } + } + + cfg.NetworkPolicy = esKubeControllersCalicoSystemPolicy(cfg) + cfg.DeprecatedNetworkPolicyName = "es-kube-controller-access" + cfg.ExtraEnv = esKubeControllersEnv(cfg) + + return rkc.NewKubeControllers(cfg) +} + +// esKubeControllersEnv builds the enterprise env vars for es-calico-kube-controllers. +func esKubeControllersEnv(cfg *rkc.KubeControllersConfiguration) []corev1.EnvVar { + var env []corev1.EnvVar + + if cfg.Tenant != nil { + env = append(env, corev1.EnvVar{Name: "TENANT_ID", Value: cfg.Tenant.Spec.ID}) + } + + // What started as a workaround is now the default behaviour. This feature uses our backend in order to + // log into Kibana for users from external identity providers, rather than configuring an authn realm + // in the Elastic stack. + env = append(env, corev1.EnvVar{Name: "ENABLE_ELASTICSEARCH_OIDC_WORKAROUND", Value: "true"}) + if cfg.Authentication != nil { + env = append(env, + corev1.EnvVar{Name: "OIDC_AUTH_USERNAME_PREFIX", Value: cfg.Authentication.Spec.UsernamePrefix}, + corev1.EnvVar{Name: "OIDC_AUTH_GROUP_PREFIX", Value: cfg.Authentication.Spec.GroupsPrefix}, + ) + } + + if cfg.TrustedBundle != nil { + env = append(env, corev1.EnvVar{Name: "MULTI_CLUSTER_FORWARDING_CA", Value: cfg.TrustedBundle.MountPath()}) + } + if cfg.Installation.CalicoNetwork != nil && cfg.Installation.CalicoNetwork.MultiInterfaceMode != nil { + env = append(env, corev1.EnvVar{Name: "MULTI_INTERFACE_MODE", Value: cfg.Installation.CalicoNetwork.MultiInterfaceMode.Value()}) + } + + if !cfg.Tenant.MultiTenant() { + _, esHost, esPort, _ := url.ParseEndpoint(relasticsearch.GatewayEndpoint(rmeta.OSTypeLinux, cfg.ClusterDomain, render.ElasticsearchNamespace)) + env = append(env, + relasticsearch.ElasticHostEnvVar(esHost), + relasticsearch.ElasticPortEnvVar(esPort), + relasticsearch.ElasticUsernameEnvVar(ElasticsearchKubeControllersUserSecret), + relasticsearch.ElasticPasswordEnvVar(ElasticsearchKubeControllersUserSecret), + relasticsearch.ElasticCAEnvVar(rmeta.OSTypeLinux), + ) + } + + return env +} + +// KubeControllersEnterpriseCommonRules are the Calico Enterprise cluster role rules +// shared by calico-kube-controllers and es-calico-kube-controllers. wafEnabled adds +// the WAF v3 (Gateway API add-on) rules; managedCluster adds the license-push rule a +// managed cluster's kube-controllers needs. +func KubeControllersEnterpriseCommonRules(wafEnabled, managedCluster bool) []rbacv1.PolicyRule { + rules := []rbacv1.PolicyRule{ + { + APIGroups: []string{""}, + Resources: []string{"configmaps"}, + Verbs: []string{"watch", "list", "get", "update", "create", "delete"}, + }, + { + // The Federated Services Controller needs access to the remote kubeconfig secret + // in order to create a remote syncer. + APIGroups: []string{""}, + Resources: []string{"secrets"}, + Verbs: []string{"watch", "list", "get"}, + }, + { + // Needed to validate the license + APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, + Resources: []string{"licensekeys"}, + Verbs: []string{"get", "watch", "list"}, + }, + { + // Needed to update the status of the LicenseKey with the result of license validation. + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"licensekeys/status"}, + Verbs: []string{"update"}, + }, + { + APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, + Resources: []string{"deeppacketinspections"}, + Verbs: []string{"get", "watch", "list"}, + }, + { + APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, + Resources: []string{"deeppacketinspections/status"}, + Verbs: []string{"update"}, + }, + { + APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, + Resources: []string{"packetcaptures"}, + Verbs: []string{"get", "list", "update"}, + }, + { + APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, + Resources: []string{"packetcaptures/status"}, + Verbs: []string{"update"}, + }, + } + + if wafEnabled { + rules = append(rules, wafRules()...) + } + + if managedCluster { + rules = append(rules, + rbacv1.PolicyRule{ + APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, + Resources: []string{"licensekeys"}, + Verbs: []string{"get", "create", "update", "list", "watch"}, + }, + ) + } + + return rules +} + +// wafRules are the WAF v3 (Gateway API add-on) cluster role rules, gated by +// GatewayAPI.spec.extensions.waf.state == Enabled. +func wafRules() []rbacv1.PolicyRule { + return []rbacv1.PolicyRule{ + // Application-layer (gateway-addons) reconcilers reconcile WAF resources + // against Gateway API targetRefs and emit events on the policy objects. + { + APIGroups: []string{"applicationlayer.projectcalico.org"}, + Resources: []string{ + "wafpolicies", "globalwafpolicies", + "wafplugins", "globalwafplugins", + "wafvalidationpolicies", "globalwafvalidationpolicies", + }, + Verbs: []string{"get", "list", "watch", "create", "update", "patch", "delete"}, + }, + { + APIGroups: []string{"applicationlayer.projectcalico.org"}, + Resources: []string{ + "wafpolicies/status", "globalwafpolicies/status", + "wafplugins/status", "globalwafplugins/status", + "wafvalidationpolicies/status", "globalwafvalidationpolicies/status", + }, + Verbs: []string{"get", "update", "patch"}, + }, + { + APIGroups: []string{"applicationlayer.projectcalico.org"}, + Resources: []string{ + "wafpolicies/finalizers", "globalwafpolicies/finalizers", + "wafplugins/finalizers", "globalwafplugins/finalizers", + "wafvalidationpolicies/finalizers", "globalwafvalidationpolicies/finalizers", + }, + Verbs: []string{"update"}, + }, + { + // Validate Gateway API targetRefs and surface attachment status. + APIGroups: []string{"gateway.networking.k8s.io"}, + Resources: []string{"gateways", "httproutes", "tcproutes", "tlsroutes", "grpcroutes"}, + Verbs: []string{"get", "list", "watch", "update", "patch"}, + }, + { + APIGroups: []string{"gateway.networking.k8s.io"}, + Resources: []string{"gateways/status", "httproutes/status", "tcproutes/status", "tlsroutes/status", "grpcroutes/status"}, + Verbs: []string{"get", "update", "patch"}, + }, + // controller-runtime Reconcilers (e.g. the applicationlayer manager) record + // events on watched objects via Recorder.Eventf; both core and events.k8s.io + // API groups are emitted depending on the kubernetes version. + { + APIGroups: []string{""}, + Resources: []string{"events"}, + Verbs: []string{"create", "patch"}, + }, + { + APIGroups: []string{"events.k8s.io"}, + Resources: []string{"events"}, + Verbs: []string{"create", "patch"}, + }, + // Application-layer reconciler replicates the WAF wasm pull Secret from + // the controller namespace (calico-system) into each WAFPolicy's + // namespace so the rendered EnvoyExtensionPolicy can reference it. Also + // replicates CA-cert ConfigMaps when WASM_CA_CERT is set. + { + APIGroups: []string{""}, + Resources: []string{"secrets", "configmaps"}, + Verbs: []string{"get", "list", "watch", "create", "update", "patch", "delete"}, + }, + // Application-layer reconciler emits one EnvoyExtensionPolicy per WAF + // targetRef to bind the Coraza wasm filter at the gateway / route. + { + APIGroups: []string{"gateway.envoyproxy.io"}, + Resources: []string{"envoyextensionpolicies"}, + Verbs: []string{"get", "list", "watch", "create", "update", "patch", "delete"}, + }, + // Application-layer reconciler stamps each namespace with its allocated WAF + // rule-id range (applicationlayer.projectcalico.org/waf-id-range annotation) + // so application operators can author in-range rules. The base role already + // grants namespaces get/list/watch; the annotation write needs patch/update. + { + APIGroups: []string{""}, + Resources: []string{"namespaces"}, + Verbs: []string{"get", "patch", "update"}, + }, + } +} + +func esKubeControllersCalicoSystemPolicy(cfg *rkc.KubeControllersConfiguration) *v3.NetworkPolicy { + if cfg.ManagementClusterConnection != nil { + return nil + } + + egressRules := []v3.Rule{} + egressRules = networkpolicy.AppendDNSEgressRules(egressRules, cfg.Installation.KubernetesProvider.IsOpenShift()) + egressRules = append(egressRules, []v3.Rule{ + { + Action: v3.Allow, + Protocol: &networkpolicy.TCPProtocol, + Destination: v3.EntityRule{ + Ports: networkpolicy.Ports(443, 6443, 12388), + }, + }, + }...) + + egressRules = append(egressRules, []v3.Rule{ + { + Action: v3.Allow, + Protocol: &networkpolicy.TCPProtocol, + Destination: networkpolicy.DefaultHelper().ESGatewayEntityRule(), + }, + }...) + + networkpolicyHelper := networkpolicy.Helper(cfg.Tenant.MultiTenant(), cfg.Namespace) + egressRules = append(egressRules, []v3.Rule{ + { + Action: v3.Allow, + Protocol: &networkpolicy.TCPProtocol, + Destination: networkpolicyHelper.ManagerEntityRule(), + }, + }...) + + return &v3.NetworkPolicy{ + TypeMeta: metav1.TypeMeta{Kind: "NetworkPolicy", APIVersion: "projectcalico.org/v3"}, + ObjectMeta: metav1.ObjectMeta{ + Name: EsKubeControllerNetworkPolicyName, + Namespace: cfg.Namespace, + }, + Spec: v3.NetworkPolicySpec{ + Order: &networkpolicy.HighPrecedenceOrder, + Tier: networkpolicy.CalicoTierName, + Selector: networkpolicy.KubernetesAppSelector(EsKubeController), + Types: []v3.PolicyType{v3.PolicyTypeEgress}, + Egress: egressRules, + }, + } +} diff --git a/pkg/enterprise/register.go b/pkg/enterprise/register.go index d8e34a4ac5..db211238c2 100644 --- a/pkg/enterprise/register.go +++ b/pkg/enterprise/register.go @@ -16,7 +16,11 @@ package enterprise import ( operatorv1 "github.com/tigera/operator/api/v1" - "github.com/tigera/operator/pkg/controller/contexts" + "github.com/tigera/operator/pkg/enterprise/apiserver" + "github.com/tigera/operator/pkg/enterprise/guardian" + "github.com/tigera/operator/pkg/enterprise/installation" + "github.com/tigera/operator/pkg/enterprise/typha" + "github.com/tigera/operator/pkg/enterprise/windows" "github.com/tigera/operator/pkg/extensions" ) @@ -24,25 +28,22 @@ import ( // controller extension, every component modifier, and the image overrides. The // operator is handed this Set at startup (the core operator is handed none). // After the monorepo split this is what calico-private's main will construct -// instead. +// instead. Each per-component subpackage registers its own controller hook and +// modifiers through its Register func. func New() *extensions.Set { s := extensions.NewSet() ent := s.Variant(operatorv1.CalicoEnterprise) - ent.Controller(contexts.InstallationController, coreControllerExtension{}) - ent.Controller(contexts.WindowsController, windowsControllerExtension{}) - ent.Controller(contexts.APIServerController, apiServerControllerExtension{}) - registerTypha(ent) - registerNode(ent) - registerWindows(ent) - registerGuardian(ent) - registerAPIServer(ent) - registerKubeControllers(ent) + typha.Register(ent) + installation.Register(ent) + windows.Register(ent) + guardian.Register(ent) + apiserver.Register(ent) // When the enterprise operator manages a Calico installation, clean up the // Enterprise objects left behind by a prior Enterprise installation. cal := s.Variant(operatorv1.Calico) - registerAPIServerCleanup(cal) + apiserver.RegisterCalicoCleanup(cal) return s } diff --git a/pkg/enterprise/typha.go b/pkg/enterprise/typha/extension.go similarity index 94% rename from pkg/enterprise/typha.go rename to pkg/enterprise/typha/extension.go index 950b66415e..203d8c6236 100644 --- a/pkg/enterprise/typha.go +++ b/pkg/enterprise/typha/extension.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package enterprise +package typha import ( appsv1 "k8s.io/api/apps/v1" @@ -24,7 +24,8 @@ import ( "github.com/tigera/operator/pkg/render" ) -func registerTypha(v *extensions.Variant) { +// Register wires the typha extension into the variant. +func Register(v *extensions.Variant) { v.Modify(render.ComponentNameTypha, modifyTypha) } diff --git a/pkg/enterprise/typha_test.go b/pkg/enterprise/typha/extension_test.go similarity index 99% rename from pkg/enterprise/typha_test.go rename to pkg/enterprise/typha/extension_test.go index 8720f2091f..c1eff29e53 100644 --- a/pkg/enterprise/typha_test.go +++ b/pkg/enterprise/typha/extension_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package enterprise_test +package typha_test import ( . "github.com/onsi/ginkgo/v2" diff --git a/pkg/enterprise/enterprise_suite_test.go b/pkg/enterprise/typha/suite_test.go similarity index 91% rename from pkg/enterprise/enterprise_suite_test.go rename to pkg/enterprise/typha/suite_test.go index 368107b5a0..960119299b 100644 --- a/pkg/enterprise/enterprise_suite_test.go +++ b/pkg/enterprise/typha/suite_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package enterprise_test +package typha_test import ( "testing" @@ -28,7 +28,7 @@ import ( // immutable once built and the specs only read it, so a single instance is safe. var ext *extensions.Set = enterprise.New() -func TestEnterprise(t *testing.T) { +func TestTypha(t *testing.T) { RegisterFailHandler(Fail) - RunSpecs(t, "pkg/enterprise Suite") + RunSpecs(t, "pkg/enterprise/typha Suite") } diff --git a/pkg/enterprise/windows.go b/pkg/enterprise/windows/extension.go similarity index 95% rename from pkg/enterprise/windows.go rename to pkg/enterprise/windows/extension.go index 7aba485341..007781f00c 100644 --- a/pkg/enterprise/windows.go +++ b/pkg/enterprise/windows/extension.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package enterprise +package windows import ( "fmt" @@ -30,6 +30,7 @@ import ( "github.com/tigera/operator/pkg/controller/utils" "github.com/tigera/operator/pkg/ctrlruntime" "github.com/tigera/operator/pkg/dns" + "github.com/tigera/operator/pkg/enterprise/installation" "github.com/tigera/operator/pkg/extensions" "github.com/tigera/operator/pkg/render" rmeta "github.com/tigera/operator/pkg/render/common/meta" @@ -41,7 +42,9 @@ import ( // felix env and node volume mounts, so they receive the same enterprise layering. var windowsNodeContainers = map[string]bool{"felix": true, "node": true, "confd": true} -func registerWindows(v *extensions.Variant) { +// Register wires the windows controller hook and modifiers into the variant. +func Register(v *extensions.Variant) { + v.Controller(contexts.WindowsController, windowsControllerExtension{}) v.Image(render.ComponentNameWindowsNodeImg, components.ComponentTigeraNodeWindows) v.Image(render.ComponentNameWindowsCNIImg, components.ComponentTigeraCNIWindows) v.Modify(render.ComponentNameWindows, modifyWindows) @@ -65,7 +68,7 @@ func windowsData(rc render.RenderContext) windowsRenderData { // Validate rejects windows installation config Calico Enterprise does not support. func (windowsControllerExtension) Validate(cc contexts.ControllerContext) error { - return validateReporterPort(cc.FelixConfiguration) + return installation.ValidateReporterPort(cc.FelixConfiguration) } // Watches registers the enterprise secrets the windows controller reconciles on. @@ -142,7 +145,7 @@ func windowsEnterpriseEnv(rc render.RenderContext) []corev1.EnvVar { tls := windowsData(rc).prometheusServerTLS env := []corev1.EnvVar{ {Name: "FELIX_PROMETHEUSREPORTERENABLED", Value: "true"}, - {Name: "FELIX_PROMETHEUSREPORTERPORT", Value: fmt.Sprintf("%d", nodeReporterPort(rc.FelixConfiguration))}, + {Name: "FELIX_PROMETHEUSREPORTERPORT", Value: fmt.Sprintf("%d", installation.NodeReporterPort(rc.FelixConfiguration))}, {Name: "FELIX_FLOWLOGSFILEENABLED", Value: "true"}, {Name: "FELIX_FLOWLOGSFILEINCLUDELABELS", Value: "true"}, {Name: "FELIX_FLOWLOGSFILEINCLUDEPOLICIES", Value: "true"}, @@ -200,7 +203,7 @@ func mountWindowsPrometheusTLS(rc render.RenderContext, ds *appsv1.DaemonSet) { // windowsNodeMetricsService builds the enterprise-only calico-node-metrics-windows // Service. func windowsNodeMetricsService(rc render.RenderContext) *corev1.Service { - reporterPort := nodeReporterPort(rc.FelixConfiguration) + reporterPort := installation.NodeReporterPort(rc.FelixConfiguration) return &corev1.Service{ TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}, ObjectMeta: metav1.ObjectMeta{ diff --git a/pkg/enterprise/windows_test.go b/pkg/enterprise/windows/extension_test.go similarity index 99% rename from pkg/enterprise/windows_test.go rename to pkg/enterprise/windows/extension_test.go index 9b68a67e32..bb89a31f94 100644 --- a/pkg/enterprise/windows_test.go +++ b/pkg/enterprise/windows/extension_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package enterprise_test +package windows_test import ( "context" diff --git a/pkg/enterprise/windows/suite_test.go b/pkg/enterprise/windows/suite_test.go new file mode 100644 index 0000000000..3c96905e80 --- /dev/null +++ b/pkg/enterprise/windows/suite_test.go @@ -0,0 +1,34 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package windows_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/tigera/operator/pkg/enterprise" + "github.com/tigera/operator/pkg/extensions" +) + +// ext is the enterprise extension Set under test, shared across the suite. It is +// immutable once built and the specs only read it, so a single instance is safe. +var ext *extensions.Set = enterprise.New() + +func TestWindows(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "pkg/enterprise/windows Suite") +} diff --git a/pkg/render/kubecontrollers/kube-controllers_test.go b/pkg/render/kubecontrollers/kube-controllers_test.go index 89cfad0940..0c21b6dd29 100644 --- a/pkg/render/kubecontrollers/kube-controllers_test.go +++ b/pkg/render/kubecontrollers/kube-controllers_test.go @@ -39,7 +39,7 @@ import ( "github.com/tigera/operator/pkg/controller/k8sapi" ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" "github.com/tigera/operator/pkg/dns" - "github.com/tigera/operator/pkg/enterprise" + entkubecontrollers "github.com/tigera/operator/pkg/enterprise/kubecontrollers" "github.com/tigera/operator/pkg/render" rmeta "github.com/tigera/operator/pkg/render/common/meta" "github.com/tigera/operator/pkg/render/common/networkpolicy" @@ -264,20 +264,20 @@ var _ = Describe("kube-controllers rendering tests", func() { version string kind string }{ - {name: enterprise.EsKubeControllerNetworkPolicyName, ns: common.CalicoNamespace, group: "projectcalico.org", version: "v3", kind: "NetworkPolicy"}, + {name: entkubecontrollers.EsKubeControllerNetworkPolicyName, ns: common.CalicoNamespace, group: "projectcalico.org", version: "v3", kind: "NetworkPolicy"}, {name: "calico-kube-controllers", ns: common.CalicoNamespace, group: "", version: "v1", kind: "ServiceAccount"}, - {name: enterprise.EsKubeControllerRole, ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRole"}, - {name: enterprise.EsKubeControllerRoleBinding, ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRoleBinding"}, - {name: enterprise.EsKubeController, ns: common.CalicoNamespace, group: "apps", version: "v1", kind: "Deployment"}, - {name: enterprise.ElasticsearchKubeControllersUserSecret, ns: common.CalicoNamespace, group: "", version: "v1", kind: "Secret"}, - {name: enterprise.EsKubeControllerMetrics, ns: common.CalicoNamespace, group: "", version: "v1", kind: "Service"}, + {name: entkubecontrollers.EsKubeControllerRole, ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRole"}, + {name: entkubecontrollers.EsKubeControllerRoleBinding, ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRoleBinding"}, + {name: entkubecontrollers.EsKubeController, ns: common.CalicoNamespace, group: "apps", version: "v1", kind: "Deployment"}, + {name: entkubecontrollers.ElasticsearchKubeControllersUserSecret, ns: common.CalicoNamespace, group: "", version: "v1", kind: "Secret"}, + {name: entkubecontrollers.EsKubeControllerMetrics, ns: common.CalicoNamespace, group: "", version: "v1", kind: "Service"}, } instance.Variant = operatorv1.CalicoEnterprise cfg.KubeControllersGatewaySecret = &testutils.KubeControllersUserSecret cfg.MetricsPort = 9094 - component := enterprise.NewElasticsearchKubeControllers(&cfg) + component := entkubecontrollers.NewElasticsearchKubeControllers(&cfg) Expect(component.ResolveImages(nil)).To(BeNil()) resources, _ := component.Objects() Expect(len(resources)).To(Equal(len(expectedResources))) @@ -290,7 +290,7 @@ var _ = Describe("kube-controllers rendering tests", func() { } // The Deployment should have the correct configuration. - dp := rtest.GetResource(resources, enterprise.EsKubeController, common.CalicoNamespace, "apps", "v1", "Deployment").(*appsv1.Deployment) + dp := rtest.GetResource(resources, entkubecontrollers.EsKubeController, common.CalicoNamespace, "apps", "v1", "Deployment").(*appsv1.Deployment) Expect(dp.Spec.Template.Spec.Containers[0].Image).To(Equal("test-reg/tigera/calico:" + components.ComponentTigeraCalico.Version)) envs := dp.Spec.Template.Spec.Containers[0].Env @@ -307,7 +307,7 @@ var _ = Describe("kube-controllers rendering tests", func() { Expect(dp.Spec.Template.Spec.Volumes[0].Name).To(Equal("tigera-ca-bundle")) Expect(dp.Spec.Template.Spec.Volumes[0].ConfigMap.Name).To(Equal("tigera-ca-bundle")) - clusterRole := rtest.GetResource(resources, enterprise.EsKubeControllerRole, "", "rbac.authorization.k8s.io", "v1", "ClusterRole").(*rbacv1.ClusterRole) + clusterRole := rtest.GetResource(resources, entkubecontrollers.EsKubeControllerRole, "", "rbac.authorization.k8s.io", "v1", "ClusterRole").(*rbacv1.ClusterRole) Expect(clusterRole.Rules).To(HaveLen(26), "cluster role should have 26 rules") Expect(clusterRole.Rules).To(ContainElement( rbacv1.PolicyRule{ @@ -397,14 +397,14 @@ var _ = Describe("kube-controllers rendering tests", func() { version string kind string }{ - {name: enterprise.EsKubeControllerNetworkPolicyName, ns: common.CalicoNamespace, group: "projectcalico.org", version: "v3", kind: "NetworkPolicy"}, + {name: entkubecontrollers.EsKubeControllerNetworkPolicyName, ns: common.CalicoNamespace, group: "projectcalico.org", version: "v3", kind: "NetworkPolicy"}, {name: "calico-kube-controllers", ns: common.CalicoNamespace, group: "", version: "v1", kind: "ServiceAccount"}, - {name: enterprise.EsKubeControllerRole, ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRole"}, - {name: enterprise.EsKubeControllerRoleBinding, ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRoleBinding"}, + {name: entkubecontrollers.EsKubeControllerRole, ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRole"}, + {name: entkubecontrollers.EsKubeControllerRoleBinding, ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRoleBinding"}, {name: kubecontrollers.ManagedClustersWatchRoleBindingName, ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRoleBinding"}, - {name: enterprise.EsKubeController, ns: common.CalicoNamespace, group: "apps", version: "v1", kind: "Deployment"}, - {name: enterprise.ElasticsearchKubeControllersUserSecret, ns: common.CalicoNamespace, group: "", version: "v1", kind: "Secret"}, - {name: enterprise.EsKubeControllerMetrics, ns: common.CalicoNamespace, group: "", version: "v1", kind: "Service"}, + {name: entkubecontrollers.EsKubeController, ns: common.CalicoNamespace, group: "apps", version: "v1", kind: "Deployment"}, + {name: entkubecontrollers.ElasticsearchKubeControllersUserSecret, ns: common.CalicoNamespace, group: "", version: "v1", kind: "Secret"}, + {name: entkubecontrollers.EsKubeControllerMetrics, ns: common.CalicoNamespace, group: "", version: "v1", kind: "Service"}, } // Override configuration to match expected Enterprise config. @@ -413,7 +413,7 @@ var _ = Describe("kube-controllers rendering tests", func() { cfg.KubeControllersGatewaySecret = &testutils.KubeControllersUserSecret cfg.MetricsPort = 9094 - component := enterprise.NewElasticsearchKubeControllers(&cfg) + component := entkubecontrollers.NewElasticsearchKubeControllers(&cfg) Expect(component.ResolveImages(nil)).To(BeNil()) resources, _ := component.Objects() Expect(len(resources)).To(Equal(len(expectedResources))) @@ -426,7 +426,7 @@ var _ = Describe("kube-controllers rendering tests", func() { } // The Deployment should have the correct configuration. - dp := rtest.GetResource(resources, enterprise.EsKubeController, common.CalicoNamespace, "apps", "v1", "Deployment").(*appsv1.Deployment) + dp := rtest.GetResource(resources, entkubecontrollers.EsKubeController, common.CalicoNamespace, "apps", "v1", "Deployment").(*appsv1.Deployment) envs := dp.Spec.Template.Spec.Containers[0].Env Expect(envs).To(ContainElement(corev1.EnvVar{ @@ -444,7 +444,7 @@ var _ = Describe("kube-controllers rendering tests", func() { Expect(dp.Spec.Template.Spec.Containers[0].Image).To(Equal("test-reg/tigera/calico:" + components.ComponentTigeraCalico.Version)) - clusterRole := rtest.GetResource(resources, enterprise.EsKubeControllerRole, "", "rbac.authorization.k8s.io", "v1", "ClusterRole").(*rbacv1.ClusterRole) + clusterRole := rtest.GetResource(resources, entkubecontrollers.EsKubeControllerRole, "", "rbac.authorization.k8s.io", "v1", "ClusterRole").(*rbacv1.ClusterRole) Expect(clusterRole.Rules).To(HaveLen(26), "cluster role should have 26 rules") Expect(clusterRole.Rules).To(ContainElement( rbacv1.PolicyRule{ @@ -575,17 +575,17 @@ var _ = Describe("kube-controllers rendering tests", func() { Openshift: &operatorv1.AuthenticationOpenshift{IssuerURL: "https://api.example.com"}, }} - component := enterprise.NewElasticsearchKubeControllers(&cfg) + component := entkubecontrollers.NewElasticsearchKubeControllers(&cfg) Expect(component.ResolveImages(nil)).To(BeNil()) resources, _ := component.Objects() - depResource := rtest.GetResource(resources, enterprise.EsKubeController, common.CalicoNamespace, "apps", "v1", "Deployment") + depResource := rtest.GetResource(resources, entkubecontrollers.EsKubeController, common.CalicoNamespace, "apps", "v1", "Deployment") Expect(depResource).ToNot(BeNil()) deployment := depResource.(*appsv1.Deployment) var usernamePrefix, groupPrefix string for _, container := range deployment.Spec.Template.Spec.Containers { - if container.Name == enterprise.EsKubeController { + if container.Name == entkubecontrollers.EsKubeController { for _, env := range container.Env { switch env.Name { case "OIDC_AUTH_USERNAME_PREFIX": @@ -835,16 +835,16 @@ var _ = Describe("kube-controllers rendering tests", func() { cfg.ManagementCluster = &operatorv1.ManagementCluster{} cfg.KubeControllersGatewaySecret = &testutils.KubeControllersUserSecret cfg.MetricsPort = 9094 - component := enterprise.NewElasticsearchKubeControllers(&cfg) + component := entkubecontrollers.NewElasticsearchKubeControllers(&cfg) resources, _ := component.Objects() - depResource := rtest.GetResource(resources, enterprise.EsKubeController, common.CalicoNamespace, "apps", "v1", "Deployment") + depResource := rtest.GetResource(resources, entkubecontrollers.EsKubeController, common.CalicoNamespace, "apps", "v1", "Deployment") Expect(depResource).ToNot(BeNil()) deployment := depResource.(*appsv1.Deployment) var esLicenseType string for _, container := range deployment.Spec.Template.Spec.Containers { - if container.Name == enterprise.EsKubeController { + if container.Name == entkubecontrollers.EsKubeController { for _, env := range container.Env { if env.Name == "ENABLE_ELASTICSEARCH_OIDC_WORKAROUND" { esLicenseType = env.Value @@ -990,7 +990,7 @@ var _ = Describe("kube-controllers rendering tests", func() { instance.Variant = operatorv1.CalicoEnterprise cfg.KubeControllersGatewaySecret = &testutils.KubeControllersUserSecret - component := enterprise.NewElasticsearchKubeControllers(&cfg) + component := entkubecontrollers.NewElasticsearchKubeControllers(&cfg) resources, _ := component.Objects() policy := testutils.GetCalicoSystemPolicyFromResources(policyName, resources) diff --git a/pkg/render/logstorage/esgateway/esgateway_test.go b/pkg/render/logstorage/esgateway/esgateway_test.go index 5a76cc2c34..ce81684e66 100644 --- a/pkg/render/logstorage/esgateway/esgateway_test.go +++ b/pkg/render/logstorage/esgateway/esgateway_test.go @@ -37,7 +37,7 @@ import ( "github.com/tigera/operator/pkg/controller/certificatemanager" ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" "github.com/tigera/operator/pkg/dns" - "github.com/tigera/operator/pkg/enterprise" + entkubecontrollers "github.com/tigera/operator/pkg/enterprise/kubecontrollers" "github.com/tigera/operator/pkg/render" relasticsearch "github.com/tigera/operator/pkg/render/common/elasticsearch" rmeta "github.com/tigera/operator/pkg/render/common/meta" @@ -81,9 +81,9 @@ var _ = Describe("ES Gateway rendering tests", func() { ESGatewayKeyPair: kp, TrustedBundle: bundle, KubeControllersUserSecrets: []*corev1.Secret{ - {ObjectMeta: metav1.ObjectMeta{Name: enterprise.ElasticsearchKubeControllersUserSecret, Namespace: common.OperatorNamespace()}}, - {ObjectMeta: metav1.ObjectMeta{Name: enterprise.ElasticsearchKubeControllersVerificationUserSecret, Namespace: render.ElasticsearchNamespace}}, - {ObjectMeta: metav1.ObjectMeta{Name: enterprise.ElasticsearchKubeControllersSecureUserSecret, Namespace: render.ElasticsearchNamespace}}, + {ObjectMeta: metav1.ObjectMeta{Name: entkubecontrollers.ElasticsearchKubeControllersUserSecret, Namespace: common.OperatorNamespace()}}, + {ObjectMeta: metav1.ObjectMeta{Name: entkubecontrollers.ElasticsearchKubeControllersVerificationUserSecret, Namespace: render.ElasticsearchNamespace}}, + {ObjectMeta: metav1.ObjectMeta{Name: entkubecontrollers.ElasticsearchKubeControllersSecureUserSecret, Namespace: render.ElasticsearchNamespace}}, }, ClusterDomain: clusterDomain, EsAdminUserName: "elastic", @@ -95,9 +95,9 @@ var _ = Describe("ES Gateway rendering tests", func() { It("should render an ES Gateway deployment and all supporting resources", func() { expectedResources := []client.Object{ &v3.NetworkPolicy{ObjectMeta: metav1.ObjectMeta{Name: PolicyName, Namespace: render.ElasticsearchNamespace}}, - &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: enterprise.ElasticsearchKubeControllersUserSecret, Namespace: common.OperatorNamespace()}}, - &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: enterprise.ElasticsearchKubeControllersVerificationUserSecret, Namespace: render.ElasticsearchNamespace}}, - &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: enterprise.ElasticsearchKubeControllersSecureUserSecret, Namespace: render.ElasticsearchNamespace}}, + &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: entkubecontrollers.ElasticsearchKubeControllersUserSecret, Namespace: common.OperatorNamespace()}}, + &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: entkubecontrollers.ElasticsearchKubeControllersVerificationUserSecret, Namespace: render.ElasticsearchNamespace}}, + &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: entkubecontrollers.ElasticsearchKubeControllersSecureUserSecret, Namespace: render.ElasticsearchNamespace}}, &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: ServiceName, Namespace: render.ElasticsearchNamespace}}, &rbacv1.Role{ObjectMeta: metav1.ObjectMeta{Name: RoleName, Namespace: render.ElasticsearchNamespace}}, &rbacv1.RoleBinding{ObjectMeta: metav1.ObjectMeta{Name: RoleName, Namespace: render.ElasticsearchNamespace}}, @@ -134,9 +134,9 @@ var _ = Describe("ES Gateway rendering tests", func() { installation.CertificateManagement = &operatorv1.CertificateManagement{CACert: secret.Data[corev1.TLSCertKey]} expectedResources := []client.Object{ &v3.NetworkPolicy{ObjectMeta: metav1.ObjectMeta{Name: PolicyName, Namespace: render.ElasticsearchNamespace}}, - &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: enterprise.ElasticsearchKubeControllersUserSecret, Namespace: common.OperatorNamespace()}}, - &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: enterprise.ElasticsearchKubeControllersVerificationUserSecret, Namespace: render.ElasticsearchNamespace}}, - &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: enterprise.ElasticsearchKubeControllersSecureUserSecret, Namespace: render.ElasticsearchNamespace}}, + &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: entkubecontrollers.ElasticsearchKubeControllersUserSecret, Namespace: common.OperatorNamespace()}}, + &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: entkubecontrollers.ElasticsearchKubeControllersVerificationUserSecret, Namespace: render.ElasticsearchNamespace}}, + &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: entkubecontrollers.ElasticsearchKubeControllersSecureUserSecret, Namespace: render.ElasticsearchNamespace}}, &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: ServiceName, Namespace: render.ElasticsearchNamespace}}, &rbacv1.Role{ObjectMeta: metav1.ObjectMeta{Name: RoleName, Namespace: render.ElasticsearchNamespace}}, &rbacv1.RoleBinding{ObjectMeta: metav1.ObjectMeta{Name: RoleName, Namespace: render.ElasticsearchNamespace}}, From b5e50ae07fec0c803a1dd4ba471b956cd8b3eae9 Mon Sep 17 00:00:00 2001 From: Casey Davenport Date: Wed, 24 Jun 2026 13:57:22 -0700 Subject: [PATCH 45/54] Move guardian enterprise render tests into pkg/enterprise/guardian The enterprise integration cases (managed-cluster resources, impersonation, SCC, the enterprise GuardianPolicy) ran base render plus the real modifier, so they pulled enterprise.New() into the render test package. Move them next to the modifier in pkg/enterprise/guardian, where they run against the real Set. What stays in render is the OSS render path: the OSS guardian-access policy and the public-CA deployment cases. Those never run the modifier, so render_test no longer depends on pkg/enterprise for guardian. --- .../guardian/guardian_render_test.go | 445 +++++++++++++++++ pkg/render/guardian_test.go | 447 ++---------------- 2 files changed, 493 insertions(+), 399 deletions(-) create mode 100644 pkg/enterprise/guardian/guardian_render_test.go diff --git a/pkg/enterprise/guardian/guardian_render_test.go b/pkg/enterprise/guardian/guardian_render_test.go new file mode 100644 index 0000000000..fdb8ab382a --- /dev/null +++ b/pkg/enterprise/guardian/guardian_render_test.go @@ -0,0 +1,445 @@ +// Copyright (c) 2020-2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package guardian_test + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + netv1 "k8s.io/api/networking/v1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + + v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" + + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/apis" + "github.com/tigera/operator/pkg/common" + "github.com/tigera/operator/pkg/components" + "github.com/tigera/operator/pkg/controller/certificatemanager" + ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" + "github.com/tigera/operator/pkg/extensions/extensionstest" + "github.com/tigera/operator/pkg/render" + rmeta "github.com/tigera/operator/pkg/render/common/meta" + "github.com/tigera/operator/pkg/render/common/networkpolicy" + rtest "github.com/tigera/operator/pkg/render/common/test" + "github.com/tigera/operator/pkg/render/testutils" +) + +// The guardian policy fixtures live next to the render package's testutils, so +// reference them relative to this enterprise subpackage. +const ( + clusterDomain = "cluster.local" + guardianPolicyJSON = "../../render/testutils/expected_policies/guardian.json" + guardianPolicyOCPJSON = "../../render/testutils/expected_policies/guardian_ocp.json" +) + +// guardianObjects renders the guardian component and applies the registered +// enterprise modifier the way the componentHandler does. +func guardianObjects(cfg *render.GuardianConfiguration) []client.Object { + g := render.Guardian(cfg) + ExpectWithOffset(1, g.ResolveImages(nil)).To(BeNil()) + objs, _ := g.Objects() + rc := render.RenderContext{Installation: cfg.Installation} + var extCtx any + if p, ok := g.(render.ExtensionContextProvider); ok { + extCtx = p.ExtensionContext() + } + out, _ := extensionstest.ApplyExtensionsWithContext(ext, render.GuardianName, rc, extCtx, objs, nil) + return out +} + +var _ = Describe("Guardian enterprise rendering tests", func() { + var cfg *render.GuardianConfiguration + var g render.Component + var resources []client.Object + var deleteResources []client.Object + + createGuardianConfig := func(i operatorv1.InstallationSpec, addr string, openshift bool) *render.GuardianConfiguration { + i.Variant = operatorv1.CalicoEnterprise + secret := &corev1.Secret{ + TypeMeta: metav1.TypeMeta{Kind: "Secret", APIVersion: "v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: render.GuardianSecretName, + Namespace: common.OperatorNamespace(), + }, + Data: map[string][]byte{ + "cert": []byte("foo"), + "key": []byte("bar"), + }, + } + scheme := runtime.NewScheme() + Expect(apis.AddToScheme(scheme, false)).NotTo(HaveOccurred()) + cli := ctrlrfake.DefaultFakeClientBuilder(scheme).Build() + + certificateManager, err := certificatemanager.Create(cli, nil, clusterDomain, common.OperatorNamespace(), certificatemanager.AllowCACreation()) + Expect(err).NotTo(HaveOccurred()) + + bundle := certificateManager.CreateTrustedBundle() + + return &render.GuardianConfiguration{ + URL: addr, + PullSecrets: []*corev1.Secret{{ + TypeMeta: metav1.TypeMeta{Kind: "Secret", APIVersion: "v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "pull-secret", + Namespace: common.OperatorNamespace(), + }, + }}, + Installation: &i, + TunnelSecret: secret, + TrustedCertBundle: bundle, + OpenShift: openshift, + ManagementClusterConnection: &operatorv1.ManagementClusterConnection{}, + IncludeEgressNetworkPolicy: true, + } + } + + Context("Guardian component", func() { + renderGuardian := func(i operatorv1.InstallationSpec) { + cfg = createGuardianConfig(i, "127.0.0.1:1234", false) + g = render.Guardian(cfg) + Expect(g.ResolveImages(nil)).To(BeNil()) + resources, deleteResources = g.Objects() + // Apply the registered enterprise modifier the way the componentHandler + // does, so these enterprise tests exercise the integrated output. + rc := render.RenderContext{Installation: cfg.Installation} + var extCtx any + if p, ok := g.(render.ExtensionContextProvider); ok { + extCtx = p.ExtensionContext() + } + resources, _ = extensionstest.ApplyExtensionsWithContext(ext, render.GuardianName, rc, extCtx, resources, nil) + } + + BeforeEach(func() { + renderGuardian(operatorv1.InstallationSpec{Registry: "my-reg/"}) + }) + + It("should render all resources for a managed cluster", func() { + expectedResources := []client.Object{ + &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: render.GuardianServiceAccountName, Namespace: render.GuardianNamespace}, TypeMeta: metav1.TypeMeta{Kind: "ServiceAccount", APIVersion: "v1"}}, + &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: render.GuardianClusterRoleName}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, + &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: render.GuardianClusterRoleBindingName}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, + &rbacv1.Role{ObjectMeta: metav1.ObjectMeta{Name: render.GuardianSecretsRole, Namespace: "tigera-operator"}, TypeMeta: metav1.TypeMeta{Kind: "Role", APIVersion: "rbac.authorization.k8s.io/v1"}}, + &rbacv1.RoleBinding{ObjectMeta: metav1.ObjectMeta{Name: render.GuardianSecretsRoleBindingName, Namespace: "tigera-operator"}, TypeMeta: metav1.TypeMeta{Kind: "RoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, + &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: render.GuardianDeploymentName, Namespace: render.GuardianNamespace}, TypeMeta: metav1.TypeMeta{Kind: "Deployment", APIVersion: "apps/v1"}}, + &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: render.GuardianServiceName, Namespace: render.GuardianNamespace}, TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: ""}}, + &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: render.GuardianSecretName, Namespace: render.GuardianNamespace}, TypeMeta: metav1.TypeMeta{Kind: "Secret", APIVersion: "v1"}}, + &v3.UISettingsGroup{ObjectMeta: metav1.ObjectMeta{Name: render.ManagerClusterSettings}, TypeMeta: metav1.TypeMeta{Kind: "UISettingsGroup", APIVersion: "projectcalico.org/v3"}}, + &v3.UISettingsGroup{ObjectMeta: metav1.ObjectMeta{Name: render.ManagerUserSettings}, TypeMeta: metav1.TypeMeta{Kind: "UISettingsGroup", APIVersion: "projectcalico.org/v3"}}, + &v3.UISettings{ObjectMeta: metav1.ObjectMeta{Name: render.ManagerClusterSettingsLayerTigera}, TypeMeta: metav1.TypeMeta{Kind: "UISettings", APIVersion: "projectcalico.org/v3"}}, + &v3.UISettings{ObjectMeta: metav1.ObjectMeta{Name: render.ManagerClusterSettingsViewDefault}, TypeMeta: metav1.TypeMeta{Kind: "UISettings", APIVersion: "projectcalico.org/v3"}}, + } + + expectedDeleteResources := []client.Object{ + &corev1.Namespace{TypeMeta: metav1.TypeMeta{Kind: "Namespace", APIVersion: "v1"}, ObjectMeta: metav1.ObjectMeta{Name: "tigera-guardian"}}, + &rbacv1.ClusterRole{TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, ObjectMeta: metav1.ObjectMeta{Name: "tigera-guardian"}}, + &rbacv1.ClusterRoleBinding{TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, ObjectMeta: metav1.ObjectMeta{Name: "tigera-guardian"}}, + &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "tigera-manager"}, TypeMeta: metav1.TypeMeta{Kind: "Namespace", APIVersion: "v1"}}, + &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: "tigera-manager", Namespace: "tigera-manager"}, TypeMeta: metav1.TypeMeta{Kind: "ServiceAccount", APIVersion: "v1"}}, + &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "tigera-manager-role"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, + &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "tigera-manager-binding"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, + &netv1.NetworkPolicy{ObjectMeta: metav1.ObjectMeta{Name: "guardian", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "NetworkPolicy", APIVersion: "networking.k8s.io/v1"}}, + } + + rtest.ExpectResources(resources, expectedResources) + rtest.ExpectResources(deleteResources, expectedDeleteResources) + + deployment := rtest.GetResource(resources, render.GuardianDeploymentName, render.GuardianNamespace, "apps", "v1", "Deployment").(*appsv1.Deployment) + Expect(deployment.Spec.Template.Spec.Containers).To(HaveLen(1)) + Expect(deployment.Spec.Template.Spec.Containers[0].Image).Should(Equal("my-reg/tigera/calico:" + components.ComponentTigeraCalico.Version)) + + Expect(*deployment.Spec.Template.Spec.Containers[0].SecurityContext.AllowPrivilegeEscalation).To(BeFalse()) + Expect(*deployment.Spec.Template.Spec.Containers[0].SecurityContext.Privileged).To(BeFalse()) + Expect(*deployment.Spec.Template.Spec.Containers[0].SecurityContext.RunAsGroup).To(BeEquivalentTo(10001)) + Expect(*deployment.Spec.Template.Spec.Containers[0].SecurityContext.RunAsNonRoot).To(BeTrue()) + Expect(*deployment.Spec.Template.Spec.Containers[0].SecurityContext.RunAsUser).To(BeEquivalentTo(10001)) + Expect(deployment.Spec.Template.Spec.Containers[0].SecurityContext.SeccompProfile).To(Equal( + &corev1.SeccompProfile{ + Type: corev1.SeccompProfileTypeRuntimeDefault, + })) + Expect(deployment.Spec.Template.Spec.Containers[0].SecurityContext.Capabilities).To(Equal( + &corev1.Capabilities{ + Drop: []corev1.Capability{"ALL"}, + }, + )) + }) + + It("should render controlPlaneTolerations", func() { + t := corev1.Toleration{ + Key: "foo", + Operator: corev1.TolerationOpEqual, + Value: "bar", + } + renderGuardian(operatorv1.InstallationSpec{ + ControlPlaneTolerations: []corev1.Toleration{t}, + }) + deployment := rtest.GetResource(resources, render.GuardianDeploymentName, render.GuardianNamespace, "apps", "v1", "Deployment").(*appsv1.Deployment) + Expect(deployment.Spec.Template.Spec.Tolerations).Should(ContainElements(append(rmeta.TolerateCriticalAddonsAndControlPlane, t))) + }) + + It("should render toleration on GKE", func() { + renderGuardian(operatorv1.InstallationSpec{ + KubernetesProvider: operatorv1.ProviderGKE, + }) + deployment := rtest.GetResource(resources, render.GuardianDeploymentName, render.GuardianNamespace, "apps", "v1", "Deployment").(*appsv1.Deployment) + Expect(deployment).NotTo(BeNil()) + Expect(deployment.Spec.Template.Spec.Tolerations).To(ContainElements(corev1.Toleration{ + Key: "kubernetes.io/arch", + Operator: corev1.TolerationOpEqual, + Value: "arm64", + Effect: corev1.TaintEffectNoSchedule, + })) + }) + + It("should render guardian with unlimited impersonation", func() { + cfg.ManagementClusterConnection = &operatorv1.ManagementClusterConnection{ + Spec: operatorv1.ManagementClusterConnectionSpec{ + Impersonation: &operatorv1.Impersonation{ + Users: []string{}, + Groups: []string{}, + ServiceAccounts: []string{}, + }, + }, + } + + resources := guardianObjects(cfg) + Expect(resources).ToNot(BeNil()) + + clusterRole, ok := rtest.GetResource(resources, render.GuardianClusterRoleName, "", "rbac.authorization.k8s.io", "v1", "ClusterRole").(*rbacv1.ClusterRole) + Expect(ok).To(BeTrue()) + + foundUserImp, foundGroupImp, foundSaImp := false, false, false + for _, rule := range clusterRole.Rules { + if rule.Verbs[0] == "impersonate" { + if rule.Resources[0] == "users" { + Expect(rule.ResourceNames).To(Equal([]string{})) + foundUserImp = true + } + if rule.Resources[0] == "groups" { + Expect(rule.ResourceNames).To(Equal([]string{})) + foundGroupImp = true + } + if rule.Resources[0] == "serviceaccounts" { + Expect(rule.ResourceNames).To(Equal([]string{})) + foundSaImp = true + } + } + } + + Expect(foundUserImp).To(BeTrue()) + Expect(foundGroupImp).To(BeTrue()) + Expect(foundSaImp).To(BeTrue()) + }) + + It("should render guardian with specific impersonation", func() { + cfg.ManagementClusterConnection = &operatorv1.ManagementClusterConnection{ + Spec: operatorv1.ManagementClusterConnectionSpec{ + Impersonation: &operatorv1.Impersonation{ + Users: []string{"foo"}, + Groups: []string{"bar"}, + ServiceAccounts: []string{"zaz"}, + }, + }, + } + + resources := guardianObjects(cfg) + Expect(resources).ToNot(BeNil()) + + clusterRole, ok := rtest.GetResource(resources, render.GuardianClusterRoleName, "", "rbac.authorization.k8s.io", "v1", "ClusterRole").(*rbacv1.ClusterRole) + Expect(ok).To(BeTrue()) + + foundUserImp, foundGroupImp, foundSaImp := false, false, false + for _, rule := range clusterRole.Rules { + if rule.Verbs[0] == "impersonate" { + if rule.Resources[0] == "users" { + Expect(rule.ResourceNames).To(Equal([]string{"foo"})) + foundUserImp = true + } + if rule.Resources[0] == "groups" { + Expect(rule.ResourceNames).To(Equal([]string{"bar"})) + foundGroupImp = true + } + if rule.Resources[0] == "serviceaccounts" { + Expect(rule.ResourceNames).To(Equal([]string{"zaz"})) + foundSaImp = true + } + } + } + + Expect(foundUserImp).To(BeTrue()) + Expect(foundGroupImp).To(BeTrue()) + Expect(foundSaImp).To(BeTrue()) + }) + + It("should render guardian with specific no sa permissions but with user and group", func() { + cfg.ManagementClusterConnection = &operatorv1.ManagementClusterConnection{ + Spec: operatorv1.ManagementClusterConnectionSpec{ + Impersonation: &operatorv1.Impersonation{ + Users: []string{}, + Groups: []string{}, + }, + }, + } + + resources := guardianObjects(cfg) + Expect(resources).ToNot(BeNil()) + + clusterRole, ok := rtest.GetResource(resources, render.GuardianClusterRoleName, "", "rbac.authorization.k8s.io", "v1", "ClusterRole").(*rbacv1.ClusterRole) + Expect(ok).To(BeTrue()) + + foundUserImp, foundGroupImp, foundSaImp := false, false, false + for _, rule := range clusterRole.Rules { + if rule.Verbs[0] == "impersonate" { + if rule.Resources[0] == "users" { + foundUserImp = true + } + if rule.Resources[0] == "groups" { + foundGroupImp = true + } + if rule.Resources[0] == "serviceaccounts" { + foundSaImp = true + } + } + } + + Expect(foundUserImp).To(BeTrue()) + Expect(foundGroupImp).To(BeTrue()) + Expect(foundSaImp).To(BeFalse()) + }) + }) + + It("should render SecurityContextConstrains properly when provider is OpenShift", func() { + cfg = createGuardianConfig(operatorv1.InstallationSpec{Registry: "my-reg/"}, "127.0.0.1:1234", false) + cfg.Installation.KubernetesProvider = operatorv1.ProviderOpenShift + cfg.OpenShift = true + resources := guardianObjects(cfg) + + role := rtest.GetResource(resources, render.GuardianClusterRoleName, "", "rbac.authorization.k8s.io", "v1", "ClusterRole").(*rbacv1.ClusterRole) + Expect(role.Rules).To(ContainElement(rbacv1.PolicyRule{ + APIGroups: []string{"security.openshift.io"}, + Resources: []string{"securitycontextconstraints"}, + Verbs: []string{"use"}, + ResourceNames: []string{"nonroot-v2"}, + })) + }) + + Context("GuardianPolicy component", func() { + guardianPolicy := testutils.GetExpectedPolicyFromFile(guardianPolicyJSON) + guardianPolicyForOCP := testutils.GetExpectedPolicyFromFile(guardianPolicyOCPJSON) + + renderGuardianPolicy := func(addr string, openshift bool, variant operatorv1.ProductVariant, includeEgressNetworkPolicy bool) { + installation := operatorv1.InstallationSpec{ + Registry: "my-reg/", + } + cfg := createGuardianConfig(installation, addr, openshift) + cfg.Installation.Variant = variant + cfg.IncludeEgressNetworkPolicy = includeEgressNetworkPolicy + g, err := render.GuardianPolicy(cfg) + Expect(err).NotTo(HaveOccurred()) + objs, _ := g.Objects() + // Apply the registered enterprise modifier the way the componentHandler + // does, so the enterprise policy is exercised. + rc := render.RenderContext{Installation: cfg.Installation} + var extCtx any + if p, ok := g.(render.ExtensionContextProvider); ok { + extCtx = p.ExtensionContext() + } + resources, _ = extensionstest.ApplyExtensionsWithContext(ext, render.ComponentNameGuardianPolicy, rc, extCtx, objs, nil) + } + + Context("policy rendering based on variant and IncludeEgressNetworkPolicy", func() { + It("should render Enterprise network policy without domain-based egress when IncludeEgressNetworkPolicy is false", func() { + // Enterprise variant with IncludeEgressNetworkPolicy=false should render a policy but skip domain-based egress rules + renderGuardianPolicy("my-management.example.com:1234", false, operatorv1.CalicoEnterprise, false) + + policyName := types.NamespacedName{Name: "calico-system.guardian-access", Namespace: "calico-system"} + policy := testutils.GetCalicoSystemPolicyFromResources(policyName, resources) + Expect(policy).NotTo(BeNil(), "Enterprise variant should always render a network policy when tier exists") + + // Verify it's the Enterprise policy (should have both Ingress and Egress types) + Expect(policy.Spec.Types).To(ConsistOf(v3.PolicyTypeIngress, v3.PolicyTypeEgress)) + Expect(policy.Spec.Egress).NotTo(BeEmpty()) + + // Verify no domain-based egress rules are present + for _, rule := range policy.Spec.Egress { + Expect(rule.Destination.Domains).To(BeEmpty(), + "Domain-based egress rules should not be present when IncludeEgressNetworkPolicy is false") + } + }) + + It("should render Enterprise network policy with domain-based egress when IncludeEgressNetworkPolicy is true", func() { + // Enterprise variant with IncludeEgressNetworkPolicy=true should render the full policy including domain-based egress + renderGuardianPolicy("my-management.example.com:1234", false, operatorv1.CalicoEnterprise, true) + + policyName := types.NamespacedName{Name: "calico-system.guardian-access", Namespace: "calico-system"} + policy := testutils.GetCalicoSystemPolicyFromResources(policyName, resources) + Expect(policy).NotTo(BeNil(), "Enterprise variant with IncludeEgressNetworkPolicy=true should render a network policy") + + // Verify it's the Enterprise policy (should have both Ingress and Egress types) + Expect(policy.Spec.Types).To(ConsistOf(v3.PolicyTypeIngress, v3.PolicyTypeEgress)) + Expect(policy.Spec.Egress).NotTo(BeEmpty()) + + // Verify domain-based egress rule is present + hasDomainRule := false + for _, rule := range policy.Spec.Egress { + if len(rule.Destination.Domains) > 0 { + hasDomainRule = true + break + } + } + Expect(hasDomainRule).To(BeTrue(), "Domain-based egress rule should be present when IncludeEgressNetworkPolicy is true") + }) + }) + + Context("calico-system rendering", func() { + policyName := types.NamespacedName{Name: "calico-system.guardian-access", Namespace: "calico-system"} + + getExpectedPolicy := func(name types.NamespacedName, scenario testutils.CalicoSystemScenario) *v3.NetworkPolicy { + if name.Name == "calico-system.guardian-access" && scenario.ManagedCluster { + return testutils.SelectPolicyByProvider(scenario, guardianPolicy, guardianPolicyForOCP) + } + + return nil + } + + DescribeTable("should render calico-system policy", + func(scenario testutils.CalicoSystemScenario) { + renderGuardianPolicy("127.0.0.1:1234", scenario.OpenShift, operatorv1.CalicoEnterprise, true) + policy := testutils.GetCalicoSystemPolicyFromResources(policyName, resources) + expectedPolicy := getExpectedPolicy(policyName, scenario) + Expect(policy).To(Equal(expectedPolicy)) + }, + Entry("for managed, kube-dns", testutils.CalicoSystemScenario{ManagedCluster: true, OpenShift: false}), + Entry("for managed, openshift-dns", testutils.CalicoSystemScenario{ManagedCluster: true, OpenShift: true}), + ) + + // The test matrix above validates against an IP-based management cluster address. + // Validate policy adaptation for domain-based management cluster address here. + It("should adapt Guardian policy if ManagementClusterAddr is domain-based", func() { + renderGuardianPolicy("mydomain.io:8080", false, operatorv1.CalicoEnterprise, true) + policy := testutils.GetCalicoSystemPolicyFromResources(policyName, resources) + managementClusterEgressRule := policy.Spec.Egress[5] + Expect(managementClusterEgressRule.Destination.Domains).To(Equal([]string{"mydomain.io"})) + Expect(managementClusterEgressRule.Destination.Ports).To(Equal(networkpolicy.Ports(8080))) + }) + }) + }) +}) diff --git a/pkg/render/guardian_test.go b/pkg/render/guardian_test.go index f54664809e..c99c1e485f 100644 --- a/pkg/render/guardian_test.go +++ b/pkg/render/guardian_test.go @@ -20,8 +20,6 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" - netv1 "k8s.io/api/networking/v1" - rbacv1 "k8s.io/api/rbac/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -33,426 +31,77 @@ import ( operatorv1 "github.com/tigera/operator/api/v1" "github.com/tigera/operator/pkg/apis" "github.com/tigera/operator/pkg/common" - "github.com/tigera/operator/pkg/components" "github.com/tigera/operator/pkg/controller/certificatemanager" ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" - "github.com/tigera/operator/pkg/extensions/extensionstest" "github.com/tigera/operator/pkg/render" - rmeta "github.com/tigera/operator/pkg/render/common/meta" - "github.com/tigera/operator/pkg/render/common/networkpolicy" rtest "github.com/tigera/operator/pkg/render/common/test" "github.com/tigera/operator/pkg/render/testutils" ) -// guardianObjects renders the guardian component and applies the registered -// enterprise modifier the way the componentHandler does. +// guardianObjects renders the base guardian component. The enterprise modifier is +// exercised in the pkg/enterprise/guardian tests; these tests cover the OSS render +// path, which never runs the modifier. func guardianObjects(cfg *render.GuardianConfiguration) []client.Object { g := render.Guardian(cfg) ExpectWithOffset(1, g.ResolveImages(nil)).To(BeNil()) objs, _ := g.Objects() - rc := render.RenderContext{Installation: cfg.Installation} - var extCtx any - if p, ok := g.(render.ExtensionContextProvider); ok { - extCtx = p.ExtensionContext() - } - out, _ := extensionstest.ApplyExtensionsWithContext(ext, render.GuardianName, rc, extCtx, objs, nil) - return out + return objs } -var _ = Describe("Rendering tests", func() { - var cfg *render.GuardianConfiguration - var g render.Component - var resources []client.Object - var deleteResources []client.Object - - createGuardianConfig := func(i operatorv1.InstallationSpec, addr string, openshift bool) *render.GuardianConfiguration { - i.Variant = operatorv1.CalicoEnterprise - secret := &corev1.Secret{ - TypeMeta: metav1.TypeMeta{Kind: "Secret", APIVersion: "v1"}, - ObjectMeta: metav1.ObjectMeta{ - Name: render.GuardianSecretName, - Namespace: common.OperatorNamespace(), - }, - Data: map[string][]byte{ - "cert": []byte("foo"), - "key": []byte("bar"), - }, - } - scheme := runtime.NewScheme() - Expect(apis.AddToScheme(scheme, false)).NotTo(HaveOccurred()) - cli := ctrlrfake.DefaultFakeClientBuilder(scheme).Build() - - certificateManager, err := certificatemanager.Create(cli, nil, clusterDomain, common.OperatorNamespace(), certificatemanager.AllowCACreation()) - Expect(err).NotTo(HaveOccurred()) - - bundle := certificateManager.CreateTrustedBundle() - - return &render.GuardianConfiguration{ - URL: addr, - PullSecrets: []*corev1.Secret{{ - TypeMeta: metav1.TypeMeta{Kind: "Secret", APIVersion: "v1"}, - ObjectMeta: metav1.ObjectMeta{ - Name: "pull-secret", - Namespace: common.OperatorNamespace(), - }, - }}, - Installation: &i, - TunnelSecret: secret, - TrustedCertBundle: bundle, - OpenShift: openshift, - ManagementClusterConnection: &operatorv1.ManagementClusterConnection{}, - IncludeEgressNetworkPolicy: true, - } +func newGuardianConfig(addr string) *render.GuardianConfiguration { + secret := &corev1.Secret{ + TypeMeta: metav1.TypeMeta{Kind: "Secret", APIVersion: "v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: render.GuardianSecretName, + Namespace: common.OperatorNamespace(), + }, + Data: map[string][]byte{ + "cert": []byte("foo"), + "key": []byte("bar"), + }, } + scheme := runtime.NewScheme() + Expect(apis.AddToScheme(scheme, false)).NotTo(HaveOccurred()) + cli := ctrlrfake.DefaultFakeClientBuilder(scheme).Build() + + certificateManager, err := certificatemanager.Create(cli, nil, clusterDomain, common.OperatorNamespace(), certificatemanager.AllowCACreation()) + Expect(err).NotTo(HaveOccurred()) + + return &render.GuardianConfiguration{ + URL: addr, + Installation: &operatorv1.InstallationSpec{Registry: "my-reg/"}, + TunnelSecret: secret, + TrustedCertBundle: certificateManager.CreateTrustedBundle(), + ManagementClusterConnection: &operatorv1.ManagementClusterConnection{}, + IncludeEgressNetworkPolicy: true, + } +} - Context("Guardian component", func() { - renderGuardian := func(i operatorv1.InstallationSpec) { - cfg = createGuardianConfig(i, "127.0.0.1:1234", false) - g = render.Guardian(cfg) - Expect(g.ResolveImages(nil)).To(BeNil()) - resources, deleteResources = g.Objects() - // Apply the registered enterprise modifier the way the componentHandler - // does, so these enterprise tests exercise the integrated output. - rc := render.RenderContext{Installation: cfg.Installation} - var extCtx any - if p, ok := g.(render.ExtensionContextProvider); ok { - extCtx = p.ExtensionContext() - } - resources, _ = extensionstest.ApplyExtensionsWithContext(ext, render.GuardianName, rc, extCtx, resources, nil) - } - - BeforeEach(func() { - renderGuardian(operatorv1.InstallationSpec{Registry: "my-reg/"}) - }) - - It("should render all resources for a managed cluster", func() { - expectedResources := []client.Object{ - &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: render.GuardianServiceAccountName, Namespace: render.GuardianNamespace}, TypeMeta: metav1.TypeMeta{Kind: "ServiceAccount", APIVersion: "v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: render.GuardianClusterRoleName}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: render.GuardianClusterRoleBindingName}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.Role{ObjectMeta: metav1.ObjectMeta{Name: render.GuardianSecretsRole, Namespace: "tigera-operator"}, TypeMeta: metav1.TypeMeta{Kind: "Role", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.RoleBinding{ObjectMeta: metav1.ObjectMeta{Name: render.GuardianSecretsRoleBindingName, Namespace: "tigera-operator"}, TypeMeta: metav1.TypeMeta{Kind: "RoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: render.GuardianDeploymentName, Namespace: render.GuardianNamespace}, TypeMeta: metav1.TypeMeta{Kind: "Deployment", APIVersion: "apps/v1"}}, - &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: render.GuardianServiceName, Namespace: render.GuardianNamespace}, TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: ""}}, - &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: render.GuardianSecretName, Namespace: render.GuardianNamespace}, TypeMeta: metav1.TypeMeta{Kind: "Secret", APIVersion: "v1"}}, - &v3.UISettingsGroup{ObjectMeta: metav1.ObjectMeta{Name: render.ManagerClusterSettings}, TypeMeta: metav1.TypeMeta{Kind: "UISettingsGroup", APIVersion: "projectcalico.org/v3"}}, - &v3.UISettingsGroup{ObjectMeta: metav1.ObjectMeta{Name: render.ManagerUserSettings}, TypeMeta: metav1.TypeMeta{Kind: "UISettingsGroup", APIVersion: "projectcalico.org/v3"}}, - &v3.UISettings{ObjectMeta: metav1.ObjectMeta{Name: render.ManagerClusterSettingsLayerTigera}, TypeMeta: metav1.TypeMeta{Kind: "UISettings", APIVersion: "projectcalico.org/v3"}}, - &v3.UISettings{ObjectMeta: metav1.ObjectMeta{Name: render.ManagerClusterSettingsViewDefault}, TypeMeta: metav1.TypeMeta{Kind: "UISettings", APIVersion: "projectcalico.org/v3"}}, - } - - expectedDeleteResources := []client.Object{ - &corev1.Namespace{TypeMeta: metav1.TypeMeta{Kind: "Namespace", APIVersion: "v1"}, ObjectMeta: metav1.ObjectMeta{Name: "tigera-guardian"}}, - &rbacv1.ClusterRole{TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, ObjectMeta: metav1.ObjectMeta{Name: "tigera-guardian"}}, - &rbacv1.ClusterRoleBinding{TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, ObjectMeta: metav1.ObjectMeta{Name: "tigera-guardian"}}, - &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "tigera-manager"}, TypeMeta: metav1.TypeMeta{Kind: "Namespace", APIVersion: "v1"}}, - &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: "tigera-manager", Namespace: "tigera-manager"}, TypeMeta: metav1.TypeMeta{Kind: "ServiceAccount", APIVersion: "v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "tigera-manager-role"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "tigera-manager-binding"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &netv1.NetworkPolicy{ObjectMeta: metav1.ObjectMeta{Name: "guardian", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "NetworkPolicy", APIVersion: "networking.k8s.io/v1"}}, - } - - rtest.ExpectResources(resources, expectedResources) - rtest.ExpectResources(deleteResources, expectedDeleteResources) - - deployment := rtest.GetResource(resources, render.GuardianDeploymentName, render.GuardianNamespace, "apps", "v1", "Deployment").(*appsv1.Deployment) - Expect(deployment.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(deployment.Spec.Template.Spec.Containers[0].Image).Should(Equal("my-reg/tigera/calico:" + components.ComponentTigeraCalico.Version)) - - Expect(*deployment.Spec.Template.Spec.Containers[0].SecurityContext.AllowPrivilegeEscalation).To(BeFalse()) - Expect(*deployment.Spec.Template.Spec.Containers[0].SecurityContext.Privileged).To(BeFalse()) - Expect(*deployment.Spec.Template.Spec.Containers[0].SecurityContext.RunAsGroup).To(BeEquivalentTo(10001)) - Expect(*deployment.Spec.Template.Spec.Containers[0].SecurityContext.RunAsNonRoot).To(BeTrue()) - Expect(*deployment.Spec.Template.Spec.Containers[0].SecurityContext.RunAsUser).To(BeEquivalentTo(10001)) - Expect(deployment.Spec.Template.Spec.Containers[0].SecurityContext.SeccompProfile).To(Equal( - &corev1.SeccompProfile{ - Type: corev1.SeccompProfileTypeRuntimeDefault, - })) - Expect(deployment.Spec.Template.Spec.Containers[0].SecurityContext.Capabilities).To(Equal( - &corev1.Capabilities{ - Drop: []corev1.Capability{"ALL"}, - }, - )) - }) - - It("should render controlPlaneTolerations", func() { - t := corev1.Toleration{ - Key: "foo", - Operator: corev1.TolerationOpEqual, - Value: "bar", - } - renderGuardian(operatorv1.InstallationSpec{ - ControlPlaneTolerations: []corev1.Toleration{t}, - }) - deployment := rtest.GetResource(resources, render.GuardianDeploymentName, render.GuardianNamespace, "apps", "v1", "Deployment").(*appsv1.Deployment) - Expect(deployment.Spec.Template.Spec.Tolerations).Should(ContainElements(append(rmeta.TolerateCriticalAddonsAndControlPlane, t))) - }) - - It("should render toleration on GKE", func() { - renderGuardian(operatorv1.InstallationSpec{ - KubernetesProvider: operatorv1.ProviderGKE, - }) - deployment := rtest.GetResource(resources, render.GuardianDeploymentName, render.GuardianNamespace, "apps", "v1", "Deployment").(*appsv1.Deployment) - Expect(deployment).NotTo(BeNil()) - Expect(deployment.Spec.Template.Spec.Tolerations).To(ContainElements(corev1.Toleration{ - Key: "kubernetes.io/arch", - Operator: corev1.TolerationOpEqual, - Value: "arm64", - Effect: corev1.TaintEffectNoSchedule, - })) - }) - - It("should render guardian with unlimited impersonation", func() { - cfg.ManagementClusterConnection = &operatorv1.ManagementClusterConnection{ - Spec: operatorv1.ManagementClusterConnectionSpec{ - Impersonation: &operatorv1.Impersonation{ - Users: []string{}, - Groups: []string{}, - ServiceAccounts: []string{}, - }, - }, - } - - resources := guardianObjects(cfg) - Expect(resources).ToNot(BeNil()) - - clusterRole, ok := rtest.GetResource(resources, render.GuardianClusterRoleName, "", "rbac.authorization.k8s.io", "v1", "ClusterRole").(*rbacv1.ClusterRole) - Expect(ok).To(BeTrue()) - - foundUserImp, foundGroupImp, foundSaImp := false, false, false - for _, rule := range clusterRole.Rules { - if rule.Verbs[0] == "impersonate" { - if rule.Resources[0] == "users" { - Expect(rule.ResourceNames).To(Equal([]string{})) - foundUserImp = true - } - if rule.Resources[0] == "groups" { - Expect(rule.ResourceNames).To(Equal([]string{})) - foundGroupImp = true - } - if rule.Resources[0] == "serviceaccounts" { - Expect(rule.ResourceNames).To(Equal([]string{})) - foundSaImp = true - } - } - } - - Expect(foundUserImp).To(BeTrue()) - Expect(foundGroupImp).To(BeTrue()) - Expect(foundSaImp).To(BeTrue()) - }) - - It("should render guardian with specific impersonation", func() { - cfg.ManagementClusterConnection = &operatorv1.ManagementClusterConnection{ - Spec: operatorv1.ManagementClusterConnectionSpec{ - Impersonation: &operatorv1.Impersonation{ - Users: []string{"foo"}, - Groups: []string{"bar"}, - ServiceAccounts: []string{"zaz"}, - }, - }, - } - - resources := guardianObjects(cfg) - Expect(resources).ToNot(BeNil()) - - clusterRole, ok := rtest.GetResource(resources, render.GuardianClusterRoleName, "", "rbac.authorization.k8s.io", "v1", "ClusterRole").(*rbacv1.ClusterRole) - Expect(ok).To(BeTrue()) - - foundUserImp, foundGroupImp, foundSaImp := false, false, false - for _, rule := range clusterRole.Rules { - if rule.Verbs[0] == "impersonate" { - if rule.Resources[0] == "users" { - Expect(rule.ResourceNames).To(Equal([]string{"foo"})) - foundUserImp = true - } - if rule.Resources[0] == "groups" { - Expect(rule.ResourceNames).To(Equal([]string{"bar"})) - foundGroupImp = true - } - if rule.Resources[0] == "serviceaccounts" { - Expect(rule.ResourceNames).To(Equal([]string{"zaz"})) - foundSaImp = true - } - } - } - - Expect(foundUserImp).To(BeTrue()) - Expect(foundGroupImp).To(BeTrue()) - Expect(foundSaImp).To(BeTrue()) - }) - - It("should render guardian with specific no sa permissions but with user and group", func() { - cfg.ManagementClusterConnection = &operatorv1.ManagementClusterConnection{ - Spec: operatorv1.ManagementClusterConnectionSpec{ - Impersonation: &operatorv1.Impersonation{ - Users: []string{}, - Groups: []string{}, - }, - }, - } - - resources := guardianObjects(cfg) - Expect(resources).ToNot(BeNil()) - - clusterRole, ok := rtest.GetResource(resources, render.GuardianClusterRoleName, "", "rbac.authorization.k8s.io", "v1", "ClusterRole").(*rbacv1.ClusterRole) - Expect(ok).To(BeTrue()) - - foundUserImp, foundGroupImp, foundSaImp := false, false, false - for _, rule := range clusterRole.Rules { - if rule.Verbs[0] == "impersonate" { - if rule.Resources[0] == "users" { - foundUserImp = true - } - if rule.Resources[0] == "groups" { - foundGroupImp = true - } - if rule.Resources[0] == "serviceaccounts" { - foundSaImp = true - } - } - } - - Expect(foundUserImp).To(BeTrue()) - Expect(foundGroupImp).To(BeTrue()) - Expect(foundSaImp).To(BeFalse()) - }) - }) - - It("should render SecurityContextConstrains properly when provider is OpenShift", func() { - cfg.Installation.KubernetesProvider = operatorv1.ProviderOpenShift - cfg.OpenShift = true - resources := guardianObjects(cfg) - - role := rtest.GetResource(resources, render.GuardianClusterRoleName, "", "rbac.authorization.k8s.io", "v1", "ClusterRole").(*rbacv1.ClusterRole) - Expect(role.Rules).To(ContainElement(rbacv1.PolicyRule{ - APIGroups: []string{"security.openshift.io"}, - Resources: []string{"securitycontextconstraints"}, - Verbs: []string{"use"}, - ResourceNames: []string{"nonroot-v2"}, - })) - }) - +var _ = Describe("Guardian OSS rendering tests", func() { Context("GuardianPolicy component", func() { - guardianPolicy := testutils.GetExpectedPolicyFromFile("./testutils/expected_policies/guardian.json") - guardianPolicyForOCP := testutils.GetExpectedPolicyFromFile("./testutils/expected_policies/guardian_ocp.json") - - renderGuardianPolicy := func(addr string, openshift bool, variant operatorv1.ProductVariant, includeEgressNetworkPolicy bool) { - installation := operatorv1.InstallationSpec{ - Registry: "my-reg/", - } - cfg := createGuardianConfig(installation, addr, openshift) - cfg.Installation.Variant = variant - cfg.IncludeEgressNetworkPolicy = includeEgressNetworkPolicy + It("should render OSS network policy regardless of IncludeEgressNetworkPolicy flag", func() { + // OSS variant should always render a network policy, even when IncludeEgressNetworkPolicy is false + cfg := newGuardianConfig("127.0.0.1:1234") + cfg.Installation.Variant = operatorv1.Calico + cfg.IncludeEgressNetworkPolicy = false g, err := render.GuardianPolicy(cfg) Expect(err).NotTo(HaveOccurred()) - objs, _ := g.Objects() - // Apply the registered enterprise modifier the way the componentHandler - // does, so the enterprise policy is exercised. For the Calico variant the - // modifier is a no-op and the OSS policy is returned. - rc := render.RenderContext{Installation: cfg.Installation} - var extCtx any - if p, ok := g.(render.ExtensionContextProvider); ok { - extCtx = p.ExtensionContext() - } - resources, _ = extensionstest.ApplyExtensionsWithContext(ext, render.ComponentNameGuardianPolicy, rc, extCtx, objs, nil) - } - - Context("policy rendering based on variant and IncludeEgressNetworkPolicy", func() { - It("should render OSS network policy regardless of IncludeEgressNetworkPolicy flag", func() { - // OSS variant should always render a network policy, even when IncludeEgressNetworkPolicy is false - renderGuardianPolicy("127.0.0.1:1234", false, operatorv1.Calico, false) - - policyName := types.NamespacedName{Name: "calico-system.guardian-access", Namespace: "calico-system"} - policy := testutils.GetCalicoSystemPolicyFromResources(policyName, resources) - Expect(policy).NotTo(BeNil(), "OSS variant should always render a network policy") - - // The OSS policy has both Ingress and Egress, ending in a Pass so the - // tunnel to the management cluster isn't dropped by the default-deny. - Expect(policy.Spec.Types).To(ConsistOf(v3.PolicyTypeIngress, v3.PolicyTypeEgress)) - Expect(policy.Spec.Egress).NotTo(BeEmpty()) - Expect(policy.Spec.Egress[len(policy.Spec.Egress)-1].Action).To(Equal(v3.Pass)) - - // OSS can't express domain-based egress rules. - for _, rule := range policy.Spec.Egress { - Expect(rule.Destination.Domains).To(BeEmpty()) - } - }) + resources, _ := g.Objects() - It("should render Enterprise network policy without domain-based egress when IncludeEgressNetworkPolicy is false", func() { - // Enterprise variant with IncludeEgressNetworkPolicy=false should render a policy but skip domain-based egress rules - renderGuardianPolicy("my-management.example.com:1234", false, operatorv1.CalicoEnterprise, false) - - policyName := types.NamespacedName{Name: "calico-system.guardian-access", Namespace: "calico-system"} - policy := testutils.GetCalicoSystemPolicyFromResources(policyName, resources) - Expect(policy).NotTo(BeNil(), "Enterprise variant should always render a network policy when tier exists") - - // Verify it's the Enterprise policy (should have both Ingress and Egress types) - Expect(policy.Spec.Types).To(ConsistOf(v3.PolicyTypeIngress, v3.PolicyTypeEgress)) - Expect(policy.Spec.Egress).NotTo(BeEmpty()) - - // Verify no domain-based egress rules are present - for _, rule := range policy.Spec.Egress { - Expect(rule.Destination.Domains).To(BeEmpty(), - "Domain-based egress rules should not be present when IncludeEgressNetworkPolicy is false") - } - }) - - It("should render Enterprise network policy with domain-based egress when IncludeEgressNetworkPolicy is true", func() { - // Enterprise variant with IncludeEgressNetworkPolicy=true should render the full policy including domain-based egress - renderGuardianPolicy("my-management.example.com:1234", false, operatorv1.CalicoEnterprise, true) - - policyName := types.NamespacedName{Name: "calico-system.guardian-access", Namespace: "calico-system"} - policy := testutils.GetCalicoSystemPolicyFromResources(policyName, resources) - Expect(policy).NotTo(BeNil(), "Enterprise variant with IncludeEgressNetworkPolicy=true should render a network policy") - - // Verify it's the Enterprise policy (should have both Ingress and Egress types) - Expect(policy.Spec.Types).To(ConsistOf(v3.PolicyTypeIngress, v3.PolicyTypeEgress)) - Expect(policy.Spec.Egress).NotTo(BeEmpty()) - - // Verify domain-based egress rule is present - hasDomainRule := false - for _, rule := range policy.Spec.Egress { - if len(rule.Destination.Domains) > 0 { - hasDomainRule = true - break - } - } - Expect(hasDomainRule).To(BeTrue(), "Domain-based egress rule should be present when IncludeEgressNetworkPolicy is true") - }) - }) - - Context("calico-system rendering", func() { policyName := types.NamespacedName{Name: "calico-system.guardian-access", Namespace: "calico-system"} + policy := testutils.GetCalicoSystemPolicyFromResources(policyName, resources) + Expect(policy).NotTo(BeNil(), "OSS variant should always render a network policy") - getExpectedPolicy := func(name types.NamespacedName, scenario testutils.CalicoSystemScenario) *v3.NetworkPolicy { - if name.Name == "calico-system.guardian-access" && scenario.ManagedCluster { - return testutils.SelectPolicyByProvider(scenario, guardianPolicy, guardianPolicyForOCP) - } + // The OSS policy has both Ingress and Egress, ending in a Pass so the + // tunnel to the management cluster isn't dropped by the default-deny. + Expect(policy.Spec.Types).To(ConsistOf(v3.PolicyTypeIngress, v3.PolicyTypeEgress)) + Expect(policy.Spec.Egress).NotTo(BeEmpty()) + Expect(policy.Spec.Egress[len(policy.Spec.Egress)-1].Action).To(Equal(v3.Pass)) - return nil + // OSS can't express domain-based egress rules. + for _, rule := range policy.Spec.Egress { + Expect(rule.Destination.Domains).To(BeEmpty()) } - - DescribeTable("should render calico-system policy", - func(scenario testutils.CalicoSystemScenario) { - renderGuardianPolicy("127.0.0.1:1234", scenario.OpenShift, operatorv1.CalicoEnterprise, true) - policy := testutils.GetCalicoSystemPolicyFromResources(policyName, resources) - expectedPolicy := getExpectedPolicy(policyName, scenario) - Expect(policy).To(Equal(expectedPolicy)) - }, - Entry("for managed, kube-dns", testutils.CalicoSystemScenario{ManagedCluster: true, OpenShift: false}), - Entry("for managed, openshift-dns", testutils.CalicoSystemScenario{ManagedCluster: true, OpenShift: true}), - ) - - // The test matrix above validates against an IP-based management cluster address. - // Validate policy adaptation for domain-based management cluster address here. - It("should adapt Guardian policy if ManagementClusterAddr is domain-based", func() { - renderGuardianPolicy("mydomain.io:8080", false, operatorv1.CalicoEnterprise, true) - policy := testutils.GetCalicoSystemPolicyFromResources(policyName, resources) - managementClusterEgressRule := policy.Spec.Egress[5] - Expect(managementClusterEgressRule.Destination.Domains).To(Equal([]string{"mydomain.io"})) - Expect(managementClusterEgressRule.Destination.Ports).To(Equal(networkpolicy.Ports(8080))) - }) }) }) }) From 808d4237c75ce301e5e68be88999f59844d3e4ea Mon Sep 17 00:00:00 2001 From: Casey Davenport Date: Wed, 24 Jun 2026 14:09:35 -0700 Subject: [PATCH 46/54] Move windows enterprise render tests into pkg/enterprise/windows The CalicoEnterprise windows cases ran base render plus the real modifier (and needed the enterprise image overrides), so they pulled enterprise.New() into the render test package. Move them next to the modifier in pkg/enterprise/windows. The OSS render tests stay put, now built with an empty imageoverride.New() instead of the enterprise Set's overrides, so render_test no longer depends on pkg/enterprise for windows. --- pkg/enterprise/windows/windows_render_test.go | 937 +++++++++++ pkg/render/windows_test.go | 1450 ++++------------- 2 files changed, 1287 insertions(+), 1100 deletions(-) create mode 100644 pkg/enterprise/windows/windows_render_test.go diff --git a/pkg/enterprise/windows/windows_render_test.go b/pkg/enterprise/windows/windows_render_test.go new file mode 100644 index 0000000000..98b1cdd6b4 --- /dev/null +++ b/pkg/enterprise/windows/windows_render_test.go @@ -0,0 +1,937 @@ +// Copyright (c) 2021-2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package windows_test + +import ( + "fmt" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/onsi/gomega/gstruct" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/intstr" + "sigs.k8s.io/controller-runtime/pkg/client" + + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/apis" + "github.com/tigera/operator/pkg/common" + "github.com/tigera/operator/pkg/components" + "github.com/tigera/operator/pkg/controller/certificatemanager" + "github.com/tigera/operator/pkg/controller/k8sapi" + ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" + "github.com/tigera/operator/pkg/extensions/extensionstest" + "github.com/tigera/operator/pkg/render" + rmeta "github.com/tigera/operator/pkg/render/common/meta" + rtest "github.com/tigera/operator/pkg/render/common/test" + "github.com/tigera/operator/pkg/tls/certificatemanagement" +) + +var ( + bgpEnabled = operatorv1.BGPEnabled + bgpDisabled = operatorv1.BGPDisabled + logSeverity = operatorv1.LogLevelDebug + logFileMaxAgeDays = uint32(5) + logFileMaxCount = uint32(5) + logFileMaxSize = resource.MustParse("1Mi") +) + +// renderWindows renders the windows component and applies the registered +// enterprise modifier the way the componentHandler does, so enterprise tests +// exercise the integrated output (image overrides come from ResolveImages; the +// metrics service, env, volumes and mounts come from the modifier). +func renderWindows(cfg *render.WindowsConfiguration) []client.Object { + comp := render.Windows(cfg) + ExpectWithOffset(1, comp.ResolveImages(nil)).To(BeNil()) + objs, _ := comp.Objects() + rc := render.RenderContext{Installation: cfg.Installation} + out, _ := extensionstest.ApplyExtensions(ext, render.ComponentNameWindows, rc, objs, nil) + return out +} + +func getTyphaNodeTLS(cli client.Client, certificateManager certificatemanager.CertificateManager) *render.TyphaNodeTLS { + nodeKeyPair, err := certificateManager.GetOrCreateKeyPair(cli, render.NodeTLSSecretName, common.OperatorNamespace(), []string{render.FelixCommonName}) + Expect(err).NotTo(HaveOccurred()) + + typhaKeyPair, err := certificateManager.GetOrCreateKeyPair(cli, render.TyphaTLSSecretName, common.OperatorNamespace(), []string{render.FelixCommonName}) + Expect(err).NotTo(HaveOccurred()) + + typhaNonClusterHostKeyPair, err := certificateManager.GetOrCreateKeyPair(cli, render.TyphaTLSSecretName+render.TyphaNonClusterHostSuffix, common.OperatorNamespace(), []string{render.FelixCommonName + render.TyphaNonClusterHostSuffix}) + Expect(err).NotTo(HaveOccurred()) + + trustedBundle := certificateManager.CreateTrustedBundle(nodeKeyPair, typhaKeyPair) + + return &render.TyphaNodeTLS{ + TrustedBundle: trustedBundle, + TyphaSecret: typhaKeyPair, + TyphaSecretNonClusterHost: typhaNonClusterHostKeyPair, + TyphaCommonName: render.TyphaCommonName, + NodeSecret: nodeKeyPair, + NodeCommonName: render.FelixCommonName, + } +} + +// verifyWindowsProbesAndLifecycle asserts the expected node liveness and readiness probe plus pod lifecycle settings. +// The unused argument is kept temporarily so existing call sites compile while the OSS/Enterprise distinction +// is being phased out. +func verifyWindowsProbesAndLifecycle(ds *appsv1.DaemonSet, _ bool) { + livenessCmd := []string{"$env:CONTAINER_SANDBOX_MOUNT_POINT/CalicoWindows/calico.exe", "component", "node", "health", "--felix-live"} + readinessCmd := []string{"$env:CONTAINER_SANDBOX_MOUNT_POINT/CalicoWindows/calico.exe", "component", "node", "health", "--felix-ready"} + preStopCmd := []string{"$env:CONTAINER_SANDBOX_MOUNT_POINT/CalicoWindows/calico.exe", "component", "node", "shutdown"} + + expectedLiveness := &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + Exec: &corev1.ExecAction{Command: livenessCmd}, + }, + InitialDelaySeconds: 10, + FailureThreshold: 6, + TimeoutSeconds: 10, + PeriodSeconds: 10, + } + ExpectWithOffset(1, rtest.GetContainer(ds.Spec.Template.Spec.Containers, "felix").LivenessProbe).To(Equal(expectedLiveness)) + + expectedReadiness := &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + Exec: &corev1.ExecAction{Command: readinessCmd}, + }, + TimeoutSeconds: 10, + PeriodSeconds: 10, + } + ExpectWithOffset(1, rtest.GetContainer(ds.Spec.Template.Spec.Containers, "felix").ReadinessProbe).To(Equal(expectedReadiness)) + + expectedLifecycle := &corev1.Lifecycle{ + PreStop: &corev1.LifecycleHandler{ + Exec: &corev1.ExecAction{Command: preStopCmd}, + }, + } + ExpectWithOffset(1, rtest.GetContainer(ds.Spec.Template.Spec.Containers, "felix").Lifecycle).To(Equal(expectedLifecycle)) +} + +var _ = Describe("Windows enterprise rendering tests", func() { + var defaultInstance *operatorv1.InstallationSpec + var typhaNodeTLS *render.TyphaNodeTLS + var k8sServiceEp k8sapi.ServiceEndpoint + one := intstr.FromInt(1) + defaultNumExpectedResources := 2 + const defaultClusterDomain = "svc.cluster.local" + var defaultMode int32 = 420 + var cfg render.WindowsConfiguration + var cli client.Client + + BeforeEach(func() { + ff := true + hp := operatorv1.HostPortsEnabled + miMode := operatorv1.MultiInterfaceModeNone + defaultInstance = &operatorv1.InstallationSpec{ + CNI: &operatorv1.CNISpec{ + Type: "Calico", + IPAM: &operatorv1.IPAMSpec{Type: "Calico"}, + }, + CalicoNetwork: &operatorv1.CalicoNetworkSpec{ + BGP: &bgpEnabled, + IPPools: []operatorv1.IPPool{}, + NodeAddressAutodetectionV4: &operatorv1.NodeAddressAutodetection{}, + NodeAddressAutodetectionV6: &operatorv1.NodeAddressAutodetection{}, + HostPorts: &hp, + MultiInterfaceMode: &miMode, + }, + NodeUpdateStrategy: appsv1.DaemonSetUpdateStrategy{ + RollingUpdate: &appsv1.RollingUpdateDaemonSet{ + MaxUnavailable: &one, + }, + }, + Logging: &operatorv1.Logging{ + CNI: &operatorv1.CNILogging{ + LogSeverity: &logSeverity, + LogFileMaxSize: &logFileMaxSize, + LogFileMaxAgeDays: &logFileMaxAgeDays, + LogFileMaxCount: &logFileMaxCount, + }, + }, + WindowsNodes: &operatorv1.WindowsNodeSpec{ + CNIBinDir: "/opt/cni/bin", + CNIConfigDir: "/etc/cni/net.d", + CNILogDir: "/var/log/calico/cni", + }, + } + defaultInstance.CalicoNetwork.IPPools = append(defaultInstance.CalicoNetwork.IPPools, operatorv1.IPPool{CIDR: "192.168.1.0/16", Encapsulation: operatorv1.EncapsulationVXLAN}) + defaultInstance.CalicoNetwork.NodeAddressAutodetectionV4 = &operatorv1.NodeAddressAutodetection{FirstFound: &ff} + defaultInstance.ServiceCIDRs = []string{"10.96.0.0/12"} + scheme := runtime.NewScheme() + Expect(apis.AddToScheme(scheme, false)).NotTo(HaveOccurred()) + cli = ctrlrfake.DefaultFakeClientBuilder(scheme).Build() + certificateManager, err := certificatemanager.Create(cli, nil, defaultClusterDomain, common.OperatorNamespace(), certificatemanager.AllowCACreation()) + Expect(err).NotTo(HaveOccurred()) + // Create a dummy secret to pass as input. + typhaNodeTLS = getTyphaNodeTLS(cli, certificateManager) + + // Dummy service endpoint for k8s API. + k8sServiceEp = k8sapi.ServiceEndpoint{ + Host: "1.2.3.4", + Port: "6443", + } + + // Create a default configuration. + cfg = render.WindowsConfiguration{ + K8sServiceEp: k8sServiceEp, + K8sDNSServers: []string{"10.96.0.10"}, + Installation: defaultInstance, + ClusterDomain: defaultClusterDomain, + TLS: typhaNodeTLS, + VXLANVNI: 4096, + ImageOverrides: ext.Images(), + } + }) + + It("should render all resources for a default configuration using CalicoEnterprise", func() { + type testConf struct { + EnableBGP bool + EnableVXLAN bool + } + for _, testConfig := range []testConf{ + {true, false}, + {false, true}, + {true, true}, + } { + enableBGP := testConfig.EnableBGP + enableVXLAN := testConfig.EnableVXLAN + + if enableBGP { + defaultInstance.CalicoNetwork.BGP = &bgpEnabled + } else { + defaultInstance.CalicoNetwork.BGP = &bgpDisabled + } + + if enableVXLAN { + defaultInstance.CalicoNetwork.IPPools[0].Encapsulation = operatorv1.EncapsulationVXLAN + } else { + defaultInstance.CalicoNetwork.IPPools[0].Encapsulation = operatorv1.EncapsulationNone + } + By(fmt.Sprintf("BGP enabled: %v, VXLAN enabled: %v", enableBGP, enableVXLAN), func() { + expectedResources := []struct { + name string + ns string + group string + version string + kind string + }{ + {name: "cni-config-windows", ns: common.CalicoNamespace, group: "", version: "v1", kind: "ConfigMap"}, + {name: common.WindowsDaemonSetName, ns: common.CalicoNamespace, group: "apps", version: "v1", kind: "DaemonSet"}, + {name: "calico-node-metrics-windows", ns: "calico-system", group: "", version: "v1", kind: "Service"}, + } + defaultInstance.Variant = operatorv1.CalicoEnterprise + + resources := renderWindows(&cfg) + Expect(len(resources)).To(Equal(len(expectedResources))) + + // Should render the correct resources. + i := 0 + for _, expectedRes := range expectedResources { + rtest.ExpectResourceTypeAndObjectMetadata(resources[i], expectedRes.name, expectedRes.ns, expectedRes.group, expectedRes.version, expectedRes.kind) + i++ + } + + // The DaemonSet should have the correct configuration. + ds := rtest.GetResource(resources, "calico-node-windows", "calico-system", "apps", "v1", "DaemonSet").(*appsv1.DaemonSet) + + // The pod template should have node critical priority + Expect(ds.Spec.Template.Spec.PriorityClassName).To(Equal(render.NodePriorityClassName)) + + // The calico-node-windows daemonset has 3 containers (felix, node and confd). + // confd is only instantiated if using BGP. + numContainers := 3 + if !enableBGP { + numContainers = 2 + } + Expect(ds.Spec.Template.Spec.Containers).To(HaveLen(numContainers)) + for _, container := range ds.Spec.Template.Spec.Containers { + + // Windows node image override results in correct image. + Expect(container.Image).To(Equal(components.TigeraRegistry + "tigera/node-windows:" + components.ComponentTigeraNodeWindows.Version)) + Expect(container.SecurityContext.Capabilities).To(BeNil()) + Expect(container.SecurityContext.Privileged).To(BeNil()) + Expect(container.SecurityContext.SELinuxOptions).To(BeNil()) + Expect(container.SecurityContext.WindowsOptions).To(Not(BeNil())) + Expect(container.SecurityContext.WindowsOptions.GMSACredentialSpecName).To(BeNil()) + Expect(container.SecurityContext.WindowsOptions.GMSACredentialSpec).To(BeNil()) + Expect(*container.SecurityContext.WindowsOptions.RunAsUserName).To(Equal("NT AUTHORITY\\system")) + Expect(*container.SecurityContext.WindowsOptions.HostProcess).To(BeTrue()) + Expect(container.SecurityContext.RunAsUser).To(BeNil()) + Expect(container.SecurityContext.RunAsGroup).To(BeNil()) + Expect(container.SecurityContext.RunAsNonRoot).To(BeNil()) + Expect(container.SecurityContext.ReadOnlyRootFilesystem).To(BeNil()) + Expect(container.SecurityContext.AllowPrivilegeEscalation).To(BeNil()) + Expect(container.SecurityContext.ProcMount).To(BeNil()) + Expect(container.SecurityContext.SeccompProfile).To(BeNil()) + } + + felixContainer := rtest.GetContainer(ds.Spec.Template.Spec.Containers, "felix") + + // Windows node image override results in correct image. + Expect(felixContainer.Image).To(Equal(components.TigeraRegistry + "tigera/node-windows:" + components.ComponentTigeraNodeWindows.Version)) + Expect(felixContainer.SecurityContext.Capabilities).To(BeNil()) + Expect(felixContainer.SecurityContext.Privileged).To(BeNil()) + Expect(felixContainer.SecurityContext.SELinuxOptions).To(BeNil()) + Expect(felixContainer.SecurityContext.WindowsOptions).To(Not(BeNil())) + Expect(felixContainer.SecurityContext.WindowsOptions.GMSACredentialSpecName).To(BeNil()) + Expect(felixContainer.SecurityContext.WindowsOptions.GMSACredentialSpec).To(BeNil()) + Expect(*felixContainer.SecurityContext.WindowsOptions.RunAsUserName).To(Equal("NT AUTHORITY\\system")) + Expect(*felixContainer.SecurityContext.WindowsOptions.HostProcess).To(BeTrue()) + Expect(felixContainer.SecurityContext.RunAsUser).To(BeNil()) + Expect(felixContainer.SecurityContext.RunAsGroup).To(BeNil()) + Expect(felixContainer.SecurityContext.RunAsNonRoot).To(BeNil()) + Expect(felixContainer.SecurityContext.ReadOnlyRootFilesystem).To(BeNil()) + Expect(felixContainer.SecurityContext.AllowPrivilegeEscalation).To(BeNil()) + Expect(felixContainer.SecurityContext.ProcMount).To(BeNil()) + Expect(felixContainer.SecurityContext.SeccompProfile).To(BeNil()) + + nodeContainer := rtest.GetContainer(ds.Spec.Template.Spec.Containers, "node") + + // Windows node image override results in correct image. + Expect(nodeContainer.Image).To(Equal(components.TigeraRegistry + "tigera/node-windows:" + components.ComponentTigeraNodeWindows.Version)) + Expect(nodeContainer.SecurityContext.Capabilities).To(BeNil()) + Expect(nodeContainer.SecurityContext.Privileged).To(BeNil()) + Expect(nodeContainer.SecurityContext.SELinuxOptions).To(BeNil()) + Expect(nodeContainer.SecurityContext.WindowsOptions).To(Not(BeNil())) + Expect(nodeContainer.SecurityContext.WindowsOptions.GMSACredentialSpecName).To(BeNil()) + Expect(nodeContainer.SecurityContext.WindowsOptions.GMSACredentialSpec).To(BeNil()) + Expect(*nodeContainer.SecurityContext.WindowsOptions.RunAsUserName).To(Equal("NT AUTHORITY\\system")) + Expect(*nodeContainer.SecurityContext.WindowsOptions.HostProcess).To(BeTrue()) + Expect(nodeContainer.SecurityContext.RunAsUser).To(BeNil()) + Expect(nodeContainer.SecurityContext.RunAsGroup).To(BeNil()) + Expect(nodeContainer.SecurityContext.RunAsNonRoot).To(BeNil()) + Expect(nodeContainer.SecurityContext.ReadOnlyRootFilesystem).To(BeNil()) + Expect(nodeContainer.SecurityContext.AllowPrivilegeEscalation).To(BeNil()) + Expect(nodeContainer.SecurityContext.ProcMount).To(BeNil()) + Expect(nodeContainer.SecurityContext.SeccompProfile).To(BeNil()) + + if enableBGP { + confdContainer := rtest.GetContainer(ds.Spec.Template.Spec.Containers, "confd") + + // Windows node image override results in correct image. + Expect(confdContainer.Image).To(Equal(components.TigeraRegistry + "tigera/node-windows:" + components.ComponentTigeraNodeWindows.Version)) + Expect(confdContainer.SecurityContext.Capabilities).To(BeNil()) + Expect(confdContainer.SecurityContext.Privileged).To(BeNil()) + Expect(confdContainer.SecurityContext.SELinuxOptions).To(BeNil()) + Expect(confdContainer.SecurityContext.WindowsOptions).To(Not(BeNil())) + Expect(confdContainer.SecurityContext.WindowsOptions.GMSACredentialSpecName).To(BeNil()) + Expect(confdContainer.SecurityContext.WindowsOptions.GMSACredentialSpec).To(BeNil()) + Expect(*confdContainer.SecurityContext.WindowsOptions.RunAsUserName).To(Equal("NT AUTHORITY\\system")) + Expect(*confdContainer.SecurityContext.WindowsOptions.HostProcess).To(BeTrue()) + Expect(confdContainer.SecurityContext.RunAsUser).To(BeNil()) + Expect(confdContainer.SecurityContext.RunAsGroup).To(BeNil()) + Expect(confdContainer.SecurityContext.RunAsNonRoot).To(BeNil()) + Expect(confdContainer.SecurityContext.ReadOnlyRootFilesystem).To(BeNil()) + Expect(confdContainer.SecurityContext.AllowPrivilegeEscalation).To(BeNil()) + Expect(confdContainer.SecurityContext.ProcMount).To(BeNil()) + Expect(confdContainer.SecurityContext.SeccompProfile).To(BeNil()) + } + + // Validate correct number of init containers. + Expect(ds.Spec.Template.Spec.InitContainers).To(HaveLen(2)) + + // CNI container uses image override. + cniContainer := rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "install-cni") + rtest.ExpectEnv(cniContainer.Env, "CNI_NET_DIR", "/etc/cni/net.d") + Expect(cniContainer.Image).To(Equal(components.TigeraRegistry + "tigera/cni-windows:" + components.ComponentTigeraCNIWindows.Version)) + + Expect(cniContainer.SecurityContext.Capabilities).To(BeNil()) + Expect(cniContainer.SecurityContext.Privileged).To(BeNil()) + Expect(cniContainer.SecurityContext.SELinuxOptions).To(BeNil()) + Expect(cniContainer.SecurityContext.WindowsOptions).To(Not(BeNil())) + Expect(cniContainer.SecurityContext.WindowsOptions.GMSACredentialSpecName).To(BeNil()) + Expect(cniContainer.SecurityContext.WindowsOptions.GMSACredentialSpec).To(BeNil()) + Expect(*cniContainer.SecurityContext.WindowsOptions.RunAsUserName).To(Equal("NT AUTHORITY\\system")) + Expect(*cniContainer.SecurityContext.WindowsOptions.HostProcess).To(BeTrue()) + Expect(cniContainer.SecurityContext.RunAsUser).To(BeNil()) + Expect(cniContainer.SecurityContext.RunAsGroup).To(BeNil()) + Expect(cniContainer.SecurityContext.RunAsNonRoot).To(BeNil()) + Expect(cniContainer.SecurityContext.ReadOnlyRootFilesystem).To(BeNil()) + Expect(cniContainer.SecurityContext.AllowPrivilegeEscalation).To(BeNil()) + Expect(cniContainer.SecurityContext.ProcMount).To(BeNil()) + Expect(cniContainer.SecurityContext.SeccompProfile).To(BeNil()) + + // uninstall container uses image override. + uninstallContainer := rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "uninstall-calico") + Expect(uninstallContainer.Image).To(Equal(components.TigeraRegistry + "tigera/node-windows:" + components.ComponentTigeraNodeWindows.Version)) + + Expect(uninstallContainer.SecurityContext.Capabilities).To(BeNil()) + Expect(uninstallContainer.SecurityContext.Privileged).To(BeNil()) + Expect(uninstallContainer.SecurityContext.SELinuxOptions).To(BeNil()) + Expect(uninstallContainer.SecurityContext.WindowsOptions).To(Not(BeNil())) + Expect(uninstallContainer.SecurityContext.WindowsOptions.GMSACredentialSpecName).To(BeNil()) + Expect(uninstallContainer.SecurityContext.WindowsOptions.GMSACredentialSpec).To(BeNil()) + Expect(*uninstallContainer.SecurityContext.WindowsOptions.RunAsUserName).To(Equal("NT AUTHORITY\\system")) + Expect(*uninstallContainer.SecurityContext.WindowsOptions.HostProcess).To(BeTrue()) + Expect(uninstallContainer.SecurityContext.RunAsUser).To(BeNil()) + Expect(uninstallContainer.SecurityContext.RunAsGroup).To(BeNil()) + Expect(uninstallContainer.SecurityContext.RunAsNonRoot).To(BeNil()) + Expect(uninstallContainer.SecurityContext.ReadOnlyRootFilesystem).To(BeNil()) + Expect(uninstallContainer.SecurityContext.AllowPrivilegeEscalation).To(BeNil()) + Expect(uninstallContainer.SecurityContext.ProcMount).To(BeNil()) + Expect(uninstallContainer.SecurityContext.SeccompProfile).To(BeNil()) + + // Verify env + expectedNodeEnv := []corev1.EnvVar{ + {Name: "CNI_PLUGIN_TYPE", Value: "Calico"}, + {Name: "DATASTORE_TYPE", Value: "kubernetes"}, + {Name: "WAIT_FOR_DATASTORE", Value: "true"}, + {Name: "CALICO_MANAGE_CNI", Value: "true"}, + {Name: "CALICO_DISABLE_FILE_LOGGING", Value: "false"}, + {Name: "FELIX_DEFAULTENDPOINTTOHOSTACTION", Value: "ACCEPT"}, + {Name: "FELIX_HEALTHENABLED", Value: "true"}, + { + Name: "NODENAME", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{FieldPath: "spec.nodeName"}, + }, + }, + { + Name: "NAMESPACE", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.namespace"}, + }, + }, + {Name: "IP", Value: "autodetect"}, + {Name: "IP_AUTODETECTION_METHOD", Value: "first-found"}, + {Name: "IP6", Value: "none"}, + {Name: "FELIX_IPV6SUPPORT", Value: "false"}, + {Name: "FELIX_TYPHAK8SNAMESPACE", Value: "calico-system"}, + {Name: "FELIX_TYPHAK8SSERVICENAME", Value: "calico-typha"}, + {Name: "FELIX_TYPHACAFILE", Value: certificatemanagement.TrustedCertBundleMountPath}, + {Name: "FELIX_TYPHACERTFILE", Value: "/node-certs/tls.crt"}, + {Name: "FELIX_TYPHACN", Value: "typha-server"}, + {Name: "FELIX_TYPHAKEYFILE", Value: "/node-certs/tls.key"}, + + {Name: "VXLAN_VNI", Value: "4096"}, + {Name: "VXLAN_ADAPTER", Value: ""}, + {Name: "KUBE_NETWORK", Value: "Calico.*"}, + {Name: "KUBERNETES_SERVICE_HOST", Value: "1.2.3.4"}, + {Name: "KUBERNETES_SERVICE_PORT", Value: "6443"}, + + // Tigera-specific envvars + {Name: "FELIX_PROMETHEUSREPORTERENABLED", Value: "true"}, + {Name: "FELIX_PROMETHEUSREPORTERPORT", Value: "9081"}, + {Name: "FELIX_FLOWLOGSFILEENABLED", Value: "true"}, + {Name: "FELIX_FLOWLOGSFILEINCLUDELABELS", Value: "true"}, + {Name: "FELIX_FLOWLOGSFILEINCLUDEPOLICIES", Value: "true"}, + {Name: "FELIX_FLOWLOGSFILEINCLUDESERVICE", Value: "true"}, + {Name: "FELIX_FLOWLOGSENABLENETWORKSETS", Value: "true"}, + {Name: "FELIX_FLOWLOGSCOLLECTPROCESSINFO", Value: "true"}, + {Name: "FELIX_DNSLOGSFILEENABLED", Value: "true"}, + {Name: "FELIX_DNSLOGSFILEPERNODELIMIT", Value: "1000"}, + } + + // Set CALICO_NETWORKING_BACKEND + if enableBGP { + expectedNodeEnv = append(expectedNodeEnv, corev1.EnvVar{Name: "CALICO_NETWORKING_BACKEND", Value: "windows-bgp"}) + } else if enableVXLAN { + expectedNodeEnv = append(expectedNodeEnv, corev1.EnvVar{Name: "CALICO_NETWORKING_BACKEND", Value: "vxlan"}) + } else { + expectedNodeEnv = append(expectedNodeEnv, corev1.EnvVar{Name: "CALICO_NETWORKING_BACKEND", Value: "none"}) + } + + // Set CLUSTER_TYPE + if enableBGP { + expectedNodeEnv = append(expectedNodeEnv, corev1.EnvVar{Name: "CLUSTER_TYPE", Value: "k8s,operator,bgp,windows"}) + } else { + expectedNodeEnv = append(expectedNodeEnv, corev1.EnvVar{Name: "CLUSTER_TYPE", Value: "k8s,operator,windows"}) + } + + Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "node").Env).To(ConsistOf(expectedNodeEnv)) + Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "felix").Env).To(ConsistOf(expectedNodeEnv)) + if enableBGP { + Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "confd").Env).To(ConsistOf(expectedNodeEnv)) + } + + // Expect the SECURITY_GROUP env variables to not be set + Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "node").Env).NotTo(ContainElement(gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{"Name": Equal("TIGERA_DEFAULT_SECURITY_GROUPS")}))) + Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "node").Env).NotTo(ContainElement(gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{"Name": Equal("TIGERA_POD_SECURITY_GROUP")}))) + Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "felix").Env).NotTo(ContainElement(gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{"Name": Equal("TIGERA_DEFAULT_SECURITY_GROUPS")}))) + Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "felix").Env).NotTo(ContainElement(gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{"Name": Equal("TIGERA_POD_SECURITY_GROUP")}))) + if enableBGP { + Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "confd").Env).NotTo(ContainElement(gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{"Name": Equal("TIGERA_DEFAULT_SECURITY_GROUPS")}))) + Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "confd").Env).NotTo(ContainElement(gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{"Name": Equal("TIGERA_POD_SECURITY_GROUP")}))) + } + + expectedCNIEnv := []corev1.EnvVar{ + {Name: "SLEEP", Value: "false"}, + {Name: "CNI_PLUGIN_TYPE", Value: "Calico"}, + {Name: "CNI_BIN_DIR", Value: "/host/opt/cni/bin"}, + {Name: "CNI_CONF_NAME", Value: "10-calico.conflist"}, + {Name: "CNI_NET_DIR", Value: "/etc/cni/net.d"}, + {Name: "VXLAN_VNI", Value: "4096"}, + { + Name: "KUBERNETES_NODE_NAME", + Value: "", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{FieldPath: "spec.nodeName"}, + }, + }, + { + Name: "CNI_NETWORK_CONFIG", + ValueFrom: &corev1.EnvVarSource{ + ConfigMapKeyRef: &corev1.ConfigMapKeySelector{ + Key: "config", + LocalObjectReference: corev1.LocalObjectReference{ + Name: "cni-config-windows", + }, + }, + }, + }, + + {Name: "KUBERNETES_SERVICE_HOST", Value: "1.2.3.4"}, + {Name: "KUBERNETES_SERVICE_PORT", Value: "6443"}, + {Name: "KUBERNETES_SERVICE_CIDRS", Value: "10.96.0.0/12"}, + {Name: "KUBERNETES_DNS_SERVERS", Value: "10.96.0.10"}, + } + Expect(rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "install-cni").Env).To(ConsistOf(expectedCNIEnv)) + + expectedUninstallEnv := []corev1.EnvVar{ + {Name: "SLEEP", Value: "false"}, + {Name: "CNI_PLUGIN_TYPE", Value: "Calico"}, + {Name: "CNI_BIN_DIR", Value: "/host/opt/cni/bin"}, + {Name: "CNI_CONF_NAME", Value: "10-calico.conflist"}, + {Name: "CNI_NET_DIR", Value: "/host/etc/cni/net.d"}, + } + Expect(rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "uninstall-calico").Env).To(ConsistOf(expectedUninstallEnv)) + + // Verify volumes. + fileOrCreate := corev1.HostPathFileOrCreate + dirOrCreate := corev1.HostPathDirectoryOrCreate + expectedVols := []corev1.Volume{ + {Name: "lib-modules", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/lib/modules"}}}, + {Name: "var-run-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/run/calico", Type: &dirOrCreate}}}, + {Name: "var-lib-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/lib/calico", Type: &dirOrCreate}}}, + {Name: "xtables-lock", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/run/xtables.lock", Type: &fileOrCreate}}}, + {Name: "cni-bin-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/opt/cni/bin", Type: &dirOrCreate}}}, + {Name: "cni-net-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/etc/cni/net.d"}}}, + {Name: "cni-log-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico/cni", Type: &dirOrCreate}}}, + {Name: "policysync", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/run/nodeagent", Type: &dirOrCreate}}}, + { + Name: "tigera-ca-bundle", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "tigera-ca-bundle", + }, + }, + }, + }, + { + Name: render.NodeTLSSecretName, + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: render.NodeTLSSecretName, + DefaultMode: &defaultMode, + }, + }, + }, + {Name: "var-log-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico", Type: &dirOrCreate}}}, + } + Expect(ds.Spec.Template.Spec.Volumes).To(ConsistOf(expectedVols)) + + // Verify volume mounts. + expectedNodeVolumeMounts := []corev1.VolumeMount{ + {MountPath: "/host/etc/cni/net.d", Name: "cni-net-dir"}, + {MountPath: "/var/run/calico", Name: "var-run-calico"}, + {MountPath: "/var/lib/calico", Name: "var-lib-calico"}, + {MountPath: "c:/etc/pki/tls/certs", Name: "tigera-ca-bundle", ReadOnly: true}, + {MountPath: "c:/node-certs", Name: render.NodeTLSSecretName, ReadOnly: true}, + {MountPath: "/var/log/calico", Name: "var-log-calico", ReadOnly: false}, + } + Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "node").VolumeMounts).To(ConsistOf(expectedNodeVolumeMounts)) + Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "felix").VolumeMounts).To(ConsistOf(expectedNodeVolumeMounts)) + if enableBGP { + Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "confd").VolumeMounts).To(ConsistOf(expectedNodeVolumeMounts)) + } + + expectedCNIVolumeMounts := []corev1.VolumeMount{ + {MountPath: "/host/opt/cni/bin", Name: "cni-bin-dir"}, + {MountPath: "/host/etc/cni/net.d", Name: "cni-net-dir"}, + } + Expect(rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "install-cni").VolumeMounts).To(ConsistOf(expectedCNIVolumeMounts)) + + expectedUninstallVolumeMounts := []corev1.VolumeMount{ + {MountPath: "/host/opt/cni/bin", Name: "cni-bin-dir"}, + {MountPath: "/host/etc/cni/net.d", Name: "cni-net-dir"}, + } + Expect(rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "uninstall-calico").VolumeMounts).To(ConsistOf(expectedUninstallVolumeMounts)) + + // Verify tolerations. + Expect(ds.Spec.Template.Spec.Tolerations).To(ConsistOf(rmeta.TolerateAll)) + + // Verify readiness and liveness probes. + verifyWindowsProbesAndLifecycle(ds, false) + }) + } + }) + + It("should render all resources when variant is CalicoEnterprise and running on openshift", func() { + expectedResources := []struct { + name string + ns string + group string + version string + kind string + }{ + {name: "cni-config-windows", ns: common.CalicoNamespace, group: "", version: "v1", kind: "ConfigMap"}, + {name: common.WindowsDaemonSetName, ns: common.CalicoNamespace, group: "apps", version: "v1", kind: "DaemonSet"}, + {name: "calico-node-metrics-windows", ns: "calico-system", group: "", version: "v1", kind: "Service"}, + } + + defaultInstance.Variant = operatorv1.CalicoEnterprise + defaultInstance.KubernetesProvider = operatorv1.ProviderOpenShift + + resources := renderWindows(&cfg) + Expect(len(resources)).To(Equal(len(expectedResources))) + + // Should render the correct resources. + i := 0 + for _, expectedRes := range expectedResources { + rtest.ExpectResourceTypeAndObjectMetadata(resources[i], expectedRes.name, expectedRes.ns, expectedRes.group, expectedRes.version, expectedRes.kind) + i++ + } + + // The DaemonSet should have the correct configuration. + ds := rtest.GetResource(resources, "calico-node-windows", "calico-system", "apps", "v1", "DaemonSet").(*appsv1.DaemonSet) + + // The pod template should have node critical priority + Expect(ds.Spec.Template.Spec.PriorityClassName).To(Equal(render.NodePriorityClassName)) + + felixContainer := rtest.GetContainer(ds.Spec.Template.Spec.Containers, "felix") + Expect(felixContainer.Image).To(Equal(fmt.Sprintf("%s%s%s:%s", components.TigeraRegistry, components.TigeraImagePath, components.ComponentTigeraNodeWindows.Image, components.ComponentTigeraNodeWindows.Version))) + nodeContainer := rtest.GetContainer(ds.Spec.Template.Spec.Containers, "node") + Expect(nodeContainer.Image).To(Equal(fmt.Sprintf("%s%s%s:%s", components.TigeraRegistry, components.TigeraImagePath, components.ComponentTigeraNodeWindows.Image, components.ComponentTigeraNodeWindows.Version))) + cniContainer := rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "install-cni") + Expect(cniContainer.Image).To(Equal(fmt.Sprintf("%s%s%s:%s", components.TigeraRegistry, components.TigeraImagePath, components.ComponentTigeraCNIWindows.Image, components.ComponentTigeraCNIWindows.Version))) + uninstallContainer := rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "uninstall-calico") + Expect(uninstallContainer.Image).To(Equal(fmt.Sprintf("%s%s%s:%s", components.TigeraRegistry, components.TigeraImagePath, components.ComponentTigeraNodeWindows.Image, components.ComponentTigeraNodeWindows.Version))) + + // FIXME: confirm openshift CNI path defaults + expectedCNIVolumeMounts := []corev1.VolumeMount{ + {MountPath: "/host/opt/cni/bin", Name: "cni-bin-dir"}, + {MountPath: "/host/etc/cni/net.d", Name: "cni-net-dir"}, + } + Expect(rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "install-cni").VolumeMounts).To(ConsistOf(expectedCNIVolumeMounts)) + + // FIXME: confirm openshift CNI path defaults + expectedUninstallVolumeMounts := []corev1.VolumeMount{ + {MountPath: "/host/opt/cni/bin", Name: "cni-bin-dir"}, + {MountPath: "/host/etc/cni/net.d", Name: "cni-net-dir"}, + } + Expect(rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "uninstall-calico").VolumeMounts).To(ConsistOf(expectedUninstallVolumeMounts)) + + // Verify volumes + // FIXME: confirm openshift CNI path defaults + fileOrCreate := corev1.HostPathFileOrCreate + dirOrCreate := corev1.HostPathDirectoryOrCreate + expectedVols := []corev1.Volume{ + {Name: "lib-modules", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/lib/modules"}}}, + {Name: "var-run-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/run/calico", Type: &dirOrCreate}}}, + {Name: "var-lib-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/lib/calico", Type: &dirOrCreate}}}, + {Name: "xtables-lock", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/run/xtables.lock", Type: &fileOrCreate}}}, + {Name: "cni-bin-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/opt/cni/bin", Type: &dirOrCreate}}}, + {Name: "cni-net-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/etc/cni/net.d"}}}, + {Name: "cni-log-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico/cni", Type: &dirOrCreate}}}, + {Name: "policysync", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/run/nodeagent", Type: &dirOrCreate}}}, + { + Name: "tigera-ca-bundle", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "tigera-ca-bundle", + }, + }, + }, + }, + { + Name: render.NodeTLSSecretName, + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: render.NodeTLSSecretName, + DefaultMode: &defaultMode, + }, + }, + }, + {Name: "var-log-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico", Type: &dirOrCreate}}}, + } + Expect(ds.Spec.Template.Spec.Volumes).To(ConsistOf(expectedVols)) + + expectedNodeEnv := []corev1.EnvVar{ + // Default envvars. + {Name: "CNI_PLUGIN_TYPE", Value: "Calico"}, + {Name: "DATASTORE_TYPE", Value: "kubernetes"}, + {Name: "WAIT_FOR_DATASTORE", Value: "true"}, + {Name: "CALICO_MANAGE_CNI", Value: "true"}, + {Name: "CALICO_NETWORKING_BACKEND", Value: "windows-bgp"}, + {Name: "CLUSTER_TYPE", Value: "k8s,operator,openshift,bgp,windows"}, + {Name: "CALICO_DISABLE_FILE_LOGGING", Value: "false"}, + {Name: "FELIX_DEFAULTENDPOINTTOHOSTACTION", Value: "ACCEPT"}, + {Name: "FELIX_HEALTHENABLED", Value: "true"}, + { + Name: "NODENAME", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{FieldPath: "spec.nodeName"}, + }, + }, + { + Name: "NAMESPACE", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.namespace"}, + }, + }, + {Name: "IP", Value: "autodetect"}, + {Name: "IP_AUTODETECTION_METHOD", Value: "first-found"}, + {Name: "IP6", Value: "none"}, + {Name: "FELIX_IPV6SUPPORT", Value: "false"}, + {Name: "FELIX_TYPHAK8SNAMESPACE", Value: "calico-system"}, + {Name: "FELIX_TYPHAK8SSERVICENAME", Value: "calico-typha"}, + {Name: "FELIX_TYPHACAFILE", Value: certificatemanagement.TrustedCertBundleMountPath}, + {Name: "FELIX_TYPHACERTFILE", Value: "/node-certs/tls.crt"}, + {Name: "FELIX_TYPHACN", Value: "typha-server"}, + {Name: "FELIX_TYPHAKEYFILE", Value: "/node-certs/tls.key"}, + + {Name: "FELIX_DNSTRUSTEDSERVERS", Value: "k8s-service:openshift-dns/dns-default"}, + + // Tigera-specific envvars + {Name: "FELIX_PROMETHEUSREPORTERENABLED", Value: "true"}, + {Name: "FELIX_PROMETHEUSREPORTERPORT", Value: "9081"}, + {Name: "FELIX_FLOWLOGSFILEENABLED", Value: "true"}, + {Name: "FELIX_FLOWLOGSFILEINCLUDELABELS", Value: "true"}, + {Name: "FELIX_FLOWLOGSFILEINCLUDEPOLICIES", Value: "true"}, + {Name: "FELIX_FLOWLOGSFILEINCLUDESERVICE", Value: "true"}, + {Name: "FELIX_FLOWLOGSENABLENETWORKSETS", Value: "true"}, + {Name: "FELIX_FLOWLOGSCOLLECTPROCESSINFO", Value: "true"}, + {Name: "FELIX_DNSLOGSFILEENABLED", Value: "true"}, + {Name: "FELIX_DNSLOGSFILEPERNODELIMIT", Value: "1000"}, + + // Calico Windows specific envvars + {Name: "VXLAN_VNI", Value: "4096"}, + {Name: "VXLAN_ADAPTER", Value: ""}, + {Name: "KUBE_NETWORK", Value: "Calico.*"}, + {Name: "KUBERNETES_SERVICE_HOST", Value: "1.2.3.4"}, + {Name: "KUBERNETES_SERVICE_PORT", Value: "6443"}, + } + Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "node").Env).To(ConsistOf(expectedNodeEnv)) + Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "felix").Env).To(ConsistOf(expectedNodeEnv)) + Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "confd").Env).To(ConsistOf(expectedNodeEnv)) + + verifyWindowsProbesAndLifecycle(ds, false) + }) + + It("should render all resources when variant is CalicoEnterprise and running on RKE2", func() { + expectedResources := []struct { + name string + ns string + group string + version string + kind string + }{ + {name: "cni-config-windows", ns: common.CalicoNamespace, group: "", version: "v1", kind: "ConfigMap"}, + {name: common.WindowsDaemonSetName, ns: common.CalicoNamespace, group: "apps", version: "v1", kind: "DaemonSet"}, + {name: "calico-node-metrics-windows", ns: "calico-system", group: "", version: "v1", kind: "Service"}, + } + + defaultInstance.Variant = operatorv1.CalicoEnterprise + defaultInstance.KubernetesProvider = operatorv1.ProviderRKE2 + + resources := renderWindows(&cfg) + Expect(len(resources)).To(Equal(len(expectedResources)), fmt.Sprintf("Actual resources: %#v", resources)) + + // Should render the correct resources. + i := 0 + for _, expectedRes := range expectedResources { + rtest.ExpectResourceTypeAndObjectMetadata(resources[i], expectedRes.name, expectedRes.ns, expectedRes.group, expectedRes.version, expectedRes.kind) + i++ + } + + // The DaemonSet should have the correct configuration. + ds := rtest.GetResource(resources, "calico-node-windows", "calico-system", "apps", "v1", "DaemonSet").(*appsv1.DaemonSet) + + // The pod template should have node critical priority + Expect(ds.Spec.Template.Spec.PriorityClassName).To(Equal(render.NodePriorityClassName)) + + felixContainer := rtest.GetContainer(ds.Spec.Template.Spec.Containers, "felix") + Expect(felixContainer.Image).To(Equal(fmt.Sprintf("%s%s%s:%s", components.TigeraRegistry, components.TigeraImagePath, components.ComponentTigeraNodeWindows.Image, components.ComponentTigeraNodeWindows.Version))) + nodeContainer := rtest.GetContainer(ds.Spec.Template.Spec.Containers, "node") + Expect(nodeContainer.Image).To(Equal(fmt.Sprintf("%s%s%s:%s", components.TigeraRegistry, components.TigeraImagePath, components.ComponentTigeraNodeWindows.Image, components.ComponentTigeraNodeWindows.Version))) + cniContainer := rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "install-cni") + Expect(cniContainer.Image).To(Equal(fmt.Sprintf("%s%s%s:%s", components.TigeraRegistry, components.TigeraImagePath, components.ComponentTigeraCNIWindows.Image, components.ComponentTigeraCNIWindows.Version))) + uninstallContainer := rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "uninstall-calico") + Expect(uninstallContainer.Image).To(Equal(fmt.Sprintf("%s%s%s:%s", components.TigeraRegistry, components.TigeraImagePath, components.ComponentTigeraNodeWindows.Image, components.ComponentTigeraNodeWindows.Version))) + + // FIXME: confirm RKE2 CNI path defaults + expectedCNIVolumeMounts := []corev1.VolumeMount{ + {MountPath: "/host/opt/cni/bin", Name: "cni-bin-dir"}, + {MountPath: "/host/etc/cni/net.d", Name: "cni-net-dir"}, + } + Expect(rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "install-cni").VolumeMounts).To(ConsistOf(expectedCNIVolumeMounts)) + + // FIXME: confirm RKE2 CNI path defaults + expectedUninstallVolumeMounts := []corev1.VolumeMount{ + {MountPath: "/host/opt/cni/bin", Name: "cni-bin-dir"}, + {MountPath: "/host/etc/cni/net.d", Name: "cni-net-dir"}, + } + Expect(rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "uninstall-calico").VolumeMounts).To(ConsistOf(expectedUninstallVolumeMounts)) + + // Verify volumes + // FIXME: confirm RKE2 CNI path defaults + fileOrCreate := corev1.HostPathFileOrCreate + dirOrCreate := corev1.HostPathDirectoryOrCreate + expectedVols := []corev1.Volume{ + {Name: "lib-modules", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/lib/modules"}}}, + {Name: "var-run-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/run/calico", Type: &dirOrCreate}}}, + {Name: "var-lib-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/lib/calico", Type: &dirOrCreate}}}, + {Name: "xtables-lock", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/run/xtables.lock", Type: &fileOrCreate}}}, + {Name: "cni-bin-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/opt/cni/bin", Type: &dirOrCreate}}}, + {Name: "cni-net-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/etc/cni/net.d"}}}, + {Name: "cni-log-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico/cni", Type: &dirOrCreate}}}, + {Name: "policysync", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/run/nodeagent", Type: &dirOrCreate}}}, + { + Name: "tigera-ca-bundle", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "tigera-ca-bundle", + }, + }, + }, + }, + { + Name: render.NodeTLSSecretName, + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: render.NodeTLSSecretName, + DefaultMode: &defaultMode, + }, + }, + }, + {Name: "var-log-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico", Type: &dirOrCreate}}}, + } + Expect(ds.Spec.Template.Spec.Volumes).To(ConsistOf(expectedVols)) + + expectedNodeEnv := []corev1.EnvVar{ + // Default envvars. + {Name: "CNI_PLUGIN_TYPE", Value: "Calico"}, + {Name: "DATASTORE_TYPE", Value: "kubernetes"}, + {Name: "WAIT_FOR_DATASTORE", Value: "true"}, + {Name: "CALICO_MANAGE_CNI", Value: "true"}, + {Name: "CALICO_NETWORKING_BACKEND", Value: "windows-bgp"}, + {Name: "CLUSTER_TYPE", Value: "k8s,operator,bgp,windows"}, + {Name: "CALICO_DISABLE_FILE_LOGGING", Value: "false"}, + {Name: "FELIX_DEFAULTENDPOINTTOHOSTACTION", Value: "ACCEPT"}, + {Name: "FELIX_HEALTHENABLED", Value: "true"}, + { + Name: "NODENAME", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{FieldPath: "spec.nodeName"}, + }, + }, + { + Name: "NAMESPACE", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.namespace"}, + }, + }, + {Name: "IP", Value: "autodetect"}, + {Name: "IP_AUTODETECTION_METHOD", Value: "first-found"}, + {Name: "IP6", Value: "none"}, + {Name: "FELIX_IPV6SUPPORT", Value: "false"}, + {Name: "FELIX_TYPHAK8SNAMESPACE", Value: "calico-system"}, + {Name: "FELIX_TYPHAK8SSERVICENAME", Value: "calico-typha"}, + {Name: "FELIX_TYPHACAFILE", Value: certificatemanagement.TrustedCertBundleMountPath}, + {Name: "FELIX_TYPHACERTFILE", Value: "/node-certs/tls.crt"}, + {Name: "FELIX_TYPHACN", Value: "typha-server"}, + {Name: "FELIX_TYPHAKEYFILE", Value: "/node-certs/tls.key"}, + + {Name: "FELIX_DNSTRUSTEDSERVERS", Value: "k8s-service:kube-system/rke2-coredns-rke2-coredns"}, + + // Tigera-specific envvars + {Name: "FELIX_PROMETHEUSREPORTERENABLED", Value: "true"}, + {Name: "FELIX_PROMETHEUSREPORTERPORT", Value: "9081"}, + {Name: "FELIX_FLOWLOGSFILEENABLED", Value: "true"}, + {Name: "FELIX_FLOWLOGSFILEINCLUDELABELS", Value: "true"}, + {Name: "FELIX_FLOWLOGSFILEINCLUDEPOLICIES", Value: "true"}, + {Name: "FELIX_FLOWLOGSFILEINCLUDESERVICE", Value: "true"}, + {Name: "FELIX_FLOWLOGSENABLENETWORKSETS", Value: "true"}, + {Name: "FELIX_FLOWLOGSCOLLECTPROCESSINFO", Value: "true"}, + {Name: "FELIX_DNSLOGSFILEENABLED", Value: "true"}, + {Name: "FELIX_DNSLOGSFILEPERNODELIMIT", Value: "1000"}, + + // Calico Windows specific envvars + {Name: "VXLAN_VNI", Value: "4096"}, + {Name: "VXLAN_ADAPTER", Value: ""}, + {Name: "KUBE_NETWORK", Value: "Calico.*"}, + {Name: "KUBERNETES_SERVICE_HOST", Value: "1.2.3.4"}, + {Name: "KUBERNETES_SERVICE_PORT", Value: "6443"}, + } + Expect(ds.Spec.Template.Spec.Containers[0].Env).To(ConsistOf(expectedNodeEnv)) + Expect(len(ds.Spec.Template.Spec.Containers[0].Env)).To(Equal(len(expectedNodeEnv))) + + verifyWindowsProbesAndLifecycle(ds, false) + + // The metrics service should have the correct configuration. + ms := rtest.GetResource(resources, "calico-node-metrics-windows", "calico-system", "", "v1", "Service").(*corev1.Service) + Expect(ms.Spec.ClusterIP).To(Equal("None"), "metrics service should be headless to prevent kube-proxy from rendering too many iptables rules") + }) + + It("should not enable prometheus metrics if NodeMetricsPort is nil", func() { + defaultInstance.Variant = operatorv1.CalicoEnterprise + defaultInstance.NodeMetricsPort = nil + + resources := renderWindows(&cfg) + Expect(len(resources)).To(Equal(defaultNumExpectedResources + 1)) + + dsResource := rtest.GetResource(resources, "calico-node-windows", "calico-system", "apps", "v1", "DaemonSet") + Expect(dsResource).ToNot(BeNil()) + + notExpectedEnvVar := corev1.EnvVar{Name: "FELIX_PROMETHEUSMETRICSPORT"} + ds := dsResource.(*appsv1.DaemonSet) + Expect(ds.Spec.Template.Spec.Containers[0].Env).ToNot(ContainElement(notExpectedEnvVar)) + + // It should have the reporter port, though. + expected := corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERPORT"} + Expect(ds.Spec.Template.Spec.Containers[0].Env).ToNot(ContainElement(expected)) + }) + + It("should set FELIX_PROMETHEUSMETRICSPORT with a custom value if NodeMetricsPort is set", func() { + var nodeMetricsPort int32 = 1234 + defaultInstance.Variant = operatorv1.CalicoEnterprise + defaultInstance.NodeMetricsPort = &nodeMetricsPort + resources := renderWindows(&cfg) + Expect(len(resources)).To(Equal(defaultNumExpectedResources + 1)) + + dsResource := rtest.GetResource(resources, "calico-node-windows", "calico-system", "apps", "v1", "DaemonSet") + Expect(dsResource).ToNot(BeNil()) + + // Assert on expected env vars. + expectedEnvVars := []corev1.EnvVar{ + {Name: "FELIX_PROMETHEUSMETRICSPORT", Value: "1234"}, + {Name: "FELIX_PROMETHEUSMETRICSENABLED", Value: "true"}, + } + ds := dsResource.(*appsv1.DaemonSet) + for _, v := range expectedEnvVars { + Expect(ds.Spec.Template.Spec.Containers[0].Env).To(ContainElement(v)) + } + + // Assert we set annotations properly. + Expect(ds.Spec.Template.Annotations["prometheus.io/scrape"]).To(Equal("true")) + Expect(ds.Spec.Template.Annotations["prometheus.io/port"]).To(Equal("1234")) + }) +}) diff --git a/pkg/render/windows_test.go b/pkg/render/windows_test.go index fed3de6eaf..868334d55d 100644 --- a/pkg/render/windows_test.go +++ b/pkg/render/windows_test.go @@ -35,26 +35,13 @@ import ( "github.com/tigera/operator/pkg/controller/certificatemanager" "github.com/tigera/operator/pkg/controller/k8sapi" ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" - "github.com/tigera/operator/pkg/extensions/extensionstest" + "github.com/tigera/operator/pkg/imageoverride" "github.com/tigera/operator/pkg/render" rmeta "github.com/tigera/operator/pkg/render/common/meta" rtest "github.com/tigera/operator/pkg/render/common/test" "github.com/tigera/operator/pkg/tls/certificatemanagement" ) -// renderWindows renders the windows component and applies the registered -// enterprise modifier the way the componentHandler does, so enterprise tests -// exercise the integrated output (image overrides come from ResolveImages; the -// metrics service, env, volumes and mounts come from the modifier). -func renderWindows(cfg *render.WindowsConfiguration) []client.Object { - comp := render.Windows(cfg) - ExpectWithOffset(1, comp.ResolveImages(nil)).To(BeNil()) - objs, _ := comp.Objects() - rc := render.RenderContext{Installation: cfg.Installation} - out, _ := extensionstest.ApplyExtensions(ext, render.ComponentNameWindows, rc, objs, nil) - return out -} - var _ = Describe("Windows rendering tests", func() { var defaultInstance *operatorv1.InstallationSpec var typhaNodeTLS *render.TyphaNodeTLS @@ -127,7 +114,7 @@ var _ = Describe("Windows rendering tests", func() { ClusterDomain: defaultClusterDomain, TLS: typhaNodeTLS, VXLANVNI: 4096, - ImageOverrides: ext.Images(), + ImageOverrides: imageoverride.New(), } }) @@ -677,774 +664,229 @@ var _ = Describe("Windows rendering tests", func() { } }) - It("should render all resources for a default configuration using CalicoEnterprise", func() { - type testConf struct { - EnableBGP bool - EnableVXLAN bool + It("should render all resources when using Calico CNI on EKS", func() { + expectedResources := []struct { + name string + ns string + group string + version string + kind string + }{ + {name: "cni-config-windows", ns: common.CalicoNamespace, group: "", version: "v1", kind: "ConfigMap"}, + {name: common.WindowsDaemonSetName, ns: common.CalicoNamespace, group: "apps", version: "v1", kind: "DaemonSet"}, } - for _, testConfig := range []testConf{ - {true, false}, - {false, true}, - {true, true}, - } { - enableBGP := testConfig.EnableBGP - enableVXLAN := testConfig.EnableVXLAN - if enableBGP { - defaultInstance.CalicoNetwork.BGP = &bgpEnabled - } else { - defaultInstance.CalicoNetwork.BGP = &bgpDisabled - } + // defaultInstance.FlexVolumePath = "/usr/libexec/kubernetes/kubelet-plugins/volume/exec/" + defaultInstance.KubernetesProvider = operatorv1.ProviderEKS + defaultInstance.CNI = &operatorv1.CNISpec{Type: operatorv1.PluginCalico, IPAM: &operatorv1.IPAMSpec{ + Type: operatorv1.IPAMPluginAmazonVPC, + }} + defaultInstance.CalicoNetwork.BGP = &bgpDisabled + defaultInstance.CalicoNetwork.IPPools[0].Encapsulation = operatorv1.EncapsulationVXLAN + component := render.Windows(&cfg) + Expect(component.ResolveImages(nil)).To(BeNil()) + resources, _ := component.Objects() + Expect(len(resources)).To(Equal(len(expectedResources))) - if enableVXLAN { - defaultInstance.CalicoNetwork.IPPools[0].Encapsulation = operatorv1.EncapsulationVXLAN - } else { - defaultInstance.CalicoNetwork.IPPools[0].Encapsulation = operatorv1.EncapsulationNone - } - By(fmt.Sprintf("BGP enabled: %v, VXLAN enabled: %v", enableBGP, enableVXLAN), func() { - expectedResources := []struct { - name string - ns string - group string - version string - kind string - }{ - {name: "cni-config-windows", ns: common.CalicoNamespace, group: "", version: "v1", kind: "ConfigMap"}, - {name: common.WindowsDaemonSetName, ns: common.CalicoNamespace, group: "apps", version: "v1", kind: "DaemonSet"}, - {name: "calico-node-metrics-windows", ns: "calico-system", group: "", version: "v1", kind: "Service"}, - } - defaultInstance.Variant = operatorv1.CalicoEnterprise + // Should render the correct resources. + i := 0 + for _, expectedRes := range expectedResources { + rtest.ExpectResourceTypeAndObjectMetadata(resources[i], expectedRes.name, expectedRes.ns, expectedRes.group, expectedRes.version, expectedRes.kind) + i++ + } - resources := renderWindows(&cfg) - Expect(len(resources)).To(Equal(len(expectedResources))) + cniCmResource := rtest.GetResource(resources, "cni-config-windows", "calico-system", "", "v1", "ConfigMap") + Expect(cniCmResource).ToNot(BeNil()) + cniCm := cniCmResource.(*corev1.ConfigMap) + Expect(cniCm.Data["config"]).To(MatchJSON(`{ +"name": "Calico", +"cniVersion": "0.3.1", +"plugins": [ + { + "DNS": { + "Nameservers": [ + "10.96.0.10" + ], + "Search": [ + "svc.cluster.local" + ] + }, + "capabilities": { + "dns": true + }, + "datastore_type": "kubernetes", + "ipam": { + "subnet": "usePodCidr", + "type": "calico-ipam" + }, + "kubernetes": { + "k8s_api_root": "https://1.2.3.4:6443", + "kubeconfig": "c:/etc/cni/net.d/calico-kubeconfig" + }, + "log_file_max_age": 5, + "log_file_max_count": 5, + "log_file_max_size": 1, + "log_file_path": "c:/var/log/calico/cni/cni.log", + "log_level": "Debug", + "mode": "vxlan", + "mtu": 0, + "name": "Calico", + "nodename": "__KUBERNETES_NODE_NAME__", + "nodename_file": "__NODENAME_FILE__", + "nodename_file_optional": true, + "policies": [ + { + "Name": "EndpointPolicy", + "Value": { + "ExceptionList": [ + "10.96.0.0/12" + ], + "Type": "OutBoundNAT" + } + }, + { + "Name": "EndpointPolicy", + "Value": { + "DestinationPrefix": "10.96.0.0/12", + "NeedEncap": true, + "Type": "SDNROUTE" + } + } + ], + "policy": { + "type": "k8s" + }, + "type": "calico", + "vxlan_mac_prefix": "0E-2A", + "vxlan_vni": 4096, + "windows_loopback_DSR": "__DSR_SUPPORT__", + "windows_use_single_network": true + } +] +}`)) - // Should render the correct resources. - i := 0 - for _, expectedRes := range expectedResources { - rtest.ExpectResourceTypeAndObjectMetadata(resources[i], expectedRes.name, expectedRes.ns, expectedRes.group, expectedRes.version, expectedRes.kind) - i++ - } + dsResource := rtest.GetResource(resources, "calico-node-windows", "calico-system", "apps", "v1", "DaemonSet") + Expect(dsResource).ToNot(BeNil()) - // The DaemonSet should have the correct configuration. - ds := rtest.GetResource(resources, "calico-node-windows", "calico-system", "apps", "v1", "DaemonSet").(*appsv1.DaemonSet) + // The DaemonSet should have the correct configuration. + ds := dsResource.(*appsv1.DaemonSet) - // The pod template should have node critical priority - Expect(ds.Spec.Template.Spec.PriorityClassName).To(Equal(render.NodePriorityClassName)) + // The pod template should have node critical priority + Expect(ds.Spec.Template.Spec.PriorityClassName).To(Equal(render.NodePriorityClassName)) - // The calico-node-windows daemonset has 3 containers (felix, node and confd). - // confd is only instantiated if using BGP. - numContainers := 3 - if !enableBGP { - numContainers = 2 - } - Expect(ds.Spec.Template.Spec.Containers).To(HaveLen(numContainers)) - for _, container := range ds.Spec.Template.Spec.Containers { + // The calico-node-windows daemonset has 2 containers (felix, node) when using VXLAN + Expect(ds.Spec.Template.Spec.Containers).To(HaveLen(2)) - // Windows node image override results in correct image. - Expect(container.Image).To(Equal(components.TigeraRegistry + "tigera/node-windows:" + components.ComponentTigeraNodeWindows.Version)) - Expect(container.SecurityContext.Capabilities).To(BeNil()) - Expect(container.SecurityContext.Privileged).To(BeNil()) - Expect(container.SecurityContext.SELinuxOptions).To(BeNil()) - Expect(container.SecurityContext.WindowsOptions).To(Not(BeNil())) - Expect(container.SecurityContext.WindowsOptions.GMSACredentialSpecName).To(BeNil()) - Expect(container.SecurityContext.WindowsOptions.GMSACredentialSpec).To(BeNil()) - Expect(*container.SecurityContext.WindowsOptions.RunAsUserName).To(Equal("NT AUTHORITY\\system")) - Expect(*container.SecurityContext.WindowsOptions.HostProcess).To(BeTrue()) - Expect(container.SecurityContext.RunAsUser).To(BeNil()) - Expect(container.SecurityContext.RunAsGroup).To(BeNil()) - Expect(container.SecurityContext.RunAsNonRoot).To(BeNil()) - Expect(container.SecurityContext.ReadOnlyRootFilesystem).To(BeNil()) - Expect(container.SecurityContext.AllowPrivilegeEscalation).To(BeNil()) - Expect(container.SecurityContext.ProcMount).To(BeNil()) - Expect(container.SecurityContext.SeccompProfile).To(BeNil()) - } + for _, container := range ds.Spec.Template.Spec.Containers { + // Windows image override results in correct image. + Expect(container.Image).To(Equal(fmt.Sprintf("quay.io/%s%s:%s", components.CalicoImagePath, components.ComponentCalicoNodeWindows.Image, components.ComponentCalicoNodeWindows.Version))) + } - felixContainer := rtest.GetContainer(ds.Spec.Template.Spec.Containers, "felix") + // Validate correct number of init containers. + Expect(len(ds.Spec.Template.Spec.InitContainers)).To(Equal(2)) - // Windows node image override results in correct image. - Expect(felixContainer.Image).To(Equal(components.TigeraRegistry + "tigera/node-windows:" + components.ComponentTigeraNodeWindows.Version)) - Expect(felixContainer.SecurityContext.Capabilities).To(BeNil()) - Expect(felixContainer.SecurityContext.Privileged).To(BeNil()) - Expect(felixContainer.SecurityContext.SELinuxOptions).To(BeNil()) - Expect(felixContainer.SecurityContext.WindowsOptions).To(Not(BeNil())) - Expect(felixContainer.SecurityContext.WindowsOptions.GMSACredentialSpecName).To(BeNil()) - Expect(felixContainer.SecurityContext.WindowsOptions.GMSACredentialSpec).To(BeNil()) - Expect(*felixContainer.SecurityContext.WindowsOptions.RunAsUserName).To(Equal("NT AUTHORITY\\system")) - Expect(*felixContainer.SecurityContext.WindowsOptions.HostProcess).To(BeTrue()) - Expect(felixContainer.SecurityContext.RunAsUser).To(BeNil()) - Expect(felixContainer.SecurityContext.RunAsGroup).To(BeNil()) - Expect(felixContainer.SecurityContext.RunAsNonRoot).To(BeNil()) - Expect(felixContainer.SecurityContext.ReadOnlyRootFilesystem).To(BeNil()) - Expect(felixContainer.SecurityContext.AllowPrivilegeEscalation).To(BeNil()) - Expect(felixContainer.SecurityContext.ProcMount).To(BeNil()) - Expect(felixContainer.SecurityContext.SeccompProfile).To(BeNil()) + cniContainer := rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "install-cni") + rtest.ExpectEnv(cniContainer.Env, "CNI_NET_DIR", "/etc/cni/net.d") - nodeContainer := rtest.GetContainer(ds.Spec.Template.Spec.Containers, "node") + // CNI container uses image override. + Expect(cniContainer.Image).To(Equal(fmt.Sprintf("quay.io/%s%s:%s", components.CalicoImagePath, components.ComponentCalicoCNIWindows.Image, components.ComponentCalicoCNIWindows.Version))) - // Windows node image override results in correct image. - Expect(nodeContainer.Image).To(Equal(components.TigeraRegistry + "tigera/node-windows:" + components.ComponentTigeraNodeWindows.Version)) - Expect(nodeContainer.SecurityContext.Capabilities).To(BeNil()) - Expect(nodeContainer.SecurityContext.Privileged).To(BeNil()) - Expect(nodeContainer.SecurityContext.SELinuxOptions).To(BeNil()) - Expect(nodeContainer.SecurityContext.WindowsOptions).To(Not(BeNil())) - Expect(nodeContainer.SecurityContext.WindowsOptions.GMSACredentialSpecName).To(BeNil()) - Expect(nodeContainer.SecurityContext.WindowsOptions.GMSACredentialSpec).To(BeNil()) - Expect(*nodeContainer.SecurityContext.WindowsOptions.RunAsUserName).To(Equal("NT AUTHORITY\\system")) - Expect(*nodeContainer.SecurityContext.WindowsOptions.HostProcess).To(BeTrue()) - Expect(nodeContainer.SecurityContext.RunAsUser).To(BeNil()) - Expect(nodeContainer.SecurityContext.RunAsGroup).To(BeNil()) - Expect(nodeContainer.SecurityContext.RunAsNonRoot).To(BeNil()) - Expect(nodeContainer.SecurityContext.ReadOnlyRootFilesystem).To(BeNil()) - Expect(nodeContainer.SecurityContext.AllowPrivilegeEscalation).To(BeNil()) - Expect(nodeContainer.SecurityContext.ProcMount).To(BeNil()) - Expect(nodeContainer.SecurityContext.SeccompProfile).To(BeNil()) + // uninstall container uses image override. + uninstallContainer := rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "uninstall-calico") + Expect(uninstallContainer.Image).To(Equal(fmt.Sprintf("quay.io/%s%s:%s", components.CalicoImagePath, components.ComponentCalicoNodeWindows.Image, components.ComponentCalicoNodeWindows.Version))) - if enableBGP { - confdContainer := rtest.GetContainer(ds.Spec.Template.Spec.Containers, "confd") + // Verify env + expectedNodeEnv := []corev1.EnvVar{ + {Name: "CNI_PLUGIN_TYPE", Value: "Calico"}, + {Name: "DATASTORE_TYPE", Value: "kubernetes"}, + {Name: "WAIT_FOR_DATASTORE", Value: "true"}, + {Name: "CALICO_MANAGE_CNI", Value: "true"}, + {Name: "CALICO_NETWORKING_BACKEND", Value: "vxlan"}, + {Name: "CALICO_DISABLE_FILE_LOGGING", Value: "false"}, + {Name: "CLUSTER_TYPE", Value: "k8s,operator,ecs,windows"}, + {Name: "FELIX_DEFAULTENDPOINTTOHOSTACTION", Value: "ACCEPT"}, + {Name: "FELIX_HEALTHENABLED", Value: "true"}, + { + Name: "NODENAME", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{FieldPath: "spec.nodeName"}, + }, + }, + { + Name: "NAMESPACE", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.namespace"}, + }, + }, + {Name: "IP", Value: "autodetect"}, + {Name: "IP_AUTODETECTION_METHOD", Value: "first-found"}, + {Name: "IP6", Value: "none"}, + {Name: "FELIX_IPV6SUPPORT", Value: "false"}, + {Name: "FELIX_TYPHAK8SNAMESPACE", Value: "calico-system"}, + {Name: "FELIX_TYPHAK8SSERVICENAME", Value: "calico-typha"}, + {Name: "FELIX_TYPHACAFILE", Value: certificatemanagement.TrustedCertBundleMountPath}, + {Name: "FELIX_TYPHACERTFILE", Value: "/node-certs/tls.crt"}, + {Name: "FELIX_TYPHACN", Value: "typha-server"}, + {Name: "FELIX_TYPHAKEYFILE", Value: "/node-certs/tls.key"}, - // Windows node image override results in correct image. - Expect(confdContainer.Image).To(Equal(components.TigeraRegistry + "tigera/node-windows:" + components.ComponentTigeraNodeWindows.Version)) - Expect(confdContainer.SecurityContext.Capabilities).To(BeNil()) - Expect(confdContainer.SecurityContext.Privileged).To(BeNil()) - Expect(confdContainer.SecurityContext.SELinuxOptions).To(BeNil()) - Expect(confdContainer.SecurityContext.WindowsOptions).To(Not(BeNil())) - Expect(confdContainer.SecurityContext.WindowsOptions.GMSACredentialSpecName).To(BeNil()) - Expect(confdContainer.SecurityContext.WindowsOptions.GMSACredentialSpec).To(BeNil()) - Expect(*confdContainer.SecurityContext.WindowsOptions.RunAsUserName).To(Equal("NT AUTHORITY\\system")) - Expect(*confdContainer.SecurityContext.WindowsOptions.HostProcess).To(BeTrue()) - Expect(confdContainer.SecurityContext.RunAsUser).To(BeNil()) - Expect(confdContainer.SecurityContext.RunAsGroup).To(BeNil()) - Expect(confdContainer.SecurityContext.RunAsNonRoot).To(BeNil()) - Expect(confdContainer.SecurityContext.ReadOnlyRootFilesystem).To(BeNil()) - Expect(confdContainer.SecurityContext.AllowPrivilegeEscalation).To(BeNil()) - Expect(confdContainer.SecurityContext.ProcMount).To(BeNil()) - Expect(confdContainer.SecurityContext.SeccompProfile).To(BeNil()) - } - - // Validate correct number of init containers. - Expect(ds.Spec.Template.Spec.InitContainers).To(HaveLen(2)) - - // CNI container uses image override. - cniContainer := rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "install-cni") - rtest.ExpectEnv(cniContainer.Env, "CNI_NET_DIR", "/etc/cni/net.d") - Expect(cniContainer.Image).To(Equal(components.TigeraRegistry + "tigera/cni-windows:" + components.ComponentTigeraCNIWindows.Version)) - - Expect(cniContainer.SecurityContext.Capabilities).To(BeNil()) - Expect(cniContainer.SecurityContext.Privileged).To(BeNil()) - Expect(cniContainer.SecurityContext.SELinuxOptions).To(BeNil()) - Expect(cniContainer.SecurityContext.WindowsOptions).To(Not(BeNil())) - Expect(cniContainer.SecurityContext.WindowsOptions.GMSACredentialSpecName).To(BeNil()) - Expect(cniContainer.SecurityContext.WindowsOptions.GMSACredentialSpec).To(BeNil()) - Expect(*cniContainer.SecurityContext.WindowsOptions.RunAsUserName).To(Equal("NT AUTHORITY\\system")) - Expect(*cniContainer.SecurityContext.WindowsOptions.HostProcess).To(BeTrue()) - Expect(cniContainer.SecurityContext.RunAsUser).To(BeNil()) - Expect(cniContainer.SecurityContext.RunAsGroup).To(BeNil()) - Expect(cniContainer.SecurityContext.RunAsNonRoot).To(BeNil()) - Expect(cniContainer.SecurityContext.ReadOnlyRootFilesystem).To(BeNil()) - Expect(cniContainer.SecurityContext.AllowPrivilegeEscalation).To(BeNil()) - Expect(cniContainer.SecurityContext.ProcMount).To(BeNil()) - Expect(cniContainer.SecurityContext.SeccompProfile).To(BeNil()) - - // uninstall container uses image override. - uninstallContainer := rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "uninstall-calico") - Expect(uninstallContainer.Image).To(Equal(components.TigeraRegistry + "tigera/node-windows:" + components.ComponentTigeraNodeWindows.Version)) - - Expect(uninstallContainer.SecurityContext.Capabilities).To(BeNil()) - Expect(uninstallContainer.SecurityContext.Privileged).To(BeNil()) - Expect(uninstallContainer.SecurityContext.SELinuxOptions).To(BeNil()) - Expect(uninstallContainer.SecurityContext.WindowsOptions).To(Not(BeNil())) - Expect(uninstallContainer.SecurityContext.WindowsOptions.GMSACredentialSpecName).To(BeNil()) - Expect(uninstallContainer.SecurityContext.WindowsOptions.GMSACredentialSpec).To(BeNil()) - Expect(*uninstallContainer.SecurityContext.WindowsOptions.RunAsUserName).To(Equal("NT AUTHORITY\\system")) - Expect(*uninstallContainer.SecurityContext.WindowsOptions.HostProcess).To(BeTrue()) - Expect(uninstallContainer.SecurityContext.RunAsUser).To(BeNil()) - Expect(uninstallContainer.SecurityContext.RunAsGroup).To(BeNil()) - Expect(uninstallContainer.SecurityContext.RunAsNonRoot).To(BeNil()) - Expect(uninstallContainer.SecurityContext.ReadOnlyRootFilesystem).To(BeNil()) - Expect(uninstallContainer.SecurityContext.AllowPrivilegeEscalation).To(BeNil()) - Expect(uninstallContainer.SecurityContext.ProcMount).To(BeNil()) - Expect(uninstallContainer.SecurityContext.SeccompProfile).To(BeNil()) - - // Verify env - expectedNodeEnv := []corev1.EnvVar{ - {Name: "CNI_PLUGIN_TYPE", Value: "Calico"}, - {Name: "DATASTORE_TYPE", Value: "kubernetes"}, - {Name: "WAIT_FOR_DATASTORE", Value: "true"}, - {Name: "CALICO_MANAGE_CNI", Value: "true"}, - {Name: "CALICO_DISABLE_FILE_LOGGING", Value: "false"}, - {Name: "FELIX_DEFAULTENDPOINTTOHOSTACTION", Value: "ACCEPT"}, - {Name: "FELIX_HEALTHENABLED", Value: "true"}, - { - Name: "NODENAME", - ValueFrom: &corev1.EnvVarSource{ - FieldRef: &corev1.ObjectFieldSelector{FieldPath: "spec.nodeName"}, - }, - }, - { - Name: "NAMESPACE", - ValueFrom: &corev1.EnvVarSource{ - FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.namespace"}, - }, - }, - {Name: "IP", Value: "autodetect"}, - {Name: "IP_AUTODETECTION_METHOD", Value: "first-found"}, - {Name: "IP6", Value: "none"}, - {Name: "FELIX_IPV6SUPPORT", Value: "false"}, - {Name: "FELIX_TYPHAK8SNAMESPACE", Value: "calico-system"}, - {Name: "FELIX_TYPHAK8SSERVICENAME", Value: "calico-typha"}, - {Name: "FELIX_TYPHACAFILE", Value: certificatemanagement.TrustedCertBundleMountPath}, - {Name: "FELIX_TYPHACERTFILE", Value: "/node-certs/tls.crt"}, - {Name: "FELIX_TYPHACN", Value: "typha-server"}, - {Name: "FELIX_TYPHAKEYFILE", Value: "/node-certs/tls.key"}, - - {Name: "VXLAN_VNI", Value: "4096"}, - {Name: "VXLAN_ADAPTER", Value: ""}, - {Name: "KUBE_NETWORK", Value: "Calico.*"}, - {Name: "KUBERNETES_SERVICE_HOST", Value: "1.2.3.4"}, - {Name: "KUBERNETES_SERVICE_PORT", Value: "6443"}, - - // Tigera-specific envvars - {Name: "FELIX_PROMETHEUSREPORTERENABLED", Value: "true"}, - {Name: "FELIX_PROMETHEUSREPORTERPORT", Value: "9081"}, - {Name: "FELIX_FLOWLOGSFILEENABLED", Value: "true"}, - {Name: "FELIX_FLOWLOGSFILEINCLUDELABELS", Value: "true"}, - {Name: "FELIX_FLOWLOGSFILEINCLUDEPOLICIES", Value: "true"}, - {Name: "FELIX_FLOWLOGSFILEINCLUDESERVICE", Value: "true"}, - {Name: "FELIX_FLOWLOGSENABLENETWORKSETS", Value: "true"}, - {Name: "FELIX_FLOWLOGSCOLLECTPROCESSINFO", Value: "true"}, - {Name: "FELIX_DNSLOGSFILEENABLED", Value: "true"}, - {Name: "FELIX_DNSLOGSFILEPERNODELIMIT", Value: "1000"}, - } - - // Set CALICO_NETWORKING_BACKEND - if enableBGP { - expectedNodeEnv = append(expectedNodeEnv, corev1.EnvVar{Name: "CALICO_NETWORKING_BACKEND", Value: "windows-bgp"}) - } else if enableVXLAN { - expectedNodeEnv = append(expectedNodeEnv, corev1.EnvVar{Name: "CALICO_NETWORKING_BACKEND", Value: "vxlan"}) - } else { - expectedNodeEnv = append(expectedNodeEnv, corev1.EnvVar{Name: "CALICO_NETWORKING_BACKEND", Value: "none"}) - } - - // Set CLUSTER_TYPE - if enableBGP { - expectedNodeEnv = append(expectedNodeEnv, corev1.EnvVar{Name: "CLUSTER_TYPE", Value: "k8s,operator,bgp,windows"}) - } else { - expectedNodeEnv = append(expectedNodeEnv, corev1.EnvVar{Name: "CLUSTER_TYPE", Value: "k8s,operator,windows"}) - } - - Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "node").Env).To(ConsistOf(expectedNodeEnv)) - Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "felix").Env).To(ConsistOf(expectedNodeEnv)) - if enableBGP { - Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "confd").Env).To(ConsistOf(expectedNodeEnv)) - } - - // Expect the SECURITY_GROUP env variables to not be set - Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "node").Env).NotTo(ContainElement(gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{"Name": Equal("TIGERA_DEFAULT_SECURITY_GROUPS")}))) - Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "node").Env).NotTo(ContainElement(gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{"Name": Equal("TIGERA_POD_SECURITY_GROUP")}))) - Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "felix").Env).NotTo(ContainElement(gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{"Name": Equal("TIGERA_DEFAULT_SECURITY_GROUPS")}))) - Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "felix").Env).NotTo(ContainElement(gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{"Name": Equal("TIGERA_POD_SECURITY_GROUP")}))) - if enableBGP { - Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "confd").Env).NotTo(ContainElement(gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{"Name": Equal("TIGERA_DEFAULT_SECURITY_GROUPS")}))) - Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "confd").Env).NotTo(ContainElement(gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{"Name": Equal("TIGERA_POD_SECURITY_GROUP")}))) - } - - expectedCNIEnv := []corev1.EnvVar{ - {Name: "SLEEP", Value: "false"}, - {Name: "CNI_PLUGIN_TYPE", Value: "Calico"}, - {Name: "CNI_BIN_DIR", Value: "/host/opt/cni/bin"}, - {Name: "CNI_CONF_NAME", Value: "10-calico.conflist"}, - {Name: "CNI_NET_DIR", Value: "/etc/cni/net.d"}, - {Name: "VXLAN_VNI", Value: "4096"}, - { - Name: "KUBERNETES_NODE_NAME", - Value: "", - ValueFrom: &corev1.EnvVarSource{ - FieldRef: &corev1.ObjectFieldSelector{FieldPath: "spec.nodeName"}, - }, - }, - { - Name: "CNI_NETWORK_CONFIG", - ValueFrom: &corev1.EnvVarSource{ - ConfigMapKeyRef: &corev1.ConfigMapKeySelector{ - Key: "config", - LocalObjectReference: corev1.LocalObjectReference{ - Name: "cni-config-windows", - }, - }, - }, - }, - - {Name: "KUBERNETES_SERVICE_HOST", Value: "1.2.3.4"}, - {Name: "KUBERNETES_SERVICE_PORT", Value: "6443"}, - {Name: "KUBERNETES_SERVICE_CIDRS", Value: "10.96.0.0/12"}, - {Name: "KUBERNETES_DNS_SERVERS", Value: "10.96.0.10"}, - } - Expect(rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "install-cni").Env).To(ConsistOf(expectedCNIEnv)) - - expectedUninstallEnv := []corev1.EnvVar{ - {Name: "SLEEP", Value: "false"}, - {Name: "CNI_PLUGIN_TYPE", Value: "Calico"}, - {Name: "CNI_BIN_DIR", Value: "/host/opt/cni/bin"}, - {Name: "CNI_CONF_NAME", Value: "10-calico.conflist"}, - {Name: "CNI_NET_DIR", Value: "/host/etc/cni/net.d"}, - } - Expect(rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "uninstall-calico").Env).To(ConsistOf(expectedUninstallEnv)) - - // Verify volumes. - fileOrCreate := corev1.HostPathFileOrCreate - dirOrCreate := corev1.HostPathDirectoryOrCreate - expectedVols := []corev1.Volume{ - {Name: "lib-modules", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/lib/modules"}}}, - {Name: "var-run-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/run/calico", Type: &dirOrCreate}}}, - {Name: "var-lib-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/lib/calico", Type: &dirOrCreate}}}, - {Name: "xtables-lock", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/run/xtables.lock", Type: &fileOrCreate}}}, - {Name: "cni-bin-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/opt/cni/bin", Type: &dirOrCreate}}}, - {Name: "cni-net-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/etc/cni/net.d"}}}, - {Name: "cni-log-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico/cni", Type: &dirOrCreate}}}, - {Name: "policysync", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/run/nodeagent", Type: &dirOrCreate}}}, - { - Name: "tigera-ca-bundle", - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: "tigera-ca-bundle", - }, - }, - }, - }, - { - Name: render.NodeTLSSecretName, - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: render.NodeTLSSecretName, - DefaultMode: &defaultMode, - }, - }, - }, - {Name: "var-log-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico", Type: &dirOrCreate}}}, - } - Expect(ds.Spec.Template.Spec.Volumes).To(ConsistOf(expectedVols)) - - // Verify volume mounts. - expectedNodeVolumeMounts := []corev1.VolumeMount{ - {MountPath: "/host/etc/cni/net.d", Name: "cni-net-dir"}, - {MountPath: "/var/run/calico", Name: "var-run-calico"}, - {MountPath: "/var/lib/calico", Name: "var-lib-calico"}, - {MountPath: "c:/etc/pki/tls/certs", Name: "tigera-ca-bundle", ReadOnly: true}, - {MountPath: "c:/node-certs", Name: render.NodeTLSSecretName, ReadOnly: true}, - {MountPath: "/var/log/calico", Name: "var-log-calico", ReadOnly: false}, - } - Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "node").VolumeMounts).To(ConsistOf(expectedNodeVolumeMounts)) - Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "felix").VolumeMounts).To(ConsistOf(expectedNodeVolumeMounts)) - if enableBGP { - Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "confd").VolumeMounts).To(ConsistOf(expectedNodeVolumeMounts)) - } - - expectedCNIVolumeMounts := []corev1.VolumeMount{ - {MountPath: "/host/opt/cni/bin", Name: "cni-bin-dir"}, - {MountPath: "/host/etc/cni/net.d", Name: "cni-net-dir"}, - } - Expect(rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "install-cni").VolumeMounts).To(ConsistOf(expectedCNIVolumeMounts)) - - expectedUninstallVolumeMounts := []corev1.VolumeMount{ - {MountPath: "/host/opt/cni/bin", Name: "cni-bin-dir"}, - {MountPath: "/host/etc/cni/net.d", Name: "cni-net-dir"}, - } - Expect(rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "uninstall-calico").VolumeMounts).To(ConsistOf(expectedUninstallVolumeMounts)) - - // Verify tolerations. - Expect(ds.Spec.Template.Spec.Tolerations).To(ConsistOf(rmeta.TolerateAll)) - - // Verify readiness and liveness probes. - verifyWindowsProbesAndLifecycle(ds, false) - }) - } - }) - - It("should render all resources when using Calico CNI on EKS", func() { - expectedResources := []struct { - name string - ns string - group string - version string - kind string - }{ - {name: "cni-config-windows", ns: common.CalicoNamespace, group: "", version: "v1", kind: "ConfigMap"}, - {name: common.WindowsDaemonSetName, ns: common.CalicoNamespace, group: "apps", version: "v1", kind: "DaemonSet"}, - } - - // defaultInstance.FlexVolumePath = "/usr/libexec/kubernetes/kubelet-plugins/volume/exec/" - defaultInstance.KubernetesProvider = operatorv1.ProviderEKS - defaultInstance.CNI = &operatorv1.CNISpec{Type: operatorv1.PluginCalico, IPAM: &operatorv1.IPAMSpec{ - Type: operatorv1.IPAMPluginAmazonVPC, - }} - defaultInstance.CalicoNetwork.BGP = &bgpDisabled - defaultInstance.CalicoNetwork.IPPools[0].Encapsulation = operatorv1.EncapsulationVXLAN - component := render.Windows(&cfg) - Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() - Expect(len(resources)).To(Equal(len(expectedResources))) - - // Should render the correct resources. - i := 0 - for _, expectedRes := range expectedResources { - rtest.ExpectResourceTypeAndObjectMetadata(resources[i], expectedRes.name, expectedRes.ns, expectedRes.group, expectedRes.version, expectedRes.kind) - i++ - } - - cniCmResource := rtest.GetResource(resources, "cni-config-windows", "calico-system", "", "v1", "ConfigMap") - Expect(cniCmResource).ToNot(BeNil()) - cniCm := cniCmResource.(*corev1.ConfigMap) - Expect(cniCm.Data["config"]).To(MatchJSON(`{ -"name": "Calico", -"cniVersion": "0.3.1", -"plugins": [ - { - "DNS": { - "Nameservers": [ - "10.96.0.10" - ], - "Search": [ - "svc.cluster.local" - ] - }, - "capabilities": { - "dns": true - }, - "datastore_type": "kubernetes", - "ipam": { - "subnet": "usePodCidr", - "type": "calico-ipam" - }, - "kubernetes": { - "k8s_api_root": "https://1.2.3.4:6443", - "kubeconfig": "c:/etc/cni/net.d/calico-kubeconfig" - }, - "log_file_max_age": 5, - "log_file_max_count": 5, - "log_file_max_size": 1, - "log_file_path": "c:/var/log/calico/cni/cni.log", - "log_level": "Debug", - "mode": "vxlan", - "mtu": 0, - "name": "Calico", - "nodename": "__KUBERNETES_NODE_NAME__", - "nodename_file": "__NODENAME_FILE__", - "nodename_file_optional": true, - "policies": [ - { - "Name": "EndpointPolicy", - "Value": { - "ExceptionList": [ - "10.96.0.0/12" - ], - "Type": "OutBoundNAT" - } - }, - { - "Name": "EndpointPolicy", - "Value": { - "DestinationPrefix": "10.96.0.0/12", - "NeedEncap": true, - "Type": "SDNROUTE" - } - } - ], - "policy": { - "type": "k8s" - }, - "type": "calico", - "vxlan_mac_prefix": "0E-2A", - "vxlan_vni": 4096, - "windows_loopback_DSR": "__DSR_SUPPORT__", - "windows_use_single_network": true - } -] -}`)) - - dsResource := rtest.GetResource(resources, "calico-node-windows", "calico-system", "apps", "v1", "DaemonSet") - Expect(dsResource).ToNot(BeNil()) - - // The DaemonSet should have the correct configuration. - ds := dsResource.(*appsv1.DaemonSet) - - // The pod template should have node critical priority - Expect(ds.Spec.Template.Spec.PriorityClassName).To(Equal(render.NodePriorityClassName)) - - // The calico-node-windows daemonset has 2 containers (felix, node) when using VXLAN - Expect(ds.Spec.Template.Spec.Containers).To(HaveLen(2)) - - for _, container := range ds.Spec.Template.Spec.Containers { - // Windows image override results in correct image. - Expect(container.Image).To(Equal(fmt.Sprintf("quay.io/%s%s:%s", components.CalicoImagePath, components.ComponentCalicoNodeWindows.Image, components.ComponentCalicoNodeWindows.Version))) - } - - // Validate correct number of init containers. - Expect(len(ds.Spec.Template.Spec.InitContainers)).To(Equal(2)) - - cniContainer := rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "install-cni") - rtest.ExpectEnv(cniContainer.Env, "CNI_NET_DIR", "/etc/cni/net.d") - - // CNI container uses image override. - Expect(cniContainer.Image).To(Equal(fmt.Sprintf("quay.io/%s%s:%s", components.CalicoImagePath, components.ComponentCalicoCNIWindows.Image, components.ComponentCalicoCNIWindows.Version))) - - // uninstall container uses image override. - uninstallContainer := rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "uninstall-calico") - Expect(uninstallContainer.Image).To(Equal(fmt.Sprintf("quay.io/%s%s:%s", components.CalicoImagePath, components.ComponentCalicoNodeWindows.Image, components.ComponentCalicoNodeWindows.Version))) - - // Verify env - expectedNodeEnv := []corev1.EnvVar{ - {Name: "CNI_PLUGIN_TYPE", Value: "Calico"}, - {Name: "DATASTORE_TYPE", Value: "kubernetes"}, - {Name: "WAIT_FOR_DATASTORE", Value: "true"}, - {Name: "CALICO_MANAGE_CNI", Value: "true"}, - {Name: "CALICO_NETWORKING_BACKEND", Value: "vxlan"}, - {Name: "CALICO_DISABLE_FILE_LOGGING", Value: "false"}, - {Name: "CLUSTER_TYPE", Value: "k8s,operator,ecs,windows"}, - {Name: "FELIX_DEFAULTENDPOINTTOHOSTACTION", Value: "ACCEPT"}, - {Name: "FELIX_HEALTHENABLED", Value: "true"}, - { - Name: "NODENAME", - ValueFrom: &corev1.EnvVarSource{ - FieldRef: &corev1.ObjectFieldSelector{FieldPath: "spec.nodeName"}, - }, - }, - { - Name: "NAMESPACE", - ValueFrom: &corev1.EnvVarSource{ - FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.namespace"}, - }, - }, - {Name: "IP", Value: "autodetect"}, - {Name: "IP_AUTODETECTION_METHOD", Value: "first-found"}, - {Name: "IP6", Value: "none"}, - {Name: "FELIX_IPV6SUPPORT", Value: "false"}, - {Name: "FELIX_TYPHAK8SNAMESPACE", Value: "calico-system"}, - {Name: "FELIX_TYPHAK8SSERVICENAME", Value: "calico-typha"}, - {Name: "FELIX_TYPHACAFILE", Value: certificatemanagement.TrustedCertBundleMountPath}, - {Name: "FELIX_TYPHACERTFILE", Value: "/node-certs/tls.crt"}, - {Name: "FELIX_TYPHACN", Value: "typha-server"}, - {Name: "FELIX_TYPHAKEYFILE", Value: "/node-certs/tls.key"}, - - {Name: "VXLAN_VNI", Value: "4096"}, - {Name: "VXLAN_ADAPTER", Value: ""}, - {Name: "KUBE_NETWORK", Value: "Calico.*"}, - {Name: "KUBERNETES_SERVICE_HOST", Value: "1.2.3.4"}, - {Name: "KUBERNETES_SERVICE_PORT", Value: "6443"}, - } - - Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "node").Env).To(ConsistOf(expectedNodeEnv)) - Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "felix").Env).To(ConsistOf(expectedNodeEnv)) - - // Expect the SECURITY_GROUP env variables to not be set - Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "node").Env).NotTo(ContainElement(gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{"Name": Equal("TIGERA_DEFAULT_SECURITY_GROUPS")}))) - Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "node").Env).NotTo(ContainElement(gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{"Name": Equal("TIGERA_POD_SECURITY_GROUP")}))) - Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "felix").Env).NotTo(ContainElement(gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{"Name": Equal("TIGERA_DEFAULT_SECURITY_GROUPS")}))) - Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "felix").Env).NotTo(ContainElement(gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{"Name": Equal("TIGERA_POD_SECURITY_GROUP")}))) - - expectedCNIEnv := []corev1.EnvVar{ - {Name: "SLEEP", Value: "false"}, - {Name: "CNI_PLUGIN_TYPE", Value: "Calico"}, - {Name: "CNI_BIN_DIR", Value: "/host/opt/cni/bin"}, - {Name: "CNI_CONF_NAME", Value: "10-calico.conflist"}, - {Name: "CNI_NET_DIR", Value: "/etc/cni/net.d"}, - {Name: "VXLAN_VNI", Value: "4096"}, - - { - Name: "KUBERNETES_NODE_NAME", - Value: "", - ValueFrom: &corev1.EnvVarSource{ - FieldRef: &corev1.ObjectFieldSelector{FieldPath: "spec.nodeName"}, - }, - }, - { - Name: "CNI_NETWORK_CONFIG", - ValueFrom: &corev1.EnvVarSource{ - ConfigMapKeyRef: &corev1.ConfigMapKeySelector{ - Key: "config", - LocalObjectReference: corev1.LocalObjectReference{ - Name: "cni-config-windows", - }, - }, - }, - }, - - {Name: "KUBERNETES_SERVICE_HOST", Value: "1.2.3.4"}, - {Name: "KUBERNETES_SERVICE_PORT", Value: "6443"}, - {Name: "KUBERNETES_SERVICE_CIDRS", Value: "10.96.0.0/12"}, - {Name: "KUBERNETES_DNS_SERVERS", Value: "10.96.0.10"}, - } - Expect(rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "install-cni").Env).To(ConsistOf(expectedCNIEnv)) - - expectedUninstallEnv := []corev1.EnvVar{ - {Name: "SLEEP", Value: "false"}, - {Name: "CNI_PLUGIN_TYPE", Value: "Calico"}, - {Name: "CNI_BIN_DIR", Value: "/host/opt/cni/bin"}, - {Name: "CNI_CONF_NAME", Value: "10-calico.conflist"}, - {Name: "CNI_NET_DIR", Value: "/host/etc/cni/net.d"}, - } - Expect(rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "uninstall-calico").Env).To(ConsistOf(expectedUninstallEnv)) - - // Verify volumes. - fileOrCreate := corev1.HostPathFileOrCreate - dirOrCreate := corev1.HostPathDirectoryOrCreate - expectedVols := []corev1.Volume{ - {Name: "lib-modules", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/lib/modules"}}}, - {Name: "var-run-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/run/calico", Type: &dirOrCreate}}}, - {Name: "var-lib-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/lib/calico", Type: &dirOrCreate}}}, - {Name: "xtables-lock", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/run/xtables.lock", Type: &fileOrCreate}}}, - {Name: "cni-bin-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/opt/cni/bin", Type: &dirOrCreate}}}, - {Name: "cni-net-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/etc/cni/net.d"}}}, - {Name: "cni-log-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico/cni", Type: &dirOrCreate}}}, - {Name: "policysync", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/run/nodeagent", Type: &dirOrCreate}}}, - { - Name: "tigera-ca-bundle", - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: "tigera-ca-bundle", - }, - }, - }, - }, - { - Name: render.NodeTLSSecretName, - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: render.NodeTLSSecretName, - DefaultMode: &defaultMode, - }, - }, - }, - } - Expect(ds.Spec.Template.Spec.Volumes).To(ConsistOf(expectedVols)) - - // Verify volume mounts. - expectedNodeVolumeMounts := []corev1.VolumeMount{ - {MountPath: "/host/etc/cni/net.d", Name: "cni-net-dir"}, - {MountPath: "/var/run/calico", Name: "var-run-calico"}, - {MountPath: "/var/lib/calico", Name: "var-lib-calico"}, - {MountPath: "c:/etc/pki/tls/certs", Name: "tigera-ca-bundle", ReadOnly: true}, - {MountPath: "c:/node-certs", Name: render.NodeTLSSecretName, ReadOnly: true}, - {MountPath: "/var/log/calico/cni", Name: "cni-log-dir", ReadOnly: false}, - } - Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "node").VolumeMounts).To(ConsistOf(expectedNodeVolumeMounts)) - Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "felix").VolumeMounts).To(ConsistOf(expectedNodeVolumeMounts)) - - expectedCNIVolumeMounts := []corev1.VolumeMount{ - {MountPath: "/host/opt/cni/bin", Name: "cni-bin-dir"}, - {MountPath: "/host/etc/cni/net.d", Name: "cni-net-dir"}, - } - Expect(rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "install-cni").VolumeMounts).To(ConsistOf(expectedCNIVolumeMounts)) - - expectedUninstallVolumeMounts := []corev1.VolumeMount{ - {MountPath: "/host/opt/cni/bin", Name: "cni-bin-dir"}, - {MountPath: "/host/etc/cni/net.d", Name: "cni-net-dir"}, - } - Expect(rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "uninstall-calico").VolumeMounts).To(ConsistOf(expectedUninstallVolumeMounts)) - - // Verify tolerations. - Expect(ds.Spec.Template.Spec.Tolerations).To(ConsistOf(rmeta.TolerateAll)) - - // Verify readiness and liveness probes. - verifyWindowsProbesAndLifecycle(ds, true) - }) - - It("should properly render a configuration using the AmazonVPC CNI plugin", func() { - // Override the installation with one configured for AmazonVPC CNI. - amazonVPCInstalllation := &operatorv1.InstallationSpec{ - KubernetesProvider: operatorv1.ProviderEKS, - CNI: &operatorv1.CNISpec{Type: operatorv1.PluginAmazonVPC}, - ServiceCIDRs: []string{"10.96.0.0/12"}, - WindowsNodes: &operatorv1.WindowsNodeSpec{ - CNIBinDir: "/opt/cni/bin", - CNIConfigDir: "/etc/cni/net.d", - CNILogDir: "/var/log/calico/cni", - }, - } - cfg.Installation = amazonVPCInstalllation - - component := render.Windows(&cfg) - Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() - Expect(len(resources)).To(Equal(defaultNumExpectedResources - 1)) - - // Should render the correct resources. - Expect(rtest.GetResource(resources, "calico-node-windows", "calico-system", "apps", "v1", "DaemonSet")).ToNot(BeNil()) - dsResource := rtest.GetResource(resources, "calico-node-windows", "calico-system", "apps", "v1", "DaemonSet") - Expect(dsResource).ToNot(BeNil()) - - // Should not render CNI configuration. - cniCmResource := rtest.GetResource(resources, "cni-config-windows", "calico-system", "", "v1", "ConfigMap") - Expect(cniCmResource).To(BeNil()) - - // The DaemonSet should have the correct configuration. - ds := dsResource.(*appsv1.DaemonSet) - - // The pod template should have node critical priority - Expect(ds.Spec.Template.Spec.PriorityClassName).To(Equal(render.NodePriorityClassName)) + {Name: "VXLAN_VNI", Value: "4096"}, + {Name: "VXLAN_ADAPTER", Value: ""}, + {Name: "KUBE_NETWORK", Value: "Calico.*"}, + {Name: "KUBERNETES_SERVICE_HOST", Value: "1.2.3.4"}, + {Name: "KUBERNETES_SERVICE_PORT", Value: "6443"}, + } - // CNI install container should not be present. - cniContainer := rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "install-cni") - Expect(cniContainer).To(BeNil()) + Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "node").Env).To(ConsistOf(expectedNodeEnv)) + Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "felix").Env).To(ConsistOf(expectedNodeEnv)) - // uninstall container should still be present. - uninstallContainer := rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "uninstall-calico") - Expect(uninstallContainer).NotTo(BeNil()) + // Expect the SECURITY_GROUP env variables to not be set + Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "node").Env).NotTo(ContainElement(gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{"Name": Equal("TIGERA_DEFAULT_SECURITY_GROUPS")}))) + Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "node").Env).NotTo(ContainElement(gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{"Name": Equal("TIGERA_POD_SECURITY_GROUP")}))) + Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "felix").Env).NotTo(ContainElement(gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{"Name": Equal("TIGERA_DEFAULT_SECURITY_GROUPS")}))) + Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "felix").Env).NotTo(ContainElement(gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{"Name": Equal("TIGERA_POD_SECURITY_GROUP")}))) - // Validate correct number of init containers. - Expect(len(ds.Spec.Template.Spec.InitContainers)).To(Equal(1)) + expectedCNIEnv := []corev1.EnvVar{ + {Name: "SLEEP", Value: "false"}, + {Name: "CNI_PLUGIN_TYPE", Value: "Calico"}, + {Name: "CNI_BIN_DIR", Value: "/host/opt/cni/bin"}, + {Name: "CNI_CONF_NAME", Value: "10-calico.conflist"}, + {Name: "CNI_NET_DIR", Value: "/etc/cni/net.d"}, + {Name: "VXLAN_VNI", Value: "4096"}, - // Verify env - expectedNodeEnv := []corev1.EnvVar{ - {Name: "CNI_PLUGIN_TYPE", Value: "AmazonVPC"}, - {Name: "DATASTORE_TYPE", Value: "kubernetes"}, - {Name: "WAIT_FOR_DATASTORE", Value: "true"}, - {Name: "CALICO_NETWORKING_BACKEND", Value: "none"}, - {Name: "CALICO_DISABLE_FILE_LOGGING", Value: "false"}, - {Name: "CLUSTER_TYPE", Value: "k8s,operator,ecs,windows"}, - {Name: "IP", Value: "none"}, - {Name: "IP6", Value: "none"}, - {Name: "CALICO_MANAGE_CNI", Value: "false"}, - {Name: "FELIX_DEFAULTENDPOINTTOHOSTACTION", Value: "ACCEPT"}, - {Name: "FELIX_IPV6SUPPORT", Value: "false"}, - {Name: "FELIX_HEALTHENABLED", Value: "true"}, { - Name: "NODENAME", + Name: "KUBERNETES_NODE_NAME", + Value: "", ValueFrom: &corev1.EnvVarSource{ FieldRef: &corev1.ObjectFieldSelector{FieldPath: "spec.nodeName"}, }, }, { - Name: "NAMESPACE", + Name: "CNI_NETWORK_CONFIG", ValueFrom: &corev1.EnvVarSource{ - FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.namespace"}, + ConfigMapKeyRef: &corev1.ConfigMapKeySelector{ + Key: "config", + LocalObjectReference: corev1.LocalObjectReference{ + Name: "cni-config-windows", + }, + }, }, }, - {Name: "FELIX_TYPHAK8SNAMESPACE", Value: "calico-system"}, - {Name: "FELIX_TYPHAK8SSERVICENAME", Value: "calico-typha"}, - {Name: "FELIX_TYPHACAFILE", Value: certificatemanagement.TrustedCertBundleMountPath}, - {Name: "FELIX_TYPHACERTFILE", Value: "/node-certs/tls.crt"}, - {Name: "FELIX_TYPHACN", Value: "typha-server"}, - {Name: "FELIX_TYPHAKEYFILE", Value: "/node-certs/tls.key"}, - {Name: "FELIX_ROUTESOURCE", Value: "WorkloadIPs"}, - {Name: "FELIX_BPFEXTTOSERVICECONNMARK", Value: "0x80"}, - {Name: "VXLAN_VNI", Value: "4096"}, - {Name: "VXLAN_ADAPTER", Value: ""}, - {Name: "KUBE_NETWORK", Value: "vpc.*"}, {Name: "KUBERNETES_SERVICE_HOST", Value: "1.2.3.4"}, {Name: "KUBERNETES_SERVICE_PORT", Value: "6443"}, + {Name: "KUBERNETES_SERVICE_CIDRS", Value: "10.96.0.0/12"}, + {Name: "KUBERNETES_DNS_SERVERS", Value: "10.96.0.10"}, } + Expect(rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "install-cni").Env).To(ConsistOf(expectedCNIEnv)) - Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "node").Env).To(ConsistOf(expectedNodeEnv)) - Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "felix").Env).To(ConsistOf(expectedNodeEnv)) - - // Expect the SECURITY_GROUP env variables to not be set - Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "node").Env).NotTo(ContainElement(gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{"Name": Equal("TIGERA_DEFAULT_SECURITY_GROUPS")}))) - Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "node").Env).NotTo(ContainElement(gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{"Name": Equal("TIGERA_POD_SECURITY_GROUP")}))) - Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "felix").Env).NotTo(ContainElement(gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{"Name": Equal("TIGERA_DEFAULT_SECURITY_GROUPS")}))) - Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "felix").Env).NotTo(ContainElement(gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{"Name": Equal("TIGERA_POD_SECURITY_GROUP")}))) + expectedUninstallEnv := []corev1.EnvVar{ + {Name: "SLEEP", Value: "false"}, + {Name: "CNI_PLUGIN_TYPE", Value: "Calico"}, + {Name: "CNI_BIN_DIR", Value: "/host/opt/cni/bin"}, + {Name: "CNI_CONF_NAME", Value: "10-calico.conflist"}, + {Name: "CNI_NET_DIR", Value: "/host/etc/cni/net.d"}, + } + Expect(rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "uninstall-calico").Env).To(ConsistOf(expectedUninstallEnv)) // Verify volumes. fileOrCreate := corev1.HostPathFileOrCreate @@ -1454,6 +896,9 @@ var _ = Describe("Windows rendering tests", func() { {Name: "var-run-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/run/calico", Type: &dirOrCreate}}}, {Name: "var-lib-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/lib/calico", Type: &dirOrCreate}}}, {Name: "xtables-lock", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/run/xtables.lock", Type: &fileOrCreate}}}, + {Name: "cni-bin-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/opt/cni/bin", Type: &dirOrCreate}}}, + {Name: "cni-net-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/etc/cni/net.d"}}}, + {Name: "cni-log-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico/cni", Type: &dirOrCreate}}}, {Name: "policysync", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/run/nodeagent", Type: &dirOrCreate}}}, { Name: "tigera-ca-bundle", @@ -1479,178 +924,93 @@ var _ = Describe("Windows rendering tests", func() { // Verify volume mounts. expectedNodeVolumeMounts := []corev1.VolumeMount{ + {MountPath: "/host/etc/cni/net.d", Name: "cni-net-dir"}, {MountPath: "/var/run/calico", Name: "var-run-calico"}, {MountPath: "/var/lib/calico", Name: "var-lib-calico"}, {MountPath: "c:/etc/pki/tls/certs", Name: "tigera-ca-bundle", ReadOnly: true}, {MountPath: "c:/node-certs", Name: render.NodeTLSSecretName, ReadOnly: true}, + {MountPath: "/var/log/calico/cni", Name: "cni-log-dir", ReadOnly: false}, } Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "node").VolumeMounts).To(ConsistOf(expectedNodeVolumeMounts)) Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "felix").VolumeMounts).To(ConsistOf(expectedNodeVolumeMounts)) - // Verify tolerations. - Expect(ds.Spec.Template.Spec.Tolerations).To(ConsistOf(rmeta.TolerateAll)) - - // Verify readiness and liveness probes. - verifyWindowsProbesAndLifecycle(ds, true) - }) - - DescribeTable("should properly render configuration using non-Calico CNI plugin", - func(cni operatorv1.CNIPluginType, ipam operatorv1.IPAMPluginType) { - installlation := &operatorv1.InstallationSpec{ - CNI: &operatorv1.CNISpec{ - Type: cni, - IPAM: &operatorv1.IPAMSpec{Type: ipam}, - }, - WindowsNodes: &operatorv1.WindowsNodeSpec{ - CNIBinDir: "/opt/cni/bin", - CNIConfigDir: "/etc/cni/net.d", - CNILogDir: "/var/log/calico/cni", - }, - } - cfg.Installation = installlation - - component := render.Windows(&cfg) - Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() - - // Should render the correct resources. - Expect(rtest.GetResource(resources, "calico-node-windows", "calico-system", "apps", "v1", "DaemonSet")).ToNot(BeNil()) - dsResource := rtest.GetResource(resources, "calico-node-windows", "calico-system", "apps", "v1", "DaemonSet") - Expect(dsResource).ToNot(BeNil()) - - // Should not render CNI configuration. - cniCmResource := rtest.GetResource(resources, "cni-config-windows", "calico-system", "", "v1", "ConfigMap") - Expect(cniCmResource).To(BeNil()) - - // The DaemonSet should have the correct configuration. - ds := dsResource.(*appsv1.DaemonSet) - - // The pod template should have node critical priority - Expect(ds.Spec.Template.Spec.PriorityClassName).To(Equal(render.NodePriorityClassName)) - - // CNI install container should not be present. - cniContainer := rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "install-cni") - Expect(cniContainer).To(BeNil()) - // Validate correct number of init containers. - Expect(len(ds.Spec.Template.Spec.InitContainers)).To(Equal(1)) - - // Verify env - expectedEnvs := []corev1.EnvVar{ - {Name: "CNI_PLUGIN_TYPE", Value: string(cni)}, - {Name: "CALICO_NETWORKING_BACKEND", Value: "none"}, - {Name: "FELIX_DEFAULTENDPOINTTOHOSTACTION", Value: "ACCEPT"}, - } - for _, expected := range expectedEnvs { - Expect(ds.Spec.Template.Spec.Containers[0].Env).To(ContainElement(expected)) - } - - // Verify readiness and liveness probes. - verifyWindowsProbesAndLifecycle(ds, true) - }, - Entry("GKE", operatorv1.PluginGKE, operatorv1.IPAMPluginHostLocal), - Entry("AmazonVPC", operatorv1.PluginAmazonVPC, operatorv1.IPAMPluginAmazonVPC), - Entry("AzureVNET", operatorv1.PluginAzureVNET, operatorv1.IPAMPluginAzureVNET), - ) - - It("should render all resources when running on openshift", func() { - expectedResources := []struct { - name string - ns string - group string - version string - kind string - }{ - {name: "cni-config-windows", ns: common.CalicoNamespace, group: "", version: "v1", kind: "ConfigMap"}, - {name: common.WindowsDaemonSetName, ns: common.CalicoNamespace, group: "apps", version: "v1", kind: "DaemonSet"}, - } - - defaultInstance.FlexVolumePath = "/etc/kubernetes/kubelet-plugins/volume/exec/" - defaultInstance.KubernetesProvider = operatorv1.ProviderOpenShift - component := render.Windows(&cfg) - Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() - Expect(len(resources)).To(Equal(len(expectedResources))) - - // Should render the correct resources. - i := 0 - for _, expectedRes := range expectedResources { - rtest.ExpectResourceTypeAndObjectMetadata(resources[i], expectedRes.name, expectedRes.ns, expectedRes.group, expectedRes.version, expectedRes.kind) - i++ - } - - // The DaemonSet should have the correct configuration. - ds := rtest.GetResource(resources, "calico-node-windows", "calico-system", "apps", "v1", "DaemonSet").(*appsv1.DaemonSet) - - // The pod template should have node critical priority - Expect(ds.Spec.Template.Spec.PriorityClassName).To(Equal(render.NodePriorityClassName)) - - felixContainer := rtest.GetContainer(ds.Spec.Template.Spec.Containers, "felix") - Expect(felixContainer.Image).To(Equal(fmt.Sprintf("quay.io/%s%s:%s", components.CalicoImagePath, components.ComponentCalicoNodeWindows.Image, components.ComponentCalicoNodeWindows.Version))) - nodeContainer := rtest.GetContainer(ds.Spec.Template.Spec.Containers, "node") - Expect(nodeContainer.Image).To(Equal(fmt.Sprintf("quay.io/%s%s:%s", components.CalicoImagePath, components.ComponentCalicoNodeWindows.Image, components.ComponentCalicoNodeWindows.Version))) - cniContainer := rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "install-cni") - Expect(cniContainer.Image).To(Equal(fmt.Sprintf("quay.io/%s%s:%s", components.CalicoImagePath, components.ComponentCalicoCNIWindows.Image, components.ComponentCalicoCNIWindows.Version))) - uninstallContainer := rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "uninstall-calico") - Expect(uninstallContainer.Image).To(Equal(fmt.Sprintf("quay.io/%s%s:%s", components.CalicoImagePath, components.ComponentCalicoNodeWindows.Image, components.ComponentCalicoNodeWindows.Version))) - - // FIXME: confirm openshift CNI path defaults expectedCNIVolumeMounts := []corev1.VolumeMount{ {MountPath: "/host/opt/cni/bin", Name: "cni-bin-dir"}, {MountPath: "/host/etc/cni/net.d", Name: "cni-net-dir"}, } Expect(rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "install-cni").VolumeMounts).To(ConsistOf(expectedCNIVolumeMounts)) - // FIXME: confirm openshift CNI path defaults expectedUninstallVolumeMounts := []corev1.VolumeMount{ {MountPath: "/host/opt/cni/bin", Name: "cni-bin-dir"}, {MountPath: "/host/etc/cni/net.d", Name: "cni-net-dir"}, } Expect(rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "uninstall-calico").VolumeMounts).To(ConsistOf(expectedUninstallVolumeMounts)) - // Verify volumes - // FIXME: confirm openshift CNI path defaults - fileOrCreate := corev1.HostPathFileOrCreate - dirOrCreate := corev1.HostPathDirectoryOrCreate - expectedVols := []corev1.Volume{ - {Name: "lib-modules", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/lib/modules"}}}, - {Name: "var-run-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/run/calico", Type: &dirOrCreate}}}, - {Name: "var-lib-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/lib/calico", Type: &dirOrCreate}}}, - {Name: "xtables-lock", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/run/xtables.lock", Type: &fileOrCreate}}}, - {Name: "cni-bin-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/opt/cni/bin", Type: &dirOrCreate}}}, - {Name: "cni-net-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/etc/cni/net.d"}}}, - {Name: "cni-log-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico/cni", Type: &dirOrCreate}}}, - {Name: "policysync", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/run/nodeagent", Type: &dirOrCreate}}}, - { - Name: "tigera-ca-bundle", - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: "tigera-ca-bundle", - }, - }, - }, - }, - { - Name: render.NodeTLSSecretName, - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: render.NodeTLSSecretName, - DefaultMode: &defaultMode, - }, - }, + // Verify tolerations. + Expect(ds.Spec.Template.Spec.Tolerations).To(ConsistOf(rmeta.TolerateAll)) + + // Verify readiness and liveness probes. + verifyWindowsProbesAndLifecycle(ds, true) + }) + + It("should properly render a configuration using the AmazonVPC CNI plugin", func() { + // Override the installation with one configured for AmazonVPC CNI. + amazonVPCInstalllation := &operatorv1.InstallationSpec{ + KubernetesProvider: operatorv1.ProviderEKS, + CNI: &operatorv1.CNISpec{Type: operatorv1.PluginAmazonVPC}, + ServiceCIDRs: []string{"10.96.0.0/12"}, + WindowsNodes: &operatorv1.WindowsNodeSpec{ + CNIBinDir: "/opt/cni/bin", + CNIConfigDir: "/etc/cni/net.d", + CNILogDir: "/var/log/calico/cni", }, } - Expect(ds.Spec.Template.Spec.Volumes).To(ConsistOf(expectedVols)) + cfg.Installation = amazonVPCInstalllation + + component := render.Windows(&cfg) + Expect(component.ResolveImages(nil)).To(BeNil()) + resources, _ := component.Objects() + Expect(len(resources)).To(Equal(defaultNumExpectedResources - 1)) + + // Should render the correct resources. + Expect(rtest.GetResource(resources, "calico-node-windows", "calico-system", "apps", "v1", "DaemonSet")).ToNot(BeNil()) + dsResource := rtest.GetResource(resources, "calico-node-windows", "calico-system", "apps", "v1", "DaemonSet") + Expect(dsResource).ToNot(BeNil()) + + // Should not render CNI configuration. + cniCmResource := rtest.GetResource(resources, "cni-config-windows", "calico-system", "", "v1", "ConfigMap") + Expect(cniCmResource).To(BeNil()) + + // The DaemonSet should have the correct configuration. + ds := dsResource.(*appsv1.DaemonSet) + + // The pod template should have node critical priority + Expect(ds.Spec.Template.Spec.PriorityClassName).To(Equal(render.NodePriorityClassName)) + + // CNI install container should not be present. + cniContainer := rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "install-cni") + Expect(cniContainer).To(BeNil()) + + // uninstall container should still be present. + uninstallContainer := rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "uninstall-calico") + Expect(uninstallContainer).NotTo(BeNil()) + + // Validate correct number of init containers. + Expect(len(ds.Spec.Template.Spec.InitContainers)).To(Equal(1)) + // Verify env expectedNodeEnv := []corev1.EnvVar{ - {Name: "CNI_PLUGIN_TYPE", Value: "Calico"}, + {Name: "CNI_PLUGIN_TYPE", Value: "AmazonVPC"}, {Name: "DATASTORE_TYPE", Value: "kubernetes"}, {Name: "WAIT_FOR_DATASTORE", Value: "true"}, - {Name: "CALICO_MANAGE_CNI", Value: "true"}, - {Name: "CALICO_NETWORKING_BACKEND", Value: "windows-bgp"}, - {Name: "CLUSTER_TYPE", Value: "k8s,operator,openshift,bgp,windows"}, + {Name: "CALICO_NETWORKING_BACKEND", Value: "none"}, {Name: "CALICO_DISABLE_FILE_LOGGING", Value: "false"}, + {Name: "CLUSTER_TYPE", Value: "k8s,operator,ecs,windows"}, + {Name: "IP", Value: "none"}, + {Name: "IP6", Value: "none"}, + {Name: "CALICO_MANAGE_CNI", Value: "false"}, {Name: "FELIX_DEFAULTENDPOINTTOHOSTACTION", Value: "ACCEPT"}, + {Name: "FELIX_IPV6SUPPORT", Value: "false"}, {Name: "FELIX_HEALTHENABLED", Value: "true"}, { Name: "NODENAME", @@ -1664,89 +1024,32 @@ var _ = Describe("Windows rendering tests", func() { FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.namespace"}, }, }, - {Name: "IP", Value: "autodetect"}, - {Name: "IP_AUTODETECTION_METHOD", Value: "first-found"}, - {Name: "IP6", Value: "none"}, - {Name: "FELIX_IPV6SUPPORT", Value: "false"}, {Name: "FELIX_TYPHAK8SNAMESPACE", Value: "calico-system"}, {Name: "FELIX_TYPHAK8SSERVICENAME", Value: "calico-typha"}, {Name: "FELIX_TYPHACAFILE", Value: certificatemanagement.TrustedCertBundleMountPath}, {Name: "FELIX_TYPHACERTFILE", Value: "/node-certs/tls.crt"}, {Name: "FELIX_TYPHACN", Value: "typha-server"}, {Name: "FELIX_TYPHAKEYFILE", Value: "/node-certs/tls.key"}, + {Name: "FELIX_ROUTESOURCE", Value: "WorkloadIPs"}, + {Name: "FELIX_BPFEXTTOSERVICECONNMARK", Value: "0x80"}, - // Calico Windows specific envvars {Name: "VXLAN_VNI", Value: "4096"}, {Name: "VXLAN_ADAPTER", Value: ""}, - {Name: "KUBE_NETWORK", Value: "Calico.*"}, + {Name: "KUBE_NETWORK", Value: "vpc.*"}, {Name: "KUBERNETES_SERVICE_HOST", Value: "1.2.3.4"}, {Name: "KUBERNETES_SERVICE_PORT", Value: "6443"}, } Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "node").Env).To(ConsistOf(expectedNodeEnv)) Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "felix").Env).To(ConsistOf(expectedNodeEnv)) - Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "confd").Env).To(ConsistOf(expectedNodeEnv)) - - verifyWindowsProbesAndLifecycle(ds, true) - }) - - It("should render all resources when variant is CalicoEnterprise and running on openshift", func() { - expectedResources := []struct { - name string - ns string - group string - version string - kind string - }{ - {name: "cni-config-windows", ns: common.CalicoNamespace, group: "", version: "v1", kind: "ConfigMap"}, - {name: common.WindowsDaemonSetName, ns: common.CalicoNamespace, group: "apps", version: "v1", kind: "DaemonSet"}, - {name: "calico-node-metrics-windows", ns: "calico-system", group: "", version: "v1", kind: "Service"}, - } - - defaultInstance.Variant = operatorv1.CalicoEnterprise - defaultInstance.KubernetesProvider = operatorv1.ProviderOpenShift - - resources := renderWindows(&cfg) - Expect(len(resources)).To(Equal(len(expectedResources))) - // Should render the correct resources. - i := 0 - for _, expectedRes := range expectedResources { - rtest.ExpectResourceTypeAndObjectMetadata(resources[i], expectedRes.name, expectedRes.ns, expectedRes.group, expectedRes.version, expectedRes.kind) - i++ - } - - // The DaemonSet should have the correct configuration. - ds := rtest.GetResource(resources, "calico-node-windows", "calico-system", "apps", "v1", "DaemonSet").(*appsv1.DaemonSet) - - // The pod template should have node critical priority - Expect(ds.Spec.Template.Spec.PriorityClassName).To(Equal(render.NodePriorityClassName)) - - felixContainer := rtest.GetContainer(ds.Spec.Template.Spec.Containers, "felix") - Expect(felixContainer.Image).To(Equal(fmt.Sprintf("%s%s%s:%s", components.TigeraRegistry, components.TigeraImagePath, components.ComponentTigeraNodeWindows.Image, components.ComponentTigeraNodeWindows.Version))) - nodeContainer := rtest.GetContainer(ds.Spec.Template.Spec.Containers, "node") - Expect(nodeContainer.Image).To(Equal(fmt.Sprintf("%s%s%s:%s", components.TigeraRegistry, components.TigeraImagePath, components.ComponentTigeraNodeWindows.Image, components.ComponentTigeraNodeWindows.Version))) - cniContainer := rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "install-cni") - Expect(cniContainer.Image).To(Equal(fmt.Sprintf("%s%s%s:%s", components.TigeraRegistry, components.TigeraImagePath, components.ComponentTigeraCNIWindows.Image, components.ComponentTigeraCNIWindows.Version))) - uninstallContainer := rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "uninstall-calico") - Expect(uninstallContainer.Image).To(Equal(fmt.Sprintf("%s%s%s:%s", components.TigeraRegistry, components.TigeraImagePath, components.ComponentTigeraNodeWindows.Image, components.ComponentTigeraNodeWindows.Version))) - - // FIXME: confirm openshift CNI path defaults - expectedCNIVolumeMounts := []corev1.VolumeMount{ - {MountPath: "/host/opt/cni/bin", Name: "cni-bin-dir"}, - {MountPath: "/host/etc/cni/net.d", Name: "cni-net-dir"}, - } - Expect(rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "install-cni").VolumeMounts).To(ConsistOf(expectedCNIVolumeMounts)) - - // FIXME: confirm openshift CNI path defaults - expectedUninstallVolumeMounts := []corev1.VolumeMount{ - {MountPath: "/host/opt/cni/bin", Name: "cni-bin-dir"}, - {MountPath: "/host/etc/cni/net.d", Name: "cni-net-dir"}, - } - Expect(rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "uninstall-calico").VolumeMounts).To(ConsistOf(expectedUninstallVolumeMounts)) + // Expect the SECURITY_GROUP env variables to not be set + Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "node").Env).NotTo(ContainElement(gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{"Name": Equal("TIGERA_DEFAULT_SECURITY_GROUPS")}))) + Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "node").Env).NotTo(ContainElement(gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{"Name": Equal("TIGERA_POD_SECURITY_GROUP")}))) + Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "felix").Env).NotTo(ContainElement(gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{"Name": Equal("TIGERA_DEFAULT_SECURITY_GROUPS")}))) + Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "felix").Env).NotTo(ContainElement(gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{"Name": Equal("TIGERA_POD_SECURITY_GROUP")}))) - // Verify volumes - // FIXME: confirm openshift CNI path defaults + // Verify volumes. fileOrCreate := corev1.HostPathFileOrCreate dirOrCreate := corev1.HostPathDirectoryOrCreate expectedVols := []corev1.Volume{ @@ -1754,9 +1057,6 @@ var _ = Describe("Windows rendering tests", func() { {Name: "var-run-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/run/calico", Type: &dirOrCreate}}}, {Name: "var-lib-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/lib/calico", Type: &dirOrCreate}}}, {Name: "xtables-lock", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/run/xtables.lock", Type: &fileOrCreate}}}, - {Name: "cni-bin-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/opt/cni/bin", Type: &dirOrCreate}}}, - {Name: "cni-net-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/etc/cni/net.d"}}}, - {Name: "cni-log-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico/cni", Type: &dirOrCreate}}}, {Name: "policysync", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/run/nodeagent", Type: &dirOrCreate}}}, { Name: "tigera-ca-bundle", @@ -1777,73 +1077,85 @@ var _ = Describe("Windows rendering tests", func() { }, }, }, - {Name: "var-log-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico", Type: &dirOrCreate}}}, } Expect(ds.Spec.Template.Spec.Volumes).To(ConsistOf(expectedVols)) - expectedNodeEnv := []corev1.EnvVar{ - // Default envvars. - {Name: "CNI_PLUGIN_TYPE", Value: "Calico"}, - {Name: "DATASTORE_TYPE", Value: "kubernetes"}, - {Name: "WAIT_FOR_DATASTORE", Value: "true"}, - {Name: "CALICO_MANAGE_CNI", Value: "true"}, - {Name: "CALICO_NETWORKING_BACKEND", Value: "windows-bgp"}, - {Name: "CLUSTER_TYPE", Value: "k8s,operator,openshift,bgp,windows"}, - {Name: "CALICO_DISABLE_FILE_LOGGING", Value: "false"}, - {Name: "FELIX_DEFAULTENDPOINTTOHOSTACTION", Value: "ACCEPT"}, - {Name: "FELIX_HEALTHENABLED", Value: "true"}, - { - Name: "NODENAME", - ValueFrom: &corev1.EnvVarSource{ - FieldRef: &corev1.ObjectFieldSelector{FieldPath: "spec.nodeName"}, + // Verify volume mounts. + expectedNodeVolumeMounts := []corev1.VolumeMount{ + {MountPath: "/var/run/calico", Name: "var-run-calico"}, + {MountPath: "/var/lib/calico", Name: "var-lib-calico"}, + {MountPath: "c:/etc/pki/tls/certs", Name: "tigera-ca-bundle", ReadOnly: true}, + {MountPath: "c:/node-certs", Name: render.NodeTLSSecretName, ReadOnly: true}, + } + Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "node").VolumeMounts).To(ConsistOf(expectedNodeVolumeMounts)) + Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "felix").VolumeMounts).To(ConsistOf(expectedNodeVolumeMounts)) + + // Verify tolerations. + Expect(ds.Spec.Template.Spec.Tolerations).To(ConsistOf(rmeta.TolerateAll)) + + // Verify readiness and liveness probes. + verifyWindowsProbesAndLifecycle(ds, true) + }) + + DescribeTable("should properly render configuration using non-Calico CNI plugin", + func(cni operatorv1.CNIPluginType, ipam operatorv1.IPAMPluginType) { + installlation := &operatorv1.InstallationSpec{ + CNI: &operatorv1.CNISpec{ + Type: cni, + IPAM: &operatorv1.IPAMSpec{Type: ipam}, }, - }, - { - Name: "NAMESPACE", - ValueFrom: &corev1.EnvVarSource{ - FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.namespace"}, + WindowsNodes: &operatorv1.WindowsNodeSpec{ + CNIBinDir: "/opt/cni/bin", + CNIConfigDir: "/etc/cni/net.d", + CNILogDir: "/var/log/calico/cni", }, - }, - {Name: "IP", Value: "autodetect"}, - {Name: "IP_AUTODETECTION_METHOD", Value: "first-found"}, - {Name: "IP6", Value: "none"}, - {Name: "FELIX_IPV6SUPPORT", Value: "false"}, - {Name: "FELIX_TYPHAK8SNAMESPACE", Value: "calico-system"}, - {Name: "FELIX_TYPHAK8SSERVICENAME", Value: "calico-typha"}, - {Name: "FELIX_TYPHACAFILE", Value: certificatemanagement.TrustedCertBundleMountPath}, - {Name: "FELIX_TYPHACERTFILE", Value: "/node-certs/tls.crt"}, - {Name: "FELIX_TYPHACN", Value: "typha-server"}, - {Name: "FELIX_TYPHAKEYFILE", Value: "/node-certs/tls.key"}, + } + cfg.Installation = installlation - {Name: "FELIX_DNSTRUSTEDSERVERS", Value: "k8s-service:openshift-dns/dns-default"}, + component := render.Windows(&cfg) + Expect(component.ResolveImages(nil)).To(BeNil()) + resources, _ := component.Objects() - // Tigera-specific envvars - {Name: "FELIX_PROMETHEUSREPORTERENABLED", Value: "true"}, - {Name: "FELIX_PROMETHEUSREPORTERPORT", Value: "9081"}, - {Name: "FELIX_FLOWLOGSFILEENABLED", Value: "true"}, - {Name: "FELIX_FLOWLOGSFILEINCLUDELABELS", Value: "true"}, - {Name: "FELIX_FLOWLOGSFILEINCLUDEPOLICIES", Value: "true"}, - {Name: "FELIX_FLOWLOGSFILEINCLUDESERVICE", Value: "true"}, - {Name: "FELIX_FLOWLOGSENABLENETWORKSETS", Value: "true"}, - {Name: "FELIX_FLOWLOGSCOLLECTPROCESSINFO", Value: "true"}, - {Name: "FELIX_DNSLOGSFILEENABLED", Value: "true"}, - {Name: "FELIX_DNSLOGSFILEPERNODELIMIT", Value: "1000"}, + // Should render the correct resources. + Expect(rtest.GetResource(resources, "calico-node-windows", "calico-system", "apps", "v1", "DaemonSet")).ToNot(BeNil()) + dsResource := rtest.GetResource(resources, "calico-node-windows", "calico-system", "apps", "v1", "DaemonSet") + Expect(dsResource).ToNot(BeNil()) - // Calico Windows specific envvars - {Name: "VXLAN_VNI", Value: "4096"}, - {Name: "VXLAN_ADAPTER", Value: ""}, - {Name: "KUBE_NETWORK", Value: "Calico.*"}, - {Name: "KUBERNETES_SERVICE_HOST", Value: "1.2.3.4"}, - {Name: "KUBERNETES_SERVICE_PORT", Value: "6443"}, - } - Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "node").Env).To(ConsistOf(expectedNodeEnv)) - Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "felix").Env).To(ConsistOf(expectedNodeEnv)) - Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "confd").Env).To(ConsistOf(expectedNodeEnv)) + // Should not render CNI configuration. + cniCmResource := rtest.GetResource(resources, "cni-config-windows", "calico-system", "", "v1", "ConfigMap") + Expect(cniCmResource).To(BeNil()) - verifyWindowsProbesAndLifecycle(ds, false) - }) + // The DaemonSet should have the correct configuration. + ds := dsResource.(*appsv1.DaemonSet) + + // The pod template should have node critical priority + Expect(ds.Spec.Template.Spec.PriorityClassName).To(Equal(render.NodePriorityClassName)) + + // CNI install container should not be present. + cniContainer := rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "install-cni") + Expect(cniContainer).To(BeNil()) + // Validate correct number of init containers. + Expect(len(ds.Spec.Template.Spec.InitContainers)).To(Equal(1)) + + // Verify env + expectedEnvs := []corev1.EnvVar{ + {Name: "CNI_PLUGIN_TYPE", Value: string(cni)}, + {Name: "CALICO_NETWORKING_BACKEND", Value: "none"}, + {Name: "FELIX_DEFAULTENDPOINTTOHOSTACTION", Value: "ACCEPT"}, + } + for _, expected := range expectedEnvs { + Expect(ds.Spec.Template.Spec.Containers[0].Env).To(ContainElement(expected)) + } + + // Verify readiness and liveness probes. + verifyWindowsProbesAndLifecycle(ds, true) + }, + Entry("GKE", operatorv1.PluginGKE, operatorv1.IPAMPluginHostLocal), + Entry("AmazonVPC", operatorv1.PluginAmazonVPC, operatorv1.IPAMPluginAmazonVPC), + Entry("AzureVNET", operatorv1.PluginAzureVNET, operatorv1.IPAMPluginAzureVNET), + ) - It("should render all resources when variant is CalicoEnterprise and running on RKE2", func() { + It("should render all resources when running on openshift", func() { expectedResources := []struct { name string ns string @@ -1853,14 +1165,14 @@ var _ = Describe("Windows rendering tests", func() { }{ {name: "cni-config-windows", ns: common.CalicoNamespace, group: "", version: "v1", kind: "ConfigMap"}, {name: common.WindowsDaemonSetName, ns: common.CalicoNamespace, group: "apps", version: "v1", kind: "DaemonSet"}, - {name: "calico-node-metrics-windows", ns: "calico-system", group: "", version: "v1", kind: "Service"}, } - defaultInstance.Variant = operatorv1.CalicoEnterprise - defaultInstance.KubernetesProvider = operatorv1.ProviderRKE2 - - resources := renderWindows(&cfg) - Expect(len(resources)).To(Equal(len(expectedResources)), fmt.Sprintf("Actual resources: %#v", resources)) + defaultInstance.FlexVolumePath = "/etc/kubernetes/kubelet-plugins/volume/exec/" + defaultInstance.KubernetesProvider = operatorv1.ProviderOpenShift + component := render.Windows(&cfg) + Expect(component.ResolveImages(nil)).To(BeNil()) + resources, _ := component.Objects() + Expect(len(resources)).To(Equal(len(expectedResources))) // Should render the correct resources. i := 0 @@ -1876,22 +1188,22 @@ var _ = Describe("Windows rendering tests", func() { Expect(ds.Spec.Template.Spec.PriorityClassName).To(Equal(render.NodePriorityClassName)) felixContainer := rtest.GetContainer(ds.Spec.Template.Spec.Containers, "felix") - Expect(felixContainer.Image).To(Equal(fmt.Sprintf("%s%s%s:%s", components.TigeraRegistry, components.TigeraImagePath, components.ComponentTigeraNodeWindows.Image, components.ComponentTigeraNodeWindows.Version))) + Expect(felixContainer.Image).To(Equal(fmt.Sprintf("quay.io/%s%s:%s", components.CalicoImagePath, components.ComponentCalicoNodeWindows.Image, components.ComponentCalicoNodeWindows.Version))) nodeContainer := rtest.GetContainer(ds.Spec.Template.Spec.Containers, "node") - Expect(nodeContainer.Image).To(Equal(fmt.Sprintf("%s%s%s:%s", components.TigeraRegistry, components.TigeraImagePath, components.ComponentTigeraNodeWindows.Image, components.ComponentTigeraNodeWindows.Version))) + Expect(nodeContainer.Image).To(Equal(fmt.Sprintf("quay.io/%s%s:%s", components.CalicoImagePath, components.ComponentCalicoNodeWindows.Image, components.ComponentCalicoNodeWindows.Version))) cniContainer := rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "install-cni") - Expect(cniContainer.Image).To(Equal(fmt.Sprintf("%s%s%s:%s", components.TigeraRegistry, components.TigeraImagePath, components.ComponentTigeraCNIWindows.Image, components.ComponentTigeraCNIWindows.Version))) + Expect(cniContainer.Image).To(Equal(fmt.Sprintf("quay.io/%s%s:%s", components.CalicoImagePath, components.ComponentCalicoCNIWindows.Image, components.ComponentCalicoCNIWindows.Version))) uninstallContainer := rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "uninstall-calico") - Expect(uninstallContainer.Image).To(Equal(fmt.Sprintf("%s%s%s:%s", components.TigeraRegistry, components.TigeraImagePath, components.ComponentTigeraNodeWindows.Image, components.ComponentTigeraNodeWindows.Version))) + Expect(uninstallContainer.Image).To(Equal(fmt.Sprintf("quay.io/%s%s:%s", components.CalicoImagePath, components.ComponentCalicoNodeWindows.Image, components.ComponentCalicoNodeWindows.Version))) - // FIXME: confirm RKE2 CNI path defaults + // FIXME: confirm openshift CNI path defaults expectedCNIVolumeMounts := []corev1.VolumeMount{ {MountPath: "/host/opt/cni/bin", Name: "cni-bin-dir"}, {MountPath: "/host/etc/cni/net.d", Name: "cni-net-dir"}, } Expect(rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "install-cni").VolumeMounts).To(ConsistOf(expectedCNIVolumeMounts)) - // FIXME: confirm RKE2 CNI path defaults + // FIXME: confirm openshift CNI path defaults expectedUninstallVolumeMounts := []corev1.VolumeMount{ {MountPath: "/host/opt/cni/bin", Name: "cni-bin-dir"}, {MountPath: "/host/etc/cni/net.d", Name: "cni-net-dir"}, @@ -1899,7 +1211,7 @@ var _ = Describe("Windows rendering tests", func() { Expect(rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "uninstall-calico").VolumeMounts).To(ConsistOf(expectedUninstallVolumeMounts)) // Verify volumes - // FIXME: confirm RKE2 CNI path defaults + // FIXME: confirm openshift CNI path defaults fileOrCreate := corev1.HostPathFileOrCreate dirOrCreate := corev1.HostPathDirectoryOrCreate expectedVols := []corev1.Volume{ @@ -1930,18 +1242,16 @@ var _ = Describe("Windows rendering tests", func() { }, }, }, - {Name: "var-log-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico", Type: &dirOrCreate}}}, } Expect(ds.Spec.Template.Spec.Volumes).To(ConsistOf(expectedVols)) expectedNodeEnv := []corev1.EnvVar{ - // Default envvars. {Name: "CNI_PLUGIN_TYPE", Value: "Calico"}, {Name: "DATASTORE_TYPE", Value: "kubernetes"}, {Name: "WAIT_FOR_DATASTORE", Value: "true"}, {Name: "CALICO_MANAGE_CNI", Value: "true"}, {Name: "CALICO_NETWORKING_BACKEND", Value: "windows-bgp"}, - {Name: "CLUSTER_TYPE", Value: "k8s,operator,bgp,windows"}, + {Name: "CLUSTER_TYPE", Value: "k8s,operator,openshift,bgp,windows"}, {Name: "CALICO_DISABLE_FILE_LOGGING", Value: "false"}, {Name: "FELIX_DEFAULTENDPOINTTOHOSTACTION", Value: "ACCEPT"}, {Name: "FELIX_HEALTHENABLED", Value: "true"}, @@ -1968,20 +1278,6 @@ var _ = Describe("Windows rendering tests", func() { {Name: "FELIX_TYPHACN", Value: "typha-server"}, {Name: "FELIX_TYPHAKEYFILE", Value: "/node-certs/tls.key"}, - {Name: "FELIX_DNSTRUSTEDSERVERS", Value: "k8s-service:kube-system/rke2-coredns-rke2-coredns"}, - - // Tigera-specific envvars - {Name: "FELIX_PROMETHEUSREPORTERENABLED", Value: "true"}, - {Name: "FELIX_PROMETHEUSREPORTERPORT", Value: "9081"}, - {Name: "FELIX_FLOWLOGSFILEENABLED", Value: "true"}, - {Name: "FELIX_FLOWLOGSFILEINCLUDELABELS", Value: "true"}, - {Name: "FELIX_FLOWLOGSFILEINCLUDEPOLICIES", Value: "true"}, - {Name: "FELIX_FLOWLOGSFILEINCLUDESERVICE", Value: "true"}, - {Name: "FELIX_FLOWLOGSENABLENETWORKSETS", Value: "true"}, - {Name: "FELIX_FLOWLOGSCOLLECTPROCESSINFO", Value: "true"}, - {Name: "FELIX_DNSLOGSFILEENABLED", Value: "true"}, - {Name: "FELIX_DNSLOGSFILEPERNODELIMIT", Value: "1000"}, - // Calico Windows specific envvars {Name: "VXLAN_VNI", Value: "4096"}, {Name: "VXLAN_ADAPTER", Value: ""}, @@ -1989,14 +1285,12 @@ var _ = Describe("Windows rendering tests", func() { {Name: "KUBERNETES_SERVICE_HOST", Value: "1.2.3.4"}, {Name: "KUBERNETES_SERVICE_PORT", Value: "6443"}, } - Expect(ds.Spec.Template.Spec.Containers[0].Env).To(ConsistOf(expectedNodeEnv)) - Expect(len(ds.Spec.Template.Spec.Containers[0].Env)).To(Equal(len(expectedNodeEnv))) - verifyWindowsProbesAndLifecycle(ds, false) + Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "node").Env).To(ConsistOf(expectedNodeEnv)) + Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "felix").Env).To(ConsistOf(expectedNodeEnv)) + Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "confd").Env).To(ConsistOf(expectedNodeEnv)) - // The metrics service should have the correct configuration. - ms := rtest.GetResource(resources, "calico-node-metrics-windows", "calico-system", "", "v1", "Service").(*corev1.Service) - Expect(ms.Spec.ClusterIP).To(Equal("None"), "metrics service should be headless to prevent kube-proxy from rendering too many iptables rules") + verifyWindowsProbesAndLifecycle(ds, true) }) Describe("AKS", func() { @@ -2139,50 +1433,6 @@ var _ = Describe("Windows rendering tests", func() { }) }) - It("should not enable prometheus metrics if NodeMetricsPort is nil", func() { - defaultInstance.Variant = operatorv1.CalicoEnterprise - defaultInstance.NodeMetricsPort = nil - - resources := renderWindows(&cfg) - Expect(len(resources)).To(Equal(defaultNumExpectedResources + 1)) - - dsResource := rtest.GetResource(resources, "calico-node-windows", "calico-system", "apps", "v1", "DaemonSet") - Expect(dsResource).ToNot(BeNil()) - - notExpectedEnvVar := corev1.EnvVar{Name: "FELIX_PROMETHEUSMETRICSPORT"} - ds := dsResource.(*appsv1.DaemonSet) - Expect(ds.Spec.Template.Spec.Containers[0].Env).ToNot(ContainElement(notExpectedEnvVar)) - - // It should have the reporter port, though. - expected := corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERPORT"} - Expect(ds.Spec.Template.Spec.Containers[0].Env).ToNot(ContainElement(expected)) - }) - - It("should set FELIX_PROMETHEUSMETRICSPORT with a custom value if NodeMetricsPort is set", func() { - var nodeMetricsPort int32 = 1234 - defaultInstance.Variant = operatorv1.CalicoEnterprise - defaultInstance.NodeMetricsPort = &nodeMetricsPort - resources := renderWindows(&cfg) - Expect(len(resources)).To(Equal(defaultNumExpectedResources + 1)) - - dsResource := rtest.GetResource(resources, "calico-node-windows", "calico-system", "apps", "v1", "DaemonSet") - Expect(dsResource).ToNot(BeNil()) - - // Assert on expected env vars. - expectedEnvVars := []corev1.EnvVar{ - {Name: "FELIX_PROMETHEUSMETRICSPORT", Value: "1234"}, - {Name: "FELIX_PROMETHEUSMETRICSENABLED", Value: "true"}, - } - ds := dsResource.(*appsv1.DaemonSet) - for _, v := range expectedEnvVars { - Expect(ds.Spec.Template.Spec.Containers[0].Env).To(ContainElement(v)) - } - - // Assert we set annotations properly. - Expect(ds.Spec.Template.Annotations["prometheus.io/scrape"]).To(Equal("true")) - Expect(ds.Spec.Template.Annotations["prometheus.io/port"]).To(Equal("1234")) - }) - It("should render MaxUnavailable if a custom value was set", func() { two := intstr.FromInt(2) defaultInstance.NodeUpdateStrategy.RollingUpdate.MaxUnavailable = &two From 59199693ab03973c71731ac5a05f9c8aeb798cbe Mon Sep 17 00:00:00 2001 From: Casey Davenport Date: Wed, 24 Jun 2026 14:20:42 -0700 Subject: [PATCH 47/54] Move node enterprise render tests into pkg/enterprise/installation node_enterprise_test.go ran the real ExtendContext hook and node/typha modifiers, so move it whole into pkg/enterprise/installation next to those. node_test.go only ever rendered base output (no modifier), so its CalicoEnterprise cases are variant-blindness checks and stay put. Swap its cfg from the enterprise Set's image overrides to an empty imageoverride.New() so render_test no longer depends on pkg/enterprise for node. --- .../installation}/node_enterprise_test.go | 25 ++++++++++++++++++- pkg/render/node_test.go | 3 ++- 2 files changed, 26 insertions(+), 2 deletions(-) rename pkg/{render => enterprise/installation}/node_enterprise_test.go (87%) diff --git a/pkg/render/node_enterprise_test.go b/pkg/enterprise/installation/node_enterprise_test.go similarity index 87% rename from pkg/render/node_enterprise_test.go rename to pkg/enterprise/installation/node_enterprise_test.go index d7afd663dd..7aab518c79 100644 --- a/pkg/render/node_enterprise_test.go +++ b/pkg/enterprise/installation/node_enterprise_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package render_test +package installation_test import ( "context" @@ -42,6 +42,29 @@ import ( "github.com/tigera/operator/pkg/tls/certificatemanagement" ) +// getTyphaNodeTLS builds the node/typha TLS bundle the node render expects. +func getTyphaNodeTLS(cli client.Client, certificateManager certificatemanager.CertificateManager) *render.TyphaNodeTLS { + nodeKeyPair, err := certificateManager.GetOrCreateKeyPair(cli, render.NodeTLSSecretName, common.OperatorNamespace(), []string{render.FelixCommonName}) + Expect(err).NotTo(HaveOccurred()) + + typhaKeyPair, err := certificateManager.GetOrCreateKeyPair(cli, render.TyphaTLSSecretName, common.OperatorNamespace(), []string{render.FelixCommonName}) + Expect(err).NotTo(HaveOccurred()) + + typhaNonClusterHostKeyPair, err := certificateManager.GetOrCreateKeyPair(cli, render.TyphaTLSSecretName+render.TyphaNonClusterHostSuffix, common.OperatorNamespace(), []string{render.FelixCommonName + render.TyphaNonClusterHostSuffix}) + Expect(err).NotTo(HaveOccurred()) + + trustedBundle := certificateManager.CreateTrustedBundle(nodeKeyPair, typhaKeyPair) + + return &render.TyphaNodeTLS{ + TrustedBundle: trustedBundle, + TyphaSecret: typhaKeyPair, + TyphaSecretNonClusterHost: typhaNonClusterHostKeyPair, + TyphaCommonName: render.TyphaCommonName, + NodeSecret: nodeKeyPair, + NodeCommonName: render.FelixCommonName, + } +} + // These tests run the real node/typha render output through the registered // enterprise modifiers. The render suite registers the enterprise extensions in // its BeforeSuite, so this exercises the same integrated behavior the operator diff --git a/pkg/render/node_test.go b/pkg/render/node_test.go index b052be0ef4..40f4712f97 100644 --- a/pkg/render/node_test.go +++ b/pkg/render/node_test.go @@ -38,6 +38,7 @@ import ( "github.com/tigera/operator/pkg/controller/certificatemanager" "github.com/tigera/operator/pkg/controller/k8sapi" ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" + "github.com/tigera/operator/pkg/imageoverride" "github.com/tigera/operator/pkg/render" rmeta "github.com/tigera/operator/pkg/render/common/meta" rtest "github.com/tigera/operator/pkg/render/common/test" @@ -140,7 +141,7 @@ var _ = Describe("Node rendering tests", func() { ClusterDomain: defaultClusterDomain, FelixHealthPort: 9099, IPPools: defaultInstance.CalicoNetwork.IPPools, - ImageOverrides: ext.Images(), + ImageOverrides: imageoverride.New(), } }) From ad7b115994d5312bc5286f664bdbf746b6b3580d Mon Sep 17 00:00:00 2001 From: Casey Davenport Date: Wed, 24 Jun 2026 14:25:05 -0700 Subject: [PATCH 48/54] Drop the enterprise dependency from the render test suite The apiserver OSS tests went through the enterprise modifier only to pick up the Calico-variant cleanup deletes, but those deletes come from the base render itself (and the cleanup modifier is covered in pkg/enterprise/apiserver). Render the base component directly instead. With apiserver no longer reaching for the enterprise Set, nothing in the render test suite uses it, so remove enterprise_setup_test.go. pkg/render no longer depends on pkg/enterprise, in production or test code. --- pkg/render/apiserver_test.go | 19 +++++-------------- pkg/render/enterprise_setup_test.go | 27 --------------------------- 2 files changed, 5 insertions(+), 41 deletions(-) delete mode 100644 pkg/render/enterprise_setup_test.go diff --git a/pkg/render/apiserver_test.go b/pkg/render/apiserver_test.go index 955e0b5357..25c3f85ad5 100644 --- a/pkg/render/apiserver_test.go +++ b/pkg/render/apiserver_test.go @@ -28,7 +28,6 @@ import ( "github.com/tigera/operator/pkg/controller/k8sapi" ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" "github.com/tigera/operator/pkg/dns" - "github.com/tigera/operator/pkg/extensions/extensionstest" "github.com/tigera/operator/pkg/render" rmeta "github.com/tigera/operator/pkg/render/common/meta" "github.com/tigera/operator/pkg/render/common/podaffinity" @@ -47,20 +46,12 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) -// apiServerObjects renders the API server component and applies the registered variant -// modifier the way the componentHandler does, so the Enterprise objects (query server, -// audit logging, Enterprise RBAC) and the Calico cleanup deletes are reflected in the -// returned create and delete lists. +// apiServerObjects renders the base (OSS) API server component. The enterprise +// modifier and the Calico-variant cleanup are exercised in pkg/enterprise/apiserver; +// these tests cover the OSS render path, which the base render handles on its own +// (including the deletes it queues for itself). func apiServerObjects(c render.Component) ([]client.Object, []client.Object) { - create, del := c.Objects() - rc := render.RenderContext{} - var extCtx any - if p, ok := c.(render.ExtensionContextProvider); ok { - ec := p.ExtensionContext().(render.APIServerExtensionContext) - rc.Installation = ec.Config.Installation - extCtx = ec - } - return extensionstest.ApplyExtensionsWithContext(ext, render.ComponentNameAPIServer, rc, extCtx, create, del) + return c.Objects() } func verifyAPIService(service *apiregv1.APIService, enterprise bool, clusterDomain string) { diff --git a/pkg/render/enterprise_setup_test.go b/pkg/render/enterprise_setup_test.go deleted file mode 100644 index e896366124..0000000000 --- a/pkg/render/enterprise_setup_test.go +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright (c) 2026 Tigera, Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package render_test - -import ( - "github.com/tigera/operator/pkg/enterprise" - "github.com/tigera/operator/pkg/extensions" -) - -// ext is the enterprise extension Set the render suite tests against. The -// Objects()-level image tests pass ext.Images() into the node/windows configs to -// pick up the enterprise images, and the enterprise modifier tests apply ext's -// modifiers explicitly to real render output to check they still match it. It is -// immutable once built and specs only read it, so a single instance is safe. -var ext *extensions.Set = enterprise.New() From 0e99b4c6fdf4058ebdb322b93f1cca4b0aec16b1 Mon Sep 17 00:00:00 2001 From: Casey Davenport Date: Wed, 24 Jun 2026 14:38:34 -0700 Subject: [PATCH 49/54] Move es-kube-controllers render tests into pkg/enterprise/kubecontrollers The es-kube-controllers assembly lives in pkg/enterprise/kubecontrollers, but its render tests sat in pkg/render/kubecontrollers and pulled the enterprise package into the render test deps. Move them next to the assembly. The calico-kube-controllers (OSS) tests stay put, and pkg/render/kubecontrollers no longer depends on pkg/enterprise. --- .../es_kube_controllers_test.go | 369 ++++++++++++++++++ pkg/enterprise/kubecontrollers/suite_test.go | 27 ++ .../kubecontrollers/kube-controllers_test.go | 285 -------------- 3 files changed, 396 insertions(+), 285 deletions(-) create mode 100644 pkg/enterprise/kubecontrollers/es_kube_controllers_test.go create mode 100644 pkg/enterprise/kubecontrollers/suite_test.go diff --git a/pkg/enterprise/kubecontrollers/es_kube_controllers_test.go b/pkg/enterprise/kubecontrollers/es_kube_controllers_test.go new file mode 100644 index 0000000000..5c682fc99e --- /dev/null +++ b/pkg/enterprise/kubecontrollers/es_kube_controllers_test.go @@ -0,0 +1,369 @@ +// Copyright (c) 2020-2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package kubecontrollers + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + + v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" + + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/apis" + "github.com/tigera/operator/pkg/common" + "github.com/tigera/operator/pkg/components" + "github.com/tigera/operator/pkg/controller/certificatemanager" + "github.com/tigera/operator/pkg/controller/k8sapi" + ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" + "github.com/tigera/operator/pkg/dns" + "github.com/tigera/operator/pkg/render" + rtest "github.com/tigera/operator/pkg/render/common/test" + rkc "github.com/tigera/operator/pkg/render/kubecontrollers" + "github.com/tigera/operator/pkg/render/testutils" + "github.com/tigera/operator/pkg/tls/certificatemanagement" +) + +var _ = Describe("es-kube-controllers rendering tests", func() { + var ( + instance *operatorv1.InstallationSpec + k8sServiceEp k8sapi.ServiceEndpoint + cfg rkc.KubeControllersConfiguration + cli client.Client + ) + + esEnvs := []corev1.EnvVar{ + {Name: "ELASTIC_HOST", Value: "tigera-secure-es-gateway-http.tigera-elasticsearch.svc"}, + {Name: "ELASTIC_PORT", Value: "9200", ValueFrom: nil}, + { + Name: "ELASTIC_USERNAME", Value: "", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "tigera-ee-kube-controllers-elasticsearch-access", + }, + Key: "username", + }, + }, + }, + { + Name: "ELASTIC_PASSWORD", Value: "", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "tigera-ee-kube-controllers-elasticsearch-access", + }, + Key: "password", + }, + }, + }, + {Name: "ELASTIC_CA", Value: certificatemanagement.TrustedCertBundleMountPath}, + } + + // The es-kube-controllers policy fixtures live next to the render package's + // testutils, so reference them relative to this enterprise subpackage. + expectedESPolicy := testutils.GetExpectedPolicyFromFile("../../render/testutils/expected_policies/es-kubecontrollers.json") + expectedESPolicyForOpenshift := testutils.GetExpectedPolicyFromFile("../../render/testutils/expected_policies/es-kubecontrollers_ocp.json") + + BeforeEach(func() { + // Initialize a default instance to use. Each test can override this to its + // desired configuration. + + miMode := operatorv1.MultiInterfaceModeNone + instance = &operatorv1.InstallationSpec{ + CalicoNetwork: &operatorv1.CalicoNetworkSpec{ + IPPools: []operatorv1.IPPool{{CIDR: "192.168.1.0/16"}}, + MultiInterfaceMode: &miMode, + }, + Registry: "test-reg/", + } + k8sServiceEp = k8sapi.ServiceEndpoint{} + + scheme := runtime.NewScheme() + Expect(apis.AddToScheme(scheme, false)).NotTo(HaveOccurred()) + cli = ctrlrfake.DefaultFakeClientBuilder(scheme).Build() + + certificateManager, err := certificatemanager.Create(cli, nil, dns.DefaultClusterDomain, common.OperatorNamespace(), certificatemanager.AllowCACreation()) + Expect(err).NotTo(HaveOccurred()) + + cfg = rkc.KubeControllersConfiguration{ + K8sServiceEp: k8sServiceEp, + Installation: instance, + ClusterDomain: dns.DefaultClusterDomain, + MetricsPort: 9094, + TrustedBundle: certificateManager.CreateTrustedBundle(), + Namespace: common.CalicoNamespace, + BindingNamespaces: []string{common.CalicoNamespace}, + } + }) + + It("should render all es-calico-kube-controllers resources for a default configuration (standalone) using CalicoEnterprise when logstorage and secrets exist", func() { + expectedResources := []struct { + name string + ns string + group string + version string + kind string + }{ + {name: EsKubeControllerNetworkPolicyName, ns: common.CalicoNamespace, group: "projectcalico.org", version: "v3", kind: "NetworkPolicy"}, + {name: "calico-kube-controllers", ns: common.CalicoNamespace, group: "", version: "v1", kind: "ServiceAccount"}, + {name: EsKubeControllerRole, ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRole"}, + {name: EsKubeControllerRoleBinding, ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRoleBinding"}, + {name: EsKubeController, ns: common.CalicoNamespace, group: "apps", version: "v1", kind: "Deployment"}, + {name: ElasticsearchKubeControllersUserSecret, ns: common.CalicoNamespace, group: "", version: "v1", kind: "Secret"}, + {name: EsKubeControllerMetrics, ns: common.CalicoNamespace, group: "", version: "v1", kind: "Service"}, + } + + instance.Variant = operatorv1.CalicoEnterprise + cfg.KubeControllersGatewaySecret = &testutils.KubeControllersUserSecret + cfg.MetricsPort = 9094 + + component := NewElasticsearchKubeControllers(&cfg) + Expect(component.ResolveImages(nil)).To(BeNil()) + resources, _ := component.Objects() + Expect(len(resources)).To(Equal(len(expectedResources))) + + // Should render the correct resources. + i := 0 + for _, expectedRes := range expectedResources { + rtest.ExpectResourceTypeAndObjectMetadata(resources[i], expectedRes.name, expectedRes.ns, expectedRes.group, expectedRes.version, expectedRes.kind) + i++ + } + + // The Deployment should have the correct configuration. + dp := rtest.GetResource(resources, EsKubeController, common.CalicoNamespace, "apps", "v1", "Deployment").(*appsv1.Deployment) + + Expect(dp.Spec.Template.Spec.Containers[0].Image).To(Equal("test-reg/tigera/calico:" + components.ComponentTigeraCalico.Version)) + envs := dp.Spec.Template.Spec.Containers[0].Env + Expect(envs).To(ContainElement(corev1.EnvVar{ + Name: "ENABLED_CONTROLLERS", Value: "authorization,elasticsearchconfiguration", + })) + Expect(envs).To(ContainElements(esEnvs)) + + Expect(dp.Spec.Template.Spec.Containers[0].VolumeMounts).To(HaveLen(1)) + Expect(dp.Spec.Template.Spec.Containers[0].VolumeMounts[0].Name).To(Equal("tigera-ca-bundle")) + Expect(dp.Spec.Template.Spec.Containers[0].VolumeMounts[0].MountPath).To(Equal("/etc/pki/tls/certs")) + + Expect(dp.Spec.Template.Spec.Volumes).To(HaveLen(1)) + Expect(dp.Spec.Template.Spec.Volumes[0].Name).To(Equal("tigera-ca-bundle")) + Expect(dp.Spec.Template.Spec.Volumes[0].ConfigMap.Name).To(Equal("tigera-ca-bundle")) + + clusterRole := rtest.GetResource(resources, EsKubeControllerRole, "", "rbac.authorization.k8s.io", "v1", "ClusterRole").(*rbacv1.ClusterRole) + Expect(clusterRole.Rules).To(HaveLen(26), "cluster role should have 26 rules") + Expect(clusterRole.Rules).To(ContainElement( + rbacv1.PolicyRule{ + APIGroups: []string{""}, + Resources: []string{"configmaps"}, + Verbs: []string{"watch", "list", "get", "update", "create", "delete"}, + })) + Expect(clusterRole.Rules).To(ContainElement( + rbacv1.PolicyRule{ + APIGroups: []string{""}, + Resources: []string{"secrets"}, + Verbs: []string{"watch", "list", "get"}, + })) + }) + + It("should render all es-calico-kube-controllers resources for a default configuration using CalicoEnterprise and ClusterType is Management", func() { + expectedResources := []struct { + name string + ns string + group string + version string + kind string + }{ + {name: EsKubeControllerNetworkPolicyName, ns: common.CalicoNamespace, group: "projectcalico.org", version: "v3", kind: "NetworkPolicy"}, + {name: "calico-kube-controllers", ns: common.CalicoNamespace, group: "", version: "v1", kind: "ServiceAccount"}, + {name: EsKubeControllerRole, ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRole"}, + {name: EsKubeControllerRoleBinding, ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRoleBinding"}, + {name: rkc.ManagedClustersWatchRoleBindingName, ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRoleBinding"}, + {name: EsKubeController, ns: common.CalicoNamespace, group: "apps", version: "v1", kind: "Deployment"}, + {name: ElasticsearchKubeControllersUserSecret, ns: common.CalicoNamespace, group: "", version: "v1", kind: "Secret"}, + {name: EsKubeControllerMetrics, ns: common.CalicoNamespace, group: "", version: "v1", kind: "Service"}, + } + + // Override configuration to match expected Enterprise config. + instance.Variant = operatorv1.CalicoEnterprise + cfg.ManagementCluster = &operatorv1.ManagementCluster{} + cfg.KubeControllersGatewaySecret = &testutils.KubeControllersUserSecret + cfg.MetricsPort = 9094 + + component := NewElasticsearchKubeControllers(&cfg) + Expect(component.ResolveImages(nil)).To(BeNil()) + resources, _ := component.Objects() + Expect(len(resources)).To(Equal(len(expectedResources))) + + // Should render the correct resources. + i := 0 + for _, expectedRes := range expectedResources { + rtest.ExpectResourceTypeAndObjectMetadata(resources[i], expectedRes.name, expectedRes.ns, expectedRes.group, expectedRes.version, expectedRes.kind) + i++ + } + + // The Deployment should have the correct configuration. + dp := rtest.GetResource(resources, EsKubeController, common.CalicoNamespace, "apps", "v1", "Deployment").(*appsv1.Deployment) + + envs := dp.Spec.Template.Spec.Containers[0].Env + Expect(envs).To(ContainElement(corev1.EnvVar{ + Name: "ENABLED_CONTROLLERS", + Value: "authorization,elasticsearchconfiguration,managedcluster", + })) + + Expect(dp.Spec.Template.Spec.Containers[0].VolumeMounts).To(HaveLen(1)) + Expect(dp.Spec.Template.Spec.Containers[0].VolumeMounts[0].Name).To(Equal("tigera-ca-bundle")) + Expect(dp.Spec.Template.Spec.Containers[0].VolumeMounts[0].MountPath).To(Equal("/etc/pki/tls/certs")) + + Expect(dp.Spec.Template.Spec.Volumes).To(HaveLen(1)) + Expect(dp.Spec.Template.Spec.Volumes[0].Name).To(Equal("tigera-ca-bundle")) + Expect(dp.Spec.Template.Spec.Volumes[0].ConfigMap.Name).To(Equal("tigera-ca-bundle")) + + Expect(dp.Spec.Template.Spec.Containers[0].Image).To(Equal("test-reg/tigera/calico:" + components.ComponentTigeraCalico.Version)) + + clusterRole := rtest.GetResource(resources, EsKubeControllerRole, "", "rbac.authorization.k8s.io", "v1", "ClusterRole").(*rbacv1.ClusterRole) + Expect(clusterRole.Rules).To(HaveLen(26), "cluster role should have 26 rules") + Expect(clusterRole.Rules).To(ContainElement( + rbacv1.PolicyRule{ + APIGroups: []string{""}, + Resources: []string{"configmaps"}, + Verbs: []string{"watch", "list", "get", "update", "create", "delete"}, + })) + Expect(clusterRole.Rules).To(ContainElement( + rbacv1.PolicyRule{ + APIGroups: []string{""}, + Resources: []string{"secrets"}, + Verbs: []string{"watch", "list", "get"}, + })) + roleBindingWatch := rtest.GetResource(resources, rkc.ManagedClustersWatchRoleBindingName, "", "rbac.authorization.k8s.io", "v1", "ClusterRoleBinding").(*rbacv1.ClusterRoleBinding) + Expect(roleBindingWatch.RoleRef.Name).To(Equal(render.ManagedClustersWatchClusterRoleName)) + Expect(roleBindingWatch.Subjects).To(ConsistOf([]rbacv1.Subject{ + { + Kind: "ServiceAccount", + Name: rkc.KubeControllerServiceAccount, + Namespace: common.CalicoNamespace, + }, + })) + }) + + It("should add the OIDC prefix env variables", func() { + instance.Variant = operatorv1.CalicoEnterprise + cfg.ManagementCluster = &operatorv1.ManagementCluster{} + cfg.KubeControllersGatewaySecret = &testutils.KubeControllersUserSecret + cfg.MetricsPort = 9094 + cfg.Authentication = &operatorv1.Authentication{Spec: operatorv1.AuthenticationSpec{ + UsernamePrefix: "uOIDC:", + GroupsPrefix: "gOIDC:", + Openshift: &operatorv1.AuthenticationOpenshift{IssuerURL: "https://api.example.com"}, + }} + + component := NewElasticsearchKubeControllers(&cfg) + Expect(component.ResolveImages(nil)).To(BeNil()) + resources, _ := component.Objects() + + depResource := rtest.GetResource(resources, EsKubeController, common.CalicoNamespace, "apps", "v1", "Deployment") + Expect(depResource).ToNot(BeNil()) + deployment := depResource.(*appsv1.Deployment) + + var usernamePrefix, groupPrefix string + for _, container := range deployment.Spec.Template.Spec.Containers { + if container.Name == EsKubeController { + for _, env := range container.Env { + switch env.Name { + case "OIDC_AUTH_USERNAME_PREFIX": + usernamePrefix = env.Value + case "OIDC_AUTH_GROUP_PREFIX": + groupPrefix = env.Value + } + } + } + } + + Expect(usernamePrefix).To(Equal("uOIDC:")) + Expect(groupPrefix).To(Equal("gOIDC:")) + }) + + When("enableESOIDCWorkaround is true", func() { + It("should set the ENABLE_ELASTICSEARCH_OIDC_WORKAROUND env variable to true", func() { + instance.Variant = operatorv1.CalicoEnterprise + cfg.ManagementCluster = &operatorv1.ManagementCluster{} + cfg.KubeControllersGatewaySecret = &testutils.KubeControllersUserSecret + cfg.MetricsPort = 9094 + component := NewElasticsearchKubeControllers(&cfg) + resources, _ := component.Objects() + + depResource := rtest.GetResource(resources, EsKubeController, common.CalicoNamespace, "apps", "v1", "Deployment") + Expect(depResource).ToNot(BeNil()) + deployment := depResource.(*appsv1.Deployment) + + var esLicenseType string + for _, container := range deployment.Spec.Template.Spec.Containers { + if container.Name == EsKubeController { + for _, env := range container.Env { + if env.Name == "ENABLE_ELASTICSEARCH_OIDC_WORKAROUND" { + esLicenseType = env.Value + } + } + } + } + + Expect(esLicenseType).To(Equal("true")) + }) + }) + + Context("es-kube-controllers calico-system rendering", func() { + policyName := types.NamespacedName{Name: "calico-system.es-kube-controller-access", Namespace: common.CalicoNamespace} + + getExpectedPolicy := func(scenario testutils.CalicoSystemScenario) *v3.NetworkPolicy { + if scenario.ManagedCluster { + return nil + } + + return testutils.SelectPolicyByProvider(scenario, expectedESPolicy, expectedESPolicyForOpenshift) + } + + DescribeTable("should render calico-system policy", + func(scenario testutils.CalicoSystemScenario) { + if scenario.OpenShift { + cfg.Installation.KubernetesProvider = operatorv1.ProviderOpenShift + } else { + cfg.Installation.KubernetesProvider = operatorv1.ProviderNone + } + if scenario.ManagedCluster { + cfg.ManagementClusterConnection = &operatorv1.ManagementClusterConnection{} + } else { + cfg.ManagementClusterConnection = nil + } + instance.Variant = operatorv1.CalicoEnterprise + cfg.KubeControllersGatewaySecret = &testutils.KubeControllersUserSecret + + component := NewElasticsearchKubeControllers(&cfg) + resources, _ := component.Objects() + + policy := testutils.GetCalicoSystemPolicyFromResources(policyName, resources) + expectedPolicy := getExpectedPolicy(scenario) + Expect(policy).To(Equal(expectedPolicy)) + }, + Entry("for management/standalone, kube-dns", testutils.CalicoSystemScenario{ManagedCluster: false, OpenShift: false}), + Entry("for management/standalone, openshift-dns", testutils.CalicoSystemScenario{ManagedCluster: false, OpenShift: true}), + Entry("for managed, kube-dns", testutils.CalicoSystemScenario{ManagedCluster: true, OpenShift: false}), + Entry("for managed, openshift-dns", testutils.CalicoSystemScenario{ManagedCluster: true, OpenShift: true}), + ) + }) +}) diff --git a/pkg/enterprise/kubecontrollers/suite_test.go b/pkg/enterprise/kubecontrollers/suite_test.go new file mode 100644 index 0000000000..a28166fba6 --- /dev/null +++ b/pkg/enterprise/kubecontrollers/suite_test.go @@ -0,0 +1,27 @@ +// Copyright (c) 2020-2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package kubecontrollers_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestKubeControllers(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "pkg/enterprise/kubecontrollers Suite") +} diff --git a/pkg/render/kubecontrollers/kube-controllers_test.go b/pkg/render/kubecontrollers/kube-controllers_test.go index 0c21b6dd29..025dbbad35 100644 --- a/pkg/render/kubecontrollers/kube-controllers_test.go +++ b/pkg/render/kubecontrollers/kube-controllers_test.go @@ -39,14 +39,11 @@ import ( "github.com/tigera/operator/pkg/controller/k8sapi" ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" "github.com/tigera/operator/pkg/dns" - entkubecontrollers "github.com/tigera/operator/pkg/enterprise/kubecontrollers" - "github.com/tigera/operator/pkg/render" rmeta "github.com/tigera/operator/pkg/render/common/meta" "github.com/tigera/operator/pkg/render/common/networkpolicy" rtest "github.com/tigera/operator/pkg/render/common/test" "github.com/tigera/operator/pkg/render/kubecontrollers" "github.com/tigera/operator/pkg/render/testutils" - "github.com/tigera/operator/pkg/tls/certificatemanagement" ) var _ = Describe("kube-controllers rendering tests", func() { @@ -57,40 +54,10 @@ var _ = Describe("kube-controllers rendering tests", func() { cli client.Client ) - esEnvs := []corev1.EnvVar{ - {Name: "ELASTIC_HOST", Value: "tigera-secure-es-gateway-http.tigera-elasticsearch.svc"}, - {Name: "ELASTIC_PORT", Value: "9200", ValueFrom: nil}, - { - Name: "ELASTIC_USERNAME", Value: "", - ValueFrom: &corev1.EnvVarSource{ - SecretKeyRef: &corev1.SecretKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: "tigera-ee-kube-controllers-elasticsearch-access", - }, - Key: "username", - }, - }, - }, - { - Name: "ELASTIC_PASSWORD", Value: "", - ValueFrom: &corev1.EnvVarSource{ - SecretKeyRef: &corev1.SecretKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: "tigera-ee-kube-controllers-elasticsearch-access", - }, - Key: "password", - }, - }, - }, - {Name: "ELASTIC_CA", Value: certificatemanagement.TrustedCertBundleMountPath}, - } - expectedPolicyForUnmanaged := testutils.GetExpectedPolicyFromFile("../testutils/expected_policies/kubecontrollers.json") expectedPolicyForUnmanagedOCP := testutils.GetExpectedPolicyFromFile("../testutils/expected_policies/kubecontrollers_ocp.json") expectedPolicyForManaged := testutils.GetExpectedPolicyFromFile("../testutils/expected_policies/kubecontrollers_managed.json") expectedPolicyForManagedOCP := testutils.GetExpectedPolicyFromFile("../testutils/expected_policies/kubecontrollers_managed_ocp.json") - expectedESPolicy := testutils.GetExpectedPolicyFromFile("../testutils/expected_policies/es-kubecontrollers.json") - expectedESPolicyForOpenshift := testutils.GetExpectedPolicyFromFile("../testutils/expected_policies/es-kubecontrollers_ocp.json") BeforeEach(func() { // Initialize a default instance to use. Each test can override this to its @@ -256,73 +223,6 @@ var _ = Describe("kube-controllers rendering tests", func() { } }) - It("should render all es-calico-kube-controllers resources for a default configuration (standalone) using CalicoEnterprise when logstorage and secrets exist", func() { - expectedResources := []struct { - name string - ns string - group string - version string - kind string - }{ - {name: entkubecontrollers.EsKubeControllerNetworkPolicyName, ns: common.CalicoNamespace, group: "projectcalico.org", version: "v3", kind: "NetworkPolicy"}, - {name: "calico-kube-controllers", ns: common.CalicoNamespace, group: "", version: "v1", kind: "ServiceAccount"}, - {name: entkubecontrollers.EsKubeControllerRole, ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRole"}, - {name: entkubecontrollers.EsKubeControllerRoleBinding, ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRoleBinding"}, - {name: entkubecontrollers.EsKubeController, ns: common.CalicoNamespace, group: "apps", version: "v1", kind: "Deployment"}, - {name: entkubecontrollers.ElasticsearchKubeControllersUserSecret, ns: common.CalicoNamespace, group: "", version: "v1", kind: "Secret"}, - {name: entkubecontrollers.EsKubeControllerMetrics, ns: common.CalicoNamespace, group: "", version: "v1", kind: "Service"}, - } - - instance.Variant = operatorv1.CalicoEnterprise - cfg.KubeControllersGatewaySecret = &testutils.KubeControllersUserSecret - cfg.MetricsPort = 9094 - - component := entkubecontrollers.NewElasticsearchKubeControllers(&cfg) - Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() - Expect(len(resources)).To(Equal(len(expectedResources))) - - // Should render the correct resources. - i := 0 - for _, expectedRes := range expectedResources { - rtest.ExpectResourceTypeAndObjectMetadata(resources[i], expectedRes.name, expectedRes.ns, expectedRes.group, expectedRes.version, expectedRes.kind) - i++ - } - - // The Deployment should have the correct configuration. - dp := rtest.GetResource(resources, entkubecontrollers.EsKubeController, common.CalicoNamespace, "apps", "v1", "Deployment").(*appsv1.Deployment) - - Expect(dp.Spec.Template.Spec.Containers[0].Image).To(Equal("test-reg/tigera/calico:" + components.ComponentTigeraCalico.Version)) - envs := dp.Spec.Template.Spec.Containers[0].Env - Expect(envs).To(ContainElement(corev1.EnvVar{ - Name: "ENABLED_CONTROLLERS", Value: "authorization,elasticsearchconfiguration", - })) - Expect(envs).To(ContainElements(esEnvs)) - - Expect(dp.Spec.Template.Spec.Containers[0].VolumeMounts).To(HaveLen(1)) - Expect(dp.Spec.Template.Spec.Containers[0].VolumeMounts[0].Name).To(Equal("tigera-ca-bundle")) - Expect(dp.Spec.Template.Spec.Containers[0].VolumeMounts[0].MountPath).To(Equal("/etc/pki/tls/certs")) - - Expect(dp.Spec.Template.Spec.Volumes).To(HaveLen(1)) - Expect(dp.Spec.Template.Spec.Volumes[0].Name).To(Equal("tigera-ca-bundle")) - Expect(dp.Spec.Template.Spec.Volumes[0].ConfigMap.Name).To(Equal("tigera-ca-bundle")) - - clusterRole := rtest.GetResource(resources, entkubecontrollers.EsKubeControllerRole, "", "rbac.authorization.k8s.io", "v1", "ClusterRole").(*rbacv1.ClusterRole) - Expect(clusterRole.Rules).To(HaveLen(26), "cluster role should have 26 rules") - Expect(clusterRole.Rules).To(ContainElement( - rbacv1.PolicyRule{ - APIGroups: []string{""}, - Resources: []string{"configmaps"}, - Verbs: []string{"watch", "list", "get", "update", "create", "delete"}, - })) - Expect(clusterRole.Rules).To(ContainElement( - rbacv1.PolicyRule{ - APIGroups: []string{""}, - Resources: []string{"secrets"}, - Verbs: []string{"watch", "list", "get"}, - })) - }) - It("should render all calico-kube-controllers resources for a default configuration using CalicoEnterprise", func() { expectedResources := []struct { name string @@ -389,86 +289,6 @@ var _ = Describe("kube-controllers rendering tests", func() { Expect(dp.Spec.Template.Spec.Containers[0].Image).To(Equal("test-reg/tigera/calico:" + components.ComponentTigeraCalico.Version)) }) - It("should render all es-calico-kube-controllers resources for a default configuration using CalicoEnterprise and ClusterType is Management", func() { - expectedResources := []struct { - name string - ns string - group string - version string - kind string - }{ - {name: entkubecontrollers.EsKubeControllerNetworkPolicyName, ns: common.CalicoNamespace, group: "projectcalico.org", version: "v3", kind: "NetworkPolicy"}, - {name: "calico-kube-controllers", ns: common.CalicoNamespace, group: "", version: "v1", kind: "ServiceAccount"}, - {name: entkubecontrollers.EsKubeControllerRole, ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRole"}, - {name: entkubecontrollers.EsKubeControllerRoleBinding, ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRoleBinding"}, - {name: kubecontrollers.ManagedClustersWatchRoleBindingName, ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRoleBinding"}, - {name: entkubecontrollers.EsKubeController, ns: common.CalicoNamespace, group: "apps", version: "v1", kind: "Deployment"}, - {name: entkubecontrollers.ElasticsearchKubeControllersUserSecret, ns: common.CalicoNamespace, group: "", version: "v1", kind: "Secret"}, - {name: entkubecontrollers.EsKubeControllerMetrics, ns: common.CalicoNamespace, group: "", version: "v1", kind: "Service"}, - } - - // Override configuration to match expected Enterprise config. - instance.Variant = operatorv1.CalicoEnterprise - cfg.ManagementCluster = &operatorv1.ManagementCluster{} - cfg.KubeControllersGatewaySecret = &testutils.KubeControllersUserSecret - cfg.MetricsPort = 9094 - - component := entkubecontrollers.NewElasticsearchKubeControllers(&cfg) - Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() - Expect(len(resources)).To(Equal(len(expectedResources))) - - // Should render the correct resources. - i := 0 - for _, expectedRes := range expectedResources { - rtest.ExpectResourceTypeAndObjectMetadata(resources[i], expectedRes.name, expectedRes.ns, expectedRes.group, expectedRes.version, expectedRes.kind) - i++ - } - - // The Deployment should have the correct configuration. - dp := rtest.GetResource(resources, entkubecontrollers.EsKubeController, common.CalicoNamespace, "apps", "v1", "Deployment").(*appsv1.Deployment) - - envs := dp.Spec.Template.Spec.Containers[0].Env - Expect(envs).To(ContainElement(corev1.EnvVar{ - Name: "ENABLED_CONTROLLERS", - Value: "authorization,elasticsearchconfiguration,managedcluster", - })) - - Expect(dp.Spec.Template.Spec.Containers[0].VolumeMounts).To(HaveLen(1)) - Expect(dp.Spec.Template.Spec.Containers[0].VolumeMounts[0].Name).To(Equal("tigera-ca-bundle")) - Expect(dp.Spec.Template.Spec.Containers[0].VolumeMounts[0].MountPath).To(Equal("/etc/pki/tls/certs")) - - Expect(dp.Spec.Template.Spec.Volumes).To(HaveLen(1)) - Expect(dp.Spec.Template.Spec.Volumes[0].Name).To(Equal("tigera-ca-bundle")) - Expect(dp.Spec.Template.Spec.Volumes[0].ConfigMap.Name).To(Equal("tigera-ca-bundle")) - - Expect(dp.Spec.Template.Spec.Containers[0].Image).To(Equal("test-reg/tigera/calico:" + components.ComponentTigeraCalico.Version)) - - clusterRole := rtest.GetResource(resources, entkubecontrollers.EsKubeControllerRole, "", "rbac.authorization.k8s.io", "v1", "ClusterRole").(*rbacv1.ClusterRole) - Expect(clusterRole.Rules).To(HaveLen(26), "cluster role should have 26 rules") - Expect(clusterRole.Rules).To(ContainElement( - rbacv1.PolicyRule{ - APIGroups: []string{""}, - Resources: []string{"configmaps"}, - Verbs: []string{"watch", "list", "get", "update", "create", "delete"}, - })) - Expect(clusterRole.Rules).To(ContainElement( - rbacv1.PolicyRule{ - APIGroups: []string{""}, - Resources: []string{"secrets"}, - Verbs: []string{"watch", "list", "get"}, - })) - roleBindingWatch := rtest.GetResource(resources, kubecontrollers.ManagedClustersWatchRoleBindingName, "", "rbac.authorization.k8s.io", "v1", "ClusterRoleBinding").(*rbacv1.ClusterRoleBinding) - Expect(roleBindingWatch.RoleRef.Name).To(Equal(render.ManagedClustersWatchClusterRoleName)) - Expect(roleBindingWatch.Subjects).To(ConsistOf([]rbacv1.Subject{ - { - Kind: "ServiceAccount", - Name: kubecontrollers.KubeControllerServiceAccount, - Namespace: common.CalicoNamespace, - }, - })) - }) - It("should include a ControlPlaneNodeSelector when specified", func() { expectedResources := []struct { name string @@ -564,43 +384,6 @@ var _ = Describe("kube-controllers rendering tests", func() { Expect(passed).To(Equal(true)) }) - It("should add the OIDC prefix env variables", func() { - instance.Variant = operatorv1.CalicoEnterprise - cfg.ManagementCluster = &operatorv1.ManagementCluster{} - cfg.KubeControllersGatewaySecret = &testutils.KubeControllersUserSecret - cfg.MetricsPort = 9094 - cfg.Authentication = &operatorv1.Authentication{Spec: operatorv1.AuthenticationSpec{ - UsernamePrefix: "uOIDC:", - GroupsPrefix: "gOIDC:", - Openshift: &operatorv1.AuthenticationOpenshift{IssuerURL: "https://api.example.com"}, - }} - - component := entkubecontrollers.NewElasticsearchKubeControllers(&cfg) - Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() - - depResource := rtest.GetResource(resources, entkubecontrollers.EsKubeController, common.CalicoNamespace, "apps", "v1", "Deployment") - Expect(depResource).ToNot(BeNil()) - deployment := depResource.(*appsv1.Deployment) - - var usernamePrefix, groupPrefix string - for _, container := range deployment.Spec.Template.Spec.Containers { - if container.Name == entkubecontrollers.EsKubeController { - for _, env := range container.Env { - switch env.Name { - case "OIDC_AUTH_USERNAME_PREFIX": - usernamePrefix = env.Value - case "OIDC_AUTH_GROUP_PREFIX": - groupPrefix = env.Value - } - } - } - } - - Expect(usernamePrefix).To(Equal("uOIDC:")) - Expect(groupPrefix).To(Equal("gOIDC:")) - }) - Context("With calico-kube-controllers overrides", func() { rr1 := corev1.ResourceRequirements{ Limits: corev1.ResourceList{ @@ -829,34 +612,6 @@ var _ = Describe("kube-controllers rendering tests", func() { }) }) - When("enableESOIDCWorkaround is true", func() { - It("should set the ENABLE_ELASTICSEARCH_OIDC_WORKAROUND env variable to true", func() { - instance.Variant = operatorv1.CalicoEnterprise - cfg.ManagementCluster = &operatorv1.ManagementCluster{} - cfg.KubeControllersGatewaySecret = &testutils.KubeControllersUserSecret - cfg.MetricsPort = 9094 - component := entkubecontrollers.NewElasticsearchKubeControllers(&cfg) - resources, _ := component.Objects() - - depResource := rtest.GetResource(resources, entkubecontrollers.EsKubeController, common.CalicoNamespace, "apps", "v1", "Deployment") - Expect(depResource).ToNot(BeNil()) - deployment := depResource.(*appsv1.Deployment) - - var esLicenseType string - for _, container := range deployment.Spec.Template.Spec.Containers { - if container.Name == entkubecontrollers.EsKubeController { - for _, env := range container.Env { - if env.Name == "ENABLE_ELASTICSEARCH_OIDC_WORKAROUND" { - esLicenseType = env.Value - } - } - } - } - - Expect(esLicenseType).To(Equal("true")) - }) - }) - It("should add the KUBERNETES_SERVICE_... variables", func() { cfg.K8sServiceEpPodNetwork = k8sapi.ServiceEndpoint{ Host: "k8shost", @@ -964,46 +719,6 @@ var _ = Describe("kube-controllers rendering tests", func() { }) }) - Context("es-kube-controllers calico-system rendering", func() { - policyName := types.NamespacedName{Name: "calico-system.es-kube-controller-access", Namespace: common.CalicoNamespace} - - getExpectedPolicy := func(scenario testutils.CalicoSystemScenario) *v3.NetworkPolicy { - if scenario.ManagedCluster { - return nil - } - - return testutils.SelectPolicyByProvider(scenario, expectedESPolicy, expectedESPolicyForOpenshift) - } - - DescribeTable("should render calico-system policy", - func(scenario testutils.CalicoSystemScenario) { - if scenario.OpenShift { - cfg.Installation.KubernetesProvider = operatorv1.ProviderOpenShift - } else { - cfg.Installation.KubernetesProvider = operatorv1.ProviderNone - } - if scenario.ManagedCluster { - cfg.ManagementClusterConnection = &operatorv1.ManagementClusterConnection{} - } else { - cfg.ManagementClusterConnection = nil - } - instance.Variant = operatorv1.CalicoEnterprise - cfg.KubeControllersGatewaySecret = &testutils.KubeControllersUserSecret - - component := entkubecontrollers.NewElasticsearchKubeControllers(&cfg) - resources, _ := component.Objects() - - policy := testutils.GetCalicoSystemPolicyFromResources(policyName, resources) - expectedPolicy := getExpectedPolicy(scenario) - Expect(policy).To(Equal(expectedPolicy)) - }, - Entry("for management/standalone, kube-dns", testutils.CalicoSystemScenario{ManagedCluster: false, OpenShift: false}), - Entry("for management/standalone, openshift-dns", testutils.CalicoSystemScenario{ManagedCluster: false, OpenShift: true}), - Entry("for managed, kube-dns", testutils.CalicoSystemScenario{ManagedCluster: true, OpenShift: false}), - Entry("for managed, openshift-dns", testutils.CalicoSystemScenario{ManagedCluster: true, OpenShift: true}), - ) - }) - It("should add egress policy with Enterprise variant and K8SServiceEndpoint defined", func() { cfg.K8sServiceEp.Host = "k8shost" cfg.K8sServiceEp.Port = "1234" From efcf22374b2fa394bc9c5487cc06dd96cde10fb9 Mon Sep 17 00:00:00 2001 From: Casey Davenport Date: Wed, 24 Jun 2026 15:08:32 -0700 Subject: [PATCH 50/54] Decouple the clusterconnection controller from the enterprise variant The ManagementClusterConnection controller renders Guardian for both OSS (Whisker) and Enterprise (managed-cluster tunnel), and had IsEnterprise branches woven through its reconcile: the management/managed mutual-exclusion check, the managed cluster version (CNX vs Calico), and the license-gated egress network policy. Add a clusterconnection ControllerExtension (pkg/enterprise/clusterconnection) with Validate (the mutual-exclusion check) and ExtendContext (CNXVersion + the license check), stashing the result as a render.GuardianRenderData in the render context. The controller reads it back generically to fill GuardianConfiguration, falling back to OSS defaults (CalicoVersion, the Whisker Guardian client keypair, egress disabled) when no extension is registered. No IsEnterprise left in the reconcile data path. Still variant-aware in the controller: the ManagementClusterConnection CR validation and defaulting (impersonation, public CA) and the ImageSet selection. Those would need a CR validator/defaulter companion to move, left as a follow-up. --- .../clusterconnection_controller.go | 63 +++++----- pkg/controller/contexts/context.go | 7 +- pkg/enterprise/clusterconnection/extension.go | 79 ++++++++++++ .../clusterconnection/extension_test.go | 114 ++++++++++++++++++ .../clusterconnection/suite_test.go | 34 ++++++ pkg/enterprise/register.go | 2 + pkg/render/guardian.go | 25 ++++ pkg/render/rendercontext.go | 11 +- 8 files changed, 298 insertions(+), 37 deletions(-) create mode 100644 pkg/enterprise/clusterconnection/extension.go create mode 100644 pkg/enterprise/clusterconnection/extension_test.go create mode 100644 pkg/enterprise/clusterconnection/suite_test.go diff --git a/pkg/controller/clusterconnection/clusterconnection_controller.go b/pkg/controller/clusterconnection/clusterconnection_controller.go index 34cba8d367..b5f5f85e96 100644 --- a/pkg/controller/clusterconnection/clusterconnection_controller.go +++ b/pkg/controller/clusterconnection/clusterconnection_controller.go @@ -43,6 +43,7 @@ import ( operatorv1 "github.com/tigera/operator/api/v1" "github.com/tigera/operator/pkg/common" "github.com/tigera/operator/pkg/controller/certificatemanager" + "github.com/tigera/operator/pkg/controller/contexts" "github.com/tigera/operator/pkg/controller/options" "github.com/tigera/operator/pkg/controller/status" "github.com/tigera/operator/pkg/controller/utils" @@ -244,21 +245,6 @@ func (r *ReconcileConnection) Reconcile(ctx context.Context, request reconcile.R } } - // Verify the cluster doesn't also have the ManagementCluster CRD installed. - if variant.IsEnterprise() { - managementCluster, err := utils.GetManagementCluster(ctx, r.cli) - if err != nil { - r.status.SetDegraded(operatorv1.ResourceReadError, "Error reading ManagementCluster", err, reqLogger) - return reconcile.Result{}, err - } - - if managementCluster != nil { - err = fmt.Errorf("having both a ManagementCluster and a ManagementClusterConnection is not supported") - r.status.SetDegraded(operatorv1.ResourceValidationError, "", err, reqLogger) - return reconcile.Result{}, err - } - } - // Validate that the cluster information watch is ready. if !r.clusterInfoWatchReady.IsReady() { r.status.SetDegraded(operatorv1.ResourceNotReady, "Waiting for clusterInfoWatchReady watch to be established", err, reqLogger) @@ -289,6 +275,29 @@ func (r *ReconcileConnection) Reconcile(ctx context.Context, request reconcile.R return reconcile.Result{}, err } + // Run the variant extension: it validates the configuration (a cluster cannot be + // both a management and a managed cluster) and produces the Enterprise-specific + // Guardian inputs the controller reads back below (the managed cluster version and + // the license-gated egress policy flag). For the core operator this is a no-op and + // the render context carries no extension data, so the OSS defaults apply. + cc := contexts.ControllerContext{ + RenderContext: render.RenderContext{Installation: installationSpec, ClusterDomain: r.opts.ClusterDomain}, + Controller: contexts.ClusterConnectionController, + Ctx: ctx, + Client: r.cli, + CertificateManager: certificateManager, + } + if err := r.opts.Extensions.Validate(cc); err != nil { + r.status.SetDegraded(operatorv1.ResourceValidationError, "Invalid ManagementClusterConnection configuration", err, reqLogger) + return reconcile.Result{}, err + } + renderCtx, _, err := r.opts.Extensions.ExtendContext(cc) + if err != nil { + r.status.SetDegraded(operatorv1.ResourceCreateError, "Error preparing the clusterconnection extension", err, reqLogger) + return reconcile.Result{}, err + } + guardianData, haveGuardianData := render.GuardianRenderDataFromContext(renderCtx) + includeSystem := false if managementClusterConnection.Spec.TLS.CA == operatorv1.CATypePublic { if variant == operatorv1.Calico { @@ -305,8 +314,11 @@ func (r *ReconcileConnection) Reconcile(ctx context.Context, request reconcile.R r.status.SetDegraded(operatorv1.ResourceCreateError, "Unable to create the trusted bundle", err, reqLogger) } + // In the OSS (Whisker) path Guardian connects with its own client keypair. The + // Enterprise path uses the tunnel secret instead, so when the extension supplied + // its Guardian inputs we skip creating this keypair. var guardianKeyPair certificatemanagement.KeyPairInterface - if !variant.IsEnterprise() { + if !haveGuardianData { guardianCertificateNames := dns.GetServiceDNSNames("guardian", render.GuardianNamespace, r.opts.ClusterDomain) guardianCertificateNames = append(guardianCertificateNames, "localhost", "127.0.0.1") guardianKeyPair, err = certificateManager.GetOrCreateKeyPair(r.cli, render.GuardianKeyPairSecret, whisker.WhiskerNamespace, guardianCertificateNames) @@ -409,8 +421,8 @@ func (r *ReconcileConnection) Reconcile(ctx context.Context, request reconcile.R r.status.SetDegraded(operatorv1.ResourceReadError, "Error querying clusterInformation", err, reqLogger) return reconcile.Result{}, err } - if variant.IsEnterprise() { - managedClusterVersion = clusterInformation.Spec.CNXVersion + if haveGuardianData { + managedClusterVersion = guardianData.Version } else { managedClusterVersion = clusterInformation.Spec.CalicoVersion } @@ -421,18 +433,9 @@ func (r *ReconcileConnection) Reconcile(ctx context.Context, request reconcile.R return reconcile.Result{RequeueAfter: utils.StandardRetry}, nil } - var includeEgressNetworkPolicy bool - if variant.IsEnterprise() { - // Ensure the license can support enterprise policy, before rendering any network policies within it. - if license, err := utils.FetchLicenseKey(ctx, r.cli); err == nil { - if utils.IsFeatureActive(license, common.EgressAccessControlFeature) { - includeEgressNetworkPolicy = true - } - } else if !k8serrors.IsNotFound(err) { - r.status.SetDegraded(operatorv1.ResourceReadError, "Error querying license", err, reqLogger) - return reconcile.Result{}, err - } - } + // The Enterprise extension gates the domain-based egress rules on the license; the + // OSS default is to leave them disabled. + includeEgressNetworkPolicy := guardianData.IncludeEgressNetworkPolicy // Ensure the calico-system tier exists, before rendering any network policies within it. var tierAvailable bool diff --git a/pkg/controller/contexts/context.go b/pkg/controller/contexts/context.go index eb5baf4eab..2436206237 100644 --- a/pkg/controller/contexts/context.go +++ b/pkg/controller/contexts/context.go @@ -34,9 +34,10 @@ import ( type ControllerName string const ( - InstallationController ControllerName = "installation" - WindowsController ControllerName = "windows" - APIServerController ControllerName = "apiserver" + InstallationController ControllerName = "installation" + WindowsController ControllerName = "windows" + APIServerController ControllerName = "apiserver" + ClusterConnectionController ControllerName = "clusterconnection" ) // ControllerContext is the controller-phase context, the corollary to the diff --git a/pkg/enterprise/clusterconnection/extension.go b/pkg/enterprise/clusterconnection/extension.go new file mode 100644 index 0000000000..d07a984162 --- /dev/null +++ b/pkg/enterprise/clusterconnection/extension.go @@ -0,0 +1,79 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package clusterconnection + +import ( + "fmt" + + k8serrors "k8s.io/apimachinery/pkg/api/errors" + + "github.com/tigera/operator/pkg/common" + "github.com/tigera/operator/pkg/controller/contexts" + "github.com/tigera/operator/pkg/controller/utils" + "github.com/tigera/operator/pkg/extensions" + "github.com/tigera/operator/pkg/render" + "github.com/tigera/operator/pkg/tls/certificatemanagement" +) + +// Register wires the clusterconnection controller hook into the variant. +func Register(v *extensions.Variant) { + v.Controller(contexts.ClusterConnectionController, clusterConnectionControllerExtension{}) +} + +// clusterConnectionControllerExtension is the Calico Enterprise controller-side hook +// for the clusterconnection (ManagementClusterConnection) controller. +type clusterConnectionControllerExtension struct{} + +// Validate rejects clusterconnection configuration Calico Enterprise does not +// support: a cluster cannot be both a management cluster and a managed cluster. +func (clusterConnectionControllerExtension) Validate(cc contexts.ControllerContext) error { + managementCluster, err := utils.GetManagementCluster(cc.Ctx, cc.Client) + if err != nil { + return fmt.Errorf("error reading ManagementCluster: %w", err) + } + if managementCluster != nil { + return fmt.Errorf("having both a ManagementCluster and a ManagementClusterConnection is not supported") + } + return nil +} + +// ExtendContext computes the Enterprise-specific Guardian inputs the controller +// reads back: the managed cluster version (CNXVersion) and whether the license +// permits the domain-based egress network policy. It creates no certificates, so it +// returns no managed keypairs. The OSS controller path supplies its own defaults +// when this hook is absent. +func (clusterConnectionControllerExtension) ExtendContext(cc contexts.ControllerContext) (render.RenderContext, []certificatemanagement.KeyPairInterface, error) { + rc := cc.RenderContext + + clusterInformation, err := utils.FetchClusterInformation(cc.Ctx, cc.Client) + if err != nil { + return rc, nil, fmt.Errorf("error querying ClusterInformation: %w", err) + } + + // Ensure the license can support enterprise policy before enabling the + // domain-based egress rules. A missing license simply leaves them disabled. + var includeEgressNetworkPolicy bool + if license, err := utils.FetchLicenseKey(cc.Ctx, cc.Client); err == nil { + includeEgressNetworkPolicy = utils.IsFeatureActive(license, common.EgressAccessControlFeature) + } else if !k8serrors.IsNotFound(err) { + return rc, nil, fmt.Errorf("error querying license: %w", err) + } + + rc.Extension = render.GuardianRenderData{ + Version: clusterInformation.Spec.CNXVersion, + IncludeEgressNetworkPolicy: includeEgressNetworkPolicy, + } + return rc, nil, nil +} diff --git a/pkg/enterprise/clusterconnection/extension_test.go b/pkg/enterprise/clusterconnection/extension_test.go new file mode 100644 index 0000000000..1a807b8545 --- /dev/null +++ b/pkg/enterprise/clusterconnection/extension_test.go @@ -0,0 +1,114 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package clusterconnection_test + +import ( + "context" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + + v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" + + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/apis" + "github.com/tigera/operator/pkg/common" + "github.com/tigera/operator/pkg/controller/contexts" + ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" + "github.com/tigera/operator/pkg/render" +) + +var _ = Describe("clusterconnection enterprise controller extension", func() { + var cli client.Client + + // controllerContext builds a ControllerContext selecting the enterprise + // clusterconnection hook against the given client. + controllerContext := func() contexts.ControllerContext { + return contexts.ControllerContext{ + RenderContext: render.RenderContext{ + Installation: &operatorv1.InstallationSpec{Variant: operatorv1.CalicoEnterprise}, + }, + Controller: contexts.ClusterConnectionController, + Ctx: context.Background(), + Client: cli, + } + } + + clusterInformation := func() *v3.ClusterInformation { + return &v3.ClusterInformation{ + ObjectMeta: metav1.ObjectMeta{Name: "default"}, + Spec: v3.ClusterInformationSpec{CNXVersion: "v3.99.0", CalicoVersion: "v3.99.0-calico"}, + } + } + + newClient := func(objs ...client.Object) client.Client { + scheme := runtime.NewScheme() + Expect(apis.AddToScheme(scheme, false)).NotTo(HaveOccurred()) + return ctrlrfake.DefaultFakeClientBuilder(scheme).WithObjects(objs...).Build() + } + + Describe("Validate", func() { + It("passes when no ManagementCluster exists", func() { + cli = newClient() + Expect(ext.Validate(controllerContext())).NotTo(HaveOccurred()) + }) + + It("rejects a cluster that is both a management and a managed cluster", func() { + cli = newClient(&operatorv1.ManagementCluster{ObjectMeta: metav1.ObjectMeta{Name: "tigera-secure"}}) + err := ext.Validate(controllerContext()) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("not supported")) + }) + }) + + Describe("ExtendContext", func() { + It("reports the managed cluster CNX version", func() { + cli = newClient(clusterInformation()) + rc, managed, err := ext.ExtendContext(controllerContext()) + Expect(err).NotTo(HaveOccurred()) + Expect(managed).To(BeEmpty()) + + data, ok := render.GuardianRenderDataFromContext(rc) + Expect(ok).To(BeTrue()) + Expect(data.Version).To(Equal("v3.99.0")) + Expect(data.IncludeEgressNetworkPolicy).To(BeFalse()) + }) + + It("enables the egress network policy when the license has the feature", func() { + license := &v3.LicenseKey{ + ObjectMeta: metav1.ObjectMeta{Name: "default"}, + Status: v3.LicenseKeyStatus{Features: []string{common.EgressAccessControlFeature}}, + } + cli = newClient(clusterInformation(), license) + rc, _, err := ext.ExtendContext(controllerContext()) + Expect(err).NotTo(HaveOccurred()) + + data, ok := render.GuardianRenderDataFromContext(rc) + Expect(ok).To(BeTrue()) + Expect(data.IncludeEgressNetworkPolicy).To(BeTrue()) + }) + + It("errors when ClusterInformation is missing", func() { + cli = newClient() + _, _, err := ext.ExtendContext(controllerContext()) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("ClusterInformation")) + }) + }) +}) diff --git a/pkg/enterprise/clusterconnection/suite_test.go b/pkg/enterprise/clusterconnection/suite_test.go new file mode 100644 index 0000000000..abd8adc498 --- /dev/null +++ b/pkg/enterprise/clusterconnection/suite_test.go @@ -0,0 +1,34 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package clusterconnection_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/tigera/operator/pkg/enterprise" + "github.com/tigera/operator/pkg/extensions" +) + +// ext is the enterprise extension Set under test, shared across the suite. It is +// immutable once built and the specs only read it, so a single instance is safe. +var ext *extensions.Set = enterprise.New() + +func TestClusterConnection(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "pkg/enterprise/clusterconnection Suite") +} diff --git a/pkg/enterprise/register.go b/pkg/enterprise/register.go index db211238c2..a58abc1b13 100644 --- a/pkg/enterprise/register.go +++ b/pkg/enterprise/register.go @@ -17,6 +17,7 @@ package enterprise import ( operatorv1 "github.com/tigera/operator/api/v1" "github.com/tigera/operator/pkg/enterprise/apiserver" + "github.com/tigera/operator/pkg/enterprise/clusterconnection" "github.com/tigera/operator/pkg/enterprise/guardian" "github.com/tigera/operator/pkg/enterprise/installation" "github.com/tigera/operator/pkg/enterprise/typha" @@ -39,6 +40,7 @@ func New() *extensions.Set { windows.Register(ent) guardian.Register(ent) apiserver.Register(ent) + clusterconnection.Register(ent) // When the enterprise operator manages a Calico installation, clean up the // Enterprise objects left behind by a prior Enterprise installation. diff --git a/pkg/render/guardian.go b/pkg/render/guardian.go index 9de89e6736..6fe327a3d0 100644 --- a/pkg/render/guardian.go +++ b/pkg/render/guardian.go @@ -149,6 +149,31 @@ type GuardianConfiguration struct { Version string } +// GuardianRenderData is the variant-specific Guardian input a controller extension +// computes during reconcile and stashes in RenderContext.Extension. The +// clusterconnection controller reads it back to fill GuardianConfiguration without +// depending on the extension: when present it carries the enterprise values +// (the management-cluster version and the license-gated egress policy flag) and +// signals that the controller should not create the OSS Guardian client keypair. +// It lives in render so the controller can read it generically. +type GuardianRenderData struct { + // Version is the managed cluster version reported by ClusterInformation + // (CNXVersion for Enterprise, CalicoVersion for the OSS default). + Version string + + // IncludeEgressNetworkPolicy enables the domain-based egress rules in the + // Guardian policy, gated on an Enterprise license feature. + IncludeEgressNetworkPolicy bool +} + +// GuardianRenderDataFromContext returns the GuardianRenderData a controller +// extension stashed in the render context, and whether it was present. Absent +// means the OSS path: the controller applies its own defaults. +func GuardianRenderDataFromContext(rc RenderContext) (GuardianRenderData, bool) { + data, ok := rc.Extension.(GuardianRenderData) + return data, ok +} + type GuardianComponent struct { cfg *GuardianConfiguration calicoImage string diff --git a/pkg/render/rendercontext.go b/pkg/render/rendercontext.go index d3821c168b..7101a0e10d 100644 --- a/pkg/render/rendercontext.go +++ b/pkg/render/rendercontext.go @@ -45,10 +45,13 @@ type RenderContext struct { TrustedBundle certificatemanagement.TrustedBundle // Extension is opaque, extension-owned data that the controller extension - // produced for its own modifiers - typically an artifact that can only be - // created controller-side because it has cluster side effects (e.g. a keypair). - // The extension that set it type-asserts it back out in its modifiers; core - // code never reads it. Nil when no extension is active. + // produced. Usually a modifier reads it back out (an artifact that can only be + // created controller-side because it has cluster side effects, e.g. a keypair), + // in which case it holds an extension-package type core code never names. When a + // controller needs the extension's reconcile-derived inputs back (e.g. the + // clusterconnection controller's variant-specific Guardian inputs), the payload + // is instead a render-package type the controller reads generically without + // depending on the extension. Nil when no extension is active. Extension any } From 5ab843e35f03b9912f4e81ed22f8ba855f7e2d96 Mon Sep 17 00:00:00 2001 From: Casey Davenport Date: Wed, 24 Jun 2026 15:19:39 -0700 Subject: [PATCH 51/54] Split the clusterconnection enterprise tests into their own file Move the Calico Enterprise cases (enterprise images, the license/tier-gated calico-system policy, and impersonation) out of clusterconnection_controller_test.go into clusterconnection_controller_enterprise_test.go. The generic controller mechanics (default reconcile, the guardian finalizer, proxy settings, and the tigerastatus conditions) stay in the main file. Same specs, just relocated; the package-scope proxy helpers are shared between the two files. --- ...erconnection_controller_enterprise_test.go | 510 ++++++++++++++++++ .../clusterconnection_controller_test.go | 353 ------------ 2 files changed, 510 insertions(+), 353 deletions(-) create mode 100644 pkg/controller/clusterconnection/clusterconnection_controller_enterprise_test.go diff --git a/pkg/controller/clusterconnection/clusterconnection_controller_enterprise_test.go b/pkg/controller/clusterconnection/clusterconnection_controller_enterprise_test.go new file mode 100644 index 0000000000..d79ec8aad7 --- /dev/null +++ b/pkg/controller/clusterconnection/clusterconnection_controller_enterprise_test.go @@ -0,0 +1,510 @@ +// Copyright (c) 2020-2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package clusterconnection_test + +import ( + "context" + "fmt" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/stretchr/testify/mock" + + appsv1 "k8s.io/api/apps/v1" + v1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" + + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/apis" + "github.com/tigera/operator/pkg/common" + "github.com/tigera/operator/pkg/components" + "github.com/tigera/operator/pkg/controller/certificatemanager" + "github.com/tigera/operator/pkg/controller/clusterconnection" + "github.com/tigera/operator/pkg/controller/status" + "github.com/tigera/operator/pkg/controller/utils" + ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" + "github.com/tigera/operator/pkg/dns" + "github.com/tigera/operator/pkg/render" + "github.com/tigera/operator/pkg/render/common/networkpolicy" + "github.com/tigera/operator/pkg/render/monitor" + "github.com/tigera/operator/test" +) + +// These tests cover the Calico Enterprise behavior of the ManagementClusterConnection +// controller: the enterprise images, the license/tier-gated calico-system network +// policy, and impersonation. The shared (variant-agnostic) controller mechanics live +// in clusterconnection_controller_test.go. +var _ = Describe("ManagementClusterConnection controller enterprise tests", func() { + var c client.Client + var ctx context.Context + var cfg *operatorv1.ManagementClusterConnection + var installation *operatorv1.Installation + var r reconcile.Reconciler + var clientScheme *runtime.Scheme + var mockStatus *status.MockStatus + var objTrackerWithCalls test.ObjectTrackerWithCalls + + notReady := &utils.ReadyFlag{} + ready := &utils.ReadyFlag{} + ready.MarkAsReady() + + BeforeEach(func() { + // Create a Kubernetes client. + clientScheme = runtime.NewScheme() + Expect(apis.AddToScheme(clientScheme, false)).ShouldNot(HaveOccurred()) + Expect(appsv1.SchemeBuilder.AddToScheme(clientScheme)).ShouldNot(HaveOccurred()) + Expect(rbacv1.SchemeBuilder.AddToScheme(clientScheme)).ShouldNot(HaveOccurred()) + err := operatorv1.SchemeBuilder.AddToScheme(clientScheme) + Expect(err).NotTo(HaveOccurred()) + objTrackerWithCalls = test.NewObjectTrackerWithCalls(clientScheme) + c = ctrlrfake.DefaultFakeClientBuilder(clientScheme).WithObjectTracker(&objTrackerWithCalls).Build() + ctx = context.Background() + mockStatus = &status.MockStatus{} + mockStatus.On("Run").Return() + mockStatus.On("AddDaemonsets", mock.Anything) + mockStatus.On("AddDeployments", mock.Anything) + mockStatus.On("AddStatefulSets", mock.Anything) + mockStatus.On("AddCronJobs", mock.Anything) + mockStatus.On("ClearDegraded", mock.Anything) + mockStatus.On("SetDegraded", mock.Anything, mock.Anything, mock.Anything, mock.Anything) + mockStatus.On("OnCRFound").Return() + mockStatus.On("ReadyToMonitor") + mockStatus.On("SetMetaData", mock.Anything).Return() + mockStatus.On("OnCRNotFound").Return() + + Expect(c.Create(ctx, &operatorv1.Monitor{ + ObjectMeta: metav1.ObjectMeta{Name: "tigera-secure"}, + })) + + Expect(c.Create(ctx, &v3.ClusterInformation{ObjectMeta: metav1.ObjectMeta{Name: "default"}})).NotTo(HaveOccurred()) + + r = clusterconnection.NewReconcilerWithShims(c, clientScheme, mockStatus, operatorv1.ProviderNone, ready, ready) + + Expect(c.Create(ctx, &v1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: render.GuardianNamespace}})) + certificateManager, err := certificatemanager.Create(c, nil, dns.DefaultClusterDomain, common.OperatorNamespace(), certificatemanager.AllowCACreation()) + Expect(err).NotTo(HaveOccurred()) + Expect(c.Create(ctx, certificateManager.KeyPair().Secret(common.OperatorNamespace()))) // Persist the root-ca in the operator namespace. + secret, err := certificateManager.GetOrCreateKeyPair(c, render.GuardianSecretName, common.OperatorNamespace(), []string{"a"}) + Expect(err).NotTo(HaveOccurred()) + + pcSecret, err := certificateManager.GetOrCreateKeyPair(c, render.PacketCaptureServerCert, common.OperatorNamespace(), []string{"a"}) + Expect(err).NotTo(HaveOccurred()) + + promSecret, err := certificateManager.GetOrCreateKeyPair(c, monitor.PrometheusServerTLSSecretName, common.OperatorNamespace(), []string{"a"}) + Expect(err).NotTo(HaveOccurred()) + + queryServerSecret, err := certificateManager.GetOrCreateKeyPair(c, render.CalicoAPIServerTLSSecretName, common.OperatorNamespace(), []string{"a"}) + Expect(err).NotTo(HaveOccurred()) + + err = c.Create(ctx, secret.Secret(common.OperatorNamespace())) + Expect(err).NotTo(HaveOccurred()) + err = c.Create(ctx, pcSecret.Secret(common.OperatorNamespace())) + Expect(err).NotTo(HaveOccurred()) + err = c.Create(ctx, promSecret.Secret(common.OperatorNamespace())) + Expect(err).NotTo(HaveOccurred()) + err = c.Create(ctx, queryServerSecret.Secret(common.OperatorNamespace())) + Expect(err).NotTo(HaveOccurred()) + + trustedBundle := certificateManager.CreateTrustedBundle() + Expect(c.Create(ctx, trustedBundle.ConfigMap(render.GuardianNamespace))).NotTo(HaveOccurred()) + + By("applying the required prerequisites") + // Create a ManagementClusterConnection in the k8s client. + cfg = &operatorv1.ManagementClusterConnection{ + ObjectMeta: metav1.ObjectMeta{Name: "tigera-secure", Generation: 3}, + Spec: operatorv1.ManagementClusterConnectionSpec{ + ManagementClusterAddr: "127.0.0.1:12345", + }, + } + err = c.Create(ctx, cfg) + Expect(err).NotTo(HaveOccurred()) + + installation = &operatorv1.Installation{ + Spec: operatorv1.InstallationSpec{ + Variant: operatorv1.CalicoEnterprise, + Registry: "some.registry.org/", + }, + ObjectMeta: metav1.ObjectMeta{Name: "default"}, + Status: operatorv1.InstallationStatus{ + Variant: operatorv1.CalicoEnterprise, + Computed: &operatorv1.InstallationSpec{ + Registry: "my-reg", + KubernetesProvider: operatorv1.ProviderNone, + }, + }, + } + err = c.Create(ctx, installation) + Expect(err).NotTo(HaveOccurred()) + }) + + Context("image reconciliation", func() { + BeforeEach(func() { + Expect(c.Create(ctx, &v3.Tier{ObjectMeta: metav1.ObjectMeta{Name: "calico-system"}})).NotTo(HaveOccurred()) + }) + + It("should use builtin images", func() { + r = clusterconnection.NewReconcilerWithShims(c, clientScheme, mockStatus, operatorv1.ProviderNone, ready, ready) + _, err := r.Reconcile(ctx, reconcile.Request{}) + Expect(err).ShouldNot(HaveOccurred()) + + d := appsv1.Deployment{ + TypeMeta: metav1.TypeMeta{Kind: "Deployment", APIVersion: "v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: render.GuardianDeploymentName, + Namespace: render.GuardianNamespace, + }, + } + Expect(test.GetResource(c, &d)).To(BeNil()) + Expect(d.Spec.Template.Spec.Containers).To(HaveLen(1)) + dexC := test.GetContainer(d.Spec.Template.Spec.Containers, render.GuardianContainerName) + Expect(dexC).ToNot(BeNil()) + Expect(dexC.Image).To(Equal( + fmt.Sprintf("some.registry.org/%s%s:%s", + components.TigeraImagePath, + components.ComponentTigeraCalico.Image, + components.ComponentTigeraCalico.Version))) + }) + It("should use images from imageset", func() { + Expect(c.Create(ctx, &operatorv1.ImageSet{ + ObjectMeta: metav1.ObjectMeta{Name: "enterprise-" + components.EnterpriseRelease}, + Spec: operatorv1.ImageSetSpec{ + Images: []operatorv1.Image{ + {Image: "tigera/calico", Digest: "sha256:guardianhash"}, + {Image: "tigera/calico", Digest: "sha256:deadbeef0123456789"}, + }, + }, + })).ToNot(HaveOccurred()) + + r = clusterconnection.NewReconcilerWithShims(c, clientScheme, mockStatus, operatorv1.ProviderNone, ready, ready) + _, err := r.Reconcile(ctx, reconcile.Request{}) + Expect(err).ShouldNot(HaveOccurred()) + + d := appsv1.Deployment{ + TypeMeta: metav1.TypeMeta{Kind: "Deployment", APIVersion: "v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: render.GuardianDeploymentName, + Namespace: render.GuardianNamespace, + }, + } + Expect(test.GetResource(c, &d)).To(BeNil()) + Expect(d.Spec.Template.Spec.Containers).To(HaveLen(1)) + apiserver := test.GetContainer(d.Spec.Template.Spec.Containers, render.GuardianContainerName) + Expect(apiserver).ToNot(BeNil()) + Expect(apiserver.Image).To(Equal( + fmt.Sprintf("some.registry.org/%s%s@%s", + components.TigeraImagePath, + components.ComponentTigeraCalico.Image, + "sha256:guardianhash"))) + }) + }) + + Context("calico-system reconciliation", func() { + var licenseKey *v3.LicenseKey + BeforeEach(func() { + licenseKey = &v3.LicenseKey{ + ObjectMeta: metav1.ObjectMeta{Name: "default"}, + Status: v3.LicenseKeyStatus{ + Features: []string{ + common.TiersFeature, + common.EgressAccessControlFeature, + }, + }, + } + Expect(c.Create(ctx, licenseKey)).NotTo(HaveOccurred()) + Expect(c.Create(ctx, &v3.Tier{ObjectMeta: metav1.ObjectMeta{Name: "calico-system"}})).NotTo(HaveOccurred()) + r = clusterconnection.NewReconcilerWithShims(c, clientScheme, mockStatus, operatorv1.ProviderNone, ready, ready) + }) + + Context("IP-based management cluster address", func() { + It("should render calico-system policy when tier and watch are ready", func() { + _, err := r.Reconcile(ctx, reconcile.Request{}) + Expect(err).ShouldNot(HaveOccurred()) + + policies := v3.NetworkPolicyList{} + Expect(c.List(ctx, &policies)).ToNot(HaveOccurred()) + + Expect(policies.Items).To(HaveLen(1)) + Expect(policies.Items[0].Name).To(Equal("calico-system.guardian-access")) + }) + + It("should omit calico-system policy and not degrade when tier is not ready", func() { + Expect(c.Delete(ctx, &v3.Tier{ObjectMeta: metav1.ObjectMeta{Name: "calico-system"}})).NotTo(HaveOccurred()) + _, err := r.Reconcile(ctx, reconcile.Request{}) + Expect(err).ShouldNot(HaveOccurred()) + + policies := v3.NetworkPolicyList{} + Expect(c.List(ctx, &policies)).ToNot(HaveOccurred()) + Expect(policies.Items).To(HaveLen(0)) + }) + + It("should degrade and wait when tier is ready, but tier watch is not ready", func() { + mockStatus = &status.MockStatus{} + mockStatus.On("Run").Return() + mockStatus.On("OnCRFound").Return() + mockStatus.On("SetMetaData", mock.Anything).Return() + + r = clusterconnection.NewReconcilerWithShims(c, clientScheme, mockStatus, operatorv1.ProviderNone, notReady, ready) + test.ExpectWaitForTierWatch(ctx, r, mockStatus) + + policies := v3.NetworkPolicyList{} + Expect(c.List(ctx, &policies)).ToNot(HaveOccurred()) + Expect(policies.Items).To(HaveLen(0)) + }) + }) + + Context("Domain-based management cluster address", func() { + BeforeEach(func() { + cfg.Spec.ManagementClusterAddr = "mydomain.io:443" + Expect(c.Update(ctx, cfg)).NotTo(HaveOccurred()) + }) + + It("should render calico-system policy when license and tier are ready", func() { + _, err := r.Reconcile(ctx, reconcile.Request{}) + Expect(err).ShouldNot(HaveOccurred()) + + policies := v3.NetworkPolicyList{} + Expect(c.List(ctx, &policies)).ToNot(HaveOccurred()) + + Expect(policies.Items).To(HaveLen(1)) + Expect(policies.Items[0].Name).To(Equal("calico-system.guardian-access")) + }) + + It("should render calico-system policy without domain-based egress when tier is ready, but license is not sufficient", func() { + licenseKey.Status.Features = []string{common.TiersFeature} + Expect(c.Update(ctx, licenseKey)).NotTo(HaveOccurred()) + + _, err := r.Reconcile(ctx, reconcile.Request{}) + Expect(err).ShouldNot(HaveOccurred()) + + policies := v3.NetworkPolicyList{} + Expect(c.List(ctx, &policies)).ToNot(HaveOccurred()) + Expect(policies.Items).To(HaveLen(1)) + Expect(policies.Items[0].Name).To(Equal("calico-system.guardian-access")) + + // Verify no domain-based egress rules are present + for _, rule := range policies.Items[0].Spec.Egress { + Expect(rule.Destination.Domains).To(BeEmpty(), + "Domain-based egress rules should not be present when license lacks EgressAccessControl") + } + }) + + It("should degrade and wait when tier and license are ready, but tier watch is not ready", func() { + mockStatus = &status.MockStatus{} + mockStatus.On("Run").Return() + mockStatus.On("OnCRFound").Return() + mockStatus.On("SetMetaData", mock.Anything).Return() + + r = clusterconnection.NewReconcilerWithShims(c, clientScheme, mockStatus, operatorv1.ProviderNone, notReady, ready) + test.ExpectWaitForTierWatch(ctx, r, mockStatus) + + policies := v3.NetworkPolicyList{} + Expect(c.List(ctx, &policies)).ToNot(HaveOccurred()) + Expect(policies.Items).To(HaveLen(0)) + }) + + It("should render calico-system policy without domain-based egress when tier is ready but license is not ready", func() { + Expect(c.Delete(ctx, &v3.LicenseKey{ObjectMeta: metav1.ObjectMeta{Name: "default"}})).NotTo(HaveOccurred()) + _, err := r.Reconcile(ctx, reconcile.Request{}) + Expect(err).ShouldNot(HaveOccurred()) + + policies := v3.NetworkPolicyList{} + Expect(c.List(ctx, &policies)).ToNot(HaveOccurred()) + Expect(policies.Items).To(HaveLen(1)) + Expect(policies.Items[0].Name).To(Equal("calico-system.guardian-access")) + + // Verify no domain-based egress rules are present + for _, rule := range policies.Items[0].Spec.Egress { + Expect(rule.Destination.Domains).To(BeEmpty(), + "Domain-based egress rules should not be present when license is not ready") + } + }) + + It("should omit calico-system policy when license is ready but tier is not ready", func() { + Expect(c.Delete(ctx, &v3.Tier{ObjectMeta: metav1.ObjectMeta{Name: "calico-system"}})).NotTo(HaveOccurred()) + _, err := r.Reconcile(ctx, reconcile.Request{}) + Expect(err).ShouldNot(HaveOccurred()) + + policies := v3.NetworkPolicyList{} + Expect(c.List(ctx, &policies)).ToNot(HaveOccurred()) + Expect(policies.Items).To(HaveLen(0)) + }) + }) + + Context("Proxy detection", func() { + // Generate test cases based on the combinations of proxy address forms and proxy settings. + // Here we specify the base targets along with the base proxy IP, domain, and port that will be used for generation. + coreCases := generateCoreProxyTestCases("voltron.io:9000", "192.168.1.2:9000", "proxy.io", "10.1.2.3", "8080") + + // In case we support multiple guardian replicas in the future, we test specific multi-pod scenarios. + multiPodCases := multiplePodCases() + + testCases := append(coreCases, multiPodCases...) + for _, testCase := range testCases { + Describe(fmt.Sprintf("Proxy detection when %+v", test.PrettyFormatProxyTestCase(testCase)), func() { + // Set up the test based on the test case. + BeforeEach(func() { + for i, proxy := range testCase.PodProxies { + createPodWithProxy(ctx, c, proxy, testCase.Lowercase, i) + } + + // Set the target + cfg.Spec.ManagementClusterAddr = testCase.Target + err := c.Update(ctx, cfg) + Expect(err).NotTo(HaveOccurred()) + }) + + It(fmt.Sprintf("detects proxy correctly when %+v", test.PrettyFormatProxyTestCase(testCase)), func() { + // First reconcile creates the guardian deployment without any availability condition. + _, err := r.Reconcile(ctx, reconcile.Request{}) + Expect(err).ShouldNot(HaveOccurred()) + + // Validate that we made no calls to get Pods at this stage. + podGVR := schema.GroupVersionResource{ + Version: "v1", + Resource: "pods", + } + Expect(objTrackerWithCalls.CallCount(podGVR, test.ObjectTrackerCallList)).To(BeZero()) + + // Set the deployment to be unavailable. We need to recreate the deployment otherwise the status update is ignored. + gd := appsv1.Deployment{} + err = c.Get(ctx, client.ObjectKey{Name: render.GuardianDeploymentName, Namespace: render.GuardianNamespace}, &gd) + Expect(err).NotTo(HaveOccurred()) + err = c.Delete(ctx, &gd) + Expect(err).NotTo(HaveOccurred()) + gd.ResourceVersion = "" + gd.Status.Conditions = []appsv1.DeploymentCondition{{ + Type: appsv1.DeploymentAvailable, + Status: v1.ConditionFalse, + LastTransitionTime: metav1.Time{Time: time.Now()}, + }} + err = c.Create(ctx, &gd) + Expect(err).NotTo(HaveOccurred()) + + // Reconcile again. We should see no calls since the deployment has not transitioned to available. + _, err = r.Reconcile(ctx, reconcile.Request{}) + Expect(err).ShouldNot(HaveOccurred()) + Expect(objTrackerWithCalls.CallCount(podGVR, test.ObjectTrackerCallList)).To(Equal(0)) + + // Set the deployment to available. + err = c.Delete(ctx, &gd) + Expect(err).NotTo(HaveOccurred()) + gd.ResourceVersion = "" + gd.Status.Conditions = []appsv1.DeploymentCondition{{ + Type: appsv1.DeploymentAvailable, + Status: v1.ConditionTrue, + LastTransitionTime: metav1.Time{Time: time.Now().Add(time.Minute)}, + }} + err = c.Create(ctx, &gd) + Expect(err).NotTo(HaveOccurred()) + + // Reconcile again. The proxy detection logic should kick in since the guardian deployment is ready. + _, err = r.Reconcile(ctx, reconcile.Request{}) + Expect(err).ShouldNot(HaveOccurred()) + Expect(objTrackerWithCalls.CallCount(podGVR, test.ObjectTrackerCallList)).To(Equal(1)) + + // Resolve the rendered rule that governs egress from guardian to voltron. + policies := v3.NetworkPolicyList{} + Expect(c.List(ctx, &policies)).ToNot(HaveOccurred()) + Expect(policies.Items).To(HaveLen(1)) + Expect(policies.Items[0].Name).To(Equal("calico-system.guardian-access")) + policy := policies.Items[0] + + // Generate the expectation based on the test case, and compare the rendered rule to our expectation. + expectedEgressRules := getExpectedEgressRulesFromCase(testCase) + Expect(policy.Spec.Egress).To(HaveLen(6 + len(expectedEgressRules))) + for i, egressRule := range expectedEgressRules { + managementClusterEgressRule := policy.Spec.Egress[5+i] + if egressRule.hostIsIP { + Expect(managementClusterEgressRule.Destination.Nets).To(HaveLen(1)) + Expect(managementClusterEgressRule.Destination.Nets[0]).To(Equal(fmt.Sprintf("%s/32", egressRule.host))) + Expect(managementClusterEgressRule.Destination.Ports).To(Equal(networkpolicy.Ports(egressRule.port))) + } else { + Expect(managementClusterEgressRule.Destination.Domains).To(Equal([]string{egressRule.host})) + Expect(managementClusterEgressRule.Destination.Ports).To(Equal(networkpolicy.Ports(egressRule.port))) + } + } + + // Reconcile again. Verify that we do not cause any additional query for pods now that we have resolved the proxy. + _, err = r.Reconcile(ctx, reconcile.Request{}) + Expect(err).ShouldNot(HaveOccurred()) + Expect(objTrackerWithCalls.CallCount(podGVR, test.ObjectTrackerCallList)).To(Equal(1)) + }) + }) + } + }) + }) + + val := []string{"some-value"} + DescribeTable("should render impersonation permissions correctly", func(impersonation *operatorv1.Impersonation, expectedUser, expectedGroup, expectedSA []string) { + By("ensuring a tigerastatus exists") + ts := &operatorv1.TigeraStatus{ + ObjectMeta: metav1.ObjectMeta{Name: "management-cluster-connection"}, + Spec: operatorv1.TigeraStatusSpec{}, + Status: operatorv1.TigeraStatusStatus{}, + } + Expect(c.Create(ctx, ts)).NotTo(HaveOccurred()) + + By("updating the CR with the impersonation settings, reconciling and fetching the results") + err := c.Get(ctx, client.ObjectKey{Name: cfg.Name, Namespace: cfg.Namespace}, cfg) + Expect(err).ShouldNot(HaveOccurred()) + cfg.Spec.Impersonation = impersonation + Expect(c.Update(ctx, cfg)).NotTo(HaveOccurred()) + _, err = r.Reconcile(ctx, reconcile.Request{NamespacedName: types.NamespacedName{ + Name: "management-cluster-connection", + Namespace: "", + }}) + Expect(err).ShouldNot(HaveOccurred()) + role := &rbacv1.ClusterRole{} + err = c.Get(ctx, client.ObjectKey{Name: render.GuardianClusterRoleName}, role) + Expect(err).NotTo(HaveOccurred()) + By("verifying the resulting RBAC") + var users, groups, sas []string + for _, rule := range role.Rules { + if len(rule.Verbs) == 1 && rule.Verbs[0] == "impersonate" { + if len(rule.Resources) == 1 { + switch rule.Resources[0] { + case "users": + users = rule.ResourceNames + case "groups": + groups = rule.ResourceNames + case "serviceaccounts": + sas = rule.ResourceNames + } + } + } + } + Expect(users).To(Equal(expectedUser)) + Expect(groups).To(Equal(expectedGroup)) + Expect(sas).To(Equal(expectedSA)) + }, + Entry("no impersonation configured", nil, nil, nil, nil), + Entry("all set", &operatorv1.Impersonation{Users: val, Groups: val, ServiceAccounts: val}, val, val, val), + Entry("all set to empty", &operatorv1.Impersonation{Users: []string{}, Groups: []string{}, ServiceAccounts: []string{}}, nil, nil, nil), + Entry("user set", &operatorv1.Impersonation{Users: val}, val, nil, nil), + Entry("groups set", &operatorv1.Impersonation{Groups: val}, nil, val, nil), + Entry("service accounts set", &operatorv1.Impersonation{ServiceAccounts: val}, nil, nil, val), + Entry("empty impersonation", &operatorv1.Impersonation{}, nil, nil, nil), + ) +}) diff --git a/pkg/controller/clusterconnection/clusterconnection_controller_test.go b/pkg/controller/clusterconnection/clusterconnection_controller_test.go index 6b45399bd0..e8ac420d82 100644 --- a/pkg/controller/clusterconnection/clusterconnection_controller_test.go +++ b/pkg/controller/clusterconnection/clusterconnection_controller_test.go @@ -21,7 +21,6 @@ import ( "net/url" "strconv" "strings" - "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -33,7 +32,6 @@ import ( rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/reconcile" @@ -43,7 +41,6 @@ import ( operatorv1 "github.com/tigera/operator/api/v1" "github.com/tigera/operator/pkg/apis" "github.com/tigera/operator/pkg/common" - "github.com/tigera/operator/pkg/components" "github.com/tigera/operator/pkg/controller/certificatemanager" "github.com/tigera/operator/pkg/controller/clusterconnection" "github.com/tigera/operator/pkg/controller/status" @@ -51,7 +48,6 @@ import ( ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" "github.com/tigera/operator/pkg/dns" "github.com/tigera/operator/pkg/render" - "github.com/tigera/operator/pkg/render/common/networkpolicy" "github.com/tigera/operator/pkg/render/monitor" "github.com/tigera/operator/test" ) @@ -67,7 +63,6 @@ var _ = Describe("ManagementClusterConnection controller tests", func() { var mockStatus *status.MockStatus var objTrackerWithCalls test.ObjectTrackerWithCalls - notReady := &utils.ReadyFlag{} ready := &utils.ReadyFlag{} ready.MarkAsReady() @@ -224,303 +219,6 @@ var _ = Describe("ManagementClusterConnection controller tests", func() { }) }) - Context("image reconciliation", func() { - BeforeEach(func() { - Expect(c.Create(ctx, &v3.Tier{ObjectMeta: metav1.ObjectMeta{Name: "calico-system"}})).NotTo(HaveOccurred()) - }) - - It("should use builtin images", func() { - r = clusterconnection.NewReconcilerWithShims(c, clientScheme, mockStatus, operatorv1.ProviderNone, ready, ready) - _, err := r.Reconcile(ctx, reconcile.Request{}) - Expect(err).ShouldNot(HaveOccurred()) - - d := appsv1.Deployment{ - TypeMeta: metav1.TypeMeta{Kind: "Deployment", APIVersion: "v1"}, - ObjectMeta: metav1.ObjectMeta{ - Name: render.GuardianDeploymentName, - Namespace: render.GuardianNamespace, - }, - } - Expect(test.GetResource(c, &d)).To(BeNil()) - Expect(d.Spec.Template.Spec.Containers).To(HaveLen(1)) - dexC := test.GetContainer(d.Spec.Template.Spec.Containers, render.GuardianContainerName) - Expect(dexC).ToNot(BeNil()) - Expect(dexC.Image).To(Equal( - fmt.Sprintf("some.registry.org/%s%s:%s", - components.TigeraImagePath, - components.ComponentTigeraCalico.Image, - components.ComponentTigeraCalico.Version))) - }) - It("should use images from imageset", func() { - Expect(c.Create(ctx, &operatorv1.ImageSet{ - ObjectMeta: metav1.ObjectMeta{Name: "enterprise-" + components.EnterpriseRelease}, - Spec: operatorv1.ImageSetSpec{ - Images: []operatorv1.Image{ - {Image: "tigera/calico", Digest: "sha256:guardianhash"}, - {Image: "tigera/calico", Digest: "sha256:deadbeef0123456789"}, - }, - }, - })).ToNot(HaveOccurred()) - - r = clusterconnection.NewReconcilerWithShims(c, clientScheme, mockStatus, operatorv1.ProviderNone, ready, ready) - _, err := r.Reconcile(ctx, reconcile.Request{}) - Expect(err).ShouldNot(HaveOccurred()) - - d := appsv1.Deployment{ - TypeMeta: metav1.TypeMeta{Kind: "Deployment", APIVersion: "v1"}, - ObjectMeta: metav1.ObjectMeta{ - Name: render.GuardianDeploymentName, - Namespace: render.GuardianNamespace, - }, - } - Expect(test.GetResource(c, &d)).To(BeNil()) - Expect(d.Spec.Template.Spec.Containers).To(HaveLen(1)) - apiserver := test.GetContainer(d.Spec.Template.Spec.Containers, render.GuardianContainerName) - Expect(apiserver).ToNot(BeNil()) - Expect(apiserver.Image).To(Equal( - fmt.Sprintf("some.registry.org/%s%s@%s", - components.TigeraImagePath, - components.ComponentTigeraCalico.Image, - "sha256:guardianhash"))) - }) - }) - - Context("calico-system reconciliation", func() { - var licenseKey *v3.LicenseKey - BeforeEach(func() { - licenseKey = &v3.LicenseKey{ - ObjectMeta: metav1.ObjectMeta{Name: "default"}, - Status: v3.LicenseKeyStatus{ - Features: []string{ - common.TiersFeature, - common.EgressAccessControlFeature, - }, - }, - } - Expect(c.Create(ctx, licenseKey)).NotTo(HaveOccurred()) - Expect(c.Create(ctx, &v3.Tier{ObjectMeta: metav1.ObjectMeta{Name: "calico-system"}})).NotTo(HaveOccurred()) - r = clusterconnection.NewReconcilerWithShims(c, clientScheme, mockStatus, operatorv1.ProviderNone, ready, ready) - }) - - Context("IP-based management cluster address", func() { - It("should render calico-system policy when tier and watch are ready", func() { - _, err := r.Reconcile(ctx, reconcile.Request{}) - Expect(err).ShouldNot(HaveOccurred()) - - policies := v3.NetworkPolicyList{} - Expect(c.List(ctx, &policies)).ToNot(HaveOccurred()) - - Expect(policies.Items).To(HaveLen(1)) - Expect(policies.Items[0].Name).To(Equal("calico-system.guardian-access")) - }) - - It("should omit calico-system policy and not degrade when tier is not ready", func() { - Expect(c.Delete(ctx, &v3.Tier{ObjectMeta: metav1.ObjectMeta{Name: "calico-system"}})).NotTo(HaveOccurred()) - _, err := r.Reconcile(ctx, reconcile.Request{}) - Expect(err).ShouldNot(HaveOccurred()) - - policies := v3.NetworkPolicyList{} - Expect(c.List(ctx, &policies)).ToNot(HaveOccurred()) - Expect(policies.Items).To(HaveLen(0)) - }) - - It("should degrade and wait when tier is ready, but tier watch is not ready", func() { - mockStatus = &status.MockStatus{} - mockStatus.On("Run").Return() - mockStatus.On("OnCRFound").Return() - mockStatus.On("SetMetaData", mock.Anything).Return() - - r = clusterconnection.NewReconcilerWithShims(c, clientScheme, mockStatus, operatorv1.ProviderNone, notReady, ready) - test.ExpectWaitForTierWatch(ctx, r, mockStatus) - - policies := v3.NetworkPolicyList{} - Expect(c.List(ctx, &policies)).ToNot(HaveOccurred()) - Expect(policies.Items).To(HaveLen(0)) - }) - }) - - Context("Domain-based management cluster address", func() { - BeforeEach(func() { - cfg.Spec.ManagementClusterAddr = "mydomain.io:443" - Expect(c.Update(ctx, cfg)).NotTo(HaveOccurred()) - }) - - It("should render calico-system policy when license and tier are ready", func() { - _, err := r.Reconcile(ctx, reconcile.Request{}) - Expect(err).ShouldNot(HaveOccurred()) - - policies := v3.NetworkPolicyList{} - Expect(c.List(ctx, &policies)).ToNot(HaveOccurred()) - - Expect(policies.Items).To(HaveLen(1)) - Expect(policies.Items[0].Name).To(Equal("calico-system.guardian-access")) - }) - - It("should render calico-system policy without domain-based egress when tier is ready, but license is not sufficient", func() { - licenseKey.Status.Features = []string{common.TiersFeature} - Expect(c.Update(ctx, licenseKey)).NotTo(HaveOccurred()) - - _, err := r.Reconcile(ctx, reconcile.Request{}) - Expect(err).ShouldNot(HaveOccurred()) - - policies := v3.NetworkPolicyList{} - Expect(c.List(ctx, &policies)).ToNot(HaveOccurred()) - Expect(policies.Items).To(HaveLen(1)) - Expect(policies.Items[0].Name).To(Equal("calico-system.guardian-access")) - - // Verify no domain-based egress rules are present - for _, rule := range policies.Items[0].Spec.Egress { - Expect(rule.Destination.Domains).To(BeEmpty(), - "Domain-based egress rules should not be present when license lacks EgressAccessControl") - } - }) - - It("should degrade and wait when tier and license are ready, but tier watch is not ready", func() { - mockStatus = &status.MockStatus{} - mockStatus.On("Run").Return() - mockStatus.On("OnCRFound").Return() - mockStatus.On("SetMetaData", mock.Anything).Return() - - r = clusterconnection.NewReconcilerWithShims(c, clientScheme, mockStatus, operatorv1.ProviderNone, notReady, ready) - test.ExpectWaitForTierWatch(ctx, r, mockStatus) - - policies := v3.NetworkPolicyList{} - Expect(c.List(ctx, &policies)).ToNot(HaveOccurred()) - Expect(policies.Items).To(HaveLen(0)) - }) - - It("should render calico-system policy without domain-based egress when tier is ready but license is not ready", func() { - Expect(c.Delete(ctx, &v3.LicenseKey{ObjectMeta: metav1.ObjectMeta{Name: "default"}})).NotTo(HaveOccurred()) - _, err := r.Reconcile(ctx, reconcile.Request{}) - Expect(err).ShouldNot(HaveOccurred()) - - policies := v3.NetworkPolicyList{} - Expect(c.List(ctx, &policies)).ToNot(HaveOccurred()) - Expect(policies.Items).To(HaveLen(1)) - Expect(policies.Items[0].Name).To(Equal("calico-system.guardian-access")) - - // Verify no domain-based egress rules are present - for _, rule := range policies.Items[0].Spec.Egress { - Expect(rule.Destination.Domains).To(BeEmpty(), - "Domain-based egress rules should not be present when license is not ready") - } - }) - - It("should omit calico-system policy when license is ready but tier is not ready", func() { - Expect(c.Delete(ctx, &v3.Tier{ObjectMeta: metav1.ObjectMeta{Name: "calico-system"}})).NotTo(HaveOccurred()) - _, err := r.Reconcile(ctx, reconcile.Request{}) - Expect(err).ShouldNot(HaveOccurred()) - - policies := v3.NetworkPolicyList{} - Expect(c.List(ctx, &policies)).ToNot(HaveOccurred()) - Expect(policies.Items).To(HaveLen(0)) - }) - }) - - Context("Proxy detection", func() { - // Generate test cases based on the combinations of proxy address forms and proxy settings. - // Here we specify the base targets along with the base proxy IP, domain, and port that will be used for generation. - coreCases := generateCoreProxyTestCases("voltron.io:9000", "192.168.1.2:9000", "proxy.io", "10.1.2.3", "8080") - - // In case we support multiple guardian replicas in the future, we test specific multi-pod scenarios. - multiPodCases := multiplePodCases() - - testCases := append(coreCases, multiPodCases...) - for _, testCase := range testCases { - Describe(fmt.Sprintf("Proxy detection when %+v", test.PrettyFormatProxyTestCase(testCase)), func() { - // Set up the test based on the test case. - BeforeEach(func() { - for i, proxy := range testCase.PodProxies { - createPodWithProxy(ctx, c, proxy, testCase.Lowercase, i) - } - - // Set the target - cfg.Spec.ManagementClusterAddr = testCase.Target - err := c.Update(ctx, cfg) - Expect(err).NotTo(HaveOccurred()) - }) - - It(fmt.Sprintf("detects proxy correctly when %+v", test.PrettyFormatProxyTestCase(testCase)), func() { - // First reconcile creates the guardian deployment without any availability condition. - _, err := r.Reconcile(ctx, reconcile.Request{}) - Expect(err).ShouldNot(HaveOccurred()) - - // Validate that we made no calls to get Pods at this stage. - podGVR := schema.GroupVersionResource{ - Version: "v1", - Resource: "pods", - } - Expect(objTrackerWithCalls.CallCount(podGVR, test.ObjectTrackerCallList)).To(BeZero()) - - // Set the deployment to be unavailable. We need to recreate the deployment otherwise the status update is ignored. - gd := appsv1.Deployment{} - err = c.Get(ctx, client.ObjectKey{Name: render.GuardianDeploymentName, Namespace: render.GuardianNamespace}, &gd) - Expect(err).NotTo(HaveOccurred()) - err = c.Delete(ctx, &gd) - Expect(err).NotTo(HaveOccurred()) - gd.ResourceVersion = "" - gd.Status.Conditions = []appsv1.DeploymentCondition{{ - Type: appsv1.DeploymentAvailable, - Status: v1.ConditionFalse, - LastTransitionTime: metav1.Time{Time: time.Now()}, - }} - err = c.Create(ctx, &gd) - Expect(err).NotTo(HaveOccurred()) - - // Reconcile again. We should see no calls since the deployment has not transitioned to available. - _, err = r.Reconcile(ctx, reconcile.Request{}) - Expect(err).ShouldNot(HaveOccurred()) - Expect(objTrackerWithCalls.CallCount(podGVR, test.ObjectTrackerCallList)).To(Equal(0)) - - // Set the deployment to available. - err = c.Delete(ctx, &gd) - Expect(err).NotTo(HaveOccurred()) - gd.ResourceVersion = "" - gd.Status.Conditions = []appsv1.DeploymentCondition{{ - Type: appsv1.DeploymentAvailable, - Status: v1.ConditionTrue, - LastTransitionTime: metav1.Time{Time: time.Now().Add(time.Minute)}, - }} - err = c.Create(ctx, &gd) - Expect(err).NotTo(HaveOccurred()) - - // Reconcile again. The proxy detection logic should kick in since the guardian deployment is ready. - _, err = r.Reconcile(ctx, reconcile.Request{}) - Expect(err).ShouldNot(HaveOccurred()) - Expect(objTrackerWithCalls.CallCount(podGVR, test.ObjectTrackerCallList)).To(Equal(1)) - - // Resolve the rendered rule that governs egress from guardian to voltron. - policies := v3.NetworkPolicyList{} - Expect(c.List(ctx, &policies)).ToNot(HaveOccurred()) - Expect(policies.Items).To(HaveLen(1)) - Expect(policies.Items[0].Name).To(Equal("calico-system.guardian-access")) - policy := policies.Items[0] - - // Generate the expectation based on the test case, and compare the rendered rule to our expectation. - expectedEgressRules := getExpectedEgressRulesFromCase(testCase) - Expect(policy.Spec.Egress).To(HaveLen(6 + len(expectedEgressRules))) - for i, egressRule := range expectedEgressRules { - managementClusterEgressRule := policy.Spec.Egress[5+i] - if egressRule.hostIsIP { - Expect(managementClusterEgressRule.Destination.Nets).To(HaveLen(1)) - Expect(managementClusterEgressRule.Destination.Nets[0]).To(Equal(fmt.Sprintf("%s/32", egressRule.host))) - Expect(managementClusterEgressRule.Destination.Ports).To(Equal(networkpolicy.Ports(egressRule.port))) - } else { - Expect(managementClusterEgressRule.Destination.Domains).To(Equal([]string{egressRule.host})) - Expect(managementClusterEgressRule.Destination.Ports).To(Equal(networkpolicy.Ports(egressRule.port))) - } - } - - // Reconcile again. Verify that we do not cause any additional query for pods now that we have resolved the proxy. - _, err = r.Reconcile(ctx, reconcile.Request{}) - Expect(err).ShouldNot(HaveOccurred()) - Expect(objTrackerWithCalls.CallCount(podGVR, test.ObjectTrackerCallList)).To(Equal(1)) - }) - }) - } - }) - }) - Context("Proxy setting", func() { DescribeTable("sets the proxy", func(http, https, noProxy bool) { installationCopy := installation.DeepCopy() @@ -774,57 +472,6 @@ var _ = Describe("ManagementClusterConnection controller tests", func() { }) }) - val := []string{"some-value"} - DescribeTable("should render impersonation permissions correctly", func(impersonation *operatorv1.Impersonation, expectedUser, expectedGroup, expectedSA []string) { - By("ensuring a tigerastatus exists") - ts := &operatorv1.TigeraStatus{ - ObjectMeta: metav1.ObjectMeta{Name: "management-cluster-connection"}, - Spec: operatorv1.TigeraStatusSpec{}, - Status: operatorv1.TigeraStatusStatus{}, - } - Expect(c.Create(ctx, ts)).NotTo(HaveOccurred()) - - By("updating the CR with the impersonation settings, reconciling and fetching the results") - err := c.Get(ctx, client.ObjectKey{Name: cfg.Name, Namespace: cfg.Namespace}, cfg) - Expect(err).ShouldNot(HaveOccurred()) - cfg.Spec.Impersonation = impersonation - Expect(c.Update(ctx, cfg)).NotTo(HaveOccurred()) - _, err = r.Reconcile(ctx, reconcile.Request{NamespacedName: types.NamespacedName{ - Name: "management-cluster-connection", - Namespace: "", - }}) - Expect(err).ShouldNot(HaveOccurred()) - role := &rbacv1.ClusterRole{} - err = c.Get(ctx, client.ObjectKey{Name: render.GuardianClusterRoleName}, role) - Expect(err).NotTo(HaveOccurred()) - By("verifying the resulting RBAC") - var users, groups, sas []string - for _, rule := range role.Rules { - if len(rule.Verbs) == 1 && rule.Verbs[0] == "impersonate" { - if len(rule.Resources) == 1 { - switch rule.Resources[0] { - case "users": - users = rule.ResourceNames - case "groups": - groups = rule.ResourceNames - case "serviceaccounts": - sas = rule.ResourceNames - } - } - } - } - Expect(users).To(Equal(expectedUser)) - Expect(groups).To(Equal(expectedGroup)) - Expect(sas).To(Equal(expectedSA)) - }, - Entry("no impersonation configured", nil, nil, nil, nil), - Entry("all set", &operatorv1.Impersonation{Users: val, Groups: val, ServiceAccounts: val}, val, val, val), - Entry("all set to empty", &operatorv1.Impersonation{Users: []string{}, Groups: []string{}, ServiceAccounts: []string{}}, nil, nil, nil), - Entry("user set", &operatorv1.Impersonation{Users: val}, val, nil, nil), - Entry("groups set", &operatorv1.Impersonation{Groups: val}, nil, val, nil), - Entry("service accounts set", &operatorv1.Impersonation{ServiceAccounts: val}, nil, nil, val), - Entry("empty impersonation", &operatorv1.Impersonation{}, nil, nil, nil), - ) }) func createPodWithProxy(ctx context.Context, c client.Client, config *test.ProxyConfig, lowercase bool, replicaNum int) { From 3f3b9c40c1e596d18f0996f0599f58bf62f03b34 Mon Sep 17 00:00:00 2001 From: Casey Davenport Date: Wed, 24 Jun 2026 15:50:50 -0700 Subject: [PATCH 52/54] Return the controller context from ExtendContext ExtendContext handed back a render context while the controller context it was given already embeds one, leaving two near-identical copies in play. Return the updated controller context instead, so a single context flows through the reconcile and callers read its embedded render context. --- .../apiserver/apiserver_controller.go | 6 +-- .../clusterconnection_controller.go | 4 +- .../installation/core_controller.go | 4 +- .../installation/windows_controller.go | 4 +- pkg/enterprise/apiserver/extension.go | 39 +++++++++---------- pkg/enterprise/apiserver/extension_test.go | 33 ++++++++++------ pkg/enterprise/clusterconnection/extension.go | 12 +++--- .../clusterconnection/extension_test.go | 6 ++- pkg/enterprise/installation/core.go | 23 ++++++----- .../installation/kubecontrollers_test.go | 18 ++++++--- .../installation/node_enterprise_test.go | 6 ++- pkg/enterprise/windows/extension.go | 9 ++--- pkg/enterprise/windows/extension_test.go | 3 +- pkg/extensions/controllerextension.go | 11 +++--- pkg/extensions/controllerextension_test.go | 7 ++-- pkg/extensions/set.go | 8 ++-- pkg/extensions/variant.go | 8 ++-- 17 files changed, 109 insertions(+), 92 deletions(-) diff --git a/pkg/controller/apiserver/apiserver_controller.go b/pkg/controller/apiserver/apiserver_controller.go index 32b9a795d7..70543c88a6 100644 --- a/pkg/controller/apiserver/apiserver_controller.go +++ b/pkg/controller/apiserver/apiserver_controller.go @@ -314,12 +314,12 @@ func (r *ReconcileAPIServer) Reconcile(ctx context.Context, request reconcile.Re r.status.SetDegraded(operatorv1.ResourceValidationError, "Invalid API server configuration", err, reqLogger) return reconcile.Result{}, err } - renderCtx, managedKeyPairs, err := r.opts.Extensions.ExtendContext(cc) + cc, managedKeyPairs, err := r.opts.Extensions.ExtendContext(cc) if err != nil { r.status.SetDegraded(operatorv1.ResourceCreateError, "Error preparing the API server extension", err, reqLogger) return reconcile.Result{}, err } - trustedBundle := renderCtx.TrustedBundle + trustedBundle := cc.TrustedBundle // The webhooks component (v3-CRD mode) needs the ManagementCluster to register the // managed-cluster webhook. Reading it requires the enterprise CRDs. @@ -369,7 +369,7 @@ func (r *ReconcileAPIServer) Reconcile(ctx context.Context, request reconcile.Re r.client, r.scheme, instance, - utils.WithRenderContext(renderCtx), + utils.WithRenderContext(cc.RenderContext), utils.WithExtensions(r.opts.Extensions), ) diff --git a/pkg/controller/clusterconnection/clusterconnection_controller.go b/pkg/controller/clusterconnection/clusterconnection_controller.go index b5f5f85e96..284f1943d7 100644 --- a/pkg/controller/clusterconnection/clusterconnection_controller.go +++ b/pkg/controller/clusterconnection/clusterconnection_controller.go @@ -291,12 +291,12 @@ func (r *ReconcileConnection) Reconcile(ctx context.Context, request reconcile.R r.status.SetDegraded(operatorv1.ResourceValidationError, "Invalid ManagementClusterConnection configuration", err, reqLogger) return reconcile.Result{}, err } - renderCtx, _, err := r.opts.Extensions.ExtendContext(cc) + cc, _, err = r.opts.Extensions.ExtendContext(cc) if err != nil { r.status.SetDegraded(operatorv1.ResourceCreateError, "Error preparing the clusterconnection extension", err, reqLogger) return reconcile.Result{}, err } - guardianData, haveGuardianData := render.GuardianRenderDataFromContext(renderCtx) + guardianData, haveGuardianData := render.GuardianRenderDataFromContext(cc.RenderContext) includeSystem := false if managementClusterConnection.Spec.TLS.CA == operatorv1.CATypePublic { diff --git a/pkg/controller/installation/core_controller.go b/pkg/controller/installation/core_controller.go index 3ef8ac48b7..442ed86622 100644 --- a/pkg/controller/installation/core_controller.go +++ b/pkg/controller/installation/core_controller.go @@ -1168,7 +1168,7 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile r.status.SetDegraded(operatorv1.ResourceValidationError, "Invalid installation configuration", err, reqLogger) return reconcile.Result{}, err } - renderCtx, managedKeyPairs, err := r.opts.Extensions.ExtendContext(cc) + cc, managedKeyPairs, err := r.opts.Extensions.ExtendContext(cc) if err != nil { r.status.SetDegraded(operatorv1.ResourceCreateError, "Error preparing installation extension", err, reqLogger) return reconcile.Result{}, err @@ -1192,7 +1192,7 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile r.client, r.scheme, instance, - utils.WithRenderContext(renderCtx), + utils.WithRenderContext(cc.RenderContext), utils.WithExtensions(r.opts.Extensions), ) diff --git a/pkg/controller/installation/windows_controller.go b/pkg/controller/installation/windows_controller.go index 87750f134d..bdd7af26f6 100644 --- a/pkg/controller/installation/windows_controller.go +++ b/pkg/controller/installation/windows_controller.go @@ -356,7 +356,7 @@ func (r *ReconcileWindows) Reconcile(ctx context.Context, request reconcile.Requ r.status.SetDegraded(operatorv1.ResourceValidationError, "Invalid installation configuration", err, reqLogger) return reconcile.Result{}, err } - renderCtx, _, err := r.opts.Extensions.ExtendContext(cc) + cc, _, err = r.opts.Extensions.ExtendContext(cc) if err != nil { r.status.SetDegraded(operatorv1.ResourceCreateError, "Error preparing windows extension", err, reqLogger) return reconcile.Result{}, err @@ -395,7 +395,7 @@ func (r *ReconcileWindows) Reconcile(ctx context.Context, request reconcile.Requ r.client, r.scheme, instance, - utils.WithRenderContext(renderCtx), + utils.WithRenderContext(cc.RenderContext), utils.WithExtensions(r.opts.Extensions), ) if err := handler.CreateOrUpdateOrDelete(ctx, component, nil); err != nil { diff --git a/pkg/enterprise/apiserver/extension.go b/pkg/enterprise/apiserver/extension.go index d27cf7fc83..43b0d153e7 100644 --- a/pkg/enterprise/apiserver/extension.go +++ b/pkg/enterprise/apiserver/extension.go @@ -152,28 +152,27 @@ func (apiServerControllerExtension) Watches(c ctrlruntime.Controller) error { // fetches the enterprise CRs, creates the query server certificate, resolves the L7 // sidecar images, and stashes them for the modifiers. The base API server render carries // none of this. -func (apiServerControllerExtension) ExtendContext(cc contexts.ControllerContext) (render.RenderContext, []certificatemanagement.KeyPairInterface, error) { - rc := cc.RenderContext +func (apiServerControllerExtension) ExtendContext(cc contexts.ControllerContext) (contexts.ControllerContext, []certificatemanagement.KeyPairInterface, error) { in := cc.Installation trustedBundle, err := cc.CertificateManager.CreateNamedTrustedBundleFromSecrets(render.APIServerResourceName, cc.Client, common.OperatorNamespace(), false) if err != nil { - return rc, nil, fmt.Errorf("unable to create the trusted bundle: %w", err) + return cc, nil, fmt.Errorf("unable to create the trusted bundle: %w", err) } applicationLayer, err := utils.GetApplicationLayer(cc.Ctx, cc.Client) if err != nil { - return rc, nil, fmt.Errorf("error reading ApplicationLayer: %w", err) + return cc, nil, fmt.Errorf("error reading ApplicationLayer: %w", err) } managementCluster, err := utils.GetManagementCluster(cc.Ctx, cc.Client) if err != nil { - return rc, nil, fmt.Errorf("error reading ManagementCluster: %w", err) + return cc, nil, fmt.Errorf("error reading ManagementCluster: %w", err) } managementClusterConnection, err := utils.GetManagementClusterConnection(cc.Ctx, cc.Client) if err != nil { - return rc, nil, fmt.Errorf("error reading ManagementClusterConnection: %w", err) + return cc, nil, fmt.Errorf("error reading ManagementClusterConnection: %w", err) } // Management cluster only: the apiserver mounts the tunnel CA secret so it can sign @@ -181,13 +180,13 @@ func (apiServerControllerExtension) ExtendContext(cc contexts.ControllerContext) // ManagementCluster.Spec.TLS is defaulted; degrade until it exists. if managementCluster != nil && managementCluster.Spec.TLS != nil && !cc.MultiTenant { if _, err := utils.GetSecret(cc.Ctx, cc.Client, managementCluster.Spec.TLS.SecretName, common.OperatorNamespace()); err != nil { - return rc, nil, fmt.Errorf("unable to fetch the tunnel secret: %w", err) + return cc, nil, fmt.Errorf("unable to fetch the tunnel secret: %w", err) } } prometheusCertificate, err := cc.CertificateManager.GetCertificate(cc.Client, monitor.PrometheusClientTLSSecretName, common.OperatorNamespace()) if err != nil { - return rc, nil, fmt.Errorf("failed to get certificate: %w", err) + return cc, nil, fmt.Errorf("failed to get certificate: %w", err) } if prometheusCertificate != nil { trustedBundle.AddCertificates(prometheusCertificate) @@ -196,7 +195,7 @@ func (apiServerControllerExtension) ExtendContext(cc contexts.ControllerContext) if managementClusterConnection != nil { voltronLinseedCert, err := cc.CertificateManager.GetCertificate(cc.Client, render.VoltronLinseedPublicCert, common.OperatorNamespace()) if err != nil { - return rc, nil, fmt.Errorf("failed to retrieve %s: %w", render.VoltronLinseedPublicCert, err) + return cc, nil, fmt.Errorf("failed to retrieve %s: %w", render.VoltronLinseedPublicCert, err) } if voltronLinseedCert != nil { trustedBundle.AddCertificates(voltronLinseedCert) @@ -208,21 +207,21 @@ func (apiServerControllerExtension) ExtendContext(cc contexts.ControllerContext) var keyValidatorConfig authentication.KeyValidatorConfig authenticationCR, err := utils.GetAuthentication(cc.Ctx, cc.Client) if err != nil && !apierrors.IsNotFound(err) { - return rc, nil, fmt.Errorf("error while fetching Authentication: %w", err) + return cc, nil, fmt.Errorf("error while fetching Authentication: %w", err) } if authenticationCR != nil && authenticationCR.Status.State == operatorv1.TigeraStatusReady { if utils.DexEnabled(authenticationCR) { certificate, err := cc.CertificateManager.GetCertificate(cc.Client, render.DexTLSSecretName, common.OperatorNamespace()) if err != nil { - return rc, nil, fmt.Errorf("failed to retrieve %s: %w", render.DexTLSSecretName, err) + return cc, nil, fmt.Errorf("failed to retrieve %s: %w", render.DexTLSSecretName, err) } else if certificate == nil { - return rc, nil, fmt.Errorf("waiting for secret %q to become available", render.DexTLSSecretName) + return cc, nil, fmt.Errorf("waiting for secret %q to become available", render.DexTLSSecretName) } trustedBundle.AddCertificates(certificate) } keyValidatorConfig, err = utils.GetKeyValidatorConfig(cc.Ctx, cc.Client, authenticationCR, cc.ClusterDomain) if err != nil { - return rc, nil, fmt.Errorf("failed to get KeyValidator config: %w", err) + return cc, nil, fmt.Errorf("failed to get KeyValidator config: %w", err) } } @@ -237,7 +236,7 @@ func (apiServerControllerExtension) ExtendContext(cc contexts.ControllerContext) dns.GetServiceDNSNames(render.APIServerServiceName, render.APIServerNamespace, cc.ClusterDomain), ) if err != nil { - return rc, nil, fmt.Errorf("unable to get or create query server tls key pair: %w", err) + return cc, nil, fmt.Errorf("unable to get or create query server tls key pair: %w", err) } } @@ -249,20 +248,20 @@ func (apiServerControllerExtension) ExtendContext(cc contexts.ControllerContext) *applicationLayer.Spec.SidecarInjection == operatorv1.SidecarEnabled { imageSet, err := imageset.GetImageSet(cc.Ctx, cc.Client, in.Variant) if err != nil { - return rc, nil, err + return cc, nil, err } l7EnvoyImage, err = components.GetReference(components.ComponentEnvoyProxy, in.Registry, in.ImagePath, in.ImagePrefix, imageSet) if err != nil { - return rc, nil, err + return cc, nil, err } dikastesImage, err = components.GetReference(components.ComponentDikastes, in.Registry, in.ImagePath, in.ImagePrefix, imageSet) if err != nil { - return rc, nil, err + return cc, nil, err } } - rc.TrustedBundle = trustedBundle - rc.Extension = apiServerRenderData{ + cc.TrustedBundle = trustedBundle + cc.Extension = apiServerRenderData{ managementCluster: managementCluster, managementClusterConnection: managementClusterConnection, applicationLayer: applicationLayer, @@ -271,7 +270,7 @@ func (apiServerControllerExtension) ExtendContext(cc contexts.ControllerContext) l7EnvoyImage: l7EnvoyImage, dikastesImage: dikastesImage, } - return rc, nil, nil + return cc, nil, nil } // RegisterCalicoCleanup registers, for the Calico variant, the cleanup that diff --git a/pkg/enterprise/apiserver/extension_test.go b/pkg/enterprise/apiserver/extension_test.go index 89f1e7b561..cb959c87c3 100644 --- a/pkg/enterprise/apiserver/extension_test.go +++ b/pkg/enterprise/apiserver/extension_test.go @@ -175,7 +175,8 @@ var _ = Describe("API server enterprise modifier", func() { It("is a no-op for the Calico variant (no enterprise objects added)", func() { cc := apiServerControllerContext(operatorv1.Calico, nil) - rc, _, err := ext.ExtendContext(cc) + ecc, _, err := ext.ExtendContext(cc) + rc := ecc.RenderContext Expect(err).NotTo(HaveOccurred()) objs, _ := renderAPIServer(cc, rc, apiServerKeyPair(cc)) @@ -190,7 +191,8 @@ var _ = Describe("API server enterprise modifier", func() { It("layers the query server, enterprise RBAC, audit policy, and query server port on", func() { cc := apiServerControllerContext(operatorv1.CalicoEnterprise, nil) - rc, _, err := ext.ExtendContext(cc) + ecc, _, err := ext.ExtendContext(cc) + rc := ecc.RenderContext Expect(err).NotTo(HaveOccurred()) objs, _ := renderAPIServer(cc, rc, apiServerKeyPair(cc)) @@ -217,7 +219,8 @@ var _ = Describe("API server enterprise modifier", func() { It("queues the enterprise RBAC for deletion when not a management cluster", func() { cc := apiServerControllerContext(operatorv1.CalicoEnterprise, nil) - rc, _, err := ext.ExtendContext(cc) + ecc, _, err := ext.ExtendContext(cc) + rc := ecc.RenderContext Expect(err).NotTo(HaveOccurred()) _, del := renderAPIServer(cc, rc, apiServerKeyPair(cc)) @@ -240,7 +243,8 @@ var _ = Describe("API server enterprise modifier", func() { Data: map[string][]byte{"cert": []byte("a"), "key": []byte("b")}, }, ) - rc, _, err := ext.ExtendContext(cc) + ecc, _, err := ext.ExtendContext(cc) + rc := ecc.RenderContext Expect(err).NotTo(HaveOccurred()) objs, _ := renderAPIServer(cc, rc, apiServerKeyPair(cc)) @@ -266,7 +270,8 @@ var _ = Describe("API server enterprise modifier", func() { ObjectMeta: metav1.ObjectMeta{Name: utils.DefaultEnterpriseInstanceKey.Name}, }, ) - rc, _, err := ext.ExtendContext(cc) + ecc, _, err := ext.ExtendContext(cc) + rc := ecc.RenderContext Expect(err).NotTo(HaveOccurred()) objs, _ := renderAPIServer(cc, rc, apiServerKeyPair(cc)) @@ -285,7 +290,8 @@ var _ = Describe("API server enterprise modifier", func() { Context("v3-CRD mode (no aggregation server)", func() { It("renders the deployment skeleton with the query server and pulls it out of the delete list", func() { cc := apiServerControllerContext(operatorv1.CalicoEnterprise, nil) - rc, _, err := ext.ExtendContext(cc) + ecc, _, err := ext.ExtendContext(cc) + rc := ecc.RenderContext Expect(err).NotTo(HaveOccurred()) cfg := &render.APIServerConfiguration{ @@ -330,7 +336,8 @@ var _ = Describe("API server enterprise modifier", func() { It("adds the L7 admission controller container, the sidecar webhook, and the L7 service port", func() { cc := apiServerControllerContext(operatorv1.CalicoEnterprise, nil, applicationLayerSidecar()) - rc, _, err := ext.ExtendContext(cc) + ecc, _, err := ext.ExtendContext(cc) + rc := ecc.RenderContext Expect(err).NotTo(HaveOccurred()) objs, _ := renderAPIServer(cc, rc, apiServerKeyPair(cc)) @@ -348,7 +355,8 @@ var _ = Describe("API server enterprise modifier", func() { It("pulls the sidecar webhook out of the delete list", func() { cc := apiServerControllerContext(operatorv1.CalicoEnterprise, nil, applicationLayerSidecar()) - rc, _, err := ext.ExtendContext(cc) + ecc, _, err := ext.ExtendContext(cc) + rc := ecc.RenderContext Expect(err).NotTo(HaveOccurred()) _, del := renderAPIServer(cc, rc, apiServerKeyPair(cc)) @@ -377,7 +385,8 @@ var _ = Describe("API server enterprise policy modifier", func() { It("leaves the egress rules as the base when no OIDC key validator is configured", func() { cc := apiServerControllerContext(operatorv1.CalicoEnterprise, nil) - rc, _, err := ext.ExtendContext(cc) + ecc, _, err := ext.ExtendContext(cc) + rc := ecc.RenderContext Expect(err).NotTo(HaveOccurred()) policy := applyPolicy(cc, rc) @@ -393,7 +402,8 @@ var _ = Describe("API server enterprise policy modifier", func() { ObjectMeta: metav1.ObjectMeta{Name: utils.DefaultEnterpriseInstanceKey.Name}, Spec: operatorv1.ApplicationLayerSpec{SidecarInjection: &enabled}, }) - rc, _, err := ext.ExtendContext(cc) + ecc, _, err := ext.ExtendContext(cc) + rc := ecc.RenderContext Expect(err).NotTo(HaveOccurred()) policy := applyPolicy(cc, rc) @@ -415,7 +425,8 @@ var _ = Describe("API server enterprise policy modifier", func() { var _ = Describe("API server Calico-variant cleanup", func() { It("queues the enterprise RBAC for deletion", func() { cc := apiServerControllerContext(operatorv1.Calico, nil) - rc, _, err := ext.ExtendContext(cc) + ecc, _, err := ext.ExtendContext(cc) + rc := ecc.RenderContext Expect(err).NotTo(HaveOccurred()) cfg := &render.APIServerConfiguration{ diff --git a/pkg/enterprise/clusterconnection/extension.go b/pkg/enterprise/clusterconnection/extension.go index d07a984162..7c81fe6748 100644 --- a/pkg/enterprise/clusterconnection/extension.go +++ b/pkg/enterprise/clusterconnection/extension.go @@ -54,12 +54,10 @@ func (clusterConnectionControllerExtension) Validate(cc contexts.ControllerConte // permits the domain-based egress network policy. It creates no certificates, so it // returns no managed keypairs. The OSS controller path supplies its own defaults // when this hook is absent. -func (clusterConnectionControllerExtension) ExtendContext(cc contexts.ControllerContext) (render.RenderContext, []certificatemanagement.KeyPairInterface, error) { - rc := cc.RenderContext - +func (clusterConnectionControllerExtension) ExtendContext(cc contexts.ControllerContext) (contexts.ControllerContext, []certificatemanagement.KeyPairInterface, error) { clusterInformation, err := utils.FetchClusterInformation(cc.Ctx, cc.Client) if err != nil { - return rc, nil, fmt.Errorf("error querying ClusterInformation: %w", err) + return cc, nil, fmt.Errorf("error querying ClusterInformation: %w", err) } // Ensure the license can support enterprise policy before enabling the @@ -68,12 +66,12 @@ func (clusterConnectionControllerExtension) ExtendContext(cc contexts.Controller if license, err := utils.FetchLicenseKey(cc.Ctx, cc.Client); err == nil { includeEgressNetworkPolicy = utils.IsFeatureActive(license, common.EgressAccessControlFeature) } else if !k8serrors.IsNotFound(err) { - return rc, nil, fmt.Errorf("error querying license: %w", err) + return cc, nil, fmt.Errorf("error querying license: %w", err) } - rc.Extension = render.GuardianRenderData{ + cc.Extension = render.GuardianRenderData{ Version: clusterInformation.Spec.CNXVersion, IncludeEgressNetworkPolicy: includeEgressNetworkPolicy, } - return rc, nil, nil + return cc, nil, nil } diff --git a/pkg/enterprise/clusterconnection/extension_test.go b/pkg/enterprise/clusterconnection/extension_test.go index 1a807b8545..e3d85604ab 100644 --- a/pkg/enterprise/clusterconnection/extension_test.go +++ b/pkg/enterprise/clusterconnection/extension_test.go @@ -80,7 +80,8 @@ var _ = Describe("clusterconnection enterprise controller extension", func() { Describe("ExtendContext", func() { It("reports the managed cluster CNX version", func() { cli = newClient(clusterInformation()) - rc, managed, err := ext.ExtendContext(controllerContext()) + ecc, managed, err := ext.ExtendContext(controllerContext()) + rc := ecc.RenderContext Expect(err).NotTo(HaveOccurred()) Expect(managed).To(BeEmpty()) @@ -96,7 +97,8 @@ var _ = Describe("clusterconnection enterprise controller extension", func() { Status: v3.LicenseKeyStatus{Features: []string{common.EgressAccessControlFeature}}, } cli = newClient(clusterInformation(), license) - rc, _, err := ext.ExtendContext(controllerContext()) + ecc, _, err := ext.ExtendContext(controllerContext()) + rc := ecc.RenderContext Expect(err).NotTo(HaveOccurred()) data, ok := render.GuardianRenderDataFromContext(rc) diff --git a/pkg/enterprise/installation/core.go b/pkg/enterprise/installation/core.go index 6e1803bc3a..d983c2d665 100644 --- a/pkg/enterprise/installation/core.go +++ b/pkg/enterprise/installation/core.go @@ -146,8 +146,7 @@ func (coreControllerExtension) Watches(c ctrlruntime.Controller) error { // fetching the certificates that feed the trusted bundle. It returns the render // context carrying the produced node prometheus keypair, and that keypair as one // the controller should manage. -func (coreControllerExtension) ExtendContext(cc contexts.ControllerContext) (render.RenderContext, []certificatemanagement.KeyPairInterface, error) { - rc := cc.RenderContext +func (coreControllerExtension) ExtendContext(cc contexts.ControllerContext) (contexts.ControllerContext, []certificatemanagement.KeyPairInterface, error) { nodePrometheusTLS, err := cc.CertificateManager.GetOrCreateKeyPair( cc.Client, @@ -156,7 +155,7 @@ func (coreControllerExtension) ExtendContext(cc contexts.ControllerContext) (ren dns.GetServiceDNSNames(render.CalicoNodeMetricsService, common.CalicoNamespace, cc.ClusterDomain), ) if err != nil { - return rc, nil, fmt.Errorf("error creating node prometheus TLS certificate: %w", err) + return cc, nil, fmt.Errorf("error creating node prometheus TLS certificate: %w", err) } if nodePrometheusTLS != nil { cc.TrustedBundle.AddCertificates(nodePrometheusTLS) @@ -172,7 +171,7 @@ func (coreControllerExtension) ExtendContext(cc contexts.ControllerContext) (ren dns.GetServiceDNSNames(kubecontrollers.KubeControllerMetrics, common.CalicoNamespace, cc.ClusterDomain), ) if err != nil { - return rc, nil, fmt.Errorf("error creating kube-controllers metrics TLS certificate: %w", err) + return cc, nil, fmt.Errorf("error creating kube-controllers metrics TLS certificate: %w", err) } if kubeControllerTLS != nil { cc.TrustedBundle.AddCertificates(kubeControllerTLS) @@ -180,7 +179,7 @@ func (coreControllerExtension) ExtendContext(cc contexts.ControllerContext) (ren logCollector, err := utils.GetLogCollector(cc.Ctx, cc.Client) if err != nil { - return rc, nil, fmt.Errorf("error reading LogCollector: %w", err) + return cc, nil, fmt.Errorf("error reading LogCollector: %w", err) } // calico-kube-controllers enterprise additions: the WAF surface, the enterprise @@ -188,14 +187,14 @@ func (coreControllerExtension) ExtendContext(cc contexts.ControllerContext) (ren // kube-controllers needs an extra license-push rule. managementClusterConnection, err := utils.GetManagementClusterConnection(cc.Ctx, cc.Client) if err != nil { - return rc, nil, fmt.Errorf("error reading ManagementClusterConnection: %w", err) + return cc, nil, fmt.Errorf("error reading ManagementClusterConnection: %w", err) } waf, wafWebhookTLS, err := buildWAFData(cc) if err != nil { - return rc, nil, fmt.Errorf("error preparing WAF configuration: %w", err) + return cc, nil, fmt.Errorf("error preparing WAF configuration: %w", err) } - rc.Extension = installationRenderData{ + cc.Extension = installationRenderData{ nodePrometheusTLS: nodePrometheusTLS, kubeControllerTLS: kubeControllerTLS, collectProcessPath: collectProcessPathEnabled(logCollector), @@ -206,7 +205,7 @@ func (coreControllerExtension) ExtendContext(cc contexts.ControllerContext) (ren prometheusClientCert, err := cc.CertificateManager.GetCertificate(cc.Client, monitor.PrometheusClientTLSSecretName, common.OperatorNamespace()) if err != nil { - return rc, nil, fmt.Errorf("unable to fetch prometheus certificate: %w", err) + return cc, nil, fmt.Errorf("unable to fetch prometheus certificate: %w", err) } if prometheusClientCert != nil { cc.TrustedBundle.AddCertificates(prometheusClientCert) @@ -214,7 +213,7 @@ func (coreControllerExtension) ExtendContext(cc contexts.ControllerContext) (ren esgwCertificate, err := cc.CertificateManager.GetCertificate(cc.Client, relasticsearch.PublicCertSecret, common.OperatorNamespace()) if err != nil { - return rc, nil, fmt.Errorf("failed to retrieve / validate %s: %w", relasticsearch.PublicCertSecret, err) + return cc, nil, fmt.Errorf("failed to retrieve / validate %s: %w", relasticsearch.PublicCertSecret, err) } if esgwCertificate != nil { cc.TrustedBundle.AddCertificates(esgwCertificate) @@ -224,7 +223,7 @@ func (coreControllerExtension) ExtendContext(cc contexts.ControllerContext) (ren // manager internal cert. managerInternalTLS, err := cc.CertificateManager.GetCertificate(cc.Client, render.ManagerInternalTLSSecretName, common.OperatorNamespace()) if err != nil { - return rc, nil, fmt.Errorf("failed to retrieve %s: %w", render.ManagerInternalTLSSecretName, err) + return cc, nil, fmt.Errorf("failed to retrieve %s: %w", render.ManagerInternalTLSSecretName, err) } if managerInternalTLS != nil { cc.TrustedBundle.AddCertificates(managerInternalTLS) @@ -240,5 +239,5 @@ func (coreControllerExtension) ExtendContext(cc contexts.ControllerContext) (ren if wafWebhookTLS != nil { managed = append(managed, wafWebhookTLS) } - return rc, managed, nil + return cc, managed, nil } diff --git a/pkg/enterprise/installation/kubecontrollers_test.go b/pkg/enterprise/installation/kubecontrollers_test.go index 3fe2034902..ad4867d610 100644 --- a/pkg/enterprise/installation/kubecontrollers_test.go +++ b/pkg/enterprise/installation/kubecontrollers_test.go @@ -66,7 +66,8 @@ var _ = Describe("kube-controllers enterprise modifier", func() { } It("mounts the metrics serving TLS keypair onto the deployment", func() { - rc, _, err := ext.ExtendContext(newControllerContext(operatorv1.CalicoEnterprise)) + ecc, _, err := ext.ExtendContext(newControllerContext(operatorv1.CalicoEnterprise)) + rc := ecc.RenderContext Expect(err).NotTo(HaveOccurred()) objs, _ := extensionstest.ApplyExtensions(ext, render.ComponentNameKubeControllers, rc, []client.Object{kubeControllersDeployment()}, nil) @@ -85,7 +86,8 @@ var _ = Describe("kube-controllers enterprise modifier", func() { }) It("adds the cert-management init container when certificate management is enabled", func() { - rc, _, err := ext.ExtendContext(certManagementControllerContext()) + ecc, _, err := ext.ExtendContext(certManagementControllerContext()) + rc := ecc.RenderContext Expect(err).NotTo(HaveOccurred()) objs, _ := extensionstest.ApplyExtensions(ext, render.ComponentNameKubeControllers, rc, []client.Object{kubeControllersDeployment()}, nil) @@ -159,7 +161,8 @@ var _ = Describe("calico-kube-controllers enterprise surface", func() { } It("layers the enterprise rules, controllers, and metrics TLS on (WAF off)", func() { - rc, _, err := ext.ExtendContext(newControllerContext(operatorv1.CalicoEnterprise)) + ecc, _, err := ext.ExtendContext(newControllerContext(operatorv1.CalicoEnterprise)) + rc := ecc.RenderContext Expect(err).NotTo(HaveOccurred()) objs := renderKubeControllers(newControllerContext(operatorv1.CalicoEnterprise), rc) @@ -181,7 +184,8 @@ var _ = Describe("calico-kube-controllers enterprise surface", func() { It("layers the full WAF surface on when the GatewayAPI extension is enabled", func() { cc := wafControllerContext() - rc, managed, err := ext.ExtendContext(cc) + ecc, managed, err := ext.ExtendContext(cc) + rc := ecc.RenderContext Expect(err).NotTo(HaveOccurred()) names := []string{} for _, kp := range managed { @@ -218,7 +222,8 @@ var _ = Describe("calico-kube-controllers enterprise surface", func() { It("deletes the WAF webhook surface when the extension is disabled", func() { cc := newControllerContext(operatorv1.CalicoEnterprise) - rc, _, err := ext.ExtendContext(cc) + ecc, _, err := ext.ExtendContext(cc) + rc := ecc.RenderContext Expect(err).NotTo(HaveOccurred()) comp := kubecontrollers.NewCalicoKubeControllers(calicoKubeControllersCfg(cc)) @@ -232,7 +237,8 @@ var _ = Describe("calico-kube-controllers enterprise surface", func() { It("adds the WAF webhook ingress rule to the network policy when enabled", func() { cc := wafControllerContext() - rc, _, err := ext.ExtendContext(cc) + ecc, _, err := ext.ExtendContext(cc) + rc := ecc.RenderContext Expect(err).NotTo(HaveOccurred()) comp := kubecontrollers.NewCalicoKubeControllersPolicy(calicoKubeControllersCfg(cc), nil) diff --git a/pkg/enterprise/installation/node_enterprise_test.go b/pkg/enterprise/installation/node_enterprise_test.go index 7aab518c79..8e8af1099e 100644 --- a/pkg/enterprise/installation/node_enterprise_test.go +++ b/pkg/enterprise/installation/node_enterprise_test.go @@ -133,8 +133,9 @@ var _ = Describe("node enterprise modifier integration", func() { Client: cli, CertificateManager: certManager, } - renderCtx, _, err = ext.ExtendContext(cc) + ecc, _, err := ext.ExtendContext(cc) Expect(err).NotTo(HaveOccurred()) + renderCtx = ecc.RenderContext }) // renderNodeObjects renders the real node component and applies the registered @@ -205,7 +206,7 @@ var _ = Describe("node enterprise modifier integration", func() { Spec: operatorv1.LogCollectorSpec{CollectProcessPath: &enable}, })).NotTo(HaveOccurred()) - rc, _, err := ext.ExtendContext(contexts.ControllerContext{ + ecc, _, err := ext.ExtendContext(contexts.ControllerContext{ RenderContext: render.RenderContext{ Installation: instance, TrustedBundle: typhaNodeTLS.TrustedBundle, @@ -217,6 +218,7 @@ var _ = Describe("node enterprise modifier integration", func() { CertificateManager: certManager, }) Expect(err).NotTo(HaveOccurred()) + rc := ecc.RenderContext ds, ok := extensions.FindObject[*appsv1.DaemonSet](renderNodeObjects(rc), common.NodeDaemonSetName) Expect(ok).To(BeTrue()) diff --git a/pkg/enterprise/windows/extension.go b/pkg/enterprise/windows/extension.go index 007781f00c..d24b84083f 100644 --- a/pkg/enterprise/windows/extension.go +++ b/pkg/enterprise/windows/extension.go @@ -86,8 +86,7 @@ func (windowsControllerExtension) Watches(c ctrlruntime.Controller) error { // ExtendContext fetches the node prometheus keypair the installation controller // created and stashes it in the render context for the windows modifier. -func (windowsControllerExtension) ExtendContext(cc contexts.ControllerContext) (render.RenderContext, []certificatemanagement.KeyPairInterface, error) { - rc := cc.RenderContext +func (windowsControllerExtension) ExtendContext(cc contexts.ControllerContext) (contexts.ControllerContext, []certificatemanagement.KeyPairInterface, error) { tls, err := cc.CertificateManager.GetKeyPair( cc.Client, render.NodePrometheusTLSServerSecret, @@ -95,10 +94,10 @@ func (windowsControllerExtension) ExtendContext(cc contexts.ControllerContext) ( dns.GetServiceDNSNames(render.WindowsNodeMetricsService, common.CalicoNamespace, cc.ClusterDomain), ) if err != nil { - return rc, nil, fmt.Errorf("error getting node prometheus TLS certificate: %w", err) + return cc, nil, fmt.Errorf("error getting node prometheus TLS certificate: %w", err) } - rc.Extension = windowsRenderData{prometheusServerTLS: tls} - return rc, nil, nil + cc.Extension = windowsRenderData{prometheusServerTLS: tls} + return cc, nil, nil } // modifyWindows layers Calico Enterprise behavior onto the rendered diff --git a/pkg/enterprise/windows/extension_test.go b/pkg/enterprise/windows/extension_test.go index bb89a31f94..3709bbbcba 100644 --- a/pkg/enterprise/windows/extension_test.go +++ b/pkg/enterprise/windows/extension_test.go @@ -151,7 +151,8 @@ var _ = Describe("windows enterprise modifier", func() { Client: cli, CertificateManager: cm, } - rc, _, err := ext.ExtendContext(cc) + ecc, _, err := ext.ExtendContext(cc) + rc := ecc.RenderContext Expect(err).NotTo(HaveOccurred()) out, _ := extensionstest.ApplyExtensions(ext, render.ComponentNameWindows, rc, newObjs(), nil) diff --git a/pkg/extensions/controllerextension.go b/pkg/extensions/controllerextension.go index 0f5025bdb7..18957a524f 100644 --- a/pkg/extensions/controllerextension.go +++ b/pkg/extensions/controllerextension.go @@ -20,7 +20,6 @@ import ( operatorv1 "github.com/tigera/operator/api/v1" "github.com/tigera/operator/pkg/controller/contexts" "github.com/tigera/operator/pkg/ctrlruntime" - "github.com/tigera/operator/pkg/render" "github.com/tigera/operator/pkg/tls/certificatemanagement" ) @@ -34,11 +33,11 @@ type ControllerExtension interface { Validate(cc contexts.ControllerContext) error // ExtendContext does the controller-side reconcile work the render phase - // cannot, returning the RenderContext the render phase consumes plus any - // keypairs the extension created that the controller should manage (add to - // certificate management and BYO-expiry warnings), or an error that aborts the - // reconcile. - ExtendContext(cc contexts.ControllerContext) (render.RenderContext, []certificatemanagement.KeyPairInterface, error) + // cannot, returning the updated ControllerContext (its embedded RenderContext is + // what the render phase consumes) plus any keypairs the extension created that the + // controller should manage (add to certificate management and BYO-expiry + // warnings), or an error that aborts the reconcile. + ExtendContext(cc contexts.ControllerContext) (contexts.ControllerContext, []certificatemanagement.KeyPairInterface, error) } // Watcher is an optional companion to ControllerExtension. A controller's Add() diff --git a/pkg/extensions/controllerextension_test.go b/pkg/extensions/controllerextension_test.go index 87b409fe70..538d23d294 100644 --- a/pkg/extensions/controllerextension_test.go +++ b/pkg/extensions/controllerextension_test.go @@ -110,11 +110,12 @@ func (f fakeController) Validate(_ contexts.ControllerContext) error { return f.validateErr } -func (f fakeController) ExtendContext(_ contexts.ControllerContext) (render.RenderContext, []certificatemanagement.KeyPairInterface, error) { +func (f fakeController) ExtendContext(cc contexts.ControllerContext) (contexts.ControllerContext, []certificatemanagement.KeyPairInterface, error) { if f.err != nil { - return render.RenderContext{}, nil, f.err + return cc, nil, f.err } - return render.RenderContext{ClusterDomain: "from-fake"}, nil, nil + cc.ClusterDomain = "from-fake" + return cc, nil, nil } // watchingController is a fakeController that also implements the Watcher diff --git a/pkg/extensions/set.go b/pkg/extensions/set.go index e7ccd405e3..d4bfa24b6f 100644 --- a/pkg/extensions/set.go +++ b/pkg/extensions/set.go @@ -94,12 +94,12 @@ func (s *Set) Validate(cc contexts.ControllerContext) error { } // ExtendContext runs the cc.Controller extension for the installation's variant -// and returns the resulting render.RenderContext plus any keypairs the extension wants -// the controller to manage, or the base render context and no keypairs when no +// and returns the updated ControllerContext plus any keypairs the extension wants +// the controller to manage, or the context unchanged and no keypairs when no // extension is registered. Nil-safe. -func (s *Set) ExtendContext(cc contexts.ControllerContext) (render.RenderContext, []certificatemanagement.KeyPairInterface, error) { +func (s *Set) ExtendContext(cc contexts.ControllerContext) (contexts.ControllerContext, []certificatemanagement.KeyPairInterface, error) { if cc.Installation == nil { - return cc.RenderContext, nil, nil + return cc, nil, nil } return s.variant(cc.Installation.Variant).extendContext(cc) } diff --git a/pkg/extensions/variant.go b/pkg/extensions/variant.go index f714404c57..a7438fdac4 100644 --- a/pkg/extensions/variant.go +++ b/pkg/extensions/variant.go @@ -118,11 +118,11 @@ func (v *Variant) validate(cc contexts.ControllerContext) error { return v.controllers[cc.Controller].Validate(cc) } -// extendContext runs the cc.Controller extension, or returns the base render -// context and no managed keypairs when the variant has none for it. Nil-safe. -func (v *Variant) extendContext(cc contexts.ControllerContext) (render.RenderContext, []certificatemanagement.KeyPairInterface, error) { +// extendContext runs the cc.Controller extension, or returns the context unchanged +// and no managed keypairs when the variant has none for it. Nil-safe. +func (v *Variant) extendContext(cc contexts.ControllerContext) (contexts.ControllerContext, []certificatemanagement.KeyPairInterface, error) { if v == nil || v.controllers[cc.Controller] == nil { - return cc.RenderContext, nil, nil + return cc, nil, nil } return v.controllers[cc.Controller].ExtendContext(cc) } From 09a8780f7cdcd9d0d5e90b85dc4802698ac61766 Mon Sep 17 00:00:00 2001 From: Casey Davenport Date: Wed, 24 Jun 2026 23:00:37 -0700 Subject: [PATCH 53/54] Compute variant controller options on the extension Set The extension Set computes its variant's controller-phase options once at startup (ComputeOptions, run from main) and carries them onto each ControllerContext, so the shared types no longer name an enterprise-only setting. The ControllerContext holds an opaque Options the variant's hooks assert back out; the enterprise multi-tenant flag and its discovery live entirely in pkg/enterprise. --- cmd/main.go | 10 ++++- .../apiserver/apiserver_controller.go | 1 - .../apiserver/apiserver_controller_test.go | 2 +- .../apiserver/apiserver_suite_test.go | 14 +++++++ pkg/controller/contexts/context.go | 9 +++-- pkg/enterprise/apiserver/extension.go | 3 +- pkg/enterprise/options/options.go | 33 ++++++++++++++++ pkg/enterprise/register.go | 18 +++++++++ pkg/extensions/set.go | 39 +++++++++++++++++++ 9 files changed, 121 insertions(+), 8 deletions(-) create mode 100644 pkg/enterprise/options/options.go diff --git a/cmd/main.go b/cmd/main.go index 60560c9e46..75d89973d3 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -510,6 +510,14 @@ If a value other than 'all' is specified, the first CRD with a prefix of the spe os.Exit(1) } + // Build the variant extensions and let them compute their controller-phase + // options (the core operator's Set computes none). + extensionSet := enterprise.New() + if err := extensionSet.ComputeOptions(ctx, clientset); err != nil { + setupLog.Error(err, "Failed to compute extension options") + os.Exit(1) + } + options := options.ControllerOptions{ DetectedProvider: provider, EnterpriseCRDExists: enterpriseCRDExists, @@ -522,7 +530,7 @@ If a value other than 'all' is specified, the first CRD with a prefix of the spe ElasticExternal: discovery.UseExternalElastic(bootConfig), UseV3CRDs: v3CRDs, APIDiscovery: apiDiscovery, - Extensions: enterprise.New(), + Extensions: extensionSet, } // Before we start any controllers, make sure our options are valid. diff --git a/pkg/controller/apiserver/apiserver_controller.go b/pkg/controller/apiserver/apiserver_controller.go index 70543c88a6..8575e023dc 100644 --- a/pkg/controller/apiserver/apiserver_controller.go +++ b/pkg/controller/apiserver/apiserver_controller.go @@ -308,7 +308,6 @@ func (r *ReconcileAPIServer) Reconcile(ctx context.Context, request reconcile.Re Ctx: ctx, Client: r.client, CertificateManager: certificateManager, - MultiTenant: r.opts.MultiTenant, } if err := r.opts.Extensions.Validate(cc); err != nil { r.status.SetDegraded(operatorv1.ResourceValidationError, "Invalid API server configuration", err, reqLogger) diff --git a/pkg/controller/apiserver/apiserver_controller_test.go b/pkg/controller/apiserver/apiserver_controller_test.go index 11a4f896de..44f5417028 100644 --- a/pkg/controller/apiserver/apiserver_controller_test.go +++ b/pkg/controller/apiserver/apiserver_controller_test.go @@ -853,7 +853,7 @@ var _ = Describe("apiserver controller tests", func() { tierWatchReady: ready, migrationWatchReady: &utils.ReadyFlag{}, opts: options.ControllerOptions{ - Extensions: testExtensions, + Extensions: multiTenantExtensions(), EnterpriseCRDExists: true, DetectedProvider: operatorv1.ProviderNone, MultiTenant: true, diff --git a/pkg/controller/apiserver/apiserver_suite_test.go b/pkg/controller/apiserver/apiserver_suite_test.go index 2e1863335b..a027083ea7 100644 --- a/pkg/controller/apiserver/apiserver_suite_test.go +++ b/pkg/controller/apiserver/apiserver_suite_test.go @@ -15,6 +15,7 @@ package apiserver import ( + "context" "testing" uzap "go.uber.org/zap" @@ -22,10 +23,12 @@ import ( "github.com/onsi/ginkgo/v2" "github.com/onsi/gomega" + "k8s.io/client-go/kubernetes" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" "github.com/tigera/operator/pkg/enterprise" + eoptions "github.com/tigera/operator/pkg/enterprise/options" "github.com/tigera/operator/pkg/extensions" ) @@ -35,6 +38,17 @@ import ( // it on their options, mirroring how main wires it in production. var testExtensions *extensions.Set = enterprise.New() +// multiTenantExtensions is an enterprise Set whose computed options report +// multi-tenant mode, for the multi-tenant API server test. +func multiTenantExtensions() *extensions.Set { + s := enterprise.New() + s.RegisterOptions(func(context.Context, kubernetes.Interface) (any, error) { + return eoptions.Options{MultiTenant: true}, nil + }) + _ = s.ComputeOptions(context.Background(), nil) + return s +} + func TestStatus(t *testing.T) { logf.SetLogger(zap.New(zap.WriteTo(ginkgo.GinkgoWriter), zap.UseDevMode(true), zap.Level(uzap.NewAtomicLevelAt(uzap.DebugLevel)))) gomega.RegisterFailHandler(ginkgo.Fail) diff --git a/pkg/controller/contexts/context.go b/pkg/controller/contexts/context.go index 2436206237..a4b75c11e9 100644 --- a/pkg/controller/contexts/context.go +++ b/pkg/controller/contexts/context.go @@ -61,8 +61,9 @@ type ControllerContext struct { Client client.Client CertificateManager certificatemanager.CertificateManager - // MultiTenant reports whether the operator runs in multi-tenant mode. It's a - // controller-phase operator mode some extensions gate on (e.g. the API server's - // management-cluster tunnel-secret check). - MultiTenant bool + // Options carries the active variant's computed controller-phase options. The + // extension Set fills it before dispatching to a hook (Validate/ExtendContext). + // It's opaque so core never names a variant-only option; the variant's hooks + // assert it back out. Nil for the core operator. + Options any } diff --git a/pkg/enterprise/apiserver/extension.go b/pkg/enterprise/apiserver/extension.go index 43b0d153e7..52fc28108a 100644 --- a/pkg/enterprise/apiserver/extension.go +++ b/pkg/enterprise/apiserver/extension.go @@ -42,6 +42,7 @@ import ( "github.com/tigera/operator/pkg/controller/utils/imageset" "github.com/tigera/operator/pkg/ctrlruntime" "github.com/tigera/operator/pkg/dns" + eoptions "github.com/tigera/operator/pkg/enterprise/options" "github.com/tigera/operator/pkg/extensions" "github.com/tigera/operator/pkg/render" "github.com/tigera/operator/pkg/render/common/authentication" @@ -178,7 +179,7 @@ func (apiServerControllerExtension) ExtendContext(cc contexts.ControllerContext) // Management cluster only: the apiserver mounts the tunnel CA secret so it can sign // certificates for managed clusters. The manager controller writes it once // ManagementCluster.Spec.TLS is defaulted; degrade until it exists. - if managementCluster != nil && managementCluster.Spec.TLS != nil && !cc.MultiTenant { + if managementCluster != nil && managementCluster.Spec.TLS != nil && !eoptions.From(cc).MultiTenant { if _, err := utils.GetSecret(cc.Ctx, cc.Client, managementCluster.Spec.TLS.SecretName, common.OperatorNamespace()); err != nil { return cc, nil, fmt.Errorf("unable to fetch the tunnel secret: %w", err) } diff --git a/pkg/enterprise/options/options.go b/pkg/enterprise/options/options.go new file mode 100644 index 0000000000..9a621be5e5 --- /dev/null +++ b/pkg/enterprise/options/options.go @@ -0,0 +1,33 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package options + +import "github.com/tigera/operator/pkg/controller/contexts" + +// Options is the Calico Enterprise controller-phase options. The extension build +// computes it once at startup (see extensions.Set.ComputeOptions) and its +// controller hooks read it back off the ControllerContext via From. It lives in +// its own leaf package so the hooks and the operator's main can both reference it. +type Options struct { + // MultiTenant reports whether the operator runs in multi-tenant mode. + MultiTenant bool +} + +// From returns the enterprise options carried on the controller context, or the +// zero value when none are set. +func From(cc contexts.ControllerContext) Options { + o, _ := cc.Options.(Options) + return o +} diff --git a/pkg/enterprise/register.go b/pkg/enterprise/register.go index a58abc1b13..12f02fd64f 100644 --- a/pkg/enterprise/register.go +++ b/pkg/enterprise/register.go @@ -15,11 +15,17 @@ package enterprise import ( + "context" + + "k8s.io/client-go/kubernetes" + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/common/discovery" "github.com/tigera/operator/pkg/enterprise/apiserver" "github.com/tigera/operator/pkg/enterprise/clusterconnection" "github.com/tigera/operator/pkg/enterprise/guardian" "github.com/tigera/operator/pkg/enterprise/installation" + eoptions "github.com/tigera/operator/pkg/enterprise/options" "github.com/tigera/operator/pkg/enterprise/typha" "github.com/tigera/operator/pkg/enterprise/windows" "github.com/tigera/operator/pkg/extensions" @@ -33,6 +39,7 @@ import ( // modifiers through its Register func. func New() *extensions.Set { s := extensions.NewSet() + s.RegisterOptions(computeOptions) ent := s.Variant(operatorv1.CalicoEnterprise) typha.Register(ent) @@ -49,3 +56,14 @@ func New() *extensions.Set { return s } + +// computeOptions discovers the Calico Enterprise controller-phase options at +// startup. extensions.Set.ComputeOptions runs it from main; the result rides on +// each ControllerContext for the enterprise hooks to read. +func computeOptions(ctx context.Context, cli kubernetes.Interface) (any, error) { + multiTenant, err := discovery.MultiTenant(ctx, cli) + if err != nil { + return nil, err + } + return eoptions.Options{MultiTenant: multiTenant}, nil +} diff --git a/pkg/extensions/set.go b/pkg/extensions/set.go index d4bfa24b6f..4b8161cae1 100644 --- a/pkg/extensions/set.go +++ b/pkg/extensions/set.go @@ -15,7 +15,10 @@ package extensions import ( + "context" + v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" + "k8s.io/client-go/kubernetes" operatorv1 "github.com/tigera/operator/api/v1" "github.com/tigera/operator/pkg/components" @@ -39,6 +42,15 @@ import ( type Set struct { variants map[operatorv1.ProductVariant]*Variant images *imageoverride.Overrides + + // computeOptions, registered by the extension build, discovers the variant's + // controller-phase options at startup. ComputeOptions runs it. The core operator + // registers none. + computeOptions func(context.Context, kubernetes.Interface) (any, error) + + // options is the result of ComputeOptions, carried onto every ControllerContext + // so a hook can read its variant's options. + options any } // NewSet returns an empty Set ready to register variant extensions into. @@ -49,6 +61,27 @@ func NewSet() *Set { } } +// RegisterOptions sets the function that computes the variant's controller-phase +// options. The extension build registers one; the core operator registers none. +func (s *Set) RegisterOptions(f func(context.Context, kubernetes.Interface) (any, error)) { + s.computeOptions = f +} + +// ComputeOptions runs the registered options computer (if any) and stores the +// result, which Validate and ExtendContext then carry onto the ControllerContext. +// It's meant to be called once at startup, before any reconcile. Nil-safe. +func (s *Set) ComputeOptions(ctx context.Context, cli kubernetes.Interface) error { + if s == nil || s.computeOptions == nil { + return nil + } + o, err := s.computeOptions(ctx, cli) + if err != nil { + return err + } + s.options = o + return nil +} + // Variant returns the extension bundle for v, creating an empty one if needed. // Used at registration time to build up a variant's extensions. func (s *Set) Variant(v operatorv1.ProductVariant) *Variant { @@ -87,6 +120,9 @@ func (s *Set) Decorate(component render.Component, ctx render.RenderContext) ren // Validate runs the cc.Controller extension's validation for the installation's // variant, or returns nil when no extension is registered. Nil-safe. func (s *Set) Validate(cc contexts.ControllerContext) error { + if s != nil { + cc.Options = s.options + } if cc.Installation == nil { return nil } @@ -98,6 +134,9 @@ func (s *Set) Validate(cc contexts.ControllerContext) error { // the controller to manage, or the context unchanged and no keypairs when no // extension is registered. Nil-safe. func (s *Set) ExtendContext(cc contexts.ControllerContext) (contexts.ControllerContext, []certificatemanagement.KeyPairInterface, error) { + if s != nil { + cc.Options = s.options + } if cc.Installation == nil { return cc, nil, nil } From 42e3bd2bdb92dc3b30a009c25cba06589165997a Mon Sep 17 00:00:00 2001 From: Casey Davenport Date: Wed, 1 Jul 2026 13:52:49 -0700 Subject: [PATCH 54/54] Preserve WAF RBAC test coverage from master Master added assertions that the tigera-ui-user and tigera-network-admin cluster roles grant WAF policy access. Those lived in the apiserver_test.go fixtures this branch removed, so assert the WAF rules directly on the rendered roles in the extension test instead. --- pkg/enterprise/apiserver/extension_test.go | 31 ++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/pkg/enterprise/apiserver/extension_test.go b/pkg/enterprise/apiserver/extension_test.go index cb959c87c3..ee6733dae0 100644 --- a/pkg/enterprise/apiserver/extension_test.go +++ b/pkg/enterprise/apiserver/extension_test.go @@ -207,6 +207,37 @@ var _ = Describe("API server enterprise modifier", func() { Expect(ok).To(BeTrue(), "expected ClusterRole %q", name) } + // The user and network-admin roles grant access to WAF policy resources. + uiUser, found := extensions.FindObject[*rbacv1.ClusterRole](objs, "tigera-ui-user") + Expect(found).To(BeTrue()) + Expect(uiUser.Rules).To(ContainElement(rbacv1.PolicyRule{ + APIGroups: []string{"applicationlayer.projectcalico.org"}, + Resources: []string{ + "globalwafpolicies", + "globalwafplugins", + "globalwafvalidationpolicies", + "wafpolicies", + "wafplugins", + "wafvalidationpolicies", + }, + Verbs: []string{"get", "watch", "list"}, + })) + + networkAdmin, found := extensions.FindObject[*rbacv1.ClusterRole](objs, "tigera-network-admin") + Expect(found).To(BeTrue()) + Expect(networkAdmin.Rules).To(ContainElement(rbacv1.PolicyRule{ + APIGroups: []string{"applicationlayer.projectcalico.org"}, + Resources: []string{ + "globalwafpolicies", + "globalwafplugins", + "globalwafvalidationpolicies", + "wafpolicies", + "wafplugins", + "wafvalidationpolicies", + }, + Verbs: []string{"create", "update", "delete", "patch", "get", "watch", "list"}, + })) + // Audit policy configmap. _, ok := extensions.FindObject[*corev1.ConfigMap](objs, "calico-audit-policy") Expect(ok).To(BeTrue())