diff --git a/cmd/main.go b/cmd/main.go index 20ebc767c6..75d89973d3 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -42,6 +42,7 @@ import ( "github.com/tigera/operator/pkg/controller/options" "github.com/tigera/operator/pkg/controller/utils" "github.com/tigera/operator/pkg/dns" + "github.com/tigera/operator/pkg/enterprise" "github.com/tigera/operator/pkg/imports/admission" "github.com/tigera/operator/pkg/imports/crds" "github.com/tigera/operator/pkg/render" @@ -509,6 +510,14 @@ If a value other than 'all' is specified, the first CRD with a prefix of the spe os.Exit(1) } + // Build the variant extensions and let them compute their controller-phase + // options (the core operator's Set computes none). + extensionSet := enterprise.New() + if err := extensionSet.ComputeOptions(ctx, clientset); err != nil { + setupLog.Error(err, "Failed to compute extension options") + os.Exit(1) + } + options := options.ControllerOptions{ DetectedProvider: provider, EnterpriseCRDExists: enterpriseCRDExists, @@ -521,6 +530,7 @@ If a value other than 'all' is specified, the first CRD with a prefix of the spe ElasticExternal: discovery.UseExternalElastic(bootConfig), UseV3CRDs: v3CRDs, APIDiscovery: apiDiscovery, + Extensions: extensionSet, } // Before we start any controllers, make sure our options are valid. diff --git a/pkg/controller/apiserver/apiserver_controller.go b/pkg/controller/apiserver/apiserver_controller.go index 668756cc10..8575e023dc 100644 --- a/pkg/controller/apiserver/apiserver_controller.go +++ b/pkg/controller/apiserver/apiserver_controller.go @@ -41,6 +41,7 @@ import ( apiserver "github.com/tigera/operator/pkg/common/validation/apiserver" webhooksvalidation "github.com/tigera/operator/pkg/common/validation/webhooks" "github.com/tigera/operator/pkg/controller/certificatemanager" + "github.com/tigera/operator/pkg/controller/contexts" "github.com/tigera/operator/pkg/controller/k8sapi" "github.com/tigera/operator/pkg/controller/migration/datastoremigration" "github.com/tigera/operator/pkg/controller/options" @@ -51,7 +52,6 @@ import ( "github.com/tigera/operator/pkg/dns" "github.com/tigera/operator/pkg/render" rcertificatemanagement "github.com/tigera/operator/pkg/render/certificatemanagement" - "github.com/tigera/operator/pkg/render/common/authentication" "github.com/tigera/operator/pkg/render/common/networkpolicy" "github.com/tigera/operator/pkg/render/monitor" "github.com/tigera/operator/pkg/render/webhooks" @@ -104,42 +104,11 @@ func Add(mgr manager.Manager, opts options.ControllerOptions) error { } if opts.EnterpriseCRDExists { - // Watch for changes to ApplicationLayer - err = c.WatchObject(&operatorv1.ApplicationLayer{ObjectMeta: metav1.ObjectMeta{Name: utils.DefaultEnterpriseInstanceKey.Name}}, &handler.EnqueueRequestForObject{}) - if err != nil { - return fmt.Errorf("apiserver-controller failed to watch ApplicationLayer resource: %v", err) - } - - // Watch for changes to primary resource ManagementCluster - err = c.WatchObject(&operatorv1.ManagementCluster{}, &handler.EnqueueRequestForObject{}) - if err != nil { - return fmt.Errorf("apiserver-controller failed to watch primary resource: %v", err) - } - - // Watch for changes to primary resource ManagementClusterConnection - err = c.WatchObject(&operatorv1.ManagementClusterConnection{}, &handler.EnqueueRequestForObject{}) - if err != nil { - return fmt.Errorf("apiserver-controller failed to watch primary resource: %v", err) - } - - for _, namespace := range []string{common.OperatorNamespace(), render.APIServerNamespace} { - for _, secretName := range []string{render.VoltronTunnelSecretName, render.ManagerTLSSecretName} { - if err = utils.AddSecretsWatch(c, secretName, namespace); err != nil { - return fmt.Errorf("apiserver-controller failed to watch the Secret resource: %v", err) - } - } + // The variant extension registers the enterprise watches it needs (the management + // cluster CRs, ApplicationLayer, Authentication, and the tunnel secrets). + if err = opts.Extensions.SetupWatches(contexts.APIServerController, c); err != nil { + return fmt.Errorf("apiserver-controller failed to set up extension watches: %w", err) } - - if err = utils.AddSecretsWatch(c, render.VoltronLinseedPublicCert, common.OperatorNamespace()); err != nil { - return fmt.Errorf("apiserver-controller failed to watch the Secret resource: %v", err) - } - - // Watch for changes to authentication - err = c.WatchObject(&operatorv1.Authentication{}, &handler.EnqueueRequestForObject{}) - if err != nil { - return fmt.Errorf("apiserver-controller failed to watch resource: %w", err) - } - } // Watch for the namespace(s) managed by this controller. @@ -318,16 +287,6 @@ func (r *ReconcileAPIServer) Reconcile(ctx context.Context, request reconcile.Re return reconcile.Result{}, err } - // Since apiserver and queryserver may have different UID:GID at run-time, we need to produce this secret in separate volumes and with different permissions. - var queryServerTLSSecretCertificateManagementOnly certificatemanagement.KeyPairInterface - if installationSpec.CertificateManagement != nil { - queryServerTLSSecretCertificateManagementOnly, err = certificateManager.GetOrCreateKeyPair(r.client, "query-server-tls", common.OperatorNamespace(), dns.GetServiceDNSNames(render.APIServerServiceName, render.APIServerNamespace, r.opts.ClusterDomain)) - if err != nil { - r.status.SetDegraded(operatorv1.ResourceCreateError, "Unable to get or create tls key pair", err, reqLogger) - return reconcile.Result{}, err - } - } - certificateManager.AddToStatusManager(r.status, render.APIServerNamespace) pullSecrets, err := utils.GetInstallationPullSecrets(installationSpec, r.client) @@ -336,114 +295,44 @@ func (r *ReconcileAPIServer) Reconcile(ctx context.Context, request reconcile.Re return reconcile.Result{}, err } - // Query enterprise-only data. - var trustedBundle certificatemanagement.TrustedBundle - var applicationLayer *operatorv1.ApplicationLayer - var managementCluster *operatorv1.ManagementCluster - var managementClusterConnection *operatorv1.ManagementClusterConnection - var keyValidatorConfig authentication.KeyValidatorConfig - includeV3NetworkPolicy := false - - if installationSpec.Variant.IsEnterprise() { - trustedBundle, err = certificateManager.CreateNamedTrustedBundleFromSecrets(render.APIServerResourceName, r.client, - common.OperatorNamespace(), false) - if err != nil { - r.status.SetDegraded(operatorv1.ResourceCreateError, "Unable to create the trusted bundle", err, reqLogger) - return reconcile.Result{}, err - } - - applicationLayer, err = utils.GetApplicationLayer(ctx, r.client) - if err != nil { - r.status.SetDegraded(operatorv1.ResourceReadError, "Error reading ApplicationLayer", err, reqLogger) - return reconcile.Result{}, err - } + // Run the variant extension: it validates the configuration and produces the + // enterprise render data (the trusted bundle, the query server certificate, the + // management cluster CRs, the OIDC config, and the resolved L7 sidecar images). For + // the core operator this is a no-op and the render context carries no extension data. + cc := contexts.ControllerContext{ + RenderContext: render.RenderContext{ + Installation: installationSpec, + ClusterDomain: r.opts.ClusterDomain, + }, + Controller: contexts.APIServerController, + Ctx: ctx, + Client: r.client, + CertificateManager: certificateManager, + } + if err := r.opts.Extensions.Validate(cc); err != nil { + r.status.SetDegraded(operatorv1.ResourceValidationError, "Invalid API server configuration", err, reqLogger) + return reconcile.Result{}, err + } + cc, managedKeyPairs, err := r.opts.Extensions.ExtendContext(cc) + if err != nil { + r.status.SetDegraded(operatorv1.ResourceCreateError, "Error preparing the API server extension", err, reqLogger) + return reconcile.Result{}, err + } + trustedBundle := cc.TrustedBundle + // The webhooks component (v3-CRD mode) needs the ManagementCluster to register the + // managed-cluster webhook. Reading it requires the enterprise CRDs. + var managementCluster *operatorv1.ManagementCluster + if r.opts.EnterpriseCRDExists { managementCluster, err = utils.GetManagementCluster(ctx, r.client) if err != nil { r.status.SetDegraded(operatorv1.ResourceReadError, "Error reading ManagementCluster", err, reqLogger) return reconcile.Result{}, err } - - managementClusterConnection, err = utils.GetManagementClusterConnection(ctx, r.client) - if err != nil { - r.status.SetDegraded(operatorv1.ResourceReadError, "Error reading ManagementClusterConnection", err, reqLogger) - return reconcile.Result{}, err - } - - if managementClusterConnection != nil && managementCluster != nil { - err = fmt.Errorf("having both a ManagementCluster and a ManagementClusterConnection is not supported") - r.status.SetDegraded(operatorv1.ResourceValidationError, "", err, reqLogger) - return reconcile.Result{}, err - } - - // Management cluster only: check if the tunnel CA secret has been created. The apiserver mounts this secret so - // it can sign certificates for managed clusters. If the managementCluster has not been defaulted then we should - // not degrade. This is because the manager_controller exits the reconcile loop if the apiserver is not available. - if managementCluster != nil && managementCluster.Spec.TLS != nil && !r.opts.MultiTenant { - tunnelSecretName := managementCluster.Spec.TLS.SecretName - // The manager_controller should have written this secret. We know this since spec.TLS has been defaulted. - // If the secret does not exist, we degrade this controller. - _, err := utils.GetSecret(ctx, r.client, tunnelSecretName, common.OperatorNamespace()) - if err != nil { - r.status.SetDegraded(operatorv1.ResourceReadError, "Unable to fetch the tunnel secret", err, reqLogger) - return reconcile.Result{}, err - } - } - - prometheusCertificate, err := certificateManager.GetCertificate(r.client, monitor.PrometheusClientTLSSecretName, common.OperatorNamespace()) - if err != nil { - r.status.SetDegraded(operatorv1.ResourceReadError, "Failed to get certificate", err, reqLogger) - return reconcile.Result{}, err - } else if prometheusCertificate != nil { - trustedBundle.AddCertificates(prometheusCertificate) - } - - if managementClusterConnection != nil { - voltronLinseedCert, err := certificateManager.GetCertificate(r.client, render.VoltronLinseedPublicCert, common.OperatorNamespace()) - if err != nil { - r.status.SetDegraded(operatorv1.ResourceReadError, fmt.Sprintf("Failed to retrieve %s", render.VoltronLinseedPublicCert), err, reqLogger) - return reconcile.Result{}, err - } - if voltronLinseedCert != nil { - trustedBundle.AddCertificates(voltronLinseedCert) - } - } - - var authenticationCR *operatorv1.Authentication - // Fetch the Authentication spec. If present, we use it to configure user authentication. - authenticationCR, err = utils.GetAuthentication(ctx, r.client) - if err != nil && !errors.IsNotFound(err) { - r.status.SetDegraded(operatorv1.ResourceReadError, "Error while fetching Authentication", err, reqLogger) - return reconcile.Result{}, err - } - - if authenticationCR != nil && authenticationCR.Status.State == operatorv1.TigeraStatusReady { - if utils.DexEnabled(authenticationCR) { - // Do not include DEX TLS Secret Name if authentication CR does not have type Dex - secret := render.DexTLSSecretName - certificate, err := certificateManager.GetCertificate(r.client, secret, common.OperatorNamespace()) - if err != nil { - r.status.SetDegraded(operatorv1.CertificateError, fmt.Sprintf("Failed to retrieve %s", secret), - err, reqLogger) - return reconcile.Result{}, err - } else if certificate == nil { - reqLogger.Info(fmt.Sprintf("Waiting for secret '%s' to become available", secret)) - r.status.SetDegraded(operatorv1.ResourceNotReady, - fmt.Sprintf("Waiting for secret '%s' to become available", secret), - nil, reqLogger) - return reconcile.Result{}, nil - } - trustedBundle.AddCertificates(certificate) - } - - keyValidatorConfig, err = utils.GetKeyValidatorConfig(ctx, r.client, authenticationCR, r.opts.ClusterDomain) - if err != nil { - r.status.SetDegraded(operatorv1.ResourceReadError, "Failed to get KeyValidator Config", err, reqLogger) - return reconcile.Result{}, err - } - } } + includeV3NetworkPolicy := false + // Ensure the calico-system tier exists, before rendering any network policies within it. // // The creation of the Tier depends on this controller to reconcile it's non-NetworkPolicy resources so that @@ -474,7 +363,14 @@ func (r *ReconcileAPIServer) Reconcile(ctx context.Context, request reconcile.Re } // Create a component handler to manage the rendered component. - handler := utils.NewComponentHandler(log, r.client, r.scheme, instance) + handler := utils.NewComponentHandler( + log, + r.client, + r.scheme, + instance, + utils.WithRenderContext(cc.RenderContext), + utils.WithExtensions(r.opts.Extensions), + ) // Render the desired objects from the CRD and create or update them. reqLogger.V(3).Info("rendering components") @@ -485,19 +381,14 @@ func (r *ReconcileAPIServer) Reconcile(ctx context.Context, request reconcile.Re Installation: installationSpec, APIServer: &instance.Spec, ForceHostNetwork: false, - ApplicationLayer: applicationLayer, - ManagementCluster: managementCluster, - ManagementClusterConnection: managementClusterConnection, TLSKeyPair: tlsSecret, PullSecrets: pullSecrets, OpenShift: r.opts.DetectedProvider.IsOpenShift(), TrustedBundle: trustedBundle, MultiTenant: r.opts.MultiTenant, - KeyValidatorConfig: keyValidatorConfig, KubernetesVersion: r.opts.KubernetesVersion, ClusterDomain: r.opts.ClusterDomain, RequiresAggregationServer: !r.opts.UseV3CRDs, - QueryServerTLSKeyPairCertificateManagementOnly: queryServerTLSSecretCertificateManagementOnly, } var components []render.Component @@ -506,6 +397,9 @@ func (r *ReconcileAPIServer) Reconcile(ctx context.Context, request reconcile.Re certKeyPairOptions := []rcertificatemanagement.KeyPairOption{ rcertificatemanagement.NewKeyPairOption(tlsSecret, true, true), } + for _, kp := range managedKeyPairs { + certKeyPairOptions = append(certKeyPairOptions, rcertificatemanagement.NewKeyPairOption(kp, true, true)) + } if r.opts.UseV3CRDs { // If using v3 CRDs, we render the webhooks component that handles various RBAC and validation // responsibilities. The ordering of resources here is important to avoid a deadlock: @@ -585,10 +479,14 @@ func (r *ReconcileAPIServer) Reconcile(ctx context.Context, request reconcile.Re } // Check BYO certificate expiry warnings. - certificatemanagement.CheckKeyPairWarnings(map[string]certificatemanagement.KeyPairInterface{ + keyPairWarnings := map[string]certificatemanagement.KeyPairInterface{ render.CalicoAPIServerTLSSecretName: tlsSecret, webhooks.WebhooksTLSSecretName: webhooksTLS, - }, r.status) + } + for _, kp := range managedKeyPairs { + keyPairWarnings[kp.GetName()] = kp + } + certificatemanagement.CheckKeyPairWarnings(keyPairWarnings, r.status) // Clear the degraded bit if we've reached this far. r.status.ClearDegraded() diff --git a/pkg/controller/apiserver/apiserver_controller_test.go b/pkg/controller/apiserver/apiserver_controller_test.go index b70bd58906..44f5417028 100644 --- a/pkg/controller/apiserver/apiserver_controller_test.go +++ b/pkg/controller/apiserver/apiserver_controller_test.go @@ -170,6 +170,7 @@ var _ = Describe("apiserver controller tests", func() { tierWatchReady: ready, migrationWatchReady: &utils.ReadyFlag{}, opts: options.ControllerOptions{ + Extensions: testExtensions, EnterpriseCRDExists: true, DetectedProvider: operatorv1.ProviderNone, }, @@ -229,6 +230,7 @@ var _ = Describe("apiserver controller tests", func() { tierWatchReady: ready, migrationWatchReady: &utils.ReadyFlag{}, opts: options.ControllerOptions{ + Extensions: testExtensions, EnterpriseCRDExists: true, DetectedProvider: operatorv1.ProviderNone, }, @@ -282,6 +284,7 @@ var _ = Describe("apiserver controller tests", func() { tierWatchReady: ready, migrationWatchReady: &utils.ReadyFlag{}, opts: options.ControllerOptions{ + Extensions: testExtensions, EnterpriseCRDExists: true, DetectedProvider: operatorv1.ProviderNone, ClusterDomain: dns.DefaultClusterDomain, @@ -307,6 +310,7 @@ var _ = Describe("apiserver controller tests", func() { tierWatchReady: ready, migrationWatchReady: &utils.ReadyFlag{}, opts: options.ControllerOptions{ + Extensions: testExtensions, EnterpriseCRDExists: true, DetectedProvider: operatorv1.ProviderNone, }, @@ -329,6 +333,7 @@ var _ = Describe("apiserver controller tests", func() { tierWatchReady: ready, migrationWatchReady: &utils.ReadyFlag{}, opts: options.ControllerOptions{ + Extensions: testExtensions, EnterpriseCRDExists: true, DetectedProvider: operatorv1.ProviderNone, }, @@ -353,6 +358,7 @@ var _ = Describe("apiserver controller tests", func() { tierWatchReady: ready, migrationWatchReady: &utils.ReadyFlag{}, opts: options.ControllerOptions{ + Extensions: testExtensions, EnterpriseCRDExists: true, DetectedProvider: operatorv1.ProviderNone, }, @@ -375,6 +381,7 @@ var _ = Describe("apiserver controller tests", func() { tierWatchReady: notReady, migrationWatchReady: &utils.ReadyFlag{}, opts: options.ControllerOptions{ + Extensions: testExtensions, EnterpriseCRDExists: true, DetectedProvider: operatorv1.ProviderNone, }, @@ -400,6 +407,7 @@ var _ = Describe("apiserver controller tests", func() { tierWatchReady: ready, migrationWatchReady: &utils.ReadyFlag{}, opts: options.ControllerOptions{ + Extensions: testExtensions, EnterpriseCRDExists: true, DetectedProvider: operatorv1.ProviderNone, }, @@ -427,6 +435,7 @@ var _ = Describe("apiserver controller tests", func() { tierWatchReady: ready, migrationWatchReady: &utils.ReadyFlag{}, opts: options.ControllerOptions{ + Extensions: testExtensions, EnterpriseCRDExists: true, DetectedProvider: operatorv1.ProviderNone, }, @@ -452,6 +461,7 @@ var _ = Describe("apiserver controller tests", func() { tierWatchReady: notReady, migrationWatchReady: &utils.ReadyFlag{}, opts: options.ControllerOptions{ + Extensions: testExtensions, EnterpriseCRDExists: true, DetectedProvider: operatorv1.ProviderNone, }, @@ -478,6 +488,7 @@ var _ = Describe("apiserver controller tests", func() { tierWatchReady: ready, migrationWatchReady: &utils.ReadyFlag{}, opts: options.ControllerOptions{ + Extensions: testExtensions, EnterpriseCRDExists: false, DetectedProvider: operatorv1.ProviderNone, }, @@ -520,6 +531,7 @@ var _ = Describe("apiserver controller tests", func() { tierWatchReady: ready, migrationWatchReady: &utils.ReadyFlag{}, opts: options.ControllerOptions{ + Extensions: testExtensions, EnterpriseCRDExists: true, DetectedProvider: operatorv1.ProviderNone, }, @@ -552,6 +564,7 @@ var _ = Describe("apiserver controller tests", func() { tierWatchReady: ready, migrationWatchReady: &utils.ReadyFlag{}, opts: options.ControllerOptions{ + Extensions: testExtensions, EnterpriseCRDExists: true, DetectedProvider: operatorv1.ProviderNone, }, @@ -604,6 +617,7 @@ var _ = Describe("apiserver controller tests", func() { tierWatchReady: ready, migrationWatchReady: &utils.ReadyFlag{}, opts: options.ControllerOptions{ + Extensions: testExtensions, EnterpriseCRDExists: true, DetectedProvider: operatorv1.ProviderNone, }, @@ -673,6 +687,7 @@ var _ = Describe("apiserver controller tests", func() { tierWatchReady: ready, migrationWatchReady: &utils.ReadyFlag{}, opts: options.ControllerOptions{ + Extensions: testExtensions, EnterpriseCRDExists: true, DetectedProvider: operatorv1.ProviderNone, }, @@ -777,6 +792,7 @@ var _ = Describe("apiserver controller tests", func() { tierWatchReady: ready, migrationWatchReady: &utils.ReadyFlag{}, opts: options.ControllerOptions{ + Extensions: testExtensions, EnterpriseCRDExists: true, DetectedProvider: operatorv1.ProviderNone, }, @@ -806,6 +822,7 @@ var _ = Describe("apiserver controller tests", func() { tierWatchReady: ready, migrationWatchReady: &utils.ReadyFlag{}, opts: options.ControllerOptions{ + Extensions: testExtensions, EnterpriseCRDExists: true, DetectedProvider: operatorv1.ProviderNone, }, @@ -836,6 +853,7 @@ var _ = Describe("apiserver controller tests", func() { tierWatchReady: ready, migrationWatchReady: &utils.ReadyFlag{}, opts: options.ControllerOptions{ + Extensions: multiTenantExtensions(), EnterpriseCRDExists: true, DetectedProvider: operatorv1.ProviderNone, MultiTenant: true, @@ -883,6 +901,7 @@ var _ = Describe("apiserver controller tests", func() { tierWatchReady: ready, migrationWatchReady: &utils.ReadyFlag{}, opts: options.ControllerOptions{ + Extensions: testExtensions, EnterpriseCRDExists: true, DetectedProvider: operatorv1.ProviderNone, UseV3CRDs: true, @@ -927,6 +946,7 @@ var _ = Describe("apiserver controller tests", func() { tierWatchReady: ready, migrationWatchReady: &utils.ReadyFlag{}, opts: options.ControllerOptions{ + Extensions: testExtensions, EnterpriseCRDExists: false, DetectedProvider: operatorv1.ProviderNone, UseV3CRDs: true, @@ -955,6 +975,7 @@ var _ = Describe("apiserver controller tests", func() { tierWatchReady: ready, migrationWatchReady: &utils.ReadyFlag{}, opts: options.ControllerOptions{ + Extensions: testExtensions, EnterpriseCRDExists: true, DetectedProvider: operatorv1.ProviderNone, UseV3CRDs: false, diff --git a/pkg/controller/apiserver/apiserver_suite_test.go b/pkg/controller/apiserver/apiserver_suite_test.go index ae0a254ba0..a027083ea7 100644 --- a/pkg/controller/apiserver/apiserver_suite_test.go +++ b/pkg/controller/apiserver/apiserver_suite_test.go @@ -15,6 +15,7 @@ package apiserver import ( + "context" "testing" uzap "go.uber.org/zap" @@ -22,10 +23,32 @@ import ( "github.com/onsi/ginkgo/v2" "github.com/onsi/gomega" + "k8s.io/client-go/kubernetes" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" + + "github.com/tigera/operator/pkg/enterprise" + eoptions "github.com/tigera/operator/pkg/enterprise/options" + "github.com/tigera/operator/pkg/extensions" ) +// testExtensions is the enterprise extension Set the API server controller tests +// reconcile with, so the componentHandler applies the API server modifier (query +// server, audit logging, Enterprise RBAC). Reconcilers built in these tests put +// it on their options, mirroring how main wires it in production. +var testExtensions *extensions.Set = enterprise.New() + +// multiTenantExtensions is an enterprise Set whose computed options report +// multi-tenant mode, for the multi-tenant API server test. +func multiTenantExtensions() *extensions.Set { + s := enterprise.New() + s.RegisterOptions(func(context.Context, kubernetes.Interface) (any, error) { + return eoptions.Options{MultiTenant: true}, nil + }) + _ = s.ComputeOptions(context.Background(), nil) + return s +} + func TestStatus(t *testing.T) { logf.SetLogger(zap.New(zap.WriteTo(ginkgo.GinkgoWriter), zap.UseDevMode(true), zap.Level(uzap.NewAtomicLevelAt(uzap.DebugLevel)))) gomega.RegisterFailHandler(ginkgo.Fail) diff --git a/pkg/controller/clusterconnection/clusterconnection_controller.go b/pkg/controller/clusterconnection/clusterconnection_controller.go index 4b1b7dc8f6..284f1943d7 100644 --- a/pkg/controller/clusterconnection/clusterconnection_controller.go +++ b/pkg/controller/clusterconnection/clusterconnection_controller.go @@ -43,6 +43,7 @@ import ( operatorv1 "github.com/tigera/operator/api/v1" "github.com/tigera/operator/pkg/common" "github.com/tigera/operator/pkg/controller/certificatemanager" + "github.com/tigera/operator/pkg/controller/contexts" "github.com/tigera/operator/pkg/controller/options" "github.com/tigera/operator/pkg/controller/status" "github.com/tigera/operator/pkg/controller/utils" @@ -173,9 +174,9 @@ func newReconciler( scheme: schema, provider: p, status: statusMgr, - clusterDomain: opts.ClusterDomain, tierWatchReady: tierWatchReady, clusterInfoWatchReady: clusterInfoWatchReady, + opts: opts, } c.status.Run(opts.ShutdownContext) return c @@ -190,11 +191,11 @@ type ReconcileConnection struct { scheme *runtime.Scheme provider operatorv1.Provider status status.StatusManager - clusterDomain string tierWatchReady *utils.ReadyFlag clusterInfoWatchReady *utils.ReadyFlag resolvedPodProxies []*httpproxy.Config lastAvailabilityTransition metav1.Time + opts options.ControllerOptions } // Reconcile reads that state of the cluster for a ManagementClusterConnection object and makes changes based on the @@ -244,21 +245,6 @@ func (r *ReconcileConnection) Reconcile(ctx context.Context, request reconcile.R } } - // Verify the cluster doesn't also have the ManagementCluster CRD installed. - if variant.IsEnterprise() { - managementCluster, err := utils.GetManagementCluster(ctx, r.cli) - if err != nil { - r.status.SetDegraded(operatorv1.ResourceReadError, "Error reading ManagementCluster", err, reqLogger) - return reconcile.Result{}, err - } - - if managementCluster != nil { - err = fmt.Errorf("having both a ManagementCluster and a ManagementClusterConnection is not supported") - r.status.SetDegraded(operatorv1.ResourceValidationError, "", err, reqLogger) - return reconcile.Result{}, err - } - } - // Validate that the cluster information watch is ready. if !r.clusterInfoWatchReady.IsReady() { r.status.SetDegraded(operatorv1.ResourceNotReady, "Waiting for clusterInfoWatchReady watch to be established", err, reqLogger) @@ -283,12 +269,35 @@ func (r *ReconcileConnection) Reconcile(ctx context.Context, request reconcile.R log.V(2).Info("Loaded ManagementClusterConnection config", "config", managementClusterConnection) - certificateManager, err := certificatemanager.Create(r.cli, installationSpec, r.clusterDomain, common.OperatorNamespace(), certificatemanager.WithLogger(reqLogger)) + certificateManager, err := certificatemanager.Create(r.cli, installationSpec, r.opts.ClusterDomain, common.OperatorNamespace(), certificatemanager.WithLogger(reqLogger)) if err != nil { r.status.SetDegraded(operatorv1.ResourceCreateError, "Unable to create the Tigera CA", err, reqLogger) return reconcile.Result{}, err } + // Run the variant extension: it validates the configuration (a cluster cannot be + // both a management and a managed cluster) and produces the Enterprise-specific + // Guardian inputs the controller reads back below (the managed cluster version and + // the license-gated egress policy flag). For the core operator this is a no-op and + // the render context carries no extension data, so the OSS defaults apply. + cc := contexts.ControllerContext{ + RenderContext: render.RenderContext{Installation: installationSpec, ClusterDomain: r.opts.ClusterDomain}, + Controller: contexts.ClusterConnectionController, + Ctx: ctx, + Client: r.cli, + CertificateManager: certificateManager, + } + if err := r.opts.Extensions.Validate(cc); err != nil { + r.status.SetDegraded(operatorv1.ResourceValidationError, "Invalid ManagementClusterConnection configuration", err, reqLogger) + return reconcile.Result{}, err + } + cc, _, err = r.opts.Extensions.ExtendContext(cc) + if err != nil { + r.status.SetDegraded(operatorv1.ResourceCreateError, "Error preparing the clusterconnection extension", err, reqLogger) + return reconcile.Result{}, err + } + guardianData, haveGuardianData := render.GuardianRenderDataFromContext(cc.RenderContext) + includeSystem := false if managementClusterConnection.Spec.TLS.CA == operatorv1.CATypePublic { if variant == operatorv1.Calico { @@ -305,9 +314,12 @@ func (r *ReconcileConnection) Reconcile(ctx context.Context, request reconcile.R r.status.SetDegraded(operatorv1.ResourceCreateError, "Unable to create the trusted bundle", err, reqLogger) } + // In the OSS (Whisker) path Guardian connects with its own client keypair. The + // Enterprise path uses the tunnel secret instead, so when the extension supplied + // its Guardian inputs we skip creating this keypair. var guardianKeyPair certificatemanagement.KeyPairInterface - if !variant.IsEnterprise() { - guardianCertificateNames := dns.GetServiceDNSNames("guardian", render.GuardianNamespace, r.clusterDomain) + if !haveGuardianData { + guardianCertificateNames := dns.GetServiceDNSNames("guardian", render.GuardianNamespace, r.opts.ClusterDomain) guardianCertificateNames = append(guardianCertificateNames, "localhost", "127.0.0.1") guardianKeyPair, err = certificateManager.GetOrCreateKeyPair(r.cli, render.GuardianKeyPairSecret, whisker.WhiskerNamespace, guardianCertificateNames) if err != nil { @@ -409,8 +421,8 @@ func (r *ReconcileConnection) Reconcile(ctx context.Context, request reconcile.R r.status.SetDegraded(operatorv1.ResourceReadError, "Error querying clusterInformation", err, reqLogger) return reconcile.Result{}, err } - if variant.IsEnterprise() { - managedClusterVersion = clusterInformation.Spec.CNXVersion + if haveGuardianData { + managedClusterVersion = guardianData.Version } else { managedClusterVersion = clusterInformation.Spec.CalicoVersion } @@ -421,18 +433,9 @@ func (r *ReconcileConnection) Reconcile(ctx context.Context, request reconcile.R return reconcile.Result{RequeueAfter: utils.StandardRetry}, nil } - var includeEgressNetworkPolicy bool - if variant.IsEnterprise() { - // Ensure the license can support enterprise policy, before rendering any network policies within it. - if license, err := utils.FetchLicenseKey(ctx, r.cli); err == nil { - if utils.IsFeatureActive(license, common.EgressAccessControlFeature) { - includeEgressNetworkPolicy = true - } - } else if !k8serrors.IsNotFound(err) { - r.status.SetDegraded(operatorv1.ResourceReadError, "Error querying license", err, reqLogger) - return reconcile.Result{}, err - } - } + // The Enterprise extension gates the domain-based egress rules on the license; the + // OSS default is to leave them disabled. + includeEgressNetworkPolicy := guardianData.IncludeEgressNetworkPolicy // Ensure the calico-system tier exists, before rendering any network policies within it. var tierAvailable bool @@ -443,7 +446,14 @@ func (r *ReconcileConnection) Reconcile(ctx context.Context, request reconcile.R return reconcile.Result{}, err } - ch := utils.NewComponentHandler(log, r.cli, r.scheme, managementClusterConnection) + ch := utils.NewComponentHandler( + log, + r.cli, + r.scheme, + managementClusterConnection, + utils.WithRenderContext(render.RenderContext{Installation: installationSpec}), + utils.WithExtensions(r.opts.Extensions), + ) guardianCfg := &render.GuardianConfiguration{ URL: managementClusterConnection.Spec.ManagementClusterAddr, PodProxies: r.resolvedPodProxies, diff --git a/pkg/controller/clusterconnection/clusterconnection_controller_enterprise_test.go b/pkg/controller/clusterconnection/clusterconnection_controller_enterprise_test.go new file mode 100644 index 0000000000..d79ec8aad7 --- /dev/null +++ b/pkg/controller/clusterconnection/clusterconnection_controller_enterprise_test.go @@ -0,0 +1,510 @@ +// Copyright (c) 2020-2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package clusterconnection_test + +import ( + "context" + "fmt" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/stretchr/testify/mock" + + appsv1 "k8s.io/api/apps/v1" + v1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" + + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/apis" + "github.com/tigera/operator/pkg/common" + "github.com/tigera/operator/pkg/components" + "github.com/tigera/operator/pkg/controller/certificatemanager" + "github.com/tigera/operator/pkg/controller/clusterconnection" + "github.com/tigera/operator/pkg/controller/status" + "github.com/tigera/operator/pkg/controller/utils" + ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" + "github.com/tigera/operator/pkg/dns" + "github.com/tigera/operator/pkg/render" + "github.com/tigera/operator/pkg/render/common/networkpolicy" + "github.com/tigera/operator/pkg/render/monitor" + "github.com/tigera/operator/test" +) + +// These tests cover the Calico Enterprise behavior of the ManagementClusterConnection +// controller: the enterprise images, the license/tier-gated calico-system network +// policy, and impersonation. The shared (variant-agnostic) controller mechanics live +// in clusterconnection_controller_test.go. +var _ = Describe("ManagementClusterConnection controller enterprise tests", func() { + var c client.Client + var ctx context.Context + var cfg *operatorv1.ManagementClusterConnection + var installation *operatorv1.Installation + var r reconcile.Reconciler + var clientScheme *runtime.Scheme + var mockStatus *status.MockStatus + var objTrackerWithCalls test.ObjectTrackerWithCalls + + notReady := &utils.ReadyFlag{} + ready := &utils.ReadyFlag{} + ready.MarkAsReady() + + BeforeEach(func() { + // Create a Kubernetes client. + clientScheme = runtime.NewScheme() + Expect(apis.AddToScheme(clientScheme, false)).ShouldNot(HaveOccurred()) + Expect(appsv1.SchemeBuilder.AddToScheme(clientScheme)).ShouldNot(HaveOccurred()) + Expect(rbacv1.SchemeBuilder.AddToScheme(clientScheme)).ShouldNot(HaveOccurred()) + err := operatorv1.SchemeBuilder.AddToScheme(clientScheme) + Expect(err).NotTo(HaveOccurred()) + objTrackerWithCalls = test.NewObjectTrackerWithCalls(clientScheme) + c = ctrlrfake.DefaultFakeClientBuilder(clientScheme).WithObjectTracker(&objTrackerWithCalls).Build() + ctx = context.Background() + mockStatus = &status.MockStatus{} + mockStatus.On("Run").Return() + mockStatus.On("AddDaemonsets", mock.Anything) + mockStatus.On("AddDeployments", mock.Anything) + mockStatus.On("AddStatefulSets", mock.Anything) + mockStatus.On("AddCronJobs", mock.Anything) + mockStatus.On("ClearDegraded", mock.Anything) + mockStatus.On("SetDegraded", mock.Anything, mock.Anything, mock.Anything, mock.Anything) + mockStatus.On("OnCRFound").Return() + mockStatus.On("ReadyToMonitor") + mockStatus.On("SetMetaData", mock.Anything).Return() + mockStatus.On("OnCRNotFound").Return() + + Expect(c.Create(ctx, &operatorv1.Monitor{ + ObjectMeta: metav1.ObjectMeta{Name: "tigera-secure"}, + })) + + Expect(c.Create(ctx, &v3.ClusterInformation{ObjectMeta: metav1.ObjectMeta{Name: "default"}})).NotTo(HaveOccurred()) + + r = clusterconnection.NewReconcilerWithShims(c, clientScheme, mockStatus, operatorv1.ProviderNone, ready, ready) + + Expect(c.Create(ctx, &v1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: render.GuardianNamespace}})) + certificateManager, err := certificatemanager.Create(c, nil, dns.DefaultClusterDomain, common.OperatorNamespace(), certificatemanager.AllowCACreation()) + Expect(err).NotTo(HaveOccurred()) + Expect(c.Create(ctx, certificateManager.KeyPair().Secret(common.OperatorNamespace()))) // Persist the root-ca in the operator namespace. + secret, err := certificateManager.GetOrCreateKeyPair(c, render.GuardianSecretName, common.OperatorNamespace(), []string{"a"}) + Expect(err).NotTo(HaveOccurred()) + + pcSecret, err := certificateManager.GetOrCreateKeyPair(c, render.PacketCaptureServerCert, common.OperatorNamespace(), []string{"a"}) + Expect(err).NotTo(HaveOccurred()) + + promSecret, err := certificateManager.GetOrCreateKeyPair(c, monitor.PrometheusServerTLSSecretName, common.OperatorNamespace(), []string{"a"}) + Expect(err).NotTo(HaveOccurred()) + + queryServerSecret, err := certificateManager.GetOrCreateKeyPair(c, render.CalicoAPIServerTLSSecretName, common.OperatorNamespace(), []string{"a"}) + Expect(err).NotTo(HaveOccurred()) + + err = c.Create(ctx, secret.Secret(common.OperatorNamespace())) + Expect(err).NotTo(HaveOccurred()) + err = c.Create(ctx, pcSecret.Secret(common.OperatorNamespace())) + Expect(err).NotTo(HaveOccurred()) + err = c.Create(ctx, promSecret.Secret(common.OperatorNamespace())) + Expect(err).NotTo(HaveOccurred()) + err = c.Create(ctx, queryServerSecret.Secret(common.OperatorNamespace())) + Expect(err).NotTo(HaveOccurred()) + + trustedBundle := certificateManager.CreateTrustedBundle() + Expect(c.Create(ctx, trustedBundle.ConfigMap(render.GuardianNamespace))).NotTo(HaveOccurred()) + + By("applying the required prerequisites") + // Create a ManagementClusterConnection in the k8s client. + cfg = &operatorv1.ManagementClusterConnection{ + ObjectMeta: metav1.ObjectMeta{Name: "tigera-secure", Generation: 3}, + Spec: operatorv1.ManagementClusterConnectionSpec{ + ManagementClusterAddr: "127.0.0.1:12345", + }, + } + err = c.Create(ctx, cfg) + Expect(err).NotTo(HaveOccurred()) + + installation = &operatorv1.Installation{ + Spec: operatorv1.InstallationSpec{ + Variant: operatorv1.CalicoEnterprise, + Registry: "some.registry.org/", + }, + ObjectMeta: metav1.ObjectMeta{Name: "default"}, + Status: operatorv1.InstallationStatus{ + Variant: operatorv1.CalicoEnterprise, + Computed: &operatorv1.InstallationSpec{ + Registry: "my-reg", + KubernetesProvider: operatorv1.ProviderNone, + }, + }, + } + err = c.Create(ctx, installation) + Expect(err).NotTo(HaveOccurred()) + }) + + Context("image reconciliation", func() { + BeforeEach(func() { + Expect(c.Create(ctx, &v3.Tier{ObjectMeta: metav1.ObjectMeta{Name: "calico-system"}})).NotTo(HaveOccurred()) + }) + + It("should use builtin images", func() { + r = clusterconnection.NewReconcilerWithShims(c, clientScheme, mockStatus, operatorv1.ProviderNone, ready, ready) + _, err := r.Reconcile(ctx, reconcile.Request{}) + Expect(err).ShouldNot(HaveOccurred()) + + d := appsv1.Deployment{ + TypeMeta: metav1.TypeMeta{Kind: "Deployment", APIVersion: "v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: render.GuardianDeploymentName, + Namespace: render.GuardianNamespace, + }, + } + Expect(test.GetResource(c, &d)).To(BeNil()) + Expect(d.Spec.Template.Spec.Containers).To(HaveLen(1)) + dexC := test.GetContainer(d.Spec.Template.Spec.Containers, render.GuardianContainerName) + Expect(dexC).ToNot(BeNil()) + Expect(dexC.Image).To(Equal( + fmt.Sprintf("some.registry.org/%s%s:%s", + components.TigeraImagePath, + components.ComponentTigeraCalico.Image, + components.ComponentTigeraCalico.Version))) + }) + It("should use images from imageset", func() { + Expect(c.Create(ctx, &operatorv1.ImageSet{ + ObjectMeta: metav1.ObjectMeta{Name: "enterprise-" + components.EnterpriseRelease}, + Spec: operatorv1.ImageSetSpec{ + Images: []operatorv1.Image{ + {Image: "tigera/calico", Digest: "sha256:guardianhash"}, + {Image: "tigera/calico", Digest: "sha256:deadbeef0123456789"}, + }, + }, + })).ToNot(HaveOccurred()) + + r = clusterconnection.NewReconcilerWithShims(c, clientScheme, mockStatus, operatorv1.ProviderNone, ready, ready) + _, err := r.Reconcile(ctx, reconcile.Request{}) + Expect(err).ShouldNot(HaveOccurred()) + + d := appsv1.Deployment{ + TypeMeta: metav1.TypeMeta{Kind: "Deployment", APIVersion: "v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: render.GuardianDeploymentName, + Namespace: render.GuardianNamespace, + }, + } + Expect(test.GetResource(c, &d)).To(BeNil()) + Expect(d.Spec.Template.Spec.Containers).To(HaveLen(1)) + apiserver := test.GetContainer(d.Spec.Template.Spec.Containers, render.GuardianContainerName) + Expect(apiserver).ToNot(BeNil()) + Expect(apiserver.Image).To(Equal( + fmt.Sprintf("some.registry.org/%s%s@%s", + components.TigeraImagePath, + components.ComponentTigeraCalico.Image, + "sha256:guardianhash"))) + }) + }) + + Context("calico-system reconciliation", func() { + var licenseKey *v3.LicenseKey + BeforeEach(func() { + licenseKey = &v3.LicenseKey{ + ObjectMeta: metav1.ObjectMeta{Name: "default"}, + Status: v3.LicenseKeyStatus{ + Features: []string{ + common.TiersFeature, + common.EgressAccessControlFeature, + }, + }, + } + Expect(c.Create(ctx, licenseKey)).NotTo(HaveOccurred()) + Expect(c.Create(ctx, &v3.Tier{ObjectMeta: metav1.ObjectMeta{Name: "calico-system"}})).NotTo(HaveOccurred()) + r = clusterconnection.NewReconcilerWithShims(c, clientScheme, mockStatus, operatorv1.ProviderNone, ready, ready) + }) + + Context("IP-based management cluster address", func() { + It("should render calico-system policy when tier and watch are ready", func() { + _, err := r.Reconcile(ctx, reconcile.Request{}) + Expect(err).ShouldNot(HaveOccurred()) + + policies := v3.NetworkPolicyList{} + Expect(c.List(ctx, &policies)).ToNot(HaveOccurred()) + + Expect(policies.Items).To(HaveLen(1)) + Expect(policies.Items[0].Name).To(Equal("calico-system.guardian-access")) + }) + + It("should omit calico-system policy and not degrade when tier is not ready", func() { + Expect(c.Delete(ctx, &v3.Tier{ObjectMeta: metav1.ObjectMeta{Name: "calico-system"}})).NotTo(HaveOccurred()) + _, err := r.Reconcile(ctx, reconcile.Request{}) + Expect(err).ShouldNot(HaveOccurred()) + + policies := v3.NetworkPolicyList{} + Expect(c.List(ctx, &policies)).ToNot(HaveOccurred()) + Expect(policies.Items).To(HaveLen(0)) + }) + + It("should degrade and wait when tier is ready, but tier watch is not ready", func() { + mockStatus = &status.MockStatus{} + mockStatus.On("Run").Return() + mockStatus.On("OnCRFound").Return() + mockStatus.On("SetMetaData", mock.Anything).Return() + + r = clusterconnection.NewReconcilerWithShims(c, clientScheme, mockStatus, operatorv1.ProviderNone, notReady, ready) + test.ExpectWaitForTierWatch(ctx, r, mockStatus) + + policies := v3.NetworkPolicyList{} + Expect(c.List(ctx, &policies)).ToNot(HaveOccurred()) + Expect(policies.Items).To(HaveLen(0)) + }) + }) + + Context("Domain-based management cluster address", func() { + BeforeEach(func() { + cfg.Spec.ManagementClusterAddr = "mydomain.io:443" + Expect(c.Update(ctx, cfg)).NotTo(HaveOccurred()) + }) + + It("should render calico-system policy when license and tier are ready", func() { + _, err := r.Reconcile(ctx, reconcile.Request{}) + Expect(err).ShouldNot(HaveOccurred()) + + policies := v3.NetworkPolicyList{} + Expect(c.List(ctx, &policies)).ToNot(HaveOccurred()) + + Expect(policies.Items).To(HaveLen(1)) + Expect(policies.Items[0].Name).To(Equal("calico-system.guardian-access")) + }) + + It("should render calico-system policy without domain-based egress when tier is ready, but license is not sufficient", func() { + licenseKey.Status.Features = []string{common.TiersFeature} + Expect(c.Update(ctx, licenseKey)).NotTo(HaveOccurred()) + + _, err := r.Reconcile(ctx, reconcile.Request{}) + Expect(err).ShouldNot(HaveOccurred()) + + policies := v3.NetworkPolicyList{} + Expect(c.List(ctx, &policies)).ToNot(HaveOccurred()) + Expect(policies.Items).To(HaveLen(1)) + Expect(policies.Items[0].Name).To(Equal("calico-system.guardian-access")) + + // Verify no domain-based egress rules are present + for _, rule := range policies.Items[0].Spec.Egress { + Expect(rule.Destination.Domains).To(BeEmpty(), + "Domain-based egress rules should not be present when license lacks EgressAccessControl") + } + }) + + It("should degrade and wait when tier and license are ready, but tier watch is not ready", func() { + mockStatus = &status.MockStatus{} + mockStatus.On("Run").Return() + mockStatus.On("OnCRFound").Return() + mockStatus.On("SetMetaData", mock.Anything).Return() + + r = clusterconnection.NewReconcilerWithShims(c, clientScheme, mockStatus, operatorv1.ProviderNone, notReady, ready) + test.ExpectWaitForTierWatch(ctx, r, mockStatus) + + policies := v3.NetworkPolicyList{} + Expect(c.List(ctx, &policies)).ToNot(HaveOccurred()) + Expect(policies.Items).To(HaveLen(0)) + }) + + It("should render calico-system policy without domain-based egress when tier is ready but license is not ready", func() { + Expect(c.Delete(ctx, &v3.LicenseKey{ObjectMeta: metav1.ObjectMeta{Name: "default"}})).NotTo(HaveOccurred()) + _, err := r.Reconcile(ctx, reconcile.Request{}) + Expect(err).ShouldNot(HaveOccurred()) + + policies := v3.NetworkPolicyList{} + Expect(c.List(ctx, &policies)).ToNot(HaveOccurred()) + Expect(policies.Items).To(HaveLen(1)) + Expect(policies.Items[0].Name).To(Equal("calico-system.guardian-access")) + + // Verify no domain-based egress rules are present + for _, rule := range policies.Items[0].Spec.Egress { + Expect(rule.Destination.Domains).To(BeEmpty(), + "Domain-based egress rules should not be present when license is not ready") + } + }) + + It("should omit calico-system policy when license is ready but tier is not ready", func() { + Expect(c.Delete(ctx, &v3.Tier{ObjectMeta: metav1.ObjectMeta{Name: "calico-system"}})).NotTo(HaveOccurred()) + _, err := r.Reconcile(ctx, reconcile.Request{}) + Expect(err).ShouldNot(HaveOccurred()) + + policies := v3.NetworkPolicyList{} + Expect(c.List(ctx, &policies)).ToNot(HaveOccurred()) + Expect(policies.Items).To(HaveLen(0)) + }) + }) + + Context("Proxy detection", func() { + // Generate test cases based on the combinations of proxy address forms and proxy settings. + // Here we specify the base targets along with the base proxy IP, domain, and port that will be used for generation. + coreCases := generateCoreProxyTestCases("voltron.io:9000", "192.168.1.2:9000", "proxy.io", "10.1.2.3", "8080") + + // In case we support multiple guardian replicas in the future, we test specific multi-pod scenarios. + multiPodCases := multiplePodCases() + + testCases := append(coreCases, multiPodCases...) + for _, testCase := range testCases { + Describe(fmt.Sprintf("Proxy detection when %+v", test.PrettyFormatProxyTestCase(testCase)), func() { + // Set up the test based on the test case. + BeforeEach(func() { + for i, proxy := range testCase.PodProxies { + createPodWithProxy(ctx, c, proxy, testCase.Lowercase, i) + } + + // Set the target + cfg.Spec.ManagementClusterAddr = testCase.Target + err := c.Update(ctx, cfg) + Expect(err).NotTo(HaveOccurred()) + }) + + It(fmt.Sprintf("detects proxy correctly when %+v", test.PrettyFormatProxyTestCase(testCase)), func() { + // First reconcile creates the guardian deployment without any availability condition. + _, err := r.Reconcile(ctx, reconcile.Request{}) + Expect(err).ShouldNot(HaveOccurred()) + + // Validate that we made no calls to get Pods at this stage. + podGVR := schema.GroupVersionResource{ + Version: "v1", + Resource: "pods", + } + Expect(objTrackerWithCalls.CallCount(podGVR, test.ObjectTrackerCallList)).To(BeZero()) + + // Set the deployment to be unavailable. We need to recreate the deployment otherwise the status update is ignored. + gd := appsv1.Deployment{} + err = c.Get(ctx, client.ObjectKey{Name: render.GuardianDeploymentName, Namespace: render.GuardianNamespace}, &gd) + Expect(err).NotTo(HaveOccurred()) + err = c.Delete(ctx, &gd) + Expect(err).NotTo(HaveOccurred()) + gd.ResourceVersion = "" + gd.Status.Conditions = []appsv1.DeploymentCondition{{ + Type: appsv1.DeploymentAvailable, + Status: v1.ConditionFalse, + LastTransitionTime: metav1.Time{Time: time.Now()}, + }} + err = c.Create(ctx, &gd) + Expect(err).NotTo(HaveOccurred()) + + // Reconcile again. We should see no calls since the deployment has not transitioned to available. + _, err = r.Reconcile(ctx, reconcile.Request{}) + Expect(err).ShouldNot(HaveOccurred()) + Expect(objTrackerWithCalls.CallCount(podGVR, test.ObjectTrackerCallList)).To(Equal(0)) + + // Set the deployment to available. + err = c.Delete(ctx, &gd) + Expect(err).NotTo(HaveOccurred()) + gd.ResourceVersion = "" + gd.Status.Conditions = []appsv1.DeploymentCondition{{ + Type: appsv1.DeploymentAvailable, + Status: v1.ConditionTrue, + LastTransitionTime: metav1.Time{Time: time.Now().Add(time.Minute)}, + }} + err = c.Create(ctx, &gd) + Expect(err).NotTo(HaveOccurred()) + + // Reconcile again. The proxy detection logic should kick in since the guardian deployment is ready. + _, err = r.Reconcile(ctx, reconcile.Request{}) + Expect(err).ShouldNot(HaveOccurred()) + Expect(objTrackerWithCalls.CallCount(podGVR, test.ObjectTrackerCallList)).To(Equal(1)) + + // Resolve the rendered rule that governs egress from guardian to voltron. + policies := v3.NetworkPolicyList{} + Expect(c.List(ctx, &policies)).ToNot(HaveOccurred()) + Expect(policies.Items).To(HaveLen(1)) + Expect(policies.Items[0].Name).To(Equal("calico-system.guardian-access")) + policy := policies.Items[0] + + // Generate the expectation based on the test case, and compare the rendered rule to our expectation. + expectedEgressRules := getExpectedEgressRulesFromCase(testCase) + Expect(policy.Spec.Egress).To(HaveLen(6 + len(expectedEgressRules))) + for i, egressRule := range expectedEgressRules { + managementClusterEgressRule := policy.Spec.Egress[5+i] + if egressRule.hostIsIP { + Expect(managementClusterEgressRule.Destination.Nets).To(HaveLen(1)) + Expect(managementClusterEgressRule.Destination.Nets[0]).To(Equal(fmt.Sprintf("%s/32", egressRule.host))) + Expect(managementClusterEgressRule.Destination.Ports).To(Equal(networkpolicy.Ports(egressRule.port))) + } else { + Expect(managementClusterEgressRule.Destination.Domains).To(Equal([]string{egressRule.host})) + Expect(managementClusterEgressRule.Destination.Ports).To(Equal(networkpolicy.Ports(egressRule.port))) + } + } + + // Reconcile again. Verify that we do not cause any additional query for pods now that we have resolved the proxy. + _, err = r.Reconcile(ctx, reconcile.Request{}) + Expect(err).ShouldNot(HaveOccurred()) + Expect(objTrackerWithCalls.CallCount(podGVR, test.ObjectTrackerCallList)).To(Equal(1)) + }) + }) + } + }) + }) + + val := []string{"some-value"} + DescribeTable("should render impersonation permissions correctly", func(impersonation *operatorv1.Impersonation, expectedUser, expectedGroup, expectedSA []string) { + By("ensuring a tigerastatus exists") + ts := &operatorv1.TigeraStatus{ + ObjectMeta: metav1.ObjectMeta{Name: "management-cluster-connection"}, + Spec: operatorv1.TigeraStatusSpec{}, + Status: operatorv1.TigeraStatusStatus{}, + } + Expect(c.Create(ctx, ts)).NotTo(HaveOccurred()) + + By("updating the CR with the impersonation settings, reconciling and fetching the results") + err := c.Get(ctx, client.ObjectKey{Name: cfg.Name, Namespace: cfg.Namespace}, cfg) + Expect(err).ShouldNot(HaveOccurred()) + cfg.Spec.Impersonation = impersonation + Expect(c.Update(ctx, cfg)).NotTo(HaveOccurred()) + _, err = r.Reconcile(ctx, reconcile.Request{NamespacedName: types.NamespacedName{ + Name: "management-cluster-connection", + Namespace: "", + }}) + Expect(err).ShouldNot(HaveOccurred()) + role := &rbacv1.ClusterRole{} + err = c.Get(ctx, client.ObjectKey{Name: render.GuardianClusterRoleName}, role) + Expect(err).NotTo(HaveOccurred()) + By("verifying the resulting RBAC") + var users, groups, sas []string + for _, rule := range role.Rules { + if len(rule.Verbs) == 1 && rule.Verbs[0] == "impersonate" { + if len(rule.Resources) == 1 { + switch rule.Resources[0] { + case "users": + users = rule.ResourceNames + case "groups": + groups = rule.ResourceNames + case "serviceaccounts": + sas = rule.ResourceNames + } + } + } + } + Expect(users).To(Equal(expectedUser)) + Expect(groups).To(Equal(expectedGroup)) + Expect(sas).To(Equal(expectedSA)) + }, + Entry("no impersonation configured", nil, nil, nil, nil), + Entry("all set", &operatorv1.Impersonation{Users: val, Groups: val, ServiceAccounts: val}, val, val, val), + Entry("all set to empty", &operatorv1.Impersonation{Users: []string{}, Groups: []string{}, ServiceAccounts: []string{}}, nil, nil, nil), + Entry("user set", &operatorv1.Impersonation{Users: val}, val, nil, nil), + Entry("groups set", &operatorv1.Impersonation{Groups: val}, nil, val, nil), + Entry("service accounts set", &operatorv1.Impersonation{ServiceAccounts: val}, nil, nil, val), + Entry("empty impersonation", &operatorv1.Impersonation{}, nil, nil, nil), + ) +}) diff --git a/pkg/controller/clusterconnection/clusterconnection_controller_test.go b/pkg/controller/clusterconnection/clusterconnection_controller_test.go index 6b45399bd0..e8ac420d82 100644 --- a/pkg/controller/clusterconnection/clusterconnection_controller_test.go +++ b/pkg/controller/clusterconnection/clusterconnection_controller_test.go @@ -21,7 +21,6 @@ import ( "net/url" "strconv" "strings" - "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -33,7 +32,6 @@ import ( rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/reconcile" @@ -43,7 +41,6 @@ import ( operatorv1 "github.com/tigera/operator/api/v1" "github.com/tigera/operator/pkg/apis" "github.com/tigera/operator/pkg/common" - "github.com/tigera/operator/pkg/components" "github.com/tigera/operator/pkg/controller/certificatemanager" "github.com/tigera/operator/pkg/controller/clusterconnection" "github.com/tigera/operator/pkg/controller/status" @@ -51,7 +48,6 @@ import ( ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" "github.com/tigera/operator/pkg/dns" "github.com/tigera/operator/pkg/render" - "github.com/tigera/operator/pkg/render/common/networkpolicy" "github.com/tigera/operator/pkg/render/monitor" "github.com/tigera/operator/test" ) @@ -67,7 +63,6 @@ var _ = Describe("ManagementClusterConnection controller tests", func() { var mockStatus *status.MockStatus var objTrackerWithCalls test.ObjectTrackerWithCalls - notReady := &utils.ReadyFlag{} ready := &utils.ReadyFlag{} ready.MarkAsReady() @@ -224,303 +219,6 @@ var _ = Describe("ManagementClusterConnection controller tests", func() { }) }) - Context("image reconciliation", func() { - BeforeEach(func() { - Expect(c.Create(ctx, &v3.Tier{ObjectMeta: metav1.ObjectMeta{Name: "calico-system"}})).NotTo(HaveOccurred()) - }) - - It("should use builtin images", func() { - r = clusterconnection.NewReconcilerWithShims(c, clientScheme, mockStatus, operatorv1.ProviderNone, ready, ready) - _, err := r.Reconcile(ctx, reconcile.Request{}) - Expect(err).ShouldNot(HaveOccurred()) - - d := appsv1.Deployment{ - TypeMeta: metav1.TypeMeta{Kind: "Deployment", APIVersion: "v1"}, - ObjectMeta: metav1.ObjectMeta{ - Name: render.GuardianDeploymentName, - Namespace: render.GuardianNamespace, - }, - } - Expect(test.GetResource(c, &d)).To(BeNil()) - Expect(d.Spec.Template.Spec.Containers).To(HaveLen(1)) - dexC := test.GetContainer(d.Spec.Template.Spec.Containers, render.GuardianContainerName) - Expect(dexC).ToNot(BeNil()) - Expect(dexC.Image).To(Equal( - fmt.Sprintf("some.registry.org/%s%s:%s", - components.TigeraImagePath, - components.ComponentTigeraCalico.Image, - components.ComponentTigeraCalico.Version))) - }) - It("should use images from imageset", func() { - Expect(c.Create(ctx, &operatorv1.ImageSet{ - ObjectMeta: metav1.ObjectMeta{Name: "enterprise-" + components.EnterpriseRelease}, - Spec: operatorv1.ImageSetSpec{ - Images: []operatorv1.Image{ - {Image: "tigera/calico", Digest: "sha256:guardianhash"}, - {Image: "tigera/calico", Digest: "sha256:deadbeef0123456789"}, - }, - }, - })).ToNot(HaveOccurred()) - - r = clusterconnection.NewReconcilerWithShims(c, clientScheme, mockStatus, operatorv1.ProviderNone, ready, ready) - _, err := r.Reconcile(ctx, reconcile.Request{}) - Expect(err).ShouldNot(HaveOccurred()) - - d := appsv1.Deployment{ - TypeMeta: metav1.TypeMeta{Kind: "Deployment", APIVersion: "v1"}, - ObjectMeta: metav1.ObjectMeta{ - Name: render.GuardianDeploymentName, - Namespace: render.GuardianNamespace, - }, - } - Expect(test.GetResource(c, &d)).To(BeNil()) - Expect(d.Spec.Template.Spec.Containers).To(HaveLen(1)) - apiserver := test.GetContainer(d.Spec.Template.Spec.Containers, render.GuardianContainerName) - Expect(apiserver).ToNot(BeNil()) - Expect(apiserver.Image).To(Equal( - fmt.Sprintf("some.registry.org/%s%s@%s", - components.TigeraImagePath, - components.ComponentTigeraCalico.Image, - "sha256:guardianhash"))) - }) - }) - - Context("calico-system reconciliation", func() { - var licenseKey *v3.LicenseKey - BeforeEach(func() { - licenseKey = &v3.LicenseKey{ - ObjectMeta: metav1.ObjectMeta{Name: "default"}, - Status: v3.LicenseKeyStatus{ - Features: []string{ - common.TiersFeature, - common.EgressAccessControlFeature, - }, - }, - } - Expect(c.Create(ctx, licenseKey)).NotTo(HaveOccurred()) - Expect(c.Create(ctx, &v3.Tier{ObjectMeta: metav1.ObjectMeta{Name: "calico-system"}})).NotTo(HaveOccurred()) - r = clusterconnection.NewReconcilerWithShims(c, clientScheme, mockStatus, operatorv1.ProviderNone, ready, ready) - }) - - Context("IP-based management cluster address", func() { - It("should render calico-system policy when tier and watch are ready", func() { - _, err := r.Reconcile(ctx, reconcile.Request{}) - Expect(err).ShouldNot(HaveOccurred()) - - policies := v3.NetworkPolicyList{} - Expect(c.List(ctx, &policies)).ToNot(HaveOccurred()) - - Expect(policies.Items).To(HaveLen(1)) - Expect(policies.Items[0].Name).To(Equal("calico-system.guardian-access")) - }) - - It("should omit calico-system policy and not degrade when tier is not ready", func() { - Expect(c.Delete(ctx, &v3.Tier{ObjectMeta: metav1.ObjectMeta{Name: "calico-system"}})).NotTo(HaveOccurred()) - _, err := r.Reconcile(ctx, reconcile.Request{}) - Expect(err).ShouldNot(HaveOccurred()) - - policies := v3.NetworkPolicyList{} - Expect(c.List(ctx, &policies)).ToNot(HaveOccurred()) - Expect(policies.Items).To(HaveLen(0)) - }) - - It("should degrade and wait when tier is ready, but tier watch is not ready", func() { - mockStatus = &status.MockStatus{} - mockStatus.On("Run").Return() - mockStatus.On("OnCRFound").Return() - mockStatus.On("SetMetaData", mock.Anything).Return() - - r = clusterconnection.NewReconcilerWithShims(c, clientScheme, mockStatus, operatorv1.ProviderNone, notReady, ready) - test.ExpectWaitForTierWatch(ctx, r, mockStatus) - - policies := v3.NetworkPolicyList{} - Expect(c.List(ctx, &policies)).ToNot(HaveOccurred()) - Expect(policies.Items).To(HaveLen(0)) - }) - }) - - Context("Domain-based management cluster address", func() { - BeforeEach(func() { - cfg.Spec.ManagementClusterAddr = "mydomain.io:443" - Expect(c.Update(ctx, cfg)).NotTo(HaveOccurred()) - }) - - It("should render calico-system policy when license and tier are ready", func() { - _, err := r.Reconcile(ctx, reconcile.Request{}) - Expect(err).ShouldNot(HaveOccurred()) - - policies := v3.NetworkPolicyList{} - Expect(c.List(ctx, &policies)).ToNot(HaveOccurred()) - - Expect(policies.Items).To(HaveLen(1)) - Expect(policies.Items[0].Name).To(Equal("calico-system.guardian-access")) - }) - - It("should render calico-system policy without domain-based egress when tier is ready, but license is not sufficient", func() { - licenseKey.Status.Features = []string{common.TiersFeature} - Expect(c.Update(ctx, licenseKey)).NotTo(HaveOccurred()) - - _, err := r.Reconcile(ctx, reconcile.Request{}) - Expect(err).ShouldNot(HaveOccurred()) - - policies := v3.NetworkPolicyList{} - Expect(c.List(ctx, &policies)).ToNot(HaveOccurred()) - Expect(policies.Items).To(HaveLen(1)) - Expect(policies.Items[0].Name).To(Equal("calico-system.guardian-access")) - - // Verify no domain-based egress rules are present - for _, rule := range policies.Items[0].Spec.Egress { - Expect(rule.Destination.Domains).To(BeEmpty(), - "Domain-based egress rules should not be present when license lacks EgressAccessControl") - } - }) - - It("should degrade and wait when tier and license are ready, but tier watch is not ready", func() { - mockStatus = &status.MockStatus{} - mockStatus.On("Run").Return() - mockStatus.On("OnCRFound").Return() - mockStatus.On("SetMetaData", mock.Anything).Return() - - r = clusterconnection.NewReconcilerWithShims(c, clientScheme, mockStatus, operatorv1.ProviderNone, notReady, ready) - test.ExpectWaitForTierWatch(ctx, r, mockStatus) - - policies := v3.NetworkPolicyList{} - Expect(c.List(ctx, &policies)).ToNot(HaveOccurred()) - Expect(policies.Items).To(HaveLen(0)) - }) - - It("should render calico-system policy without domain-based egress when tier is ready but license is not ready", func() { - Expect(c.Delete(ctx, &v3.LicenseKey{ObjectMeta: metav1.ObjectMeta{Name: "default"}})).NotTo(HaveOccurred()) - _, err := r.Reconcile(ctx, reconcile.Request{}) - Expect(err).ShouldNot(HaveOccurred()) - - policies := v3.NetworkPolicyList{} - Expect(c.List(ctx, &policies)).ToNot(HaveOccurred()) - Expect(policies.Items).To(HaveLen(1)) - Expect(policies.Items[0].Name).To(Equal("calico-system.guardian-access")) - - // Verify no domain-based egress rules are present - for _, rule := range policies.Items[0].Spec.Egress { - Expect(rule.Destination.Domains).To(BeEmpty(), - "Domain-based egress rules should not be present when license is not ready") - } - }) - - It("should omit calico-system policy when license is ready but tier is not ready", func() { - Expect(c.Delete(ctx, &v3.Tier{ObjectMeta: metav1.ObjectMeta{Name: "calico-system"}})).NotTo(HaveOccurred()) - _, err := r.Reconcile(ctx, reconcile.Request{}) - Expect(err).ShouldNot(HaveOccurred()) - - policies := v3.NetworkPolicyList{} - Expect(c.List(ctx, &policies)).ToNot(HaveOccurred()) - Expect(policies.Items).To(HaveLen(0)) - }) - }) - - Context("Proxy detection", func() { - // Generate test cases based on the combinations of proxy address forms and proxy settings. - // Here we specify the base targets along with the base proxy IP, domain, and port that will be used for generation. - coreCases := generateCoreProxyTestCases("voltron.io:9000", "192.168.1.2:9000", "proxy.io", "10.1.2.3", "8080") - - // In case we support multiple guardian replicas in the future, we test specific multi-pod scenarios. - multiPodCases := multiplePodCases() - - testCases := append(coreCases, multiPodCases...) - for _, testCase := range testCases { - Describe(fmt.Sprintf("Proxy detection when %+v", test.PrettyFormatProxyTestCase(testCase)), func() { - // Set up the test based on the test case. - BeforeEach(func() { - for i, proxy := range testCase.PodProxies { - createPodWithProxy(ctx, c, proxy, testCase.Lowercase, i) - } - - // Set the target - cfg.Spec.ManagementClusterAddr = testCase.Target - err := c.Update(ctx, cfg) - Expect(err).NotTo(HaveOccurred()) - }) - - It(fmt.Sprintf("detects proxy correctly when %+v", test.PrettyFormatProxyTestCase(testCase)), func() { - // First reconcile creates the guardian deployment without any availability condition. - _, err := r.Reconcile(ctx, reconcile.Request{}) - Expect(err).ShouldNot(HaveOccurred()) - - // Validate that we made no calls to get Pods at this stage. - podGVR := schema.GroupVersionResource{ - Version: "v1", - Resource: "pods", - } - Expect(objTrackerWithCalls.CallCount(podGVR, test.ObjectTrackerCallList)).To(BeZero()) - - // Set the deployment to be unavailable. We need to recreate the deployment otherwise the status update is ignored. - gd := appsv1.Deployment{} - err = c.Get(ctx, client.ObjectKey{Name: render.GuardianDeploymentName, Namespace: render.GuardianNamespace}, &gd) - Expect(err).NotTo(HaveOccurred()) - err = c.Delete(ctx, &gd) - Expect(err).NotTo(HaveOccurred()) - gd.ResourceVersion = "" - gd.Status.Conditions = []appsv1.DeploymentCondition{{ - Type: appsv1.DeploymentAvailable, - Status: v1.ConditionFalse, - LastTransitionTime: metav1.Time{Time: time.Now()}, - }} - err = c.Create(ctx, &gd) - Expect(err).NotTo(HaveOccurred()) - - // Reconcile again. We should see no calls since the deployment has not transitioned to available. - _, err = r.Reconcile(ctx, reconcile.Request{}) - Expect(err).ShouldNot(HaveOccurred()) - Expect(objTrackerWithCalls.CallCount(podGVR, test.ObjectTrackerCallList)).To(Equal(0)) - - // Set the deployment to available. - err = c.Delete(ctx, &gd) - Expect(err).NotTo(HaveOccurred()) - gd.ResourceVersion = "" - gd.Status.Conditions = []appsv1.DeploymentCondition{{ - Type: appsv1.DeploymentAvailable, - Status: v1.ConditionTrue, - LastTransitionTime: metav1.Time{Time: time.Now().Add(time.Minute)}, - }} - err = c.Create(ctx, &gd) - Expect(err).NotTo(HaveOccurred()) - - // Reconcile again. The proxy detection logic should kick in since the guardian deployment is ready. - _, err = r.Reconcile(ctx, reconcile.Request{}) - Expect(err).ShouldNot(HaveOccurred()) - Expect(objTrackerWithCalls.CallCount(podGVR, test.ObjectTrackerCallList)).To(Equal(1)) - - // Resolve the rendered rule that governs egress from guardian to voltron. - policies := v3.NetworkPolicyList{} - Expect(c.List(ctx, &policies)).ToNot(HaveOccurred()) - Expect(policies.Items).To(HaveLen(1)) - Expect(policies.Items[0].Name).To(Equal("calico-system.guardian-access")) - policy := policies.Items[0] - - // Generate the expectation based on the test case, and compare the rendered rule to our expectation. - expectedEgressRules := getExpectedEgressRulesFromCase(testCase) - Expect(policy.Spec.Egress).To(HaveLen(6 + len(expectedEgressRules))) - for i, egressRule := range expectedEgressRules { - managementClusterEgressRule := policy.Spec.Egress[5+i] - if egressRule.hostIsIP { - Expect(managementClusterEgressRule.Destination.Nets).To(HaveLen(1)) - Expect(managementClusterEgressRule.Destination.Nets[0]).To(Equal(fmt.Sprintf("%s/32", egressRule.host))) - Expect(managementClusterEgressRule.Destination.Ports).To(Equal(networkpolicy.Ports(egressRule.port))) - } else { - Expect(managementClusterEgressRule.Destination.Domains).To(Equal([]string{egressRule.host})) - Expect(managementClusterEgressRule.Destination.Ports).To(Equal(networkpolicy.Ports(egressRule.port))) - } - } - - // Reconcile again. Verify that we do not cause any additional query for pods now that we have resolved the proxy. - _, err = r.Reconcile(ctx, reconcile.Request{}) - Expect(err).ShouldNot(HaveOccurred()) - Expect(objTrackerWithCalls.CallCount(podGVR, test.ObjectTrackerCallList)).To(Equal(1)) - }) - }) - } - }) - }) - Context("Proxy setting", func() { DescribeTable("sets the proxy", func(http, https, noProxy bool) { installationCopy := installation.DeepCopy() @@ -774,57 +472,6 @@ var _ = Describe("ManagementClusterConnection controller tests", func() { }) }) - val := []string{"some-value"} - DescribeTable("should render impersonation permissions correctly", func(impersonation *operatorv1.Impersonation, expectedUser, expectedGroup, expectedSA []string) { - By("ensuring a tigerastatus exists") - ts := &operatorv1.TigeraStatus{ - ObjectMeta: metav1.ObjectMeta{Name: "management-cluster-connection"}, - Spec: operatorv1.TigeraStatusSpec{}, - Status: operatorv1.TigeraStatusStatus{}, - } - Expect(c.Create(ctx, ts)).NotTo(HaveOccurred()) - - By("updating the CR with the impersonation settings, reconciling and fetching the results") - err := c.Get(ctx, client.ObjectKey{Name: cfg.Name, Namespace: cfg.Namespace}, cfg) - Expect(err).ShouldNot(HaveOccurred()) - cfg.Spec.Impersonation = impersonation - Expect(c.Update(ctx, cfg)).NotTo(HaveOccurred()) - _, err = r.Reconcile(ctx, reconcile.Request{NamespacedName: types.NamespacedName{ - Name: "management-cluster-connection", - Namespace: "", - }}) - Expect(err).ShouldNot(HaveOccurred()) - role := &rbacv1.ClusterRole{} - err = c.Get(ctx, client.ObjectKey{Name: render.GuardianClusterRoleName}, role) - Expect(err).NotTo(HaveOccurred()) - By("verifying the resulting RBAC") - var users, groups, sas []string - for _, rule := range role.Rules { - if len(rule.Verbs) == 1 && rule.Verbs[0] == "impersonate" { - if len(rule.Resources) == 1 { - switch rule.Resources[0] { - case "users": - users = rule.ResourceNames - case "groups": - groups = rule.ResourceNames - case "serviceaccounts": - sas = rule.ResourceNames - } - } - } - } - Expect(users).To(Equal(expectedUser)) - Expect(groups).To(Equal(expectedGroup)) - Expect(sas).To(Equal(expectedSA)) - }, - Entry("no impersonation configured", nil, nil, nil, nil), - Entry("all set", &operatorv1.Impersonation{Users: val, Groups: val, ServiceAccounts: val}, val, val, val), - Entry("all set to empty", &operatorv1.Impersonation{Users: []string{}, Groups: []string{}, ServiceAccounts: []string{}}, nil, nil, nil), - Entry("user set", &operatorv1.Impersonation{Users: val}, val, nil, nil), - Entry("groups set", &operatorv1.Impersonation{Groups: val}, nil, val, nil), - Entry("service accounts set", &operatorv1.Impersonation{ServiceAccounts: val}, nil, nil, val), - Entry("empty impersonation", &operatorv1.Impersonation{}, nil, nil, nil), - ) }) func createPodWithProxy(ctx context.Context, c client.Client, config *test.ProxyConfig, lowercase bool, replicaNum int) { diff --git a/pkg/controller/clusterconnection/clusterconnection_suite_test.go b/pkg/controller/clusterconnection/clusterconnection_suite_test.go index 8967498282..2e6b6feb43 100644 --- a/pkg/controller/clusterconnection/clusterconnection_suite_test.go +++ b/pkg/controller/clusterconnection/clusterconnection_suite_test.go @@ -27,6 +27,7 @@ import ( func TestStatus(t *testing.T) { logf.SetLogger(zap.New(zap.WriteTo(ginkgo.GinkgoWriter))) gomega.RegisterFailHandler(ginkgo.Fail) + suiteConfig, reporterConfig := ginkgo.GinkgoConfiguration() reporterConfig.JUnitReport = "../../../report/ut/clusterconnection_controller_suite.xml" ginkgo.RunSpecs(t, "pkg/controller/Management Cluster Connection Suite", suiteConfig, reporterConfig) diff --git a/pkg/controller/clusterconnection/shim_test.go b/pkg/controller/clusterconnection/shim_test.go index 7446d229ac..bd5993478d 100644 --- a/pkg/controller/clusterconnection/shim_test.go +++ b/pkg/controller/clusterconnection/shim_test.go @@ -25,6 +25,7 @@ import ( operatorv1 "github.com/tigera/operator/api/v1" "github.com/tigera/operator/pkg/controller/options" "github.com/tigera/operator/pkg/controller/status" + "github.com/tigera/operator/pkg/enterprise" "k8s.io/apimachinery/pkg/runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/reconcile" @@ -40,6 +41,7 @@ func NewReconcilerWithShims( ) reconcile.Reconciler { opts := options.ControllerOptions{ ShutdownContext: context.Background(), + Extensions: enterprise.New(), } return newReconciler(cli, schema, status, provider, tierWatchReady, clusterInfoWatchReady, opts) diff --git a/pkg/controller/contexts/context.go b/pkg/controller/contexts/context.go new file mode 100644 index 0000000000..a4b75c11e9 --- /dev/null +++ b/pkg/controller/contexts/context.go @@ -0,0 +1,69 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package contexts holds the controller-phase context types passed between a +// controller's reconcile and a variant extension. They live here, not in the +// extensions package, because they are controller concepts (the data and +// machinery a controller gathers), not part of the extension mechanism itself - +// the extensions package consumes them. +package contexts + +import ( + "context" + + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/tigera/operator/pkg/controller/certificatemanager" + "github.com/tigera/operator/pkg/render" +) + +// ControllerName identifies the controller a ControllerExtension extends, so a +// variant can register a different hook per controller. Use the constants below +// rather than bare strings so registration and lookup stay in sync. +type ControllerName string + +const ( + InstallationController ControllerName = "installation" + WindowsController ControllerName = "windows" + APIServerController ControllerName = "apiserver" + ClusterConnectionController ControllerName = "clusterconnection" +) + +// ControllerContext is the controller-phase context, the corollary to the +// render-phase render.RenderContext. It is the embedded RenderContext (the same +// data the render phase sees) plus the controller-side machinery a +// ControllerExtension needs to produce artifacts: a client, a certificate +// manager, a context. Those deps live here, not on RenderContext, so the +// modifiers that read RenderContext can't do I/O - they only transform objects. +// +// Controller names which controller is reconciling, selecting that controller's +// extension hook. The controller fills the embedded RenderContext's data fields, +// the deps, and Controller; ExtendContext does its work, sets the produced +// artifacts on the embedded context, and returns it. +type ControllerContext struct { + render.RenderContext + + // Controller identifies the reconciling controller, selecting its hook. + Controller ControllerName + + Ctx context.Context + Client client.Client + CertificateManager certificatemanager.CertificateManager + + // Options carries the active variant's computed controller-phase options. The + // extension Set fills it before dispatching to a hook (Validate/ExtendContext). + // It's opaque so core never names a variant-only option; the variant's hooks + // assert it back out. Nil for the core operator. + Options any +} diff --git a/pkg/controller/gatewayapi/gatewayapi_controller.go b/pkg/controller/gatewayapi/gatewayapi_controller.go index 432ba36b53..5745eafde7 100644 --- a/pkg/controller/gatewayapi/gatewayapi_controller.go +++ b/pkg/controller/gatewayapi/gatewayapi_controller.go @@ -174,7 +174,7 @@ type ReconcileGatewayAPI struct { status status.StatusManager clusterDomain string multiTenant bool - newComponentHandler func(log logr.Logger, client client.Client, scheme *runtime.Scheme, cr metav1.Object) utils.ComponentHandler + newComponentHandler func(log logr.Logger, client client.Client, scheme *runtime.Scheme, cr metav1.Object, opts ...utils.ComponentHandlerOption) utils.ComponentHandler watchEnvoyProxy func(namespacedName operatorv1.NamespacedName) error watchEnvoyGateway func(namespacedName operatorv1.NamespacedName) error watchGateways func() error diff --git a/pkg/controller/gatewayapi/gatewayapi_controller_test.go b/pkg/controller/gatewayapi/gatewayapi_controller_test.go index 391160c534..bd02010623 100644 --- a/pkg/controller/gatewayapi/gatewayapi_controller_test.go +++ b/pkg/controller/gatewayapi/gatewayapi_controller_test.go @@ -779,7 +779,7 @@ var _ = Describe("Gateway API controller tests", func() { var fakeComponentHandlers []*fakeComponentHandler -func FakeComponentHandler(log logr.Logger, client client.Client, scheme *runtime.Scheme, cr metav1.Object) utils.ComponentHandler { +func FakeComponentHandler(log logr.Logger, client client.Client, scheme *runtime.Scheme, cr metav1.Object, opts ...utils.ComponentHandlerOption) utils.ComponentHandler { h := &fakeComponentHandler{ client: client, scheme: scheme, diff --git a/pkg/controller/installation/core_controller.go b/pkg/controller/installation/core_controller.go index c4dcdd6aaa..442ed86622 100644 --- a/pkg/controller/installation/core_controller.go +++ b/pkg/controller/installation/core_controller.go @@ -43,7 +43,6 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" - "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "k8s.io/client-go/tools/cache" @@ -64,7 +63,7 @@ import ( "github.com/tigera/operator/pkg/common/discovery" "github.com/tigera/operator/pkg/components" "github.com/tigera/operator/pkg/controller/certificatemanager" - "github.com/tigera/operator/pkg/controller/gatewayapi" + "github.com/tigera/operator/pkg/controller/contexts" "github.com/tigera/operator/pkg/controller/ippool" "github.com/tigera/operator/pkg/controller/k8sapi" "github.com/tigera/operator/pkg/controller/migration" @@ -75,29 +74,19 @@ import ( "github.com/tigera/operator/pkg/controller/utils" "github.com/tigera/operator/pkg/controller/utils/imageset" "github.com/tigera/operator/pkg/ctrlruntime" - "github.com/tigera/operator/pkg/dns" "github.com/tigera/operator/pkg/imports/admission" "github.com/tigera/operator/pkg/imports/crds" "github.com/tigera/operator/pkg/render" - "github.com/tigera/operator/pkg/render/applicationlayer" rcertificatemanagement "github.com/tigera/operator/pkg/render/certificatemanagement" - relasticsearch "github.com/tigera/operator/pkg/render/common/elasticsearch" "github.com/tigera/operator/pkg/render/common/networkpolicy" "github.com/tigera/operator/pkg/render/common/resourcequota" "github.com/tigera/operator/pkg/render/goldmane" "github.com/tigera/operator/pkg/render/kubecontrollers" - "github.com/tigera/operator/pkg/render/monitor" "github.com/tigera/operator/pkg/tls/certificatemanagement" ) const ( techPreviewFeatureSeccompApparmor = "tech-preview.operator.tigera.io/node-apparmor-profile" - - // The default port used by calico/node to report Calico Enterprise internal metrics. - // This is separate from the calico/node prometheus metrics port, which is user configurable. - defaultNodeReporterPort = 9081 - - defaultFelixMetricsDefaultPort = 9091 ) const InstallationName string = "calico" @@ -215,13 +204,6 @@ func Add(mgr manager.Manager, opts options.ControllerOptions) error { } // Watch for changes to KubeControllersConfiguration. - // Watch GatewayAPI: spec.extensions.waf.state gates the WAF v3 surface on - // calico-kube-controllers. See design tigera/designs#25 (PMREQ-384) §Gating. - if err := c.WatchObject(&operatorv1.GatewayAPI{}, &handler.EnqueueRequestForObject{}); err != nil { - log.V(5).Info("Failed to create GatewayAPI watch", "err", err) - return fmt.Errorf("core-controller failed to watch operator GatewayAPI resource: %w", err) - } - err = c.WatchObject(&v3.KubeControllersConfiguration{}, &handler.EnqueueRequestForObject{}) if err != nil { return fmt.Errorf("tigera-installation-controller failed to watch KubeControllersConfiguration resource: %w", err) @@ -240,27 +222,8 @@ func Add(mgr manager.Manager, opts options.ControllerOptions) error { } if opts.EnterpriseCRDExists { - // Watch for changes to primary resource ManagementCluster - err = c.WatchObject(&operatorv1.ManagementCluster{}, &handler.EnqueueRequestForObject{}) - if err != nil { - return fmt.Errorf("tigera-installation-controller failed to watch primary resource: %v", err) - } - - // Watch for changes to primary resource ManagementClusterConnection - err = c.WatchObject(&operatorv1.ManagementClusterConnection{}, &handler.EnqueueRequestForObject{}) - if err != nil { - return fmt.Errorf("tigera-installation-controller failed to watch primary resource: %v", err) - } - - // watch for change to primary resource LogCollector - err = c.WatchObject(&operatorv1.LogCollector{}, &handler.EnqueueRequestForObject{}) - if err != nil { - return fmt.Errorf("tigera-installation-controller failed to watch primary resource: %v", err) - } - - // Watch the internal manager TLS secret in the operator namespace, which included in the bundle for es-kube-controllers. - if err = utils.AddSecretsWatch(c, render.ManagerInternalTLSSecretName, common.OperatorNamespace()); err != nil { - return fmt.Errorf("tigera-installation-controller failed to watch secret: %v", err) + if err = opts.Extensions.SetupWatches(contexts.InstallationController, c); err != nil { + return fmt.Errorf("tigera-installation-controller failed to set up extension watches: %w", err) } if opts.ManageCRDs { @@ -336,25 +299,17 @@ func newReconciler(mgr manager.Manager, opts options.ControllerOptions) (*Reconc typhaScaler := newTyphaAutoscaler(opts.K8sClientset, nodeIndexInformer, typhaListWatch, statusManager) r := &ReconcileInstallation{ - config: mgr.GetConfig(), - client: mgr.GetClient(), - clientset: opts.K8sClientset, - scheme: mgr.GetScheme(), - shutdownContext: opts.ShutdownContext, - watches: make(map[runtime.Object]struct{}), - autoDetectedProvider: opts.DetectedProvider, - status: statusManager, - typhaAutoscaler: typhaScaler, - namespaceMigration: nm, - enterpriseCRDsExist: opts.EnterpriseCRDExists, - clusterDomain: opts.ClusterDomain, - manageCRDs: opts.ManageCRDs, - tierWatchReady: &utils.ReadyFlag{}, - migrationWatchReady: &utils.ReadyFlag{}, - newComponentHandler: utils.NewComponentHandler, - v3CRDs: opts.UseV3CRDs, - kubernetesVersion: opts.KubernetesVersion, - apiDiscovery: opts.APIDiscovery, + config: mgr.GetConfig(), + client: mgr.GetClient(), + scheme: mgr.GetScheme(), + watches: make(map[runtime.Object]struct{}), + status: statusManager, + typhaAutoscaler: typhaScaler, + namespaceMigration: nm, + tierWatchReady: &utils.ReadyFlag{}, + migrationWatchReady: &utils.ReadyFlag{}, + newComponentHandler: utils.NewComponentHandler, + opts: opts, } r.status.Run(opts.ShutdownContext) r.typhaAutoscaler.start(opts.ShutdownContext) @@ -396,27 +351,19 @@ type ReconcileInstallation struct { // that reads objects from the cache and writes to the apiserver config *rest.Config client client.Client - clientset *kubernetes.Clientset scheme *runtime.Scheme - shutdownContext context.Context watches map[runtime.Object]struct{} - autoDetectedProvider operatorv1.Provider status status.StatusManager typhaAutoscaler *typhaAutoscaler typhaAutoscalerNonClusterHost *typhaAutoscaler namespaceMigration migration.NamespaceMigration - enterpriseCRDsExist bool migrationChecked bool - clusterDomain string - manageCRDs bool tierWatchReady *utils.ReadyFlag migrationWatchReady *utils.ReadyFlag - v3CRDs bool - kubernetesVersion *common.VersionInfo - apiDiscovery *discovery.APIDiscovery + opts options.ControllerOptions // newComponentHandler returns a new component handler. Useful stub for unit testing. - newComponentHandler func(log logr.Logger, client client.Client, scheme *runtime.Scheme, cr metav1.Object) utils.ComponentHandler + newComponentHandler func(log logr.Logger, client client.Client, scheme *runtime.Scheme, cr metav1.Object, opts ...utils.ComponentHandlerOption) utils.ComponentHandler } // GetActivePools returns the full set of enabled IP pools in the cluster. @@ -857,7 +804,7 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile } // update Installation with defaults - if err := updateInstallationWithDefaults(ctx, r.client, instance, r.autoDetectedProvider); err != nil { + if err := updateInstallationWithDefaults(ctx, r.client, instance, r.opts.DetectedProvider); err != nil { r.status.SetDegraded(operatorv1.ResourceReadError, "Error querying installation", err, reqLogger) return reconcile.Result{}, err } @@ -1021,10 +968,10 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile // The operator supports running in a "Calico only" mode so that it doesn't need to run enterprise-specific controllers. // If we are switching from this mode to one that enables enterprise, we need to restart the operator to enable the other controllers. - if !r.enterpriseCRDsExist && instance.Spec.Variant.IsEnterprise() { + if !r.opts.EnterpriseCRDExists && instance.Spec.Variant.IsEnterprise() { // Perform an API discovery to determine if the necessary APIs exist. If they do, we can reboot into enterprise mode. // if they do not, we need to notify the user that the requested configuration is invalid. - b, err := discovery.RequiresTigeraSecure(r.clientset) + b, err := discovery.RequiresTigeraSecure(r.opts.K8sClientset) if b { log.Info("Rebooting to enable TigeraSecure controllers") os.Exit(0) @@ -1049,16 +996,7 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile var managementCluster *operatorv1.ManagementCluster var managementClusterConnection *operatorv1.ManagementClusterConnection - var logCollector *operatorv1.LogCollector - if r.enterpriseCRDsExist { - logCollector, err = utils.GetLogCollector(ctx, r.client) - if logCollector != nil { - if err != nil { - r.status.SetDegraded(operatorv1.ResourceReadError, "Error reading LogCollector", err, reqLogger) - return reconcile.Result{}, err - } - } - + if r.opts.EnterpriseCRDExists { managementCluster, err = utils.GetManagementCluster(ctx, r.client) if err != nil { r.status.SetDegraded(operatorv1.ResourceReadError, "Error reading ManagementCluster", err, reqLogger) @@ -1096,7 +1034,7 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile } } - certificateManager, err := certificatemanager.Create(r.client, &instance.Spec, r.clusterDomain, common.OperatorNamespace(), certificatemanager.WithLogger(reqLogger)) + certificateManager, err := certificatemanager.Create(r.client, &instance.Spec, r.opts.ClusterDomain, common.OperatorNamespace(), certificatemanager.WithLogger(reqLogger)) if err != nil { r.status.SetDegraded(operatorv1.ResourceCreateError, "Unable to create the Tigera CA", err, reqLogger) return reconcile.Result{}, err @@ -1109,18 +1047,6 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile return reconcile.Result{}, err } - if instance.Spec.Variant.IsEnterprise() { - managerInternalTLSSecret, err := certificateManager.GetCertificate(r.client, render.ManagerInternalTLSSecretName, common.OperatorNamespace()) - if err != nil { - r.status.SetDegraded(operatorv1.ResourceReadError, fmt.Sprintf("Error fetching TLS secret %s in namespace %s", render.ManagerInternalTLSSecretName, common.OperatorNamespace()), err, reqLogger) - return reconcile.Result{}, nil - } else if managerInternalTLSSecret != nil { - // It may seem odd to add the manager internal TLS secret to the trusted bundle for Typha / calico-node, but this bundle is also used - // for other components in this namespace such as es-kube-controllers, who communicates with Voltron and thus needs to trust this certificate. - typhaNodeTLS.TrustedBundle.AddCertificates(managerInternalTLSSecret) - } - } - birdTemplates, err := getBirdTemplates(r.client) if err != nil { r.status.SetDegraded(operatorv1.ResourceReadError, "Error retrieving confd templates", err, reqLogger) @@ -1221,93 +1147,37 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile return reconcile.Result{}, err } - // nodeReporterMetricsPort is a port used in Enterprise to host internal metrics. - // Operator is responsible for creating a service which maps to that port. - // Here, we'll check the default felixconfiguration to see if the user is specifying - // a non-default port, and use that value if they are. - nodeReporterMetricsPort := defaultNodeReporterPort - var nodePrometheusTLS certificatemanagement.KeyPairInterface calicoVersion := components.CalicoRelease - - felixPrometheusMetricsPort := defaultFelixMetricsDefaultPort - if instance.Spec.Variant.IsEnterprise() { - - // Determine the port to use for nodeReporter metrics. - if felixConfiguration.Spec.PrometheusReporterPort != nil { - nodeReporterMetricsPort = *felixConfiguration.Spec.PrometheusReporterPort - } - if nodeReporterMetricsPort == 0 { - err := errors.New("felixConfiguration prometheusReporterPort=0 not supported") - r.status.SetDegraded(operatorv1.InvalidConfigurationError, "invalid metrics port", err, reqLogger) - return reconcile.Result{}, err - } - - if felixConfiguration.Spec.PrometheusMetricsPort != nil { - felixPrometheusMetricsPort = *felixConfiguration.Spec.PrometheusMetricsPort - } - - nodePrometheusTLS, err = certificateManager.GetOrCreateKeyPair(r.client, render.NodePrometheusTLSServerSecret, common.OperatorNamespace(), dns.GetServiceDNSNames(render.CalicoNodeMetricsService, common.CalicoNamespace, r.clusterDomain)) - if err != nil { - r.status.SetDegraded(operatorv1.ResourceCreateError, "Error creating TLS certificate", err, reqLogger) - return reconcile.Result{}, err - } - if nodePrometheusTLS != nil { - typhaNodeTLS.TrustedBundle.AddCertificates(nodePrometheusTLS) - } - prometheusClientCert, err := certificateManager.GetCertificate(r.client, monitor.PrometheusClientTLSSecretName, common.OperatorNamespace()) - if err != nil { - r.status.SetDegraded(operatorv1.CertificateError, "Unable to fetch prometheus certificate", err, reqLogger) - return reconcile.Result{}, err - } - if prometheusClientCert != nil { - typhaNodeTLS.TrustedBundle.AddCertificates(prometheusClientCert) - } - - // es-kube-controllers needs to trust the ESGW certificate. We'll fetch it here and add it to the trusted bundle. - // Note that although we're adding this to the typhaNodeTLS trusted bundle, it will be used by es-kube-controllers. This is because - // all components within this namespace share a trusted CA bundle. This is necessary because prior to v3.13 secrets were not signed by - // a single CA so we need to include each individually. - esgwCertificate, err := certificateManager.GetCertificate(r.client, relasticsearch.PublicCertSecret, common.OperatorNamespace()) - if err != nil { - r.status.SetDegraded(operatorv1.CertificateError, fmt.Sprintf("Failed to retrieve / validate %s", relasticsearch.PublicCertSecret), err, reqLogger) - return reconcile.Result{}, err - } - if esgwCertificate != nil { - typhaNodeTLS.TrustedBundle.AddCertificates(esgwCertificate) - } - calicoVersion = components.EnterpriseRelease } - kubeControllersMetricsPort, err := utils.GetKubeControllerMetricsPort(ctx, r.client) + cc := contexts.ControllerContext{ + RenderContext: render.RenderContext{ + Installation: &instance.Spec, + FelixConfiguration: felixConfiguration, + ClusterDomain: r.opts.ClusterDomain, + TrustedBundle: typhaNodeTLS.TrustedBundle, + }, + Controller: contexts.InstallationController, + Ctx: ctx, + Client: r.client, + CertificateManager: certificateManager, + } + if err := r.opts.Extensions.Validate(cc); err != nil { + r.status.SetDegraded(operatorv1.ResourceValidationError, "Invalid installation configuration", err, reqLogger) + return reconcile.Result{}, err + } + cc, managedKeyPairs, err := r.opts.Extensions.ExtendContext(cc) if err != nil { - r.status.SetDegraded(operatorv1.ResourceReadError, "Unable to read KubeControllersConfiguration", err, reqLogger) + r.status.SetDegraded(operatorv1.ResourceCreateError, "Error preparing installation extension", err, reqLogger) return reconcile.Result{}, err } - // Secure calico kube controller metrics. - var kubeControllerTLS certificatemanagement.KeyPairInterface - if instance.Spec.Variant.IsEnterprise() { - // Create or Get TLS certificates for kube controller. - kubeControllerTLS, err = certificateManager.GetOrCreateKeyPair( - r.client, - kubecontrollers.KubeControllerPrometheusTLSSecret, - common.OperatorNamespace(), - dns.GetServiceDNSNames(kubecontrollers.KubeControllerMetrics, common.CalicoNamespace, r.clusterDomain)) - if err != nil { - r.status.SetDegraded(operatorv1.ResourceReadError, "Error finding or creating TLS certificate kube controllers metric", err, reqLogger) - return reconcile.Result{}, err - } - - // Add prometheus client certificate to Trusted bundle. - kubeControllerPrometheusTLS, err := certificateManager.GetCertificate(r.client, monitor.PrometheusClientTLSSecretName, common.OperatorNamespace()) - if err != nil { - r.status.SetDegraded(operatorv1.ResourceReadError, "Failed to get certificate for kube controllers", err, reqLogger) - return reconcile.Result{}, err - } else if kubeControllerPrometheusTLS != nil { - typhaNodeTLS.TrustedBundle.AddCertificates(kubeControllerTLS, kubeControllerPrometheusTLS) - } + kubeControllersMetricsPort, err := utils.GetKubeControllerMetricsPort(ctx, r.client) + if err != nil { + r.status.SetDegraded(operatorv1.ResourceReadError, "Unable to read KubeControllersConfiguration", err, reqLogger) + return reconcile.Result{}, err } nodeAppArmorProfile := "" @@ -1317,7 +1187,14 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile } // Create a component handler to create or update the rendered components. - handler := r.newComponentHandler(log, r.client, r.scheme, instance) + handler := r.newComponentHandler( + log, + r.client, + r.scheme, + instance, + utils.WithRenderContext(cc.RenderContext), + utils.WithExtensions(r.opts.Extensions), + ) // Render namespaces first - this ensures that any other controllers blocked on namespace existence can proceed. namespaceCfg := &render.NamespaceConfiguration{ @@ -1374,49 +1251,14 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile } - // Read the GatewayAPI CR (if present) to decide whether to render the WAF - // v3 (Gateway API add-on) surface — env vars, RBAC, applicationlayer - // reconciler, and the in-process admission webhook — on - // calico-kube-controllers. Default-off: if no GatewayAPI CR exists or - // spec.extensions.waf.state != Enabled, the WAF surface is not rendered. - // See design tigera/designs#25 (PMREQ-384) §Gating. - wafGatewayExtensionEnabled := false - if gatewayAPI, msg, err := gatewayapi.GetGatewayAPI(ctx, r.client); err == nil { - wafGatewayExtensionEnabled = gatewayAPI.Spec.IsWAFGatewayExtensionEnabled() - } else if !apierrors.IsNotFound(err) { - // Mirrors the GatewayAPI controller's handling: a read error or a - // duplicate default/tigera-secure pair degrades rather than guessing. - r.status.SetDegraded(operatorv1.ResourceReadError, msg, err, reqLogger) - return reconcile.Result{}, err - } - - // When the WAF v3 surface is enabled, issue the serving cert for the - // in-process WAF admission webhook (hosted by calico-kube-controllers, - // fronted by the tigera-waf-webhook Service). It is materialized into - // calico-system alongside the other kube-controllers certs below and mounted - // into the Pod by the kube-controllers render. - var wafWebhookTLS certificatemanagement.KeyPairInterface - if wafGatewayExtensionEnabled { - wafWebhookTLS, err = certificateManager.GetOrCreateKeyPair( - r.client, - applicationlayer.WAFWebhookServerTLSSecretName, - common.OperatorNamespace(), - dns.GetServiceDNSNames(applicationlayer.WAFWebhookServiceName, common.CalicoNamespace, r.clusterDomain)) - if err != nil { - r.status.SetDegraded(operatorv1.ResourceCreateError, "Error creating WAF admission webhook TLS certificate", err, reqLogger) - return reconcile.Result{}, err - } - } - keyPairOptions := []rcertificatemanagement.KeyPairOption{ rcertificatemanagement.NewKeyPairOption(typhaNodeTLS.NodeSecret, true, true), - rcertificatemanagement.NewKeyPairOption(nodePrometheusTLS, true, true), rcertificatemanagement.NewKeyPairOption(typhaNodeTLS.TyphaSecret, true, true), rcertificatemanagement.NewKeyPairOption(typhaNodeTLS.TyphaSecretNonClusterHost, true, true), - rcertificatemanagement.NewKeyPairOption(kubeControllerTLS, true, true), - // Nil when the WAF v3 surface is disabled; the certificate-management - // render skips nil key pairs. - rcertificatemanagement.NewKeyPairOption(wafWebhookTLS, true, true), + } + // Manage any key pairs the variant extension created controller-side. + for _, kp := range managedKeyPairs { + keyPairOptions = append(keyPairOptions, rcertificatemanagement.NewKeyPairOption(kp, true, true)) } components = append(components, @@ -1460,11 +1302,11 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile hepListWatch := cache.NewListWatchFromClient(calicoClient.ProjectcalicoV3().RESTClient(), "hostendpoints", corev1.NamespaceAll, fields.Everything()) hepIndexInformer := cache.NewSharedIndexInformer(hepListWatch, &v3.HostEndpoint{}, 0, cache.Indexers{}) - go hepIndexInformer.Run(r.shutdownContext.Done()) + go hepIndexInformer.Run(r.opts.ShutdownContext.Done()) - typhaNonClusterHostWatch := cache.NewListWatchFromClient(r.clientset.AppsV1().RESTClient(), "deployments", "calico-system", fields.OneTermEqualSelector("metadata.name", "calico-typha"+render.TyphaNonClusterHostSuffix)) - r.typhaAutoscalerNonClusterHost = newTyphaAutoscaler(r.clientset, hepIndexInformer, typhaNonClusterHostWatch, r.status, typhaAutoscalerOptionNonclusterHost(true)) - r.typhaAutoscalerNonClusterHost.start(r.shutdownContext) + typhaNonClusterHostWatch := cache.NewListWatchFromClient(r.opts.K8sClientset.AppsV1().RESTClient(), "deployments", "calico-system", fields.OneTermEqualSelector("metadata.name", "calico-typha"+render.TyphaNonClusterHostSuffix)) + r.typhaAutoscalerNonClusterHost = newTyphaAutoscaler(r.opts.K8sClientset, hepIndexInformer, typhaNonClusterHostWatch, r.status, typhaAutoscalerOptionNonclusterHost(true)) + r.typhaAutoscalerNonClusterHost.start(r.opts.ShutdownContext) } } } @@ -1476,7 +1318,7 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile Installation: &instance.Spec, TLS: typhaNodeTLS, MigrateNamespaces: needsNamespaceMigration, - ClusterDomain: r.clusterDomain, + ClusterDomain: r.opts.ClusterDomain, NonClusterHost: nonclusterhost, FelixHealthPort: *felixConfiguration.Spec.HealthPort, } @@ -1592,28 +1434,24 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile // Build a configuration for rendering calico/node. nodeCfg := render.NodeConfiguration{ - GoldmaneRunning: goldmaneRunning, - K8sServiceEp: k8sapi.Endpoint, - Installation: &instance.Spec, - IPPools: crdPoolsToOperator(currentPools.Items), - LogCollector: logCollector, - BirdTemplates: birdTemplates, - TLS: typhaNodeTLS, - ClusterDomain: r.clusterDomain, - DefaultDNSPolicy: defaultDNSPolicy, - DefaultDNSConfig: defaultDNSConfig, - GoldmaneIP: goldmaneIP, - NodeReporterMetricsPort: nodeReporterMetricsPort, - BGPLayouts: bgpLayout, - NodeAppArmorProfile: nodeAppArmorProfile, - MigrateNamespaces: needsNamespaceMigration, - CanRemoveCNIFinalizer: canRemoveCNI, - PrometheusServerTLS: nodePrometheusTLS, - FelixHealthPort: *felixConfiguration.Spec.HealthPort, - NodeCgroupV2Path: felixConfiguration.Spec.CgroupV2Path, - FelixPrometheusMetricsEnabled: utils.IsFelixPrometheusMetricsEnabled(felixConfiguration), - FelixPrometheusMetricsPort: felixPrometheusMetricsPort, - V3CRDs: r.v3CRDs, + GoldmaneRunning: goldmaneRunning, + K8sServiceEp: k8sapi.Endpoint, + Installation: &instance.Spec, + IPPools: crdPoolsToOperator(currentPools.Items), + BirdTemplates: birdTemplates, + TLS: typhaNodeTLS, + ClusterDomain: r.opts.ClusterDomain, + DefaultDNSPolicy: defaultDNSPolicy, + DefaultDNSConfig: defaultDNSConfig, + GoldmaneIP: goldmaneIP, + BGPLayouts: bgpLayout, + NodeAppArmorProfile: nodeAppArmorProfile, + MigrateNamespaces: needsNamespaceMigration, + CanRemoveCNIFinalizer: canRemoveCNI, + FelixHealthPort: *felixConfiguration.Spec.HealthPort, + NodeCgroupV2Path: felixConfiguration.Spec.CgroupV2Path, + V3CRDs: r.opts.UseV3CRDs, + ImageOverrides: r.opts.Extensions.Images(), } if bgpConfiguration.Spec.BindMode != nil { @@ -1657,55 +1495,21 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile } components = append(components, render.CSI(&csiCfg)) - // Build a configuration for rendering calico/kube-controllers. - // Provision a dedicated WAF wasm pull secret so the WAF reconciler - // replicates it into tenant namespaces without clashing with the - // operator-managed tigera-pull-secret the GatewayAPI render also copies - // there (EV-6386). The EnvoyExtensionPolicy image source takes a single - // pullSecretRef, so the registry auths of all Installation pull secrets - // are merged into it rather than picking one. - var wasmPullSecret *corev1.Secret - if wafGatewayExtensionEnabled && len(pullSecrets) > 0 { - var skipped []string - wasmPullSecret, skipped = kubecontrollers.MergeWAFPullSecret(pullSecrets) - if len(skipped) > 0 { - reqLogger.Info("Skipped unparseable imagePullSecrets when building the WAF wasm pull secret", "skipped", skipped) - } - } - // Provision the dedicated WAF wasm CA-bundle ConfigMap as a renamed copy of - // the trusted CA bundle, so the WAF reconciler replicates it into tenant - // namespaces for the Coraza wasm OCI registry TLS check without clashing with - // the operator-managed tigera-ca-bundle the GatewayAPI render also copies - // there (EV-6386). The dedicated source was previously a TODO; the full - // TrustedBundle (not the RO interface the kube-controllers render sees) is - // available here, so build it in the core controller. - var wasmCACert *corev1.ConfigMap - if wafGatewayExtensionEnabled { - wasmCACert = typhaNodeTLS.TrustedBundle.ConfigMap(common.CalicoNamespace) - wasmCACert.Name = kubecontrollers.WASMCACertName - } + // Build a configuration for rendering calico/kube-controllers. The Calico + // Enterprise surface (extra RBAC, enterprise controllers, metrics TLS, and the + // WAF v3 / Gateway API add-on) is layered on by the enterprise extension. kubeControllersCfg := kubecontrollers.KubeControllersConfiguration{ K8sServiceEp: k8sapi.Endpoint, K8sServiceEpPodNetwork: k8sapi.PodNetworkEndpoint, Installation: &instance.Spec, ManagementCluster: managementCluster, ManagementClusterConnection: managementClusterConnection, - ClusterDomain: r.clusterDomain, + ClusterDomain: r.opts.ClusterDomain, MetricsPort: kubeControllersMetricsPort, Terminating: installationMarkedForDeletion, - MetricsServerTLS: kubeControllerTLS, TrustedBundle: typhaNodeTLS.TrustedBundle, Namespace: common.CalicoNamespace, BindingNamespaces: []string{common.CalicoNamespace}, - WAFGatewayExtensionEnabled: wafGatewayExtensionEnabled, - WAFWebhookServerTLS: wafWebhookTLS, - WASMPullSecret: wasmPullSecret, - WASMCACert: wasmCACert, - // The webhook Service + ValidatingWebhookConfiguration are rendered by - // the kube-controllers component (and deleted when the WAF extension is - // disabled); the caBundle is the operator CA that issued the serving - // cert above. - WAFWebhookCABundle: certificateManager.KeyPair().GetCertificatePEM(), } components = append(components, kubecontrollers.NewCalicoKubeControllers(&kubeControllersCfg)) @@ -1841,13 +1645,15 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile r.status.ReadyToMonitor() // Check BYO certificate expiry warnings and propagate them to the status manager. - certificatemanagement.CheckKeyPairWarnings(map[string]certificatemanagement.KeyPairInterface{ + keyPairWarnings := map[string]certificatemanagement.KeyPairInterface{ render.TyphaTLSSecretName: typhaNodeTLS.TyphaSecret, render.NodeTLSSecretName: typhaNodeTLS.NodeSecret, render.TyphaTLSSecretName + render.TyphaNonClusterHostSuffix: typhaNodeTLS.TyphaSecretNonClusterHost, - render.NodePrometheusTLSServerSecret: nodePrometheusTLS, - kubecontrollers.KubeControllerPrometheusTLSSecret: kubeControllerTLS, - }, r.status) + } + for _, kp := range managedKeyPairs { + keyPairWarnings[kp.GetName()] = kp + } + certificatemanagement.CheckKeyPairWarnings(keyPairWarnings, r.status) // We can clear the degraded state now since as far as we know everything is in order. r.status.ClearDegraded() @@ -2095,36 +1901,13 @@ func (r *ReconcileInstallation) setDefaultsOnFelixConfiguration(ctx context.Cont updated = true } - if install.Spec.Variant.IsEnterprise() { - // Some platforms need a different default setting for dnsTrustedServers, because their DNS service is not named "kube-dns". - dnsService := "" - switch install.Spec.KubernetesProvider { - case operatorv1.ProviderOpenShift: - dnsService = "k8s-service:openshift-dns/dns-default" - case operatorv1.ProviderRKE2: - dnsService = "k8s-service:kube-system/rke2-coredns-rke2-coredns" - } - if dnsService != "" { - felixDefault := "k8s-service:kube-dns" - trustedServers := []string{dnsService} - // Keep any other values that are already configured, excepting the value - // that we are setting and the kube-dns default. - existingSetting := "" - if fc.Spec.DNSTrustedServers != nil { - existingSetting = strings.Join(*(fc.Spec.DNSTrustedServers), ",") - for _, server := range *(fc.Spec.DNSTrustedServers) { - if server != felixDefault && server != dnsService { - trustedServers = append(trustedServers, server) - } - } - } - newSetting := strings.Join(trustedServers, ",") - if newSetting != existingSetting { - fc.Spec.DNSTrustedServers = &trustedServers - updated = true - } - } + // Variant-specific FelixConfiguration defaults (e.g. the Enterprise + // provider-specific dnsTrustedServers) are owned by the variant extension. + extUpdated, err := r.opts.Extensions.DefaultFelixConfiguration(contexts.InstallationController, &install.Spec, fc) + if err != nil { + return updated, err } + updated = updated || extUpdated // If BPF is enabled, but not set on FelixConfiguration, do so here. This could happen when an older // version of operator is replaced by the new one. Older versions of the operator used an @@ -2135,7 +1918,7 @@ func (r *ReconcileInstallation) setDefaultsOnFelixConfiguration(ctx context.Cont // If calico-node daemonset exists, we need to check the ENV VAR and set FelixConfiguration accordingly. // Otherwise, this is a fresh install in eBPF mode, set the felix config. ds := &appsv1.DaemonSet{} - err := r.client.Get(ctx, types.NamespacedName{Namespace: common.CalicoNamespace, Name: common.NodeDaemonSetName}, ds) + err = r.client.Get(ctx, types.NamespacedName{Namespace: common.CalicoNamespace, Name: common.NodeDaemonSetName}, ds) if err != nil { if !apierrors.IsNotFound(err) { reqLogger.Error(err, "An error occurred when getting the Daemonset resource") @@ -2348,10 +2131,10 @@ func (r *ReconcileInstallation) checkActive(log logr.Logger) (*corev1.ConfigMap, } func (r *ReconcileInstallation) updateCRDs(ctx context.Context, variant operatorv1.ProductVariant, log logr.Logger) error { - if !r.manageCRDs { + if !r.opts.ManageCRDs { return nil } - crdComponent := render.NewCreationPassthrough(crds.ToRuntimeObjects(crds.GetCRDs(variant, r.v3CRDs)...)...) + crdComponent := render.NewCreationPassthrough(crds.ToRuntimeObjects(crds.GetCRDs(variant, r.opts.UseV3CRDs)...)...) // Specify nil for the CR so no ownership is put on the CRDs. We do this so removing the // Installation CR will not remove the CRDs. handler := r.newComponentHandler(log, r.client, r.scheme, nil) @@ -2363,19 +2146,19 @@ func (r *ReconcileInstallation) updateCRDs(ctx context.Context, variant operator } func (r *ReconcileInstallation) updateMutatingAdmissionPolicies(ctx context.Context, install *operatorv1.Installation, log logr.Logger) error { - if !r.manageCRDs || !r.v3CRDs { + if !r.opts.ManageCRDs || !r.opts.UseV3CRDs { return nil } // MutatingAdmissionPolicy served version was discovered once at startup (v1 was promoted to GA // in k8s 1.36 and v1beta1 (introduced in 1.32) is scheduled for removal in 1.37). - mapAPIVersion := r.apiDiscovery.ServedVersion(admission.APIGroup, admission.KindPolicy) + mapAPIVersion := r.opts.APIDiscovery.ServedVersion(admission.APIGroup, admission.KindPolicy) if mapAPIVersion == "" { r.status.SetDegraded(operatorv1.ResourceNotReady, "Kubernetes cluster does not serve MutatingAdmissionPolicy (requires v1.32+); policy defaulting will not be available", nil, log) return nil } - desired := admission.GetMutatingAdmissionPolicies(install.Spec.Variant, r.v3CRDs, mapAPIVersion) + desired := admission.GetMutatingAdmissionPolicies(install.Spec.Variant, r.opts.UseV3CRDs, mapAPIVersion) existingMAPs, existingMAPBs, err := admission.ListManaged(ctx, r.client, mapAPIVersion) if err != nil { r.status.SetDegraded(operatorv1.ResourceReadError, "Error listing managed MutatingAdmissionPolicy resources", err, log) @@ -2386,20 +2169,20 @@ func (r *ReconcileInstallation) updateMutatingAdmissionPolicies(ctx context.Cont } func (r *ReconcileInstallation) updateValidatingAdmissionPolicies(ctx context.Context, install *operatorv1.Installation, log logr.Logger) error { - if !r.manageCRDs || !r.v3CRDs { + if !r.opts.ManageCRDs || !r.opts.UseV3CRDs { return nil } // ValidatingAdmissionPolicy reached GA (v1) well before MutatingAdmissionPolicy, so it has its own // served version and is reconciled independently of whether the cluster serves MAPs. If the cluster // doesn't serve it at all there's nothing to do, so skip rather than degrade. - vapAPIVersion := r.apiDiscovery.ServedVersion(admission.APIGroup, admission.KindValidatingPolicy) + vapAPIVersion := r.opts.APIDiscovery.ServedVersion(admission.APIGroup, admission.KindValidatingPolicy) if vapAPIVersion == "" { log.Info("Kubernetes cluster does not serve ValidatingAdmissionPolicy, skipping") return nil } - desired := admission.GetValidatingAdmissionPolicies(install.Spec.Variant, r.v3CRDs, vapAPIVersion) + desired := admission.GetValidatingAdmissionPolicies(install.Spec.Variant, r.opts.UseV3CRDs, vapAPIVersion) existingVAPs, existingVAPBs, err := admission.ListManagedValidating(ctx, r.client, vapAPIVersion) if err != nil { r.status.SetDegraded(operatorv1.ResourceReadError, "Error listing managed ValidatingAdmissionPolicy resources", err, log) diff --git a/pkg/controller/installation/core_controller_test.go b/pkg/controller/installation/core_controller_test.go index bda4f6065b..3fcd559551 100644 --- a/pkg/controller/installation/core_controller_test.go +++ b/pkg/controller/installation/core_controller_test.go @@ -56,6 +56,7 @@ import ( "github.com/tigera/operator/pkg/common/discovery" "github.com/tigera/operator/pkg/components" "github.com/tigera/operator/pkg/controller/certificatemanager" + "github.com/tigera/operator/pkg/controller/options" "github.com/tigera/operator/pkg/controller/status" "github.com/tigera/operator/pkg/controller/utils" ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" @@ -190,18 +191,21 @@ var _ = Describe("Testing core-controller installation", func() { // As the parameters in the client changes, we expect the outcomes of the reconcile loops to change. r = ReconcileInstallation{ - config: nil, // there is no fake for config - client: c, - scheme: scheme, - autoDetectedProvider: operator.ProviderNone, - status: mockStatus, - typhaAutoscaler: newTyphaAutoscaler(cs, nodeIndexInformer, test.NewTyphaListWatch(cs), mockStatus), - namespaceMigration: &fakeNamespaceMigration{}, - enterpriseCRDsExist: true, - migrationChecked: true, - tierWatchReady: ready, - migrationWatchReady: &utils.ReadyFlag{}, - newComponentHandler: utils.NewComponentHandler, + opts: options.ControllerOptions{ + Extensions: testExtensions, + DetectedProvider: operator.ProviderNone, + EnterpriseCRDExists: true, + }, + config: nil, // there is no fake for config + client: c, + scheme: scheme, + status: mockStatus, + typhaAutoscaler: newTyphaAutoscaler(cs, nodeIndexInformer, test.NewTyphaListWatch(cs), mockStatus), + namespaceMigration: &fakeNamespaceMigration{}, + migrationChecked: true, + tierWatchReady: ready, + migrationWatchReady: &utils.ReadyFlag{}, + newComponentHandler: utils.NewComponentHandler, } r.typhaAutoscaler.start(ctx) @@ -819,19 +823,22 @@ var _ = Describe("Testing core-controller installation", func() { // As the parameters in the client changes, we expect the outcomes of the reconcile loops to change. r = ReconcileInstallation{ - config: nil, // there is no fake for config - client: c, - scheme: scheme, - autoDetectedProvider: operator.ProviderNone, - status: mockStatus, - typhaAutoscaler: newTyphaAutoscaler(cs, nodeIndexInformer, test.NewTyphaListWatch(cs), mockStatus), - namespaceMigration: &fakeNamespaceMigration{}, - enterpriseCRDsExist: true, - migrationChecked: true, - clusterDomain: dns.DefaultClusterDomain, - tierWatchReady: ready, - migrationWatchReady: &utils.ReadyFlag{}, - newComponentHandler: utils.NewComponentHandler, + opts: options.ControllerOptions{ + Extensions: testExtensions, + DetectedProvider: operator.ProviderNone, + EnterpriseCRDExists: true, + ClusterDomain: dns.DefaultClusterDomain, + }, + config: nil, // there is no fake for config + client: c, + scheme: scheme, + status: mockStatus, + typhaAutoscaler: newTyphaAutoscaler(cs, nodeIndexInformer, test.NewTyphaListWatch(cs), mockStatus), + namespaceMigration: &fakeNamespaceMigration{}, + migrationChecked: true, + tierWatchReady: ready, + migrationWatchReady: &utils.ReadyFlag{}, + newComponentHandler: utils.NewComponentHandler, } r.typhaAutoscaler.start(ctx) @@ -1041,18 +1048,21 @@ var _ = Describe("Testing core-controller installation", func() { // As the parameters in the client changes, we expect the outcomes of the reconcile loops to change. r = ReconcileInstallation{ - config: nil, // there is no fake for config - client: c, - scheme: scheme, - autoDetectedProvider: operator.ProviderNone, - status: mockStatus, - typhaAutoscaler: newTyphaAutoscaler(cs, nodeIndexInformer, test.NewTyphaListWatch(cs), mockStatus), - namespaceMigration: &fakeNamespaceMigration{}, - enterpriseCRDsExist: true, - migrationChecked: true, - tierWatchReady: ready, - migrationWatchReady: &utils.ReadyFlag{}, - newComponentHandler: utils.NewComponentHandler, + opts: options.ControllerOptions{ + Extensions: testExtensions, + DetectedProvider: operator.ProviderNone, + EnterpriseCRDExists: true, + }, + config: nil, // there is no fake for config + client: c, + scheme: scheme, + status: mockStatus, + typhaAutoscaler: newTyphaAutoscaler(cs, nodeIndexInformer, test.NewTyphaListWatch(cs), mockStatus), + namespaceMigration: &fakeNamespaceMigration{}, + migrationChecked: true, + tierWatchReady: ready, + migrationWatchReady: &utils.ReadyFlag{}, + newComponentHandler: utils.NewComponentHandler, } r.typhaAutoscaler.start(ctx) @@ -2217,7 +2227,7 @@ var _ = Describe("Testing core-controller installation", func() { cr.Spec.Variant = operator.Calico cr.Status.Variant = operator.Calico Expect(c.Create(ctx, cr)).NotTo(HaveOccurred()) - r.enterpriseCRDsExist = false + r.opts.EnterpriseCRDExists = false Expect(c.Delete(ctx, &v3.Tier{ObjectMeta: metav1.ObjectMeta{Name: "calico-system"}})).NotTo(HaveOccurred()) _, err := r.Reconcile(ctx, reconcile.Request{}) @@ -2329,19 +2339,22 @@ var _ = Describe("Testing core-controller installation", func() { // As the parameters in the client changes, we expect the outcomes of the reconcile loops to change. r = ReconcileInstallation{ - config: nil, // there is no fake for config - client: c, - scheme: scheme, - autoDetectedProvider: operator.ProviderNone, - status: mockStatus, - typhaAutoscaler: newTyphaAutoscaler(cs, nodeIndexInformer, test.NewTyphaListWatch(cs), mockStatus), - namespaceMigration: &fakeNamespaceMigration{}, - enterpriseCRDsExist: true, - migrationChecked: true, - clusterDomain: dns.DefaultClusterDomain, - tierWatchReady: ready, - migrationWatchReady: &utils.ReadyFlag{}, - newComponentHandler: utils.NewComponentHandler, + opts: options.ControllerOptions{ + Extensions: testExtensions, + DetectedProvider: operator.ProviderNone, + EnterpriseCRDExists: true, + ClusterDomain: dns.DefaultClusterDomain, + }, + config: nil, // there is no fake for config + client: c, + scheme: scheme, + status: mockStatus, + typhaAutoscaler: newTyphaAutoscaler(cs, nodeIndexInformer, test.NewTyphaListWatch(cs), mockStatus), + namespaceMigration: &fakeNamespaceMigration{}, + migrationChecked: true, + tierWatchReady: ready, + migrationWatchReady: &utils.ReadyFlag{}, + newComponentHandler: utils.NewComponentHandler, } r.typhaAutoscaler.start(ctx) @@ -2466,18 +2479,21 @@ var _ = Describe("Testing core-controller installation", func() { componentHandler = newFakeComponentHandler() r = ReconcileInstallation{ - config: nil, // there is no fake for config - client: c, - scheme: scheme, - autoDetectedProvider: operator.ProviderNone, - status: mockStatus, - typhaAutoscaler: newTyphaAutoscaler(cs, nodeIndexInformer, test.NewTyphaListWatch(cs), mockStatus), - namespaceMigration: &fakeNamespaceMigration{}, - enterpriseCRDsExist: true, - migrationChecked: true, - tierWatchReady: ready, - migrationWatchReady: &utils.ReadyFlag{}, - newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object) utils.ComponentHandler { + opts: options.ControllerOptions{ + Extensions: testExtensions, + DetectedProvider: operator.ProviderNone, + EnterpriseCRDExists: true, + }, + config: nil, // there is no fake for config + client: c, + scheme: scheme, + status: mockStatus, + typhaAutoscaler: newTyphaAutoscaler(cs, nodeIndexInformer, test.NewTyphaListWatch(cs), mockStatus), + namespaceMigration: &fakeNamespaceMigration{}, + migrationChecked: true, + tierWatchReady: ready, + migrationWatchReady: &utils.ReadyFlag{}, + newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object, ...utils.ComponentHandlerOption) utils.ComponentHandler { return componentHandler }, } @@ -2629,13 +2645,16 @@ var _ = Describe("updateMutatingAdmissionPolicies", func() { It("should create v1 MAPs when v1 is served", func() { r = ReconcileInstallation{ - client: clientFor(), - scheme: scheme, - status: mockStatus, - manageCRDs: true, - v3CRDs: true, - apiDiscovery: discoveryFor(admission.VersionV1), - newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object) utils.ComponentHandler { + opts: options.ControllerOptions{ + Extensions: testExtensions, + ManageCRDs: true, + UseV3CRDs: true, + APIDiscovery: discoveryFor(admission.VersionV1), + }, + client: clientFor(), + scheme: scheme, + status: mockStatus, + newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object, ...utils.ComponentHandlerOption) utils.ComponentHandler { return componentHandler }, } @@ -2660,13 +2679,16 @@ var _ = Describe("updateMutatingAdmissionPolicies", func() { It("should create v1beta1 MAPs when only v1beta1 is served", func() { r = ReconcileInstallation{ - client: clientFor(), - scheme: scheme, - status: mockStatus, - manageCRDs: true, - v3CRDs: true, - apiDiscovery: discoveryFor(admission.VersionV1Beta1), - newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object) utils.ComponentHandler { + opts: options.ControllerOptions{ + Extensions: testExtensions, + ManageCRDs: true, + UseV3CRDs: true, + APIDiscovery: discoveryFor(admission.VersionV1Beta1), + }, + client: clientFor(), + scheme: scheme, + status: mockStatus, + newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object, ...utils.ComponentHandlerOption) utils.ComponentHandler { return componentHandler }, } @@ -2689,13 +2711,16 @@ var _ = Describe("updateMutatingAdmissionPolicies", func() { It("should create v1alpha1 MAPs when only v1alpha1 is served", func() { r = ReconcileInstallation{ - client: clientFor(), - scheme: scheme, - status: mockStatus, - manageCRDs: true, - v3CRDs: true, - apiDiscovery: discoveryFor(admission.VersionV1Alpha1), - newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object) utils.ComponentHandler { + opts: options.ControllerOptions{ + Extensions: testExtensions, + ManageCRDs: true, + UseV3CRDs: true, + APIDiscovery: discoveryFor(admission.VersionV1Alpha1), + }, + client: clientFor(), + scheme: scheme, + status: mockStatus, + newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object, ...utils.ComponentHandlerOption) utils.ComponentHandler { return componentHandler }, } @@ -2718,13 +2743,16 @@ var _ = Describe("updateMutatingAdmissionPolicies", func() { It("should not create MAPs when no served version exists and should set degraded", func() { r = ReconcileInstallation{ - client: clientFor(), - scheme: scheme, - status: mockStatus, - manageCRDs: true, - v3CRDs: true, - apiDiscovery: discoveryFor(""), - newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object) utils.ComponentHandler { + opts: options.ControllerOptions{ + Extensions: testExtensions, + ManageCRDs: true, + UseV3CRDs: true, + APIDiscovery: discoveryFor(""), + }, + client: clientFor(), + scheme: scheme, + status: mockStatus, + newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object, ...utils.ComponentHandlerOption) utils.ComponentHandler { return componentHandler }, } @@ -2736,13 +2764,16 @@ var _ = Describe("updateMutatingAdmissionPolicies", func() { It("should not create MAPs when v3CRDs=false", func() { r = ReconcileInstallation{ - client: clientFor(), - scheme: scheme, - status: mockStatus, - manageCRDs: true, - v3CRDs: false, - apiDiscovery: discoveryFor(admission.VersionV1), - newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object) utils.ComponentHandler { + opts: options.ControllerOptions{ + Extensions: testExtensions, + ManageCRDs: true, + UseV3CRDs: false, + APIDiscovery: discoveryFor(admission.VersionV1), + }, + client: clientFor(), + scheme: scheme, + status: mockStatus, + newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object, ...utils.ComponentHandlerOption) utils.ComponentHandler { return componentHandler }, } @@ -2753,13 +2784,16 @@ var _ = Describe("updateMutatingAdmissionPolicies", func() { It("should not create MAPs when manageCRDs=false", func() { r = ReconcileInstallation{ - client: clientFor(), - scheme: scheme, - status: mockStatus, - manageCRDs: false, - v3CRDs: true, - apiDiscovery: discoveryFor(admission.VersionV1), - newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object) utils.ComponentHandler { + opts: options.ControllerOptions{ + Extensions: testExtensions, + ManageCRDs: false, + UseV3CRDs: true, + APIDiscovery: discoveryFor(admission.VersionV1), + }, + client: clientFor(), + scheme: scheme, + status: mockStatus, + newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object, ...utils.ComponentHandlerOption) utils.ComponentHandler { return componentHandler }, } @@ -2783,13 +2817,16 @@ var _ = Describe("updateMutatingAdmissionPolicies", func() { } r = ReconcileInstallation{ - client: clientFor(staleMAP, staleMAPB), - scheme: scheme, - status: mockStatus, - manageCRDs: true, - v3CRDs: true, - apiDiscovery: discoveryFor(admission.VersionV1), - newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object) utils.ComponentHandler { + opts: options.ControllerOptions{ + Extensions: testExtensions, + ManageCRDs: true, + UseV3CRDs: true, + APIDiscovery: discoveryFor(admission.VersionV1), + }, + client: clientFor(staleMAP, staleMAPB), + scheme: scheme, + status: mockStatus, + newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object, ...utils.ComponentHandlerOption) utils.ComponentHandler { return componentHandler }, } @@ -2825,13 +2862,16 @@ var _ = Describe("updateMutatingAdmissionPolicies", func() { } r = ReconcileInstallation{ - client: clientFor(initial...), - scheme: scheme, - status: mockStatus, - manageCRDs: true, - v3CRDs: true, - apiDiscovery: discoveryFor(admission.VersionV1), - newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object) utils.ComponentHandler { + opts: options.ControllerOptions{ + Extensions: testExtensions, + ManageCRDs: true, + UseV3CRDs: true, + APIDiscovery: discoveryFor(admission.VersionV1), + }, + client: clientFor(initial...), + scheme: scheme, + status: mockStatus, + newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object, ...utils.ComponentHandlerOption) utils.ComponentHandler { return componentHandler }, } @@ -2843,13 +2883,16 @@ var _ = Describe("updateMutatingAdmissionPolicies", func() { It("should work with Enterprise variant", func() { r = ReconcileInstallation{ - client: clientFor(), - scheme: scheme, - status: mockStatus, - manageCRDs: true, - v3CRDs: true, - apiDiscovery: discoveryFor(admission.VersionV1), - newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object) utils.ComponentHandler { + opts: options.ControllerOptions{ + Extensions: testExtensions, + ManageCRDs: true, + UseV3CRDs: true, + APIDiscovery: discoveryFor(admission.VersionV1), + }, + client: clientFor(), + scheme: scheme, + status: mockStatus, + newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object, ...utils.ComponentHandlerOption) utils.ComponentHandler { return componentHandler }, } @@ -2914,13 +2957,16 @@ var _ = Describe("updateValidatingAdmissionPolicies", func() { It("should create v1 VAPs when v1 is served", func() { r = ReconcileInstallation{ - client: clientFor(), - scheme: scheme, - status: mockStatus, - manageCRDs: true, - v3CRDs: true, - apiDiscovery: discoveryFor(admission.VersionV1), - newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object) utils.ComponentHandler { + opts: options.ControllerOptions{ + Extensions: testExtensions, + ManageCRDs: true, + UseV3CRDs: true, + APIDiscovery: discoveryFor(admission.VersionV1), + }, + client: clientFor(), + scheme: scheme, + status: mockStatus, + newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object, ...utils.ComponentHandlerOption) utils.ComponentHandler { return componentHandler }, } @@ -2945,13 +2991,16 @@ var _ = Describe("updateValidatingAdmissionPolicies", func() { It("should create v1beta1 VAPs when only v1beta1 is served", func() { r = ReconcileInstallation{ - client: clientFor(), - scheme: scheme, - status: mockStatus, - manageCRDs: true, - v3CRDs: true, - apiDiscovery: discoveryFor(admission.VersionV1Beta1), - newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object) utils.ComponentHandler { + opts: options.ControllerOptions{ + Extensions: testExtensions, + ManageCRDs: true, + UseV3CRDs: true, + APIDiscovery: discoveryFor(admission.VersionV1Beta1), + }, + client: clientFor(), + scheme: scheme, + status: mockStatus, + newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object, ...utils.ComponentHandlerOption) utils.ComponentHandler { return componentHandler }, } @@ -2974,13 +3023,16 @@ var _ = Describe("updateValidatingAdmissionPolicies", func() { It("should create v1alpha1 VAPs when only v1alpha1 is served", func() { r = ReconcileInstallation{ - client: clientFor(), - scheme: scheme, - status: mockStatus, - manageCRDs: true, - v3CRDs: true, - apiDiscovery: discoveryFor(admission.VersionV1Alpha1), - newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object) utils.ComponentHandler { + opts: options.ControllerOptions{ + Extensions: testExtensions, + ManageCRDs: true, + UseV3CRDs: true, + APIDiscovery: discoveryFor(admission.VersionV1Alpha1), + }, + client: clientFor(), + scheme: scheme, + status: mockStatus, + newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object, ...utils.ComponentHandlerOption) utils.ComponentHandler { return componentHandler }, } @@ -2991,13 +3043,16 @@ var _ = Describe("updateValidatingAdmissionPolicies", func() { It("should skip without degrading when no served version exists", func() { r = ReconcileInstallation{ - client: clientFor(), - scheme: scheme, - status: mockStatus, - manageCRDs: true, - v3CRDs: true, - apiDiscovery: discoveryFor(""), - newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object) utils.ComponentHandler { + opts: options.ControllerOptions{ + Extensions: testExtensions, + ManageCRDs: true, + UseV3CRDs: true, + APIDiscovery: discoveryFor(""), + }, + client: clientFor(), + scheme: scheme, + status: mockStatus, + newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object, ...utils.ComponentHandlerOption) utils.ComponentHandler { return componentHandler }, } @@ -3009,13 +3064,16 @@ var _ = Describe("updateValidatingAdmissionPolicies", func() { It("should not create VAPs when v3CRDs=false", func() { r = ReconcileInstallation{ - client: clientFor(), - scheme: scheme, - status: mockStatus, - manageCRDs: true, - v3CRDs: false, - apiDiscovery: discoveryFor(admission.VersionV1), - newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object) utils.ComponentHandler { + opts: options.ControllerOptions{ + Extensions: testExtensions, + ManageCRDs: true, + UseV3CRDs: false, + APIDiscovery: discoveryFor(admission.VersionV1), + }, + client: clientFor(), + scheme: scheme, + status: mockStatus, + newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object, ...utils.ComponentHandlerOption) utils.ComponentHandler { return componentHandler }, } @@ -3039,13 +3097,16 @@ var _ = Describe("updateValidatingAdmissionPolicies", func() { } r = ReconcileInstallation{ - client: clientFor(staleVAP, staleVAPB), - scheme: scheme, - status: mockStatus, - manageCRDs: true, - v3CRDs: true, - apiDiscovery: discoveryFor(admission.VersionV1), - newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object) utils.ComponentHandler { + opts: options.ControllerOptions{ + Extensions: testExtensions, + ManageCRDs: true, + UseV3CRDs: true, + APIDiscovery: discoveryFor(admission.VersionV1), + }, + client: clientFor(staleVAP, staleVAPB), + scheme: scheme, + status: mockStatus, + newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object, ...utils.ComponentHandlerOption) utils.ComponentHandler { return componentHandler }, } @@ -3063,13 +3124,16 @@ var _ = Describe("updateValidatingAdmissionPolicies", func() { It("should work with Enterprise variant", func() { r = ReconcileInstallation{ - client: clientFor(), - scheme: scheme, - status: mockStatus, - manageCRDs: true, - v3CRDs: true, - apiDiscovery: discoveryFor(admission.VersionV1), - newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object) utils.ComponentHandler { + opts: options.ControllerOptions{ + Extensions: testExtensions, + ManageCRDs: true, + UseV3CRDs: true, + APIDiscovery: discoveryFor(admission.VersionV1), + }, + client: clientFor(), + scheme: scheme, + status: mockStatus, + newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object, ...utils.ComponentHandlerOption) utils.ComponentHandler { return componentHandler }, } diff --git a/pkg/controller/installation/installation_controller_suite_test.go b/pkg/controller/installation/installation_controller_suite_test.go index 4924f3468b..f589361c5a 100644 --- a/pkg/controller/installation/installation_controller_suite_test.go +++ b/pkg/controller/installation/installation_controller_suite_test.go @@ -25,8 +25,17 @@ import ( clientfeaturestesting "k8s.io/client-go/features/testing" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" + + "github.com/tigera/operator/pkg/enterprise" + "github.com/tigera/operator/pkg/extensions" ) +// testExtensions is the enterprise extension Set the installation controller +// tests reconcile with, mirroring how main wires it in production. Reconcilers +// built in these tests put it on their options so the node image overrides and +// modifiers apply. +var testExtensions *extensions.Set = enterprise.New() + func TestInstallation(t *testing.T) { // Disable WatchListClient for tests. In client-go v0.35+, this feature defaults to true and // causes informers to wait for bookmark events that fake clients never send, leading to timeouts. diff --git a/pkg/controller/installation/windows_controller.go b/pkg/controller/installation/windows_controller.go index e23bf55db7..bdd7af26f6 100644 --- a/pkg/controller/installation/windows_controller.go +++ b/pkg/controller/installation/windows_controller.go @@ -16,7 +16,6 @@ package installation import ( "context" - "errors" "fmt" "reflect" @@ -44,16 +43,14 @@ import ( "github.com/tigera/operator/pkg/active" "github.com/tigera/operator/pkg/common" "github.com/tigera/operator/pkg/controller/certificatemanager" + "github.com/tigera/operator/pkg/controller/contexts" "github.com/tigera/operator/pkg/controller/k8sapi" "github.com/tigera/operator/pkg/controller/options" "github.com/tigera/operator/pkg/controller/status" "github.com/tigera/operator/pkg/controller/utils" "github.com/tigera/operator/pkg/controller/utils/imageset" "github.com/tigera/operator/pkg/ctrlruntime" - "github.com/tigera/operator/pkg/dns" "github.com/tigera/operator/pkg/render" - "github.com/tigera/operator/pkg/render/monitor" - "github.com/tigera/operator/pkg/tls/certificatemanagement" ) var logw = logf.Log.WithName("controller_windows") @@ -82,7 +79,7 @@ func AddWindowsController(mgr manager.Manager, opts options.ControllerOptions) e return fmt.Errorf("tigera-windows-controller failed to watch calico Tigerastatus: %w", err) } - if ri.autoDetectedProvider.IsOpenShift() { + if ri.opts.DetectedProvider.IsOpenShift() { // Watch for openshift network configuration as well. If we're running in OpenShift, we need to // merge this configuration with our own and the write back the status object. err = c.WatchObject(&configv1.Network{}, &handler.EnqueueRequestForObject{}) @@ -150,14 +147,9 @@ func AddWindowsController(mgr manager.Manager, opts options.ControllerOptions) e // Watch for changes to IPAMConfiguration. go utils.WaitToAddResourceWatch(c, opts.K8sClientset, logw, ri.ipamConfigWatchReady, []client.Object{&v3.IPAMConfiguration{TypeMeta: metav1.TypeMeta{Kind: v3.KindIPAMConfiguration}}}) - if ri.enterpriseCRDsExist { - for _, ns := range []string{common.CalicoNamespace, common.OperatorNamespace()} { - if err = utils.AddSecretsWatch(c, render.NodePrometheusTLSServerSecret, ns); err != nil { - return fmt.Errorf("tigera-windows-controller failed to watch secret '%s' in '%s' namespace: %w", render.NodePrometheusTLSServerSecret, ns, err) - } - if err = utils.AddSecretsWatch(c, monitor.PrometheusClientTLSSecretName, ns); err != nil { - return fmt.Errorf("tigera-windows-controller failed to watch secret '%s' in '%s' namespace: %w", monitor.PrometheusClientTLSSecretName, ns, err) - } + if ri.opts.EnterpriseCRDExists { + if err = ri.opts.Extensions.SetupWatches(contexts.WindowsController, c); err != nil { + return fmt.Errorf("tigera-windows-controller failed to set up extension watches: %w", err) } } @@ -176,11 +168,9 @@ type ReconcileWindows struct { client client.Client scheme *runtime.Scheme watches map[runtime.Object]struct{} - autoDetectedProvider operatorv1.Provider status status.StatusManager - enterpriseCRDsExist bool - clusterDomain string ipamConfigWatchReady *utils.ReadyFlag + opts options.ControllerOptions } // newWindowsReconciler returns a new reconcile.Reconciler @@ -192,11 +182,9 @@ func newWindowsReconciler(mgr manager.Manager, opts options.ControllerOptions) ( client: mgr.GetClient(), scheme: mgr.GetScheme(), watches: make(map[runtime.Object]struct{}), - autoDetectedProvider: opts.DetectedProvider, status: statusManager, - enterpriseCRDsExist: opts.EnterpriseCRDExists, - clusterDomain: opts.ClusterDomain, ipamConfigWatchReady: &utils.ReadyFlag{}, + opts: opts, } r.status.Run(opts.ShutdownContext) return r, nil @@ -288,7 +276,7 @@ func (r *ReconcileWindows) Reconcile(ctx context.Context, request reconcile.Requ return reconcile.Result{}, err } - certificateManager, err := certificatemanager.Create(r.client, &instance.Spec, r.clusterDomain, common.OperatorNamespace()) + certificateManager, err := certificatemanager.Create(r.client, &instance.Spec, r.opts.ClusterDomain, common.OperatorNamespace()) if err != nil { r.status.SetDegraded(operatorv1.ResourceCreateError, "Unable to create the Tigera CA", err, reqLogger) return reconcile.Result{}, err @@ -328,36 +316,9 @@ func (r *ReconcileWindows) Reconcile(ctx context.Context, request reconcile.Requ } } - // nodeReporterMetricsPort is a port used in Enterprise to host internal metrics. - // Operator is responsible for creating a service which maps to that port. - // Here, we'll check the default felixconfiguration to see if the user is specifying - // a non-default port, and use that value if they are. - nodeReporterMetricsPort := defaultNodeReporterPort - var nodePrometheusTLS certificatemanagement.KeyPairInterface - if instance.Spec.Variant.IsEnterprise() { - - // Determine the port to use for nodeReporter metrics. - if felixConfiguration.Spec.PrometheusReporterPort != nil { - nodeReporterMetricsPort = *felixConfiguration.Spec.PrometheusReporterPort - } - - if nodeReporterMetricsPort == 0 { - err := errors.New("felixConfiguration prometheusReporterPort=0 not supported") - r.status.SetDegraded(operatorv1.InvalidConfigurationError, "invalid metrics port", err, reqLogger) - return reconcile.Result{}, err - } - - // The key pair is created by the core controller, so if it isn't set, requeue to wait until it is - nodePrometheusTLS, err = certificateManager.GetKeyPair(r.client, render.NodePrometheusTLSServerSecret, common.OperatorNamespace(), dns.GetServiceDNSNames(render.WindowsNodeMetricsService, common.CalicoNamespace, r.clusterDomain)) - if err != nil { - r.status.SetDegraded(operatorv1.ResourceCreateError, "Error getting TLS certificate", err, reqLogger) - return reconcile.Result{}, err - } - } - var component render.Component - kubeDNSServiceName := utils.GetDNSServiceName(r.autoDetectedProvider) + kubeDNSServiceName := utils.GetDNSServiceName(r.opts.DetectedProvider) kubeDNSService := &corev1.Service{} err = r.client.Get(ctx, kubeDNSServiceName, kubeDNSService) if err != nil { @@ -377,15 +338,38 @@ func (r *ReconcileWindows) Reconcile(ctx context.Context, request reconcile.Requ return reconcile.Result{}, err } + // Run the variant's windows controller extension to build the render context + // (creating no enterprise artifacts in core). + cc := contexts.ControllerContext{ + RenderContext: render.RenderContext{ + Installation: &instance.Spec, + FelixConfiguration: felixConfiguration, + ClusterDomain: r.opts.ClusterDomain, + TrustedBundle: typhaNodeTLS.TrustedBundle, + }, + Controller: contexts.WindowsController, + Ctx: ctx, + Client: r.client, + CertificateManager: certificateManager, + } + if err := r.opts.Extensions.Validate(cc); err != nil { + r.status.SetDegraded(operatorv1.ResourceValidationError, "Invalid installation configuration", err, reqLogger) + return reconcile.Result{}, err + } + cc, _, err = r.opts.Extensions.ExtendContext(cc) + if err != nil { + r.status.SetDegraded(operatorv1.ResourceCreateError, "Error preparing windows extension", err, reqLogger) + return reconcile.Result{}, err + } + windowsCfg := render.WindowsConfiguration{ - K8sServiceEp: k8sapi.Endpoint, - K8sDNSServers: kubeDNSIPs, - Installation: &instance.Spec, - ClusterDomain: r.clusterDomain, - TLS: typhaNodeTLS, - PrometheusServerTLS: nodePrometheusTLS, - NodeReporterMetricsPort: nodeReporterMetricsPort, - VXLANVNI: *felixConfiguration.Spec.VXLANVNI, + K8sServiceEp: k8sapi.Endpoint, + K8sDNSServers: kubeDNSIPs, + Installation: &instance.Spec, + ClusterDomain: r.opts.ClusterDomain, + TLS: typhaNodeTLS, + VXLANVNI: *felixConfiguration.Spec.VXLANVNI, + ImageOverrides: r.opts.Extensions.Images(), } component = render.Windows(&windowsCfg) @@ -406,7 +390,14 @@ func (r *ReconcileWindows) Reconcile(ctx context.Context, request reconcile.Requ } // Create a component handler to create or update the rendered components. - handler := utils.NewComponentHandler(logw, r.client, r.scheme, instance) + handler := utils.NewComponentHandler( + logw, + r.client, + r.scheme, + instance, + utils.WithRenderContext(cc.RenderContext), + utils.WithExtensions(r.opts.Extensions), + ) if err := handler.CreateOrUpdateOrDelete(ctx, component, nil); err != nil { r.status.SetDegraded(operatorv1.ResourceUpdateError, "Error creating / updating resource", err, reqLogger) return reconcile.Result{}, err diff --git a/pkg/controller/installation/windows_controller_test.go b/pkg/controller/installation/windows_controller_test.go index ae5866bfa5..6fa1d2cf0d 100644 --- a/pkg/controller/installation/windows_controller_test.go +++ b/pkg/controller/installation/windows_controller_test.go @@ -29,6 +29,7 @@ import ( "github.com/tigera/operator/pkg/common" "github.com/tigera/operator/pkg/components" "github.com/tigera/operator/pkg/controller/certificatemanager" + "github.com/tigera/operator/pkg/controller/options" "github.com/tigera/operator/pkg/controller/status" "github.com/tigera/operator/pkg/controller/utils" ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" @@ -119,12 +120,15 @@ var _ = Describe("windows-controller installation tests", func() { // As the parameters in the client changes, we expect the outcomes of the reconcile loops to change. r = ReconcileWindows{ + opts: options.ControllerOptions{ + Extensions: testExtensions, + DetectedProvider: operator.ProviderNone, + EnterpriseCRDExists: true, + }, config: nil, // there is no fake for config client: c, scheme: scheme, - autoDetectedProvider: operator.ProviderNone, status: mockStatus, - enterpriseCRDsExist: true, ipamConfigWatchReady: &utils.ReadyFlag{}, } r.ipamConfigWatchReady.MarkAsReady() @@ -155,7 +159,7 @@ var _ = Describe("windows-controller installation tests", func() { }, }, } - Expect(updateInstallationWithDefaults(ctx, r.client, cr, r.autoDetectedProvider)).NotTo(HaveOccurred()) + Expect(updateInstallationWithDefaults(ctx, r.client, cr, r.opts.DetectedProvider)).NotTo(HaveOccurred()) certificateManager, err := certificatemanager.Create(c, nil, "", common.OperatorNamespace(), certificatemanager.AllowCACreation()) Expect(err).NotTo(HaveOccurred()) prometheusTLS, err := certificateManager.GetOrCreateKeyPair(c, monitor.PrometheusClientTLSSecretName, common.OperatorNamespace(), []string{monitor.PrometheusClientTLSSecretName}) @@ -194,7 +198,7 @@ var _ = Describe("windows-controller installation tests", func() { cr.Status = operator.InstallationStatus{ Variant: operator.Calico, } - Expect(updateInstallationWithDefaults(ctx, r.client, cr, r.autoDetectedProvider)).NotTo(HaveOccurred()) + Expect(updateInstallationWithDefaults(ctx, r.client, cr, r.opts.DetectedProvider)).NotTo(HaveOccurred()) // Set serviceCIDRs in the installation (required for Calico for Windows) cr.Spec.ServiceCIDRs = []string{"10.96.0.0/12"} @@ -609,12 +613,15 @@ var _ = Describe("windows-controller installation tests", func() { // As the parameters in the client changes, we expect the outcomes of the reconcile loops to change. r = ReconcileWindows{ + opts: options.ControllerOptions{ + Extensions: testExtensions, + DetectedProvider: operator.ProviderNone, + EnterpriseCRDExists: true, + }, config: nil, // there is no fake for config client: c, scheme: scheme, - autoDetectedProvider: operator.ProviderNone, status: mockStatus, - enterpriseCRDsExist: true, ipamConfigWatchReady: &utils.ReadyFlag{}, } r.ipamConfigWatchReady.MarkAsReady() @@ -663,7 +670,7 @@ var _ = Describe("windows-controller installation tests", func() { }, }, } - Expect(updateInstallationWithDefaults(ctx, r.client, instance, r.autoDetectedProvider)).NotTo(HaveOccurred()) + Expect(updateInstallationWithDefaults(ctx, r.client, instance, r.opts.DetectedProvider)).NotTo(HaveOccurred()) Expect(c.Create(ctx, instance)).NotTo(HaveOccurred()) }) AfterEach(func() { diff --git a/pkg/controller/logstorage/common/common.go b/pkg/controller/logstorage/common/common.go index 2c070439d7..92f34494a6 100644 --- a/pkg/controller/logstorage/common/common.go +++ b/pkg/controller/logstorage/common/common.go @@ -26,7 +26,7 @@ import ( "github.com/tigera/operator/pkg/controller/utils" "github.com/tigera/operator/pkg/crypto" - "github.com/tigera/operator/pkg/render/kubecontrollers" + entkubecontrollers "github.com/tigera/operator/pkg/enterprise/kubecontrollers" ) const ( @@ -45,7 +45,7 @@ const ( // the gateway credentials, and a secret containing real admin level credentials is created and stored in the tigera-elasticsearch namespace to be swapped in once // ES Gateway has confirmed that the gateway credentials match. func CreateKubeControllersSecrets(ctx context.Context, esAdminUserSecret *corev1.Secret, esAdminUserName string, cli client.Client, h utils.NamespaceHelper) (*corev1.Secret, *corev1.Secret, *corev1.Secret, error) { - kubeControllersGatewaySecret, err := utils.GetSecret(ctx, cli, kubecontrollers.ElasticsearchKubeControllersUserSecret, h.TruthNamespace()) + kubeControllersGatewaySecret, err := utils.GetSecret(ctx, cli, entkubecontrollers.ElasticsearchKubeControllersUserSecret, h.TruthNamespace()) if err != nil { return nil, nil, nil, err } @@ -53,11 +53,11 @@ func CreateKubeControllersSecrets(ctx context.Context, esAdminUserSecret *corev1 password := crypto.GeneratePassword(16) kubeControllersGatewaySecret = &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ - Name: kubecontrollers.ElasticsearchKubeControllersUserSecret, + Name: entkubecontrollers.ElasticsearchKubeControllersUserSecret, Namespace: h.TruthNamespace(), }, Data: map[string][]byte{ - "username": []byte(kubecontrollers.ElasticsearchKubeControllersUserName), + "username": []byte(entkubecontrollers.ElasticsearchKubeControllersUserName), "password": []byte(password), }, } @@ -67,34 +67,34 @@ func CreateKubeControllersSecrets(ctx context.Context, esAdminUserSecret *corev1 return nil, nil, nil, err } - kubeControllersVerificationSecret, err := utils.GetSecret(ctx, cli, kubecontrollers.ElasticsearchKubeControllersVerificationUserSecret, h.InstallNamespace()) + kubeControllersVerificationSecret, err := utils.GetSecret(ctx, cli, entkubecontrollers.ElasticsearchKubeControllersVerificationUserSecret, h.InstallNamespace()) if err != nil { return nil, nil, nil, err } if kubeControllersVerificationSecret == nil { kubeControllersVerificationSecret = &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ - Name: kubecontrollers.ElasticsearchKubeControllersVerificationUserSecret, + Name: entkubecontrollers.ElasticsearchKubeControllersVerificationUserSecret, Namespace: h.InstallNamespace(), Labels: map[string]string{ ESGatewaySelectorLabel: ESGatewaySelectorLabelValue, }, }, Data: map[string][]byte{ - "username": []byte(kubecontrollers.ElasticsearchKubeControllersUserName), + "username": []byte(entkubecontrollers.ElasticsearchKubeControllersUserName), "password": hashedPassword, }, } } - kubeControllersSecureUserSecret, err := utils.GetSecret(ctx, cli, kubecontrollers.ElasticsearchKubeControllersSecureUserSecret, h.InstallNamespace()) + kubeControllersSecureUserSecret, err := utils.GetSecret(ctx, cli, entkubecontrollers.ElasticsearchKubeControllersSecureUserSecret, h.InstallNamespace()) if err != nil { return nil, nil, nil, err } if kubeControllersSecureUserSecret == nil { kubeControllersSecureUserSecret = &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ - Name: kubecontrollers.ElasticsearchKubeControllersSecureUserSecret, + Name: entkubecontrollers.ElasticsearchKubeControllersSecureUserSecret, Namespace: h.InstallNamespace(), Labels: map[string]string{ ESGatewaySelectorLabel: ESGatewaySelectorLabelValue, diff --git a/pkg/controller/logstorage/kubecontrollers/es_kube_controllers.go b/pkg/controller/logstorage/kubecontrollers/es_kube_controllers.go index c51cbc076a..5e218d4978 100644 --- a/pkg/controller/logstorage/kubecontrollers/es_kube_controllers.go +++ b/pkg/controller/logstorage/kubecontrollers/es_kube_controllers.go @@ -44,6 +44,7 @@ import ( "github.com/tigera/operator/pkg/controller/utils" "github.com/tigera/operator/pkg/controller/utils/imageset" "github.com/tigera/operator/pkg/ctrlruntime" + entkubecontrollers "github.com/tigera/operator/pkg/enterprise/kubecontrollers" "github.com/tigera/operator/pkg/render" "github.com/tigera/operator/pkg/render/common/networkpolicy" "github.com/tigera/operator/pkg/render/kubecontrollers" @@ -140,7 +141,7 @@ func Add(mgr manager.Manager, opts options.ControllerOptions) error { if err := utils.AddDeploymentWatch(c, esgateway.DeploymentName, esKubeControllersNamespace.InstallNamespace()); err != nil { return fmt.Errorf("log-storage-access-controller failed to watch the Service resource: %w", err) } - if err := utils.AddDeploymentWatch(c, kubecontrollers.EsKubeController, esKubeControllersNamespace.InstallNamespace()); err != nil { + if err := utils.AddDeploymentWatch(c, entkubecontrollers.EsKubeController, esKubeControllersNamespace.InstallNamespace()); err != nil { return fmt.Errorf("log-storage-access-controller failed to watch the Service resource: %w", err) } @@ -168,7 +169,7 @@ func Add(mgr manager.Manager, opts options.ControllerOptions) error { // Start goroutines to establish watches against projectcalico.org/v3 resources. go utils.WaitToAddTierWatch(networkpolicy.CalicoTierName, c, opts.K8sClientset, log, r.tierWatchReady) go utils.WaitToAddNetworkPolicyWatches(c, opts.K8sClientset, log, []types.NamespacedName{ - {Name: kubecontrollers.EsKubeControllerNetworkPolicyName, Namespace: esKubeControllersNamespace.InstallNamespace()}, + {Name: entkubecontrollers.EsKubeControllerNetworkPolicyName, Namespace: esKubeControllersNamespace.InstallNamespace()}, }) return nil @@ -262,7 +263,7 @@ func (r *ESKubeControllersController) Reconcile(ctx context.Context, request rec // Get secrets needed for kube-controllers to talk to elastic. This is needed for zero-tenants and single-tenants // that deploy es-kube-controllers and need to talk to es-gateway var kubeControllersUserSecret *core.Secret - kubeControllersUserSecret, err = utils.GetSecret(ctx, r.client, kubecontrollers.ElasticsearchKubeControllersUserSecret, helper.TruthNamespace()) + kubeControllersUserSecret, err = utils.GetSecret(ctx, r.client, entkubecontrollers.ElasticsearchKubeControllersUserSecret, helper.TruthNamespace()) if err != nil { r.status.SetDegraded(operatorv1.ResourceReadError, "Failed to get kube controllers gateway secret", err, reqLogger) return reconcile.Result{}, err @@ -338,13 +339,12 @@ func (r *ESKubeControllersController) Reconcile(ctx context.Context, request rec ClusterDomain: r.clusterDomain, Authentication: authentication, KubeControllersGatewaySecret: kubeControllersUserSecret, - LogStorageExists: logStorage != nil, TrustedBundle: trustedBundle, Namespace: helper.InstallNamespace(), BindingNamespaces: namespaces, Tenant: nil, } - esKubeControllerComponents := kubecontrollers.NewElasticsearchKubeControllers(&kubeControllersCfg) + esKubeControllerComponents := entkubecontrollers.NewElasticsearchKubeControllers(&kubeControllersCfg) imageSet, err := imageset.GetImageSet(ctx, r.client, variant) if err != nil { diff --git a/pkg/controller/logstorage/kubecontrollers/es_kube_controllers_test.go b/pkg/controller/logstorage/kubecontrollers/es_kube_controllers_test.go index b35d072664..7fc4168dd7 100644 --- a/pkg/controller/logstorage/kubecontrollers/es_kube_controllers_test.go +++ b/pkg/controller/logstorage/kubecontrollers/es_kube_controllers_test.go @@ -46,8 +46,8 @@ import ( "github.com/tigera/operator/pkg/controller/utils" ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" "github.com/tigera/operator/pkg/dns" + entkubecontrollers "github.com/tigera/operator/pkg/enterprise/kubecontrollers" "github.com/tigera/operator/pkg/render" - "github.com/tigera/operator/pkg/render/kubecontrollers" "github.com/tigera/operator/pkg/render/logstorage" "github.com/tigera/operator/pkg/render/logstorage/esgateway" "github.com/tigera/operator/pkg/tls/certificatemanagement" @@ -235,7 +235,7 @@ var _ = Describe("LogStorage ES kube-controllers controller", func() { dep := appsv1.Deployment{ TypeMeta: metav1.TypeMeta{Kind: "Deployment", APIVersion: "apps/v1"}, ObjectMeta: metav1.ObjectMeta{ - Name: kubecontrollers.EsKubeController, + Name: entkubecontrollers.EsKubeController, Namespace: common.CalicoNamespace, }, } @@ -275,12 +275,12 @@ var _ = Describe("LogStorage ES kube-controllers controller", func() { dep := appsv1.Deployment{ TypeMeta: metav1.TypeMeta{Kind: "Deployment", APIVersion: "apps/v1"}, ObjectMeta: metav1.ObjectMeta{ - Name: kubecontrollers.EsKubeController, + Name: entkubecontrollers.EsKubeController, Namespace: common.CalicoNamespace, }, } Expect(test.GetResource(cli, &dep)).To(BeNil()) - kc := test.GetContainer(dep.Spec.Template.Spec.Containers, kubecontrollers.EsKubeController) + kc := test.GetContainer(dep.Spec.Template.Spec.Containers, entkubecontrollers.EsKubeController) Expect(kc).ToNot(BeNil()) Expect(kc.Image).To(Equal(fmt.Sprintf("some.registry.org/%s%s@%s", components.TigeraImagePath, components.ComponentTigeraCalico.Image, "sha256:kubecontrollershash"))) }) @@ -325,7 +325,7 @@ var _ = Describe("LogStorage ES kube-controllers controller", func() { dep := appsv1.Deployment{ TypeMeta: metav1.TypeMeta{Kind: "Deployment", APIVersion: "apps/v1"}, ObjectMeta: metav1.ObjectMeta{ - Name: kubecontrollers.EsKubeController, + Name: entkubecontrollers.EsKubeController, Namespace: common.CalicoNamespace, }, } diff --git a/pkg/controller/options/options.go b/pkg/controller/options/options.go index 14acd47230..404c399378 100644 --- a/pkg/controller/options/options.go +++ b/pkg/controller/options/options.go @@ -20,6 +20,7 @@ import ( v1 "github.com/tigera/operator/api/v1" "github.com/tigera/operator/pkg/common" "github.com/tigera/operator/pkg/common/discovery" + "github.com/tigera/operator/pkg/extensions" "k8s.io/client-go/kubernetes" ) @@ -55,4 +56,10 @@ type ControllerOptions struct { // the operator cares about. Populated once at startup so controllers can branch on API // availability without issuing further discovery requests at reconcile time. APIDiscovery *discovery.APIDiscovery + + // Extensions are the variant extensions (modifiers, image overrides, setups) + // the operator runs with. The core operator leaves this nil; an extension + // build (Calico Enterprise) sets it once at startup and controllers thread it + // into their render handlers and component configs. + Extensions *extensions.Set } diff --git a/pkg/controller/utils/component.go b/pkg/controller/utils/component.go index f330ea3028..9bfcc7c8ed 100644 --- a/pkg/controller/utils/component.go +++ b/pkg/controller/utils/component.go @@ -47,6 +47,7 @@ import ( "github.com/tigera/operator/pkg/apigroup" "github.com/tigera/operator/pkg/common" "github.com/tigera/operator/pkg/controller/status" + "github.com/tigera/operator/pkg/extensions" "github.com/tigera/operator/pkg/render" rmeta "github.com/tigera/operator/pkg/render/common/meta" ) @@ -74,16 +75,36 @@ type ComponentHandler interface { SetCreateOnly() } +// ComponentHandlerOption configures a componentHandler. +type ComponentHandlerOption func(*componentHandler) + +// WithRenderContext supplies the render.RenderContext passed to registered +// render modifiers. +func WithRenderContext(ctx render.RenderContext) ComponentHandlerOption { + return func(c *componentHandler) { c.renderCtx = ctx } +} + +// WithExtensions supplies the operator's extension Set, whose modifiers the +// handler applies to extensible components. A handler that renders an +// extensible component must be given the Set; one that doesn't can omit it. +func WithExtensions(e *extensions.Set) ComponentHandlerOption { + return func(c *componentHandler) { c.extensions = e } +} + // cr is allowed to be nil in the case we don't want to put ownership on a resource, // this is useful for CRD management so that they are not removed automatically. -func NewComponentHandler(log logr.Logger, cli client.Client, scheme *runtime.Scheme, cr metav1.Object) ComponentHandler { - return &componentHandler{ +func NewComponentHandler(log logr.Logger, cli client.Client, scheme *runtime.Scheme, cr metav1.Object, opts ...ComponentHandlerOption) ComponentHandler { + h := &componentHandler{ client: cli, scheme: scheme, cr: cr, log: log, apiGroupEnvs: apigroup.EnvVars(), } + for _, o := range opts { + o(h) + } + return h } type componentHandler struct { @@ -93,6 +114,8 @@ type componentHandler struct { log logr.Logger createOnly bool apiGroupEnvs []v1.EnvVar + renderCtx render.RenderContext + extensions *extensions.Set } func (c *componentHandler) SetCreateOnly() { @@ -435,6 +458,11 @@ func resetMetadataForCreate(obj client.Object) { } func (c *componentHandler) CreateOrUpdateOrDelete(ctx context.Context, component render.Component, status status.StatusManager) error { + if ext, ok := component.(render.Extensible); ok && ext.ModifierKey() != "" && c.extensions == nil { + c.log.Info("BUG: extensible component rendered by a handler with no extension Set; extensions will not be applied", "component", ext.ModifierKey()) + } + component = c.extensions.Decorate(component, c.renderCtx) + // Before creating the component, make sure that it is ready. This provides a hook to do // dependency checking for the component. cmpLog := c.log.WithValues("component", reflect.TypeOf(component)) @@ -1139,7 +1167,6 @@ func addComponentLabel(obj metav1.Object, cr metav1.Object) { owner, ok := cr.(runtime.Object) if ok && owner.GetObjectKind() != nil && owner.GetObjectKind() != nil { obj.GetLabels()["app.kubernetes.io/component"] = sanitizeLabel(owner.GetObjectKind().GroupVersionKind().GroupKind().String()) - } } } diff --git a/pkg/controller/utils/component_enterprise_test.go b/pkg/controller/utils/component_enterprise_test.go new file mode 100644 index 0000000000..cc749be8c2 --- /dev/null +++ b/pkg/controller/utils/component_enterprise_test.go @@ -0,0 +1,83 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package utils_test + +import ( + "context" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + rbacv1 "k8s.io/api/rbac/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + logf "sigs.k8s.io/controller-runtime/pkg/log" + + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/apis" + "github.com/tigera/operator/pkg/common" + "github.com/tigera/operator/pkg/controller/certificatemanager" + "github.com/tigera/operator/pkg/controller/k8sapi" + "github.com/tigera/operator/pkg/controller/utils" + ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" + "github.com/tigera/operator/pkg/dns" + "github.com/tigera/operator/pkg/enterprise" + "github.com/tigera/operator/pkg/render" +) + +// This exercises the full path comment-by-comment: a real render component goes +// through CreateOrUpdateOrDelete with an enterprise RenderContext, and the +// registered modifier must match the real render output by name. If render ever +// renames the typha ClusterRole, the modifier silently no-ops and this fails. +var _ = Describe("componentHandler enterprise modifier integration", func() { + It("applies the enterprise typha modifier to real render output", func() { + scheme := runtime.NewScheme() + Expect(apis.AddToScheme(scheme, false)).NotTo(HaveOccurred()) + cli := ctrlrfake.DefaultFakeClientBuilder(scheme).Build() + + certManager, err := certificatemanager.Create(cli, nil, "", common.OperatorNamespace(), certificatemanager.AllowCACreation()) + Expect(err).NotTo(HaveOccurred()) + nodeKeyPair, err := certManager.GetOrCreateKeyPair(cli, render.NodeTLSSecretName, common.OperatorNamespace(), []string{render.FelixCommonName}) + Expect(err).NotTo(HaveOccurred()) + typhaKeyPair, err := certManager.GetOrCreateKeyPair(cli, render.TyphaTLSSecretName, common.OperatorNamespace(), []string{render.TyphaCommonName}) + Expect(err).NotTo(HaveOccurred()) + + instance := &operatorv1.InstallationSpec{ + Variant: operatorv1.CalicoEnterprise, + CNI: &operatorv1.CNISpec{Type: operatorv1.PluginCalico}, + } + comp := render.Typha(&render.TyphaConfiguration{ + K8sServiceEp: k8sapi.ServiceEndpoint{}, + Installation: instance, + ClusterDomain: dns.DefaultClusterDomain, + FelixHealthPort: 9099, + TLS: &render.TyphaNodeTLS{ + TrustedBundle: certManager.CreateTrustedBundle(), + TyphaSecret: typhaKeyPair, + TyphaCommonName: render.TyphaCommonName, + NodeSecret: nodeKeyPair, + NodeCommonName: render.FelixCommonName, + }, + }) + + renderCtx := render.RenderContext{Installation: instance} + handler := utils.NewComponentHandler(logf.Log, cli, scheme, nil, utils.WithRenderContext(renderCtx), utils.WithExtensions(enterprise.New())) + Expect(handler.CreateOrUpdateOrDelete(context.Background(), comp, nil)).NotTo(HaveOccurred()) + + role := &rbacv1.ClusterRole{} + Expect(cli.Get(context.Background(), client.ObjectKey{Name: "calico-typha"}, role)).NotTo(HaveOccurred()) + Expect(role.Rules).To(ContainElement(HaveField("Resources", ContainElement("licensekeys")))) + }) +}) diff --git a/pkg/controller/utils/component_test.go b/pkg/controller/utils/component_test.go index 5936ce98bd..8bf522013f 100644 --- a/pkg/controller/utils/component_test.go +++ b/pkg/controller/utils/component_test.go @@ -46,6 +46,7 @@ import ( "github.com/tigera/operator/pkg/common" "github.com/tigera/operator/pkg/controller/status" ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" + "github.com/tigera/operator/pkg/extensions" "github.com/tigera/operator/pkg/render" rmeta "github.com/tigera/operator/pkg/render/common/meta" ) @@ -2575,3 +2576,45 @@ func (mc *mockClient) RESTMapper() restMeta.RESTMapper { func (mc *mockClient) SubResource(subResource string) client.SubResourceClient { panic("SubResource not implemented in mockClient") } + +var _ = Describe("componentHandler modifier application", func() { + It("applies registered modifiers to a named component before create", func() { + ext := extensions.NewSet() + ext.Variant(operatorv1.CalicoEnterprise).Modify("fake", func(ctx render.RenderContext, objs, del []client.Object) ([]client.Object, []client.Object) { + cm := objs[0].(*corev1.ConfigMap) + cm.Data = map[string]string{"patched": "yes"} + return objs, del + }) + + s := runtime.NewScheme() + Expect(apis.AddToScheme(s, false)).NotTo(HaveOccurred()) + Expect(corev1.SchemeBuilder.AddToScheme(s)).NotTo(HaveOccurred()) + + c := ctrlrfake.DefaultFakeClientBuilder(s).Build() + renderCtx := render.RenderContext{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.CalicoEnterprise}} + handler := NewComponentHandler(logf.Log, c, s, nil, WithRenderContext(renderCtx), WithExtensions(ext)) + comp := &namedFakeComponent{name: "fake", obj: &corev1.ConfigMap{ + TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"}, + ObjectMeta: metav1.ObjectMeta{Name: "cm", Namespace: "default"}, + }} + + Expect(handler.CreateOrUpdateOrDelete(context.Background(), comp, nil)).NotTo(HaveOccurred()) + + got := &corev1.ConfigMap{} + Expect(c.Get(context.Background(), client.ObjectKey{Name: "cm", Namespace: "default"}, got)).NotTo(HaveOccurred()) + Expect(got.Data).To(HaveKeyWithValue("patched", "yes")) + }) +}) + +type namedFakeComponent struct { + name string + obj client.Object +} + +func (f *namedFakeComponent) ModifierKey() string { return f.name } +func (f *namedFakeComponent) ResolveImages(*operatorv1.ImageSet) error { return nil } +func (f *namedFakeComponent) Objects() ([]client.Object, []client.Object) { + return []client.Object{f.obj}, nil +} +func (f *namedFakeComponent) Ready() bool { return true } +func (f *namedFakeComponent) SupportedOSType() rmeta.OSType { return rmeta.OSTypeLinux } diff --git a/pkg/enterprise/apiserver/extension.go b/pkg/enterprise/apiserver/extension.go new file mode 100644 index 0000000000..01028e9056 --- /dev/null +++ b/pkg/enterprise/apiserver/extension.go @@ -0,0 +1,1638 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package apiserver + +import ( + "fmt" + "net/url" + "slices" + "strings" + + admregv1 "k8s.io/api/admissionregistration/v1" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/handler" + + v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" + "github.com/tigera/api/pkg/lib/numorstring" + + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/common" + "github.com/tigera/operator/pkg/components" + "github.com/tigera/operator/pkg/controller/contexts" + "github.com/tigera/operator/pkg/controller/utils" + "github.com/tigera/operator/pkg/controller/utils/imageset" + "github.com/tigera/operator/pkg/ctrlruntime" + "github.com/tigera/operator/pkg/dns" + eoptions "github.com/tigera/operator/pkg/enterprise/options" + "github.com/tigera/operator/pkg/extensions" + "github.com/tigera/operator/pkg/render" + "github.com/tigera/operator/pkg/render/common/authentication" + rcomp "github.com/tigera/operator/pkg/render/common/components" + relasticsearch "github.com/tigera/operator/pkg/render/common/elasticsearch" + rmeta "github.com/tigera/operator/pkg/render/common/meta" + "github.com/tigera/operator/pkg/render/common/networkpolicy" + "github.com/tigera/operator/pkg/render/common/securitycontext" + "github.com/tigera/operator/pkg/render/monitor" + "github.com/tigera/operator/pkg/tls/certificatemanagement" +) + +const ( + auditLogsVolumeName = "calico-audit-logs" + auditPolicyVolumeName = "calico-audit-policy" +) + +// apiServerRenderData is the controller-produced data the API server hook hands to its +// modifiers through RenderContext.Extension. It carries the enterprise inputs the base +// render no longer knows about: the management cluster / managed cluster CRs, the +// ApplicationLayer (which drives the L7 sidecar), the cert-management-only query server +// keypair, the OIDC key validator config, and the resolved L7 sidecar images. +type apiServerRenderData struct { + managementCluster *operatorv1.ManagementCluster + managementClusterConnection *operatorv1.ManagementClusterConnection + applicationLayer *operatorv1.ApplicationLayer + queryServerTLS certificatemanagement.KeyPairInterface + keyValidatorConfig authentication.KeyValidatorConfig + l7EnvoyImage string + dikastesImage string +} + +// apiServerData pulls the API server hook's render data back out of the render context, +// returning the zero value when none is set. +func apiServerData(rc render.RenderContext) apiServerRenderData { + return render.ExtractExtensionData[apiServerRenderData](rc) +} + +// apiServer carries the rendered API server configuration, the resolved image, and the +// controller-produced render data so the enterprise builders can construct the +// Enterprise-only objects and deployment additions. +type apiServer struct { + cfg *render.APIServerConfiguration + calicoImage string + data apiServerRenderData +} + +// Register wires the API server controller hook and modifiers into the variant. +func Register(v *extensions.Variant) { + v.Controller(contexts.APIServerController, apiServerControllerExtension{}) + extensions.RegisterModifier(v, render.ComponentNameAPIServer, modifyAPIServer) + extensions.RegisterModifier(v, render.ComponentNameAPIServerPolicy, modifyAPIServerPolicy) +} + +func (c *apiServer) isSidecarInjectionEnabled() bool { + al := c.data.applicationLayer + return al != nil && + al.Spec.SidecarInjection != nil && + *al.Spec.SidecarInjection == operatorv1.SidecarEnabled +} + +// apiServerControllerExtension is the Calico Enterprise controller-side hook for the API +// server controller. It does the enterprise reconcile work the render phase can't: +// fetching the enterprise CRs, creating the trusted bundle and the query server cert, +// and resolving the L7 sidecar images. +type apiServerControllerExtension struct{} + +// Validate rejects an API server configuration Calico Enterprise does not support: a +// cluster cannot be both a management cluster and a managed cluster. +func (apiServerControllerExtension) Validate(cc contexts.ControllerContext) error { + managementCluster, err := utils.GetManagementCluster(cc.Ctx, cc.Client) + if err != nil { + return fmt.Errorf("error reading ManagementCluster: %w", err) + } + managementClusterConnection, err := utils.GetManagementClusterConnection(cc.Ctx, cc.Client) + if err != nil { + return fmt.Errorf("error reading ManagementClusterConnection: %w", err) + } + if managementCluster != nil && managementClusterConnection != nil { + return fmt.Errorf("having both a ManagementCluster and a ManagementClusterConnection is not supported") + } + return nil +} + +// Watches registers the enterprise resources the API server controller reconciles on. +func (apiServerControllerExtension) Watches(c ctrlruntime.Controller) error { + for _, obj := range []client.Object{ + &operatorv1.ApplicationLayer{ObjectMeta: metav1.ObjectMeta{Name: utils.DefaultEnterpriseInstanceKey.Name}}, + &operatorv1.ManagementCluster{}, + &operatorv1.ManagementClusterConnection{}, + &operatorv1.Authentication{}, + } { + if err := c.WatchObject(obj, &handler.EnqueueRequestForObject{}); err != nil { + return err + } + } + for _, namespace := range []string{common.OperatorNamespace(), render.APIServerNamespace} { + for _, secretName := range []string{render.VoltronTunnelSecretName, render.ManagerTLSSecretName} { + if err := utils.AddSecretsWatch(c, secretName, namespace); err != nil { + return err + } + } + } + return utils.AddSecretsWatch(c, render.VoltronLinseedPublicCert, common.OperatorNamespace()) +} + +// ExtendContext does the enterprise controller-side work: it builds the trusted bundle, +// fetches the enterprise CRs, creates the query server certificate, resolves the L7 +// sidecar images, and stashes them for the modifiers. The base API server render carries +// none of this. +func (apiServerControllerExtension) ExtendContext(cc contexts.ControllerContext) (contexts.ControllerContext, []certificatemanagement.KeyPairInterface, error) { + in := cc.Installation + + trustedBundle, err := cc.CertificateManager.CreateNamedTrustedBundleFromSecrets(render.APIServerResourceName, cc.Client, common.OperatorNamespace(), false) + if err != nil { + return cc, nil, fmt.Errorf("unable to create the trusted bundle: %w", err) + } + + applicationLayer, err := utils.GetApplicationLayer(cc.Ctx, cc.Client) + if err != nil { + return cc, nil, fmt.Errorf("error reading ApplicationLayer: %w", err) + } + + managementCluster, err := utils.GetManagementCluster(cc.Ctx, cc.Client) + if err != nil { + return cc, nil, fmt.Errorf("error reading ManagementCluster: %w", err) + } + + managementClusterConnection, err := utils.GetManagementClusterConnection(cc.Ctx, cc.Client) + if err != nil { + return cc, nil, fmt.Errorf("error reading ManagementClusterConnection: %w", err) + } + + // Management cluster only: the apiserver mounts the tunnel CA secret so it can sign + // certificates for managed clusters. The manager controller writes it once + // ManagementCluster.Spec.TLS is defaulted; degrade until it exists. + if managementCluster != nil && managementCluster.Spec.TLS != nil && !eoptions.From(cc).MultiTenant { + if _, err := utils.GetSecret(cc.Ctx, cc.Client, managementCluster.Spec.TLS.SecretName, common.OperatorNamespace()); err != nil { + return cc, nil, fmt.Errorf("unable to fetch the tunnel secret: %w", err) + } + } + + prometheusCertificate, err := cc.CertificateManager.GetCertificate(cc.Client, monitor.PrometheusClientTLSSecretName, common.OperatorNamespace()) + if err != nil { + return cc, nil, fmt.Errorf("failed to get certificate: %w", err) + } + if prometheusCertificate != nil { + trustedBundle.AddCertificates(prometheusCertificate) + } + + if managementClusterConnection != nil { + voltronLinseedCert, err := cc.CertificateManager.GetCertificate(cc.Client, render.VoltronLinseedPublicCert, common.OperatorNamespace()) + if err != nil { + return cc, nil, fmt.Errorf("failed to retrieve %s: %w", render.VoltronLinseedPublicCert, err) + } + if voltronLinseedCert != nil { + trustedBundle.AddCertificates(voltronLinseedCert) + } + } + + // Authentication: when a Dex-backed Authentication CR is ready, add its cert to the + // bundle and build the key validator config for the query server and the policy. + var keyValidatorConfig authentication.KeyValidatorConfig + authenticationCR, err := utils.GetAuthentication(cc.Ctx, cc.Client) + if err != nil && !apierrors.IsNotFound(err) { + return cc, nil, fmt.Errorf("error while fetching Authentication: %w", err) + } + if authenticationCR != nil && authenticationCR.Status.State == operatorv1.TigeraStatusReady { + if utils.DexEnabled(authenticationCR) { + certificate, err := cc.CertificateManager.GetCertificate(cc.Client, render.DexTLSSecretName, common.OperatorNamespace()) + if err != nil { + return cc, nil, fmt.Errorf("failed to retrieve %s: %w", render.DexTLSSecretName, err) + } else if certificate == nil { + return cc, nil, fmt.Errorf("waiting for secret %q to become available", render.DexTLSSecretName) + } + trustedBundle.AddCertificates(certificate) + } + keyValidatorConfig, err = utils.GetKeyValidatorConfig(cc.Ctx, cc.Client, authenticationCR, cc.ClusterDomain) + if err != nil { + return cc, nil, fmt.Errorf("failed to get KeyValidator config: %w", err) + } + } + + // Under certificate management, the query server needs its own keypair so it can run + // with different permissions than the apiserver. + var queryServerTLS certificatemanagement.KeyPairInterface + if in.CertificateManagement != nil { + queryServerTLS, err = cc.CertificateManager.GetOrCreateKeyPair( + cc.Client, + "query-server-tls", + common.OperatorNamespace(), + dns.GetServiceDNSNames(render.APIServerServiceName, render.APIServerNamespace, cc.ClusterDomain), + ) + if err != nil { + return cc, nil, fmt.Errorf("unable to get or create query server tls key pair: %w", err) + } + } + + // Resolve the L7 sidecar images when sidecar injection is enabled. The modifier runs + // after image resolution, so the hook resolves them here. + var l7EnvoyImage, dikastesImage string + if applicationLayer != nil && + applicationLayer.Spec.SidecarInjection != nil && + *applicationLayer.Spec.SidecarInjection == operatorv1.SidecarEnabled { + imageSet, err := imageset.GetImageSet(cc.Ctx, cc.Client, in.Variant) + if err != nil { + return cc, nil, err + } + l7EnvoyImage, err = components.GetReference(components.ComponentEnvoyProxy, in.Registry, in.ImagePath, in.ImagePrefix, imageSet) + if err != nil { + return cc, nil, err + } + dikastesImage, err = components.GetReference(components.ComponentDikastes, in.Registry, in.ImagePath, in.ImagePrefix, imageSet) + if err != nil { + return cc, nil, err + } + } + + cc.TrustedBundle = trustedBundle + cc.Extension = apiServerRenderData{ + managementCluster: managementCluster, + managementClusterConnection: managementClusterConnection, + applicationLayer: applicationLayer, + queryServerTLS: queryServerTLS, + keyValidatorConfig: keyValidatorConfig, + l7EnvoyImage: l7EnvoyImage, + dikastesImage: dikastesImage, + } + return cc, nil, nil +} + +// RegisterCalicoCleanup registers, for the Calico variant, the cleanup that +// deletes the Enterprise API server objects left behind by a prior Enterprise +// installation. +func RegisterCalicoCleanup(v *extensions.Variant) { + extensions.RegisterModifier(v, render.ComponentNameAPIServer, cleanupAPIServer) +} + +// modifyAPIServer layers Calico Enterprise behavior onto the rendered API server objects: +// the query server container and its volumes, audit logging on the aggregation API server +// container, the Enterprise RBAC objects, and the query server port on the Service. +func modifyAPIServer(rc render.RenderContext, ec render.APIServerExtensionContext, create, del []client.Object) ([]client.Object, []client.Object) { + c := &apiServer{cfg: ec.Config, calicoImage: ec.CalicoImage, data: apiServerData(rc)} + + // Ensure the deployment and its supporting objects exist. The base renders them when + // running an aggregation API server; in v3-CRD mode it queues them for deletion, but + // Enterprise always runs a query server, so render the skeleton ourselves and pull + // those objects back out of the delete list. + create, del = c.ensureDeployment(create, del) + + if dep, ok := extensions.FindObject[*appsv1.Deployment](create, render.APIServerName); ok { + c.layerDeployment(dep) + } + if svc, ok := extensions.FindObject[*corev1.Service](create, render.APIServerServiceName); ok { + c.addServicePorts(svc) + } + // Enterprise serves staged policies through the tiered-policy passthrough role. + if role, ok := extensions.FindObject[*rbacv1.ClusterRole](create, "calico-tiered-policy-passthrough"); ok { + for i := range role.Rules { + if slices.Contains(role.Rules[i].Resources, "networkpolicies") { + role.Rules[i].Resources = append(role.Rules[i].Resources, "stagednetworkpolicies", "stagedglobalnetworkpolicies") + } + } + } + + // The L7 sidecar mutating webhook is driven by ApplicationLayer. The base always + // queues it for deletion; when sidecar injection is on, render it and pull it back + // out of the delete list. + if c.isSidecarInjectionEnabled() { + create = append(create, c.sidecarMutatingWebhookConfig()) + del = removeByRef(del, &admregv1.MutatingWebhookConfiguration{ObjectMeta: metav1.ObjectMeta{Name: common.SidecarMutatingWebhookConfigName}}) + } + + // Global Enterprise RBAC. + create = append(create, c.tigeraAPIServerClusterRole(), c.tigeraAPIServerClusterRoleBinding()) + if !c.cfg.MultiTenant { + // These resources are only installed in zero-tenant clusters. + create = append(create, c.tigeraUserClusterRole(), c.tigeraNetworkAdminClusterRole()) + } + if c.data.managementCluster != nil { + create = append(create, c.managedClusterWatchClusterRole()) + if c.cfg.MultiTenant { + create = append(create, c.multiTenantSecretsRBAC()...) + create = append(create, c.multiTenantManagedClusterAccessClusterRoles()...) + } else { + create = append(create, c.secretsRBAC()...) + } + } else { + // If we're not a management cluster, the API server doesn't need permissions to access secrets. + del = append(del, c.multiTenantSecretsRBAC()...) + del = append(del, c.secretsRBAC()...) + del = append(del, c.multiTenantManagedClusterAccessClusterRoles()...) + del = append(del, c.managedClusterWatchClusterRole()) + } + + // Namespaced Enterprise objects. + if c.cfg.TrustedBundle != nil { + create = append(create, c.cfg.TrustedBundle.ConfigMap(render.QueryserverNamespace)) + } + if c.data.managementClusterConnection != nil { + create = append(create, c.externalLinseedRoleBinding()) + } + + // Objects that only exist alongside the aggregation API server. + aggregationObjects := []client.Object{ + c.uiSettingsGroupGetterClusterRole(), + c.kubeControllerManagerUISettingsGroupGetterClusterRoleBinding(), + c.uiSettingsPassthruClusterRole(), + c.uiSettingsPassthruClusterRolebinding(), + c.auditPolicyConfigMap(), + } + if c.cfg.RequiresAggregationServer { + create = append(create, aggregationObjects...) + } else { + del = append(del, aggregationObjects...) + } + + // Clean up cluster-scoped resources that were created with the 'tigera' prefix. + del = append(del, c.deprecatedResources()...) + + // Re-apply deployment overrides so the modifier-added query server container picks up + // any per-container overrides. The override appliers use replace/merge semantics, so + // re-running over the render-applied containers is idempotent. + if dep, ok := extensions.FindObject[*appsv1.Deployment](create, render.APIServerName); ok { + if overrides := c.cfg.APIServer.APIServerDeployment; overrides != nil { + rcomp.ApplyDeploymentOverrides(dep, overrides) + } + } + + return create, del +} + +// cleanupAPIServer deletes the Enterprise API server objects when running Calico, so a +// cluster switched from Enterprise to Calico does not leave them behind. +func cleanupAPIServer(rc render.RenderContext, ec render.APIServerExtensionContext, create, del []client.Object) ([]client.Object, []client.Object) { + c := &apiServer{cfg: ec.Config} + + del = append(del, c.tigeraAPIServerClusterRole(), c.tigeraAPIServerClusterRoleBinding()) + if !c.cfg.MultiTenant { + del = append(del, c.tigeraUserClusterRole(), c.tigeraNetworkAdminClusterRole()) + } + del = append(del, c.multiTenantSecretsRBAC()...) + del = append(del, c.secretsRBAC()...) + del = append(del, c.multiTenantManagedClusterAccessClusterRoles()...) + del = append(del, c.managedClusterWatchClusterRole()) + + return create, del +} + +// layerDeployment adds the Enterprise additions to the rendered API server deployment: +// the query server container (and, under certificate management, its init container and +// volume), audit logging and the management-cluster tunnel args on the aggregation API +// server container, the L7 admission controller sidecar, and the Linseed token and +// trusted bundle volumes. +func (c *apiServer) layerDeployment(d *appsv1.Deployment) { + spec := &d.Spec.Template.Spec + if d.Spec.Template.Annotations == nil { + d.Spec.Template.Annotations = map[string]string{} + } + + // Audit logging and the management-cluster tunnel args are layered onto the + // aggregation API server container, which is only present when that server runs. + if c.cfg.RequiresAggregationServer { + for i := range spec.Containers { + ctr := &spec.Containers[i] + if ctr.Name != string(render.APIServerContainerName) { + continue + } + ctr.VolumeMounts = append(ctr.VolumeMounts, + corev1.VolumeMount{Name: auditLogsVolumeName, MountPath: "/var/log/calico/audit"}, + corev1.VolumeMount{Name: auditPolicyVolumeName, MountPath: "/etc/tigera/audit"}, + ) + ctr.Args = append(ctr.Args, + "--audit-policy-file=/etc/tigera/audit/policy.conf", + "--audit-log-path=/var/log/calico/audit/tsee-audit.log", + ) + ctr.Args = append(ctr.Args, c.managementClusterArgs()...) + // In case of OpenShift, apiserver needs privileged access to write audit logs to the + // host path volume. Audit logs are owned by root on hosts so we need to be root. + ctr.SecurityContext = securitycontext.NewRootContext(c.cfg.OpenShift) + } + + spec.Volumes = append(spec.Volumes, c.auditVolumes()...) + } + + spec.Containers = append(spec.Containers, c.queryServerContainer()) + if c.isSidecarInjectionEnabled() { + spec.Containers = append(spec.Containers, c.l7AdmissionControllerContainer()) + } + + // Under certificate management the query server gets its own cert init container and + // volume, since apiserver and queryserver may run with different UID:GID. + if c.data.queryServerTLS != nil { + init := c.data.queryServerTLS.InitContainer(render.APIServerNamespace, securitycontext.NewNonRootContext()) + spec.InitContainers = append(spec.InitContainers, init) + spec.Volumes = append(spec.Volumes, c.data.queryServerTLS.Volume()) + d.Spec.Template.Annotations[c.data.queryServerTLS.HashAnnotationKey()] = c.data.queryServerTLS.HashAnnotationValue() + } + + if c.data.managementClusterConnection != nil { + // Optional: the Secret is delivered over the Guardian tunnel, which can't be + // established until calico-apiserver is Ready. + spec.Volumes = append(spec.Volumes, corev1.Volume{ + Name: render.LinseedTokenVolumeName, + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: fmt.Sprintf(render.LinseedTokenSecret, "calico-apiserver"), + Items: []corev1.KeyToPath{{Key: render.LinseedTokenKey, Path: render.LinseedTokenSubPath}}, + Optional: ptr.To(true), + }, + }, + }) + } + + if c.cfg.TrustedBundle != nil { + spec.Volumes = append(spec.Volumes, c.cfg.TrustedBundle.Volume()) + for k, v := range c.cfg.TrustedBundle.HashAnnotations() { + d.Spec.Template.Annotations[k] = v + } + } +} + +// managementClusterArgs returns the aggregation API server tunnel args for a management +// cluster, or nil when this isn't one. +func (c *apiServer) managementClusterArgs() []string { + mc := c.data.managementCluster + if mc == nil { + return nil + } + args := []string{"--enable-managed-clusters-create-api=true"} + if mc.Spec.Address != "" { + args = append(args, fmt.Sprintf("--managementClusterAddr=%s", mc.Spec.Address)) + } + if mc.Spec.TLS != nil && mc.Spec.TLS.SecretName != "" { + if mc.Spec.TLS.SecretName == render.ManagerTLSSecretName { + args = append(args, "--managementClusterCAType=Public") + } + args = append(args, fmt.Sprintf("--tunnelSecretName=%s", mc.Spec.TLS.SecretName)) + } + return args +} + +// ensureDeployment makes sure the API server Deployment and its supporting objects are +// in the create list. The base renders them when running an aggregation API server; when +// it doesn't (v3-CRD mode), it queues them for deletion, so render the skeleton and pull +// those objects back out of the delete list. +func (c *apiServer) ensureDeployment(create, del []client.Object) ([]client.Object, []client.Object) { + if _, ok := extensions.FindObject[*appsv1.Deployment](create, render.APIServerName); ok { + return create, del + } + skeleton := render.APIServerDeploymentObjects(c.cfg, c.calicoImage) + create = append(create, skeleton...) + for _, obj := range render.APIServerDeploymentObjectMeta() { + del = removeByRef(del, obj) + } + return create, del +} + +// removeByRef returns del with any object matching ref's kind, namespace, and name +// removed. +func removeByRef(del []client.Object, ref client.Object) []client.Object { + out := del[:0:0] + for _, o := range del { + if o.GetObjectKind().GroupVersionKind().Kind == ref.GetObjectKind().GroupVersionKind().Kind && + o.GetNamespace() == ref.GetNamespace() && + o.GetName() == ref.GetName() { + continue + } + out = append(out, o) + } + return out +} + +// addServicePorts adds the query server port and, when sidecar injection is enabled, the +// L7 admission controller port to the API server Service. +func (c *apiServer) addServicePorts(s *corev1.Service) { + queryServerTargetPort := render.GetContainerPort(c.cfg, render.TigeraAPIServerQueryServerContainerName) + s.Spec.Ports = append(s.Spec.Ports, corev1.ServicePort{ + Name: render.QueryServerPortName, + Port: render.QueryServerPort, + Protocol: corev1.ProtocolTCP, + TargetPort: intstr.FromInt32(queryServerTargetPort.ContainerPort), + }) + if c.isSidecarInjectionEnabled() { + l7Port := render.GetContainerPort(c.cfg, render.L7AdmissionControllerContainerName) + s.Spec.Ports = append(s.Spec.Ports, corev1.ServicePort{ + Name: render.L7AdmissionControllerPortName, + Port: render.L7AdmissionControllerPort, + Protocol: corev1.ProtocolTCP, + TargetPort: intstr.FromInt32(l7Port.ContainerPort), + }) + } +} + +// l7AdmissionControllerContainer is the L7 admission controller sidecar, rendered when +// ApplicationLayer sidecar injection is enabled. +func (c *apiServer) l7AdmissionControllerContainer() corev1.Container { + volumeMounts := []corev1.VolumeMount{ + c.cfg.TLSKeyPair.VolumeMount(rmeta.OSTypeLinux), + } + + l7Port := render.GetContainerPort(c.cfg, render.L7AdmissionControllerContainerName).ContainerPort + + dataplane := "iptables" + if c.cfg.Installation.IsNftables() { + dataplane = "nftables" + } + + return corev1.Container{ + Name: string(render.L7AdmissionControllerContainerName), + Image: c.calicoImage, + Command: []string{components.CalicoBinaryPath, "component", "l7-admission-controller"}, + Env: []corev1.EnvVar{ + {Name: "L7ADMCTRL_TLSCERTPATH", Value: c.cfg.TLSKeyPair.VolumeMountCertificateFilePath()}, + {Name: "L7ADMCTRL_TLSKEYPATH", Value: c.cfg.TLSKeyPair.VolumeMountKeyFilePath()}, + {Name: "L7ADMCTRL_ENVOYIMAGE", Value: c.data.l7EnvoyImage}, + {Name: "L7ADMCTRL_DIKASTESIMAGE", Value: c.data.dikastesImage}, + {Name: "L7ADMCTRL_LISTENADDR", Value: fmt.Sprintf(":%d", l7Port)}, + {Name: "DATAPLANE", Value: dataplane}, + }, + VolumeMounts: volumeMounts, + LivenessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/live", + Port: intstr.FromInt32(l7Port), + Scheme: corev1.URISchemeHTTPS, + }, + }, + }, + } +} + +// sidecarMutatingWebhookConfig is the L7 sidecar injection webhook, rendered when +// ApplicationLayer sidecar injection is enabled. +func (c *apiServer) sidecarMutatingWebhookConfig() *admregv1.MutatingWebhookConfiguration { + var cacert []byte + svcPort := render.GetContainerPort(c.cfg, render.L7AdmissionControllerContainerName).ContainerPort + + svcpath := "/sidecar-webhook" + svcref := admregv1.ServiceReference{ + Name: render.QueryserverServiceName, + Namespace: render.QueryserverNamespace, + Path: &svcpath, + Port: &svcPort, + } + failpol := admregv1.Fail + labelsel := metav1.LabelSelector{ + MatchLabels: map[string]string{ + "applicationlayer.projectcalico.org/sidecar": "true", + }, + } + rules := []admregv1.RuleWithOperations{ + { + Rule: admregv1.Rule{ + APIGroups: []string{""}, + APIVersions: []string{"v1"}, + Resources: []string{"pods"}, + }, + Operations: []admregv1.OperationType{admregv1.Create}, + }, + } + sidefx := admregv1.SideEffectClassNone + if !c.cfg.TLSKeyPair.UseCertificateManagement() { + cacert = c.cfg.TLSKeyPair.GetIssuer().GetCertificatePEM() + } else { + cacert = c.cfg.Installation.CertificateManagement.CACert + } + return &admregv1.MutatingWebhookConfiguration{ + TypeMeta: metav1.TypeMeta{ + Kind: "MutatingWebhookConfiguration", + APIVersion: "admissionregistration.k8s.io/v1", + }, + ObjectMeta: metav1.ObjectMeta{Name: common.SidecarMutatingWebhookConfigName}, + Webhooks: []admregv1.MutatingWebhook{ + { + AdmissionReviewVersions: []string{"v1"}, + ClientConfig: admregv1.WebhookClientConfig{ + Service: &svcref, + CABundle: cacert, + }, + Name: "sidecar.projectcalico.org", + FailurePolicy: &failpol, + ObjectSelector: &labelsel, + Rules: rules, + SideEffects: &sidefx, + }, + }, + } +} + +// modifyAPIServerPolicy adds the enterprise additions to the API server network policy: +// the OIDC egress rule (when an OIDC key validator is configured) and the L7 admission +// controller ingress port (when sidecar injection is enabled). The base policy carries +// neither. +func modifyAPIServerPolicy(rc render.RenderContext, ec render.APIServerExtensionContext, create, del []client.Object) ([]client.Object, []client.Object) { + c := &apiServer{cfg: ec.Config, data: apiServerData(rc)} + + policy, ok := extensions.FindObject[*v3.NetworkPolicy](create, render.APIServerPolicyName) + if !ok { + return create, del + } + + // Insert the OIDC egress rule before the trailing Pass rule so it is evaluated. + if c.data.keyValidatorConfig != nil { + if parsedURL, err := url.Parse(c.data.keyValidatorConfig.Issuer()); err == nil { + oidc := networkpolicy.GetOIDCEgressRule(parsedURL) + egress := policy.Spec.Egress + if n := len(egress); n > 0 && egress[n-1].Action == v3.Pass { + policy.Spec.Egress = append(egress[:n-1:n-1], oidc, egress[n-1]) + } else { + policy.Spec.Egress = append(egress, oidc) + } + } + } + + // Allow the kube-apiserver to reach the L7 admission controller. + if c.isSidecarInjectionEnabled() { + l7Port := render.GetContainerPort(c.cfg, render.L7AdmissionControllerContainerName).ContainerPort + for i := range policy.Spec.Ingress { + policy.Spec.Ingress[i].Destination.Ports = append(policy.Spec.Ingress[i].Destination.Ports, + numorstring.Port{MinPort: uint16(l7Port), MaxPort: uint16(l7Port)}) + } + } + + return create, del +} + +// auditVolumes are the host-path audit log and audit policy volumes used by the +// aggregation API server container. +func (c *apiServer) auditVolumes() []corev1.Volume { + return []corev1.Volume{ + { + Name: auditLogsVolumeName, + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: "/var/log/calico/audit", + Type: ptr.To(corev1.HostPathDirectoryOrCreate), + }, + }, + }, + { + Name: auditPolicyVolumeName, + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{Name: auditPolicyVolumeName}, + Items: []corev1.KeyToPath{ + { + Key: "config", + Path: "policy.conf", + }, + }, + }, + }, + }, + } +} + +func (c *apiServer) multiTenantSecretsRBAC() []client.Object { + return render.TunnelSecretRBAC(render.APIServerSecretsRBACName, render.APIServerServiceAccountName, c.data.managementCluster, true) +} + +func (c *apiServer) secretsRBAC() []client.Object { + return render.TunnelSecretRBAC(render.APIServerSecretsRBACName, render.APIServerServiceAccountName, c.data.managementCluster, false) +} + +func (c *apiServer) queryServerContainer() corev1.Container { + queryServerTargetPort := render.GetContainerPort(c.cfg, render.TigeraAPIServerQueryServerContainerName).ContainerPort + + var tlsSecret certificatemanagement.KeyPairInterface + if c.data.queryServerTLS != nil { + tlsSecret = c.data.queryServerTLS + } else { + tlsSecret = c.cfg.TLSKeyPair + } + env := []corev1.EnvVar{ + {Name: "DATASTORE_TYPE", Value: "kubernetes"}, + {Name: "LISTEN_ADDR", Value: fmt.Sprintf(":%d", queryServerTargetPort)}, + {Name: "TLS_CERT", Value: fmt.Sprintf("/%s/tls.crt", tlsSecret.GetName())}, + {Name: "TLS_KEY", Value: fmt.Sprintf("/%s/tls.key", tlsSecret.GetName())}, + } + if c.cfg.TrustedBundle != nil { + env = append(env, corev1.EnvVar{Name: "TRUSTED_BUNDLE_PATH", Value: c.cfg.TrustedBundle.MountPath()}) + } + + if render.HostNetwork(c.cfg) { + env = append(env, c.cfg.K8SServiceEndpoint.EnvVars()...) + } else { + env = append(env, c.cfg.K8SServiceEndpointPodNetwork.EnvVars()...) + } + + if c.cfg.Installation.CalicoNetwork != nil && c.cfg.Installation.CalicoNetwork.MultiInterfaceMode != nil { + env = append(env, corev1.EnvVar{Name: "MULTI_INTERFACE_MODE", Value: c.cfg.Installation.CalicoNetwork.MultiInterfaceMode.Value()}) + } + + if c.data.keyValidatorConfig != nil { + env = append(env, c.data.keyValidatorConfig.RequiredEnv("")...) + } + + linseedURL := relasticsearch.LinseedEndpoint(rmeta.OSTypeLinux, c.cfg.ClusterDomain, render.ElasticsearchNamespace, c.data.managementClusterConnection != nil, false) + env = append(env, + corev1.EnvVar{Name: "LINSEED_URL", Value: linseedURL}, + corev1.EnvVar{Name: "LINSEED_CLIENT_CERT", Value: fmt.Sprintf("/%s/tls.crt", tlsSecret.GetName())}, + corev1.EnvVar{Name: "LINSEED_CLIENT_KEY", Value: fmt.Sprintf("/%s/tls.key", tlsSecret.GetName())}, + ) + if c.data.managementClusterConnection != nil { + env = append(env, + corev1.EnvVar{Name: "CLUSTER_ID", Value: ""}, + corev1.EnvVar{Name: "LINSEED_TOKEN", Value: render.GetLinseedTokenPath(true)}, + ) + } + if c.cfg.TrustedBundle != nil { + env = append(env, corev1.EnvVar{Name: "LINSEED_CA", Value: c.cfg.TrustedBundle.MountPath()}) + } + + // set LogLEVEL for queryserver container + if logging := c.cfg.APIServer.Logging; logging != nil && + logging.QueryServerLogging != nil && logging.QueryServerLogging.LogSeverity != nil { + env = append(env, + corev1.EnvVar{Name: "LOGLEVEL", Value: strings.ToLower(string(*logging.QueryServerLogging.LogSeverity))}) + } else { + // set default LOGLEVEL to info when not set by the user + env = append(env, corev1.EnvVar{Name: "LOGLEVEL", Value: "info"}) + } + + volumeMounts := []corev1.VolumeMount{ + tlsSecret.VolumeMount(rmeta.OSTypeLinux), + } + if c.cfg.TrustedBundle != nil { + volumeMounts = append(volumeMounts, c.cfg.TrustedBundle.VolumeMounts(rmeta.OSTypeLinux)...) + } + if c.data.managementClusterConnection != nil { + volumeMounts = append(volumeMounts, corev1.VolumeMount{ + Name: render.LinseedTokenVolumeName, + MountPath: render.LinseedVolumeMountPath, + }) + } + + container := corev1.Container{ + Name: string(render.TigeraAPIServerQueryServerContainerName), + Image: c.calicoImage, + Command: []string{components.CalicoBinaryPath, "component", "queryserver"}, + Env: env, + LivenessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/version", + Port: intstr.FromInt32(queryServerTargetPort), + Scheme: corev1.URISchemeHTTPS, + }, + }, + InitialDelaySeconds: 90, + }, + SecurityContext: securitycontext.NewNonRootContext(), + VolumeMounts: volumeMounts, + } + return container +} + +func (c *apiServer) externalLinseedRoleBinding() *rbacv1.RoleBinding { + return &rbacv1.RoleBinding{ + TypeMeta: metav1.TypeMeta{Kind: "RoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "tigera-linseed", + Namespace: render.APIServerNamespace, + }, + RoleRef: rbacv1.RoleRef{ + APIGroup: "rbac.authorization.k8s.io", + Kind: "ClusterRole", + Name: render.TigeraLinseedSecretsClusterRole, + }, + Subjects: []rbacv1.Subject{ + { + Kind: "ServiceAccount", + Name: render.GuardianServiceAccountName, + Namespace: render.GuardianNamespace, + }, + }, + } +} + +func (c *apiServer) tigeraAPIServerClusterRole() *rbacv1.ClusterRole { + rules := []rbacv1.PolicyRule{ + { + // Read access to Linseed policy activity data for queryserver enrichment. + APIGroups: []string{"linseed.tigera.io"}, + Resources: []string{"policyactivity"}, + Verbs: []string{"get"}, + }, + { + // Calico Enterprise backing storage. + APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, + Resources: []string{ + "alertexceptions", + "bfdconfigurations", + "deeppacketinspections", + "deeppacketinspections/status", + "egressgatewaypolicies", + "externalnetworks", + "globalalerts", + "globalalerts/status", + "globalalerttemplates", + "globalreports", + "globalreports/status", + "globalreporttypes", + "globalthreatfeeds", + "globalthreatfeeds/status", + "licensekeys", + "managedclusters", + "managedclusters/status", + "networks", + "packetcaptures", + "packetcaptures/status", + "policyrecommendationscopes", + "policyrecommendationscopes/status", + "remoteclusterconfigurations", + "securityeventwebhooks", + "securityeventwebhooks/status", + "uisettings", + "uisettingsgroups", + }, + Verbs: []string{ + "get", + "list", + "watch", + "create", + "update", + "delete", + "patch", + }, + }, + { + // The queryserver's RBAC calculator needs to list tiers, + // uisettingsgroups, and managedclusters via the aggregated + // API to evaluate user permissions for the /policies endpoint. + APIGroups: []string{"projectcalico.org"}, + Resources: []string{ + "tiers", + "uisettingsgroups", + "managedclusters", + }, + Verbs: []string{"get", "list", "watch"}, + }, + { + // Required by the AuthorizationReview calculator in queryserver to evaluate + // RBAC permissions for users. + APIGroups: []string{"rbac.authorization.k8s.io"}, + Resources: []string{ + "clusterroles", + "clusterrolebindings", + "roles", + "rolebindings", + }, + Verbs: []string{"get", "list", "watch"}, + }, + } + + return &rbacv1.ClusterRole{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: render.APIServerName, + }, + Rules: rules, + } +} + +func (c *apiServer) tigeraAPIServerClusterRoleBinding() *rbacv1.ClusterRoleBinding { + return &rbacv1.ClusterRoleBinding{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: render.APIServerName, + }, + Subjects: []rbacv1.Subject{ + { + Kind: "ServiceAccount", + Name: render.APIServerServiceAccountName, + Namespace: render.APIServerNamespace, + }, + }, + RoleRef: rbacv1.RoleRef{ + Kind: "ClusterRole", + Name: render.APIServerName, + APIGroup: "rbac.authorization.k8s.io", + }, + } +} + +func (c *apiServer) uiSettingsGroupGetterClusterRole() *rbacv1.ClusterRole { + return &rbacv1.ClusterRole{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "calico-uisettingsgroup-getter", + }, + Rules: []rbacv1.PolicyRule{ + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{ + "uisettingsgroups", + }, + Verbs: []string{"get"}, + }, + }, + } +} + +func (c *apiServer) kubeControllerManagerUISettingsGroupGetterClusterRoleBinding() *rbacv1.ClusterRoleBinding { + return &rbacv1.ClusterRoleBinding{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "calico-uisettingsgroup-getter", + }, + RoleRef: rbacv1.RoleRef{ + Kind: "ClusterRole", + Name: "calico-uisettingsgroup-getter", + APIGroup: "rbac.authorization.k8s.io", + }, + Subjects: []rbacv1.Subject{ + { + Kind: "User", + Name: "system:kube-controller-manager", + APIGroup: "rbac.authorization.k8s.io", + }, + }, + } +} + +func (c *apiServer) tigeraUserClusterRole() *rbacv1.ClusterRole { + rules := []rbacv1.PolicyRule{ + // List requests that the Tigera manager needs. + { + APIGroups: []string{ + "projectcalico.org", + "networking.k8s.io", + "extensions", + "", + }, + // Use both the networkpolicies and tier.networkpolicies resource types to ensure identical behavior + // irrespective of the Calico RBAC scheme (see the ClusterRole "calico-tiered-policy-passthrough" for + // more details). Similar for all tiered policy resource types. + Resources: []string{ + "tiers", + "networkpolicies", + "tier.networkpolicies", + "globalnetworkpolicies", + "tier.globalnetworkpolicies", + "namespaces", + "globalnetworksets", + "networksets", + "managedclusters", + "stagedglobalnetworkpolicies", + "tier.stagedglobalnetworkpolicies", + "stagednetworkpolicies", + "tier.stagednetworkpolicies", + "stagedkubernetesnetworkpolicies", + "policyrecommendationscopes", + }, + Verbs: []string{"watch", "list"}, + }, + { + APIGroups: []string{"policy.networking.k8s.io"}, + Resources: []string{ + "clusternetworkpolicies", + "adminnetworkpolicies", + "baselineadminnetworkpolicies", + }, + Verbs: []string{"watch", "list"}, + }, + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"packetcaptures/files"}, + Verbs: []string{"get"}, + }, + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"packetcaptures"}, + Verbs: []string{"get", "list", "watch"}, + }, + // Allow the user to view Networks. + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"networks"}, + Verbs: []string{"get", "list", "watch"}, + }, + // Additional "list" requests required to view flows. + { + APIGroups: []string{""}, + Resources: []string{"pods"}, + Verbs: []string{"list"}, + }, + // Additional "list" requests required to view serviceaccount labels. + { + APIGroups: []string{""}, + Resources: []string{"serviceaccounts"}, + Verbs: []string{"list"}, + }, + // Access for WAF API to read in coreruleset configmap + { + APIGroups: []string{""}, + Resources: []string{"configmaps"}, + ResourceNames: []string{"coreruleset-default"}, + Verbs: []string{"get"}, + }, + // Access to statistics. + { + APIGroups: []string{""}, + Resources: []string{"services/proxy"}, + ResourceNames: []string{ + "https:calico-api:8080", "calico-node-prometheus:9090", + }, + Verbs: []string{"get", "create"}, + }, + // Access to policies in all tiers + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"tiers"}, + Verbs: []string{"get"}, + }, + // List and download the reports in the Tigera Secure manager. + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"globalreports"}, + Verbs: []string{"get", "list"}, + }, + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"globalreporttypes"}, + Verbs: []string{"get"}, + }, + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"clusterinformations"}, + Verbs: []string{"get", "list"}, + }, + // Access to hostendpoints from the UI ServiceGraph. + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"hostendpoints"}, + Verbs: []string{"get", "list"}, + }, + // List and view the threat defense configuration + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{ + "alertexceptions", + "globalalerts", + "globalalerts/status", + "globalalerttemplates", + "globalthreatfeeds", + "globalthreatfeeds/status", + "securityeventwebhooks", + }, + Verbs: []string{"get", "watch", "list"}, + }, + // User can: + // - read UISettings in the cluster-settings group + // - read and write UISettings in the user-settings group + // Default settings group and settings are created in manager.go. + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"uisettingsgroups"}, + Verbs: []string{"get"}, + ResourceNames: []string{"cluster-settings", "user-settings"}, + }, + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"uisettingsgroups/data"}, + Verbs: []string{"get", "list", "watch"}, + ResourceNames: []string{"cluster-settings"}, + }, + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"uisettingsgroups/data"}, + Verbs: []string{"*"}, + ResourceNames: []string{"user-settings"}, + }, + // Allow the user to read applicationlayers to detect if WAF is enabled/disabled. + { + APIGroups: []string{"operator.tigera.io"}, + Resources: []string{"applicationlayers", "packetcaptureapis", "compliances", "intrusiondetections"}, + Verbs: []string{"get"}, + }, + // Allow the user to view WAF policies, plugins, and validation policies. + { + APIGroups: []string{"applicationlayer.projectcalico.org"}, + Resources: []string{ + "globalwafpolicies", + "globalwafplugins", + "globalwafvalidationpolicies", + "wafpolicies", + "wafplugins", + "wafvalidationpolicies", + }, + Verbs: []string{"get", "watch", "list"}, + }, + { + APIGroups: []string{"apps"}, + Resources: []string{"deployments"}, + Verbs: []string{"get", "list", "watch"}, + }, + // Allow the user to read services to view WAF configuration. + { + APIGroups: []string{""}, + Resources: []string{"services"}, + Verbs: []string{"get", "list", "watch"}, + }, + // Allow the user to read felixconfigurations to detect if wireguard and/or other features are enabled. + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"felixconfigurations"}, + Verbs: []string{"get", "list"}, + }, + // Allow the user to only view securityeventwebhooks. + { + APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, + Resources: []string{"securityeventwebhooks"}, + Verbs: []string{"get", "list"}, + }, + } + + // Privileges for lma.tigera.io have no effect on managed clusters. + if c.data.managementClusterConnection == nil { + // Access to flow logs, audit logs, and statistics. + // Access to log into Kibana for oidc users. + rules = append(rules, rbacv1.PolicyRule{ + APIGroups: []string{"lma.tigera.io"}, + Resources: []string{"*"}, + ResourceNames: []string{ + "flows", "audit*", "l7", "events", "dns", "waf", "kibana_login", "recommendations", + }, + Verbs: []string{"get"}, + }) + } + + return &rbacv1.ClusterRole{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "tigera-ui-user", + }, + Rules: rules, + } +} + +func (c *apiServer) tigeraNetworkAdminClusterRole() *rbacv1.ClusterRole { + rules := []rbacv1.PolicyRule{ + // Full access to all network policies + { + APIGroups: []string{ + "projectcalico.org", + "networking.k8s.io", + "extensions", + }, + // Use both the networkpolicies and tier.networkpolicies resource types to ensure identical behavior + // irrespective of the Calico RBAC scheme (see the ClusterRole "calico-tiered-policy-passthrough" for + // more details). Similar for all tiered policy resource types. + Resources: []string{ + "tiers", + "networkpolicies", + "tier.networkpolicies", + "globalnetworkpolicies", + "tier.globalnetworkpolicies", + "stagedglobalnetworkpolicies", + "tier.stagedglobalnetworkpolicies", + "stagednetworkpolicies", + "tier.stagednetworkpolicies", + "stagedkubernetesnetworkpolicies", + "globalnetworksets", + "networksets", + "managedclusters", + "packetcaptures", + "policyrecommendationscopes", + }, + Verbs: []string{"create", "update", "delete", "patch", "get", "watch", "list"}, + }, + { + APIGroups: []string{ + "policy.networking.k8s.io", + }, + Resources: []string{ + "clusternetworkpolicies", + "adminnetworkpolicies", + "baselineadminnetworkpolicies", + }, + Verbs: []string{"create", "update", "delete", "patch", "get", "watch", "list"}, + }, + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"packetcaptures/files"}, + Verbs: []string{"get", "delete"}, + }, + // Allow the user to CRUD Networks. + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"networks"}, + Verbs: []string{"create", "update", "delete", "patch", "get", "watch", "list"}, + }, + // Additional "list" requests that the Tigera Secure manager needs + { + APIGroups: []string{""}, + Resources: []string{"namespaces"}, + Verbs: []string{"watch", "list"}, + }, + // Additional "list" requests required to view flows. + { + APIGroups: []string{""}, + Resources: []string{"pods"}, + Verbs: []string{"list"}, + }, + // Additional "list" requests required to view serviceaccount labels. + { + APIGroups: []string{""}, + Resources: []string{"serviceaccounts"}, + Verbs: []string{"list"}, + }, + // Access for WAF API to read in coreruleset configmap + { + APIGroups: []string{""}, + Resources: []string{"configmaps"}, + ResourceNames: []string{"coreruleset-default"}, + Verbs: []string{"get"}, + }, + // Access to statistics. + { + APIGroups: []string{""}, + Resources: []string{"services/proxy"}, + ResourceNames: []string{ + "https:calico-api:8080", "calico-node-prometheus:9090", + }, + Verbs: []string{"get", "create"}, + }, + // Manage globalreport configuration, view report generation status, and list reports in the Tigera Secure manager. + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"globalreports"}, + Verbs: []string{"*"}, + }, + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"globalreports/status"}, + Verbs: []string{"get", "list", "watch"}, + }, + // List and download the reports in the Tigera Secure manager. + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"globalreporttypes"}, + Verbs: []string{"get"}, + }, + // Access to cluster information containing Calico and EE versions from the UI. + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"clusterinformations"}, + Verbs: []string{"get", "list"}, + }, + // Access to hostendpoints from the UI ServiceGraph. + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"hostendpoints"}, + Verbs: []string{"get", "list"}, + }, + // Manage the threat defense configuration + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{ + "alertexceptions", + "globalalerts", + "globalalerts/status", + "globalalerttemplates", + "globalthreatfeeds", + "globalthreatfeeds/status", + "securityeventwebhooks", + }, + Verbs: []string{"create", "update", "delete", "patch", "get", "watch", "list"}, + }, + // User can: + // - read and write UISettings in the cluster-settings group, and rename the group + // - read and write UISettings in the user-settings group, and rename the group + // Default settings group and settings are created in manager.go. + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"uisettingsgroups"}, + Verbs: []string{"get", "patch", "update"}, + ResourceNames: []string{"cluster-settings", "user-settings"}, + }, + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"uisettingsgroups/data"}, + Verbs: []string{"*"}, + ResourceNames: []string{"cluster-settings", "user-settings"}, + }, + // Allow the user to read and write applicationlayers to enable/disable WAF. + { + APIGroups: []string{"operator.tigera.io"}, + Resources: []string{"applicationlayers", "packetcaptureapis", "compliances", "intrusiondetections"}, + Verbs: []string{"get", "update", "patch", "create", "delete"}, + }, + // Allow the user to manage WAF policies, plugins, and validation policies. + { + APIGroups: []string{"applicationlayer.projectcalico.org"}, + Resources: []string{ + "globalwafpolicies", + "globalwafplugins", + "globalwafvalidationpolicies", + "wafpolicies", + "wafplugins", + "wafvalidationpolicies", + }, + Verbs: []string{"create", "update", "delete", "patch", "get", "watch", "list"}, + }, + // Allow the user to read deployments to view WAF configuration. + { + APIGroups: []string{"apps"}, + Resources: []string{"deployments"}, + Verbs: []string{"get", "list", "watch", "patch"}, + }, + { + APIGroups: []string{""}, + Resources: []string{"services"}, + Verbs: []string{"get", "list", "watch", "patch"}, + }, + // Allow the user to read felixconfigurations to detect if wireguard and/or other features are enabled. + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"felixconfigurations"}, + Verbs: []string{"get", "list"}, + }, + // Allow the user to perform CRUD operations on securityeventwebhooks. + { + APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, + Resources: []string{"securityeventwebhooks"}, + Verbs: []string{"get", "list", "update", "patch", "create", "delete"}, + }, + // Allow the user to create secrets. + { + APIGroups: []string{""}, + Resources: []string{ + "secrets", + }, + Verbs: []string{"create"}, + }, + // Allow the user to patch webhooks-secret secret. + { + APIGroups: []string{""}, + Resources: []string{ + "secrets", + }, + ResourceNames: []string{ + "webhooks-secret", + }, + Verbs: []string{"patch"}, + }, + } + + // Privileges for lma.tigera.io have no effect on managed clusters. + if c.data.managementClusterConnection == nil { + // Access to flow logs, audit logs, and statistics. + // Elasticsearch superuser access once logged into Kibana. + rules = append(rules, rbacv1.PolicyRule{ + APIGroups: []string{"lma.tigera.io"}, + Resources: []string{"*"}, + ResourceNames: []string{ + "flows", "audit*", "l7", "events", "dns", "waf", "kibana_login", "elasticsearch_superuser", "recommendations", + }, + Verbs: []string{"get"}, + }) + } + + // In v3 CRD / webhooks mode there is no aggregated apiserver, and the + // calico-uisettings-passthrough ClusterRole that normally grants the broad + // uisettings permission isn't deployed. Grant write verbs here so the + // calico-webhooks UISettings handler (which narrows access via a SAR on + // uisettingsgroups/data) gets invoked instead of being short-circuited by + // kube-apiserver RBAC. + if !c.cfg.RequiresAggregationServer { + rules = append(rules, rbacv1.PolicyRule{ + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"uisettings"}, + Verbs: []string{"create", "update", "delete", "patch"}, + }) + } + + return &rbacv1.ClusterRole{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "tigera-network-admin", + }, + Rules: rules, + } +} + +func (c *apiServer) uiSettingsPassthruClusterRole() *rbacv1.ClusterRole { + return &rbacv1.ClusterRole{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "calico-uisettings-passthrough", + }, + Rules: []rbacv1.PolicyRule{ + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"uisettings"}, + Verbs: []string{"*"}, + }, + }, + } +} + +func (c *apiServer) uiSettingsPassthruClusterRolebinding() *rbacv1.ClusterRoleBinding { + return &rbacv1.ClusterRoleBinding{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "calico-uisettings-passthrough", + }, + Subjects: []rbacv1.Subject{ + { + Kind: "Group", + Name: "system:authenticated", + APIGroup: "rbac.authorization.k8s.io", + }, + }, + RoleRef: rbacv1.RoleRef{ + Kind: "ClusterRole", + Name: "calico-uisettings-passthrough", + APIGroup: "rbac.authorization.k8s.io", + }, + } +} + +func (c *apiServer) auditPolicyConfigMap() *corev1.ConfigMap { + const defaultAuditPolicy = `apiVersion: audit.k8s.io/v1 +kind: Policy +rules: +- level: RequestResponse + omitStages: + - RequestReceived + verbs: + - create + - patch + - update + - delete + resources: + - group: projectcalico.org + resources: + - globalnetworkpolicies + - networkpolicies + - stagedglobalnetworkpolicies + - stagednetworkpolicies + - stagedkubernetesnetworkpolicies + - globalnetworksets + - networksets + - tiers + - hostendpoints` + + return &corev1.ConfigMap{ + TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"}, + ObjectMeta: metav1.ObjectMeta{ + // This object is for Enterprise only, so pass it explicitly. + Namespace: render.APIServerNamespace, + Name: auditPolicyVolumeName, + }, + Data: map[string]string{ + "config": defaultAuditPolicy, + }, + } +} + +func (c *apiServer) multiTenantManagedClusterAccessClusterRoles() []client.Object { + var objects []client.Object + objects = append(objects, &rbacv1.ClusterRole{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{Name: render.MultiTenantManagedClustersAccessClusterRoleName}, + Rules: []rbacv1.PolicyRule{ + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"managedclusters"}, + Verbs: []string{ + // The Authentication Proxy in Voltron checks if Enterprise Components (using impersonation headers for + // the service in the canonical namespace) can get a managed clusters before sending the request down the tunnel. + // This ClusterRole will be assigned to each component using a RoleBinding in the canonical or tenant namespace. + "get", + }, + }, + }, + }) + + return objects +} + +func (c *apiServer) managedClusterWatchClusterRole() client.Object { + return &rbacv1.ClusterRole{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{Name: render.ManagedClustersWatchClusterRoleName}, + Rules: []rbacv1.PolicyRule{ + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"managedclusters"}, + Verbs: []string{ + "get", "list", "watch", + }, + }, + }, + } +} + +func (c *apiServer) deprecatedResources() []client.Object { + return []client.Object{ + &rbacv1.ClusterRole{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{Name: "tigera-extension-apiserver-secrets-access"}, + }, + &rbacv1.ClusterRoleBinding{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{Name: "tigera-extension-apiserver-secrets-access"}, + }, + + // delegateAuthClusterRoleBinding + &rbacv1.ClusterRoleBinding{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{Name: "tigera-apiserver-delegate-auth"}, + }, + + // authClusterRole + &rbacv1.ClusterRole{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{Name: "tigera-extension-apiserver-auth-access"}, + }, + + // authClusterRoleBinding + &rbacv1.ClusterRoleBinding{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{Name: "tigera-extension-apiserver-auth-access"}, + }, + // authReaderRoleBinding - need clean up in diff namespace kube-system + &rbacv1.RoleBinding{ + TypeMeta: metav1.TypeMeta{Kind: "RoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "tigera-auth-reader", + Namespace: "kube-system", + }, + }, + // webhookReaderClusterRole + &rbacv1.ClusterRole{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{Name: "tigera-webhook-reader"}, + }, + + // webhookReaderClusterRoleBinding + &rbacv1.ClusterRoleBinding{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{Name: "tigera-apiserver-webhook-reader"}, + }, + + // calico-apiserver CR and CRB + &rbacv1.ClusterRole{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{Name: "tigera-apiserver"}, + }, + &rbacv1.ClusterRoleBinding{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{Name: "tigera-apiserver"}, + }, + + &rbacv1.ClusterRole{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{Name: "tigera-uisettingsgroup-getter"}, + }, + &rbacv1.ClusterRoleBinding{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{Name: "tigera-uisettingsgroup-getter"}, + }, + + &rbacv1.ClusterRole{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{Name: "tigera-tiered-policy-passthrough"}, + }, + &rbacv1.ClusterRoleBinding{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{Name: "tigera-tiered-policy-passthrough"}, + }, + + &rbacv1.ClusterRole{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{Name: "tigera-uisettings-passthrough"}, + }, + &rbacv1.ClusterRoleBinding{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{Name: "tigera-uisettings-passthrough"}, + }, + + // Clean up legacy secrets in the tigera-operator namespace + &corev1.Secret{ + TypeMeta: metav1.TypeMeta{Kind: "Secret", APIVersion: "v1"}, + ObjectMeta: metav1.ObjectMeta{Name: "tigera-api-cert", Namespace: common.OperatorNamespace()}, + }, + } +} diff --git a/pkg/enterprise/apiserver/extension_test.go b/pkg/enterprise/apiserver/extension_test.go new file mode 100644 index 0000000000..ee6733dae0 --- /dev/null +++ b/pkg/enterprise/apiserver/extension_test.go @@ -0,0 +1,483 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package apiserver_test + +import ( + "context" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" + admregv1 "k8s.io/api/admissionregistration/v1" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/apis" + "github.com/tigera/operator/pkg/common" + "github.com/tigera/operator/pkg/controller/certificatemanager" + "github.com/tigera/operator/pkg/controller/contexts" + "github.com/tigera/operator/pkg/controller/k8sapi" + "github.com/tigera/operator/pkg/controller/utils" + ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" + "github.com/tigera/operator/pkg/dns" + "github.com/tigera/operator/pkg/extensions" + "github.com/tigera/operator/pkg/extensions/extensionstest" + "github.com/tigera/operator/pkg/render" + "github.com/tigera/operator/pkg/tls/certificatemanagement" +) + +const apiServerClusterDomain = "cluster.local" + +// apiServerControllerContext builds a controller context for the API server controller, +// seeded with a fake client that holds objs. The returned context carries a real +// certificate manager and trusted bundle, so ExtendContext can create the query server +// cert and the bundle the modifiers consume. +func apiServerControllerContext(variant operatorv1.ProductVariant, install *operatorv1.InstallationSpec, objs ...client.Object) contexts.ControllerContext { + scheme := runtime.NewScheme() + Expect(apis.AddToScheme(scheme, false)).NotTo(HaveOccurred()) + c := ctrlrfake.DefaultFakeClientBuilder(scheme).Build() + + for _, o := range objs { + Expect(c.Create(context.Background(), o)).NotTo(HaveOccurred()) + } + + if install == nil { + install = &operatorv1.InstallationSpec{Variant: variant} + } + + certManager, err := certificatemanager.Create(c, install, apiServerClusterDomain, common.OperatorNamespace(), certificatemanager.AllowCACreation()) + Expect(err).NotTo(HaveOccurred()) + + return contexts.ControllerContext{ + RenderContext: render.RenderContext{ + Installation: install, + ClusterDomain: apiServerClusterDomain, + TrustedBundle: certManager.CreateTrustedBundle(), + }, + Controller: contexts.APIServerController, + Ctx: context.Background(), + Client: c, + CertificateManager: certManager, + } +} + +// apiServerKeyPair issues the API server TLS keypair from the context's certificate +// manager, the way the controller does before rendering. +func apiServerKeyPair(cc contexts.ControllerContext) certificatemanagement.KeyPairInterface { + dnsNames := dns.GetServiceDNSNames(render.APIServerServiceName, render.APIServerNamespace, cc.ClusterDomain) + kp, err := cc.CertificateManager.GetOrCreateKeyPair(cc.Client, render.CalicoAPIServerTLSSecretName, common.OperatorNamespace(), dnsNames) + Expect(err).NotTo(HaveOccurred()) + return kp +} + +var _ = Describe("API server enterprise controller extension", func() { + managementCluster := func() *operatorv1.ManagementCluster { + return &operatorv1.ManagementCluster{ + ObjectMeta: metav1.ObjectMeta{Name: utils.DefaultEnterpriseInstanceKey.Name}, + Spec: operatorv1.ManagementClusterSpec{ + Address: "example.com:1234", + TLS: &operatorv1.TLS{SecretName: render.VoltronTunnelSecretName}, + }, + } + } + + tunnelSecret := func() *corev1.Secret { + return &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: render.VoltronTunnelSecretName, Namespace: common.OperatorNamespace()}, + Data: map[string][]byte{"cert": []byte("a"), "key": []byte("b")}, + } + } + + managementClusterConnection := func() *operatorv1.ManagementClusterConnection { + return &operatorv1.ManagementClusterConnection{ + ObjectMeta: metav1.ObjectMeta{Name: utils.DefaultEnterpriseInstanceKey.Name}, + } + } + + Describe("Validate", func() { + It("accepts a cluster with neither a ManagementCluster nor a ManagementClusterConnection", func() { + cc := apiServerControllerContext(operatorv1.CalicoEnterprise, nil) + Expect(ext.Validate(cc)).NotTo(HaveOccurred()) + }) + + It("accepts a management cluster", func() { + cc := apiServerControllerContext(operatorv1.CalicoEnterprise, nil, managementCluster(), tunnelSecret()) + Expect(ext.Validate(cc)).NotTo(HaveOccurred()) + }) + + It("accepts a managed cluster", func() { + cc := apiServerControllerContext(operatorv1.CalicoEnterprise, nil, managementClusterConnection()) + Expect(ext.Validate(cc)).NotTo(HaveOccurred()) + }) + + It("rejects a cluster that is both a management cluster and a managed cluster", func() { + cc := apiServerControllerContext(operatorv1.CalicoEnterprise, nil, managementCluster(), tunnelSecret(), managementClusterConnection()) + Expect(ext.Validate(cc)).To(HaveOccurred()) + }) + }) +}) + +var _ = Describe("API server enterprise modifier", func() { + // renderAPIServer builds the base API server objects and applies the enterprise + // modifier, the way the component handler does. It returns the create and delete + // lists after the modifier ran. + renderAPIServer := func(cc contexts.ControllerContext, rc render.RenderContext, kp certificatemanagement.KeyPairInterface) ([]client.Object, []client.Object) { + cfg := &render.APIServerConfiguration{ + RequiresAggregationServer: true, + K8SServiceEndpoint: k8sapi.ServiceEndpoint{}, + Installation: cc.Installation, + APIServer: &operatorv1.APIServerSpec{}, + TLSKeyPair: kp, + TrustedBundle: rc.TrustedBundle, + KubernetesVersion: &common.VersionInfo{Major: 1, Minor: 31}, + } + comp, err := render.APIServer(cfg) + Expect(err).NotTo(HaveOccurred()) + Expect(comp.ResolveImages(nil)).NotTo(HaveOccurred()) + create, del := comp.Objects() + + ec := comp.(render.ExtensionContextProvider).ExtensionContext() + return extensionstest.ApplyExtensionsWithContext(ext, render.ComponentNameAPIServer, rc, ec, create, del) + } + + apiServerDeployment := func(objs []client.Object) *appsv1.Deployment { + dp, ok := extensions.FindObject[*appsv1.Deployment](objs, render.APIServerName) + Expect(ok).To(BeTrue()) + return dp + } + + container := func(dp *appsv1.Deployment, name string) *corev1.Container { + for i := range dp.Spec.Template.Spec.Containers { + if dp.Spec.Template.Spec.Containers[i].Name == name { + return &dp.Spec.Template.Spec.Containers[i] + } + } + return nil + } + + It("is a no-op for the Calico variant (no enterprise objects added)", func() { + cc := apiServerControllerContext(operatorv1.Calico, nil) + ecc, _, err := ext.ExtendContext(cc) + rc := ecc.RenderContext + Expect(err).NotTo(HaveOccurred()) + + objs, _ := renderAPIServer(cc, rc, apiServerKeyPair(cc)) + + // No enterprise objects. (The Calico variant cleanup is registered on the base + // render component, not exercised here.) + _, ok := extensions.FindObject[*rbacv1.ClusterRole](objs, "tigera-ui-user") + Expect(ok).To(BeFalse()) + dp := apiServerDeployment(objs) + Expect(container(dp, string(render.TigeraAPIServerQueryServerContainerName))).To(BeNil()) + }) + + It("layers the query server, enterprise RBAC, audit policy, and query server port on", func() { + cc := apiServerControllerContext(operatorv1.CalicoEnterprise, nil) + ecc, _, err := ext.ExtendContext(cc) + rc := ecc.RenderContext + Expect(err).NotTo(HaveOccurred()) + + objs, _ := renderAPIServer(cc, rc, apiServerKeyPair(cc)) + + // The query server container is layered onto the deployment. + dp := apiServerDeployment(objs) + Expect(container(dp, string(render.TigeraAPIServerQueryServerContainerName))).NotTo(BeNil()) + + // Enterprise RBAC. + for _, name := range []string{"calico-apiserver", "tigera-ui-user", "tigera-network-admin", "calico-uisettingsgroup-getter", "calico-uisettings-passthrough"} { + _, ok := extensions.FindObject[*rbacv1.ClusterRole](objs, name) + Expect(ok).To(BeTrue(), "expected ClusterRole %q", name) + } + + // The user and network-admin roles grant access to WAF policy resources. + uiUser, found := extensions.FindObject[*rbacv1.ClusterRole](objs, "tigera-ui-user") + Expect(found).To(BeTrue()) + Expect(uiUser.Rules).To(ContainElement(rbacv1.PolicyRule{ + APIGroups: []string{"applicationlayer.projectcalico.org"}, + Resources: []string{ + "globalwafpolicies", + "globalwafplugins", + "globalwafvalidationpolicies", + "wafpolicies", + "wafplugins", + "wafvalidationpolicies", + }, + Verbs: []string{"get", "watch", "list"}, + })) + + networkAdmin, found := extensions.FindObject[*rbacv1.ClusterRole](objs, "tigera-network-admin") + Expect(found).To(BeTrue()) + Expect(networkAdmin.Rules).To(ContainElement(rbacv1.PolicyRule{ + APIGroups: []string{"applicationlayer.projectcalico.org"}, + Resources: []string{ + "globalwafpolicies", + "globalwafplugins", + "globalwafvalidationpolicies", + "wafpolicies", + "wafplugins", + "wafvalidationpolicies", + }, + Verbs: []string{"create", "update", "delete", "patch", "get", "watch", "list"}, + })) + + // Audit policy configmap. + _, ok := extensions.FindObject[*corev1.ConfigMap](objs, "calico-audit-policy") + Expect(ok).To(BeTrue()) + + // The query server port is added to the Service. + svc, ok := extensions.FindObject[*corev1.Service](objs, render.APIServerServiceName) + Expect(ok).To(BeTrue()) + Expect(svc.Spec.Ports).To(ContainElement(HaveField("Name", render.QueryServerPortName))) + }) + + It("queues the enterprise RBAC for deletion when not a management cluster", func() { + cc := apiServerControllerContext(operatorv1.CalicoEnterprise, nil) + ecc, _, err := ext.ExtendContext(cc) + rc := ecc.RenderContext + Expect(err).NotTo(HaveOccurred()) + + _, del := renderAPIServer(cc, rc, apiServerKeyPair(cc)) + _, ok := extensions.FindObject[*rbacv1.ClusterRole](del, render.ManagedClustersWatchClusterRoleName) + Expect(ok).To(BeTrue()) + }) + + Context("management cluster", func() { + It("adds the tunnel args and the managed-cluster-watch and secrets RBAC", func() { + cc := apiServerControllerContext(operatorv1.CalicoEnterprise, nil, + &operatorv1.ManagementCluster{ + ObjectMeta: metav1.ObjectMeta{Name: utils.DefaultEnterpriseInstanceKey.Name}, + Spec: operatorv1.ManagementClusterSpec{ + Address: "example.com:1234", + TLS: &operatorv1.TLS{SecretName: render.VoltronTunnelSecretName}, + }, + }, + &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: render.VoltronTunnelSecretName, Namespace: common.OperatorNamespace()}, + Data: map[string][]byte{"cert": []byte("a"), "key": []byte("b")}, + }, + ) + ecc, _, err := ext.ExtendContext(cc) + rc := ecc.RenderContext + Expect(err).NotTo(HaveOccurred()) + + objs, _ := renderAPIServer(cc, rc, apiServerKeyPair(cc)) + + dp := apiServerDeployment(objs) + apiCtr := container(dp, string(render.APIServerContainerName)) + Expect(apiCtr).NotTo(BeNil()) + Expect(apiCtr.Args).To(ContainElement("--enable-managed-clusters-create-api=true")) + Expect(apiCtr.Args).To(ContainElement("--managementClusterAddr=example.com:1234")) + Expect(apiCtr.Args).To(ContainElement("--tunnelSecretName=" + render.VoltronTunnelSecretName)) + + _, ok := extensions.FindObject[*rbacv1.ClusterRole](objs, render.ManagedClustersWatchClusterRoleName) + Expect(ok).To(BeTrue()) + _, ok = extensions.FindObject[*rbacv1.Role](objs, render.APIServerSecretsRBACName) + Expect(ok).To(BeTrue()) + }) + }) + + Context("managed cluster", func() { + It("adds the external Linseed rolebinding and the query server token volume", func() { + cc := apiServerControllerContext(operatorv1.CalicoEnterprise, nil, + &operatorv1.ManagementClusterConnection{ + ObjectMeta: metav1.ObjectMeta{Name: utils.DefaultEnterpriseInstanceKey.Name}, + }, + ) + ecc, _, err := ext.ExtendContext(cc) + rc := ecc.RenderContext + Expect(err).NotTo(HaveOccurred()) + + objs, _ := renderAPIServer(cc, rc, apiServerKeyPair(cc)) + + _, ok := extensions.FindObject[*rbacv1.RoleBinding](objs, "tigera-linseed") + Expect(ok).To(BeTrue()) + + dp := apiServerDeployment(objs) + Expect(dp.Spec.Template.Spec.Volumes).To(ContainElement(HaveField("Name", render.LinseedTokenVolumeName))) + qs := container(dp, string(render.TigeraAPIServerQueryServerContainerName)) + Expect(qs).NotTo(BeNil()) + Expect(qs.Env).To(ContainElement(HaveField("Name", "LINSEED_TOKEN"))) + }) + }) + + Context("v3-CRD mode (no aggregation server)", func() { + It("renders the deployment skeleton with the query server and pulls it out of the delete list", func() { + cc := apiServerControllerContext(operatorv1.CalicoEnterprise, nil) + ecc, _, err := ext.ExtendContext(cc) + rc := ecc.RenderContext + Expect(err).NotTo(HaveOccurred()) + + cfg := &render.APIServerConfiguration{ + RequiresAggregationServer: false, + K8SServiceEndpoint: k8sapi.ServiceEndpoint{}, + Installation: cc.Installation, + APIServer: &operatorv1.APIServerSpec{}, + TLSKeyPair: apiServerKeyPair(cc), + TrustedBundle: rc.TrustedBundle, + KubernetesVersion: &common.VersionInfo{Major: 1, Minor: 31}, + } + comp, err := render.APIServer(cfg) + Expect(err).NotTo(HaveOccurred()) + Expect(comp.ResolveImages(nil)).NotTo(HaveOccurred()) + create, del := comp.Objects() + + // The base queues the deployment objects for deletion in v3-CRD mode. + _, ok := extensions.FindObject[*appsv1.Deployment](del, render.APIServerName) + Expect(ok).To(BeTrue()) + + ec := comp.(render.ExtensionContextProvider).ExtensionContext() + create, del = extensionstest.ApplyExtensionsWithContext(ext, render.ComponentNameAPIServer, rc, ec, create, del) + + // After the modifier, the deployment (with the query server container) is in the + // create list and out of the delete list. + dp, ok := extensions.FindObject[*appsv1.Deployment](create, render.APIServerName) + Expect(ok).To(BeTrue()) + Expect(container(dp, string(render.TigeraAPIServerQueryServerContainerName))).NotTo(BeNil()) + _, ok = extensions.FindObject[*appsv1.Deployment](del, render.APIServerName) + Expect(ok).To(BeFalse()) + }) + }) + + Context("sidecar / L7 injection", func() { + applicationLayerSidecar := func() *operatorv1.ApplicationLayer { + enabled := operatorv1.SidecarEnabled + return &operatorv1.ApplicationLayer{ + ObjectMeta: metav1.ObjectMeta{Name: utils.DefaultEnterpriseInstanceKey.Name}, + Spec: operatorv1.ApplicationLayerSpec{SidecarInjection: &enabled}, + } + } + + It("adds the L7 admission controller container, the sidecar webhook, and the L7 service port", func() { + cc := apiServerControllerContext(operatorv1.CalicoEnterprise, nil, applicationLayerSidecar()) + ecc, _, err := ext.ExtendContext(cc) + rc := ecc.RenderContext + Expect(err).NotTo(HaveOccurred()) + + objs, _ := renderAPIServer(cc, rc, apiServerKeyPair(cc)) + + dp := apiServerDeployment(objs) + Expect(container(dp, string(render.L7AdmissionControllerContainerName))).NotTo(BeNil()) + + _, ok := extensions.FindObject[*admregv1.MutatingWebhookConfiguration](objs, common.SidecarMutatingWebhookConfigName) + Expect(ok).To(BeTrue()) + + svc, ok := extensions.FindObject[*corev1.Service](objs, render.APIServerServiceName) + Expect(ok).To(BeTrue()) + Expect(svc.Spec.Ports).To(ContainElement(HaveField("Name", render.L7AdmissionControllerPortName))) + }) + + It("pulls the sidecar webhook out of the delete list", func() { + cc := apiServerControllerContext(operatorv1.CalicoEnterprise, nil, applicationLayerSidecar()) + ecc, _, err := ext.ExtendContext(cc) + rc := ecc.RenderContext + Expect(err).NotTo(HaveOccurred()) + + _, del := renderAPIServer(cc, rc, apiServerKeyPair(cc)) + _, ok := extensions.FindObject[*admregv1.MutatingWebhookConfiguration](del, common.SidecarMutatingWebhookConfigName) + Expect(ok).To(BeFalse()) + }) + }) +}) + +var _ = Describe("API server enterprise policy modifier", func() { + applyPolicy := func(cc contexts.ControllerContext, rc render.RenderContext) *v3.NetworkPolicy { + cfg := &render.APIServerConfiguration{ + RequiresAggregationServer: true, + K8SServiceEndpoint: k8sapi.ServiceEndpoint{}, + Installation: cc.Installation, + APIServer: &operatorv1.APIServerSpec{}, + } + comp := render.APIServerPolicy(cfg) + create, del := comp.Objects() + ec := comp.(render.ExtensionContextProvider).ExtensionContext() + objs, _ := extensionstest.ApplyExtensionsWithContext(ext, render.ComponentNameAPIServerPolicy, rc, ec, create, del) + policy, ok := extensions.FindObject[*v3.NetworkPolicy](objs, render.APIServerPolicyName) + Expect(ok).To(BeTrue()) + return policy + } + + It("leaves the egress rules as the base when no OIDC key validator is configured", func() { + cc := apiServerControllerContext(operatorv1.CalicoEnterprise, nil) + ecc, _, err := ext.ExtendContext(cc) + rc := ecc.RenderContext + Expect(err).NotTo(HaveOccurred()) + + policy := applyPolicy(cc, rc) + // The trailing rule remains the Pass rule (no OIDC egress rule inserted). + n := len(policy.Spec.Egress) + Expect(n).To(BeNumerically(">", 0)) + Expect(policy.Spec.Egress[n-1].Action).To(Equal(v3.Pass)) + }) + + It("adds the L7 admission controller ingress port when sidecar injection is enabled", func() { + enabled := operatorv1.SidecarEnabled + cc := apiServerControllerContext(operatorv1.CalicoEnterprise, nil, &operatorv1.ApplicationLayer{ + ObjectMeta: metav1.ObjectMeta{Name: utils.DefaultEnterpriseInstanceKey.Name}, + Spec: operatorv1.ApplicationLayerSpec{SidecarInjection: &enabled}, + }) + ecc, _, err := ext.ExtendContext(cc) + rc := ecc.RenderContext + Expect(err).NotTo(HaveOccurred()) + + policy := applyPolicy(cc, rc) + // Every ingress rule should carry the L7 admission controller port. + found := false + for _, rule := range policy.Spec.Ingress { + for _, p := range rule.Destination.Ports { + if p.MinPort == uint16(render.L7AdmissionControllerPort) { + found = true + } + } + } + Expect(found).To(BeTrue(), "expected the L7 admission controller ingress port") + }) +}) + +// cleanupAPIServer behaviour for the Calico variant: the base render component carries +// the enterprise cleanup modifier, which queues the enterprise RBAC for deletion. +var _ = Describe("API server Calico-variant cleanup", func() { + It("queues the enterprise RBAC for deletion", func() { + cc := apiServerControllerContext(operatorv1.Calico, nil) + ecc, _, err := ext.ExtendContext(cc) + rc := ecc.RenderContext + Expect(err).NotTo(HaveOccurred()) + + cfg := &render.APIServerConfiguration{ + RequiresAggregationServer: true, + K8SServiceEndpoint: k8sapi.ServiceEndpoint{}, + Installation: cc.Installation, + APIServer: &operatorv1.APIServerSpec{}, + TLSKeyPair: apiServerKeyPair(cc), + KubernetesVersion: &common.VersionInfo{Major: 1, Minor: 31}, + } + comp, err := render.APIServer(cfg) + Expect(err).NotTo(HaveOccurred()) + Expect(comp.ResolveImages(nil)).NotTo(HaveOccurred()) + create, del := comp.Objects() + ec := comp.(render.ExtensionContextProvider).ExtensionContext() + _, del = extensionstest.ApplyExtensionsWithContext(ext, render.ComponentNameAPIServer, rc, ec, create, del) + + _, ok := extensions.FindObject[*rbacv1.ClusterRole](del, "tigera-ui-user") + Expect(ok).To(BeTrue()) + _, ok = extensions.FindObject[*rbacv1.ClusterRole](del, "calico-apiserver") + Expect(ok).To(BeTrue()) + }) +}) diff --git a/pkg/enterprise/apiserver/suite_test.go b/pkg/enterprise/apiserver/suite_test.go new file mode 100644 index 0000000000..eb46a99362 --- /dev/null +++ b/pkg/enterprise/apiserver/suite_test.go @@ -0,0 +1,34 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package apiserver_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/tigera/operator/pkg/enterprise" + "github.com/tigera/operator/pkg/extensions" +) + +// ext is the enterprise extension Set under test, shared across the suite. It is +// immutable once built and the specs only read it, so a single instance is safe. +var ext *extensions.Set = enterprise.New() + +func TestAPIServer(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "pkg/enterprise/apiserver Suite") +} diff --git a/pkg/enterprise/clusterconnection/extension.go b/pkg/enterprise/clusterconnection/extension.go new file mode 100644 index 0000000000..7c81fe6748 --- /dev/null +++ b/pkg/enterprise/clusterconnection/extension.go @@ -0,0 +1,77 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package clusterconnection + +import ( + "fmt" + + k8serrors "k8s.io/apimachinery/pkg/api/errors" + + "github.com/tigera/operator/pkg/common" + "github.com/tigera/operator/pkg/controller/contexts" + "github.com/tigera/operator/pkg/controller/utils" + "github.com/tigera/operator/pkg/extensions" + "github.com/tigera/operator/pkg/render" + "github.com/tigera/operator/pkg/tls/certificatemanagement" +) + +// Register wires the clusterconnection controller hook into the variant. +func Register(v *extensions.Variant) { + v.Controller(contexts.ClusterConnectionController, clusterConnectionControllerExtension{}) +} + +// clusterConnectionControllerExtension is the Calico Enterprise controller-side hook +// for the clusterconnection (ManagementClusterConnection) controller. +type clusterConnectionControllerExtension struct{} + +// Validate rejects clusterconnection configuration Calico Enterprise does not +// support: a cluster cannot be both a management cluster and a managed cluster. +func (clusterConnectionControllerExtension) Validate(cc contexts.ControllerContext) error { + managementCluster, err := utils.GetManagementCluster(cc.Ctx, cc.Client) + if err != nil { + return fmt.Errorf("error reading ManagementCluster: %w", err) + } + if managementCluster != nil { + return fmt.Errorf("having both a ManagementCluster and a ManagementClusterConnection is not supported") + } + return nil +} + +// ExtendContext computes the Enterprise-specific Guardian inputs the controller +// reads back: the managed cluster version (CNXVersion) and whether the license +// permits the domain-based egress network policy. It creates no certificates, so it +// returns no managed keypairs. The OSS controller path supplies its own defaults +// when this hook is absent. +func (clusterConnectionControllerExtension) ExtendContext(cc contexts.ControllerContext) (contexts.ControllerContext, []certificatemanagement.KeyPairInterface, error) { + clusterInformation, err := utils.FetchClusterInformation(cc.Ctx, cc.Client) + if err != nil { + return cc, nil, fmt.Errorf("error querying ClusterInformation: %w", err) + } + + // Ensure the license can support enterprise policy before enabling the + // domain-based egress rules. A missing license simply leaves them disabled. + var includeEgressNetworkPolicy bool + if license, err := utils.FetchLicenseKey(cc.Ctx, cc.Client); err == nil { + includeEgressNetworkPolicy = utils.IsFeatureActive(license, common.EgressAccessControlFeature) + } else if !k8serrors.IsNotFound(err) { + return cc, nil, fmt.Errorf("error querying license: %w", err) + } + + cc.Extension = render.GuardianRenderData{ + Version: clusterInformation.Spec.CNXVersion, + IncludeEgressNetworkPolicy: includeEgressNetworkPolicy, + } + return cc, nil, nil +} diff --git a/pkg/enterprise/clusterconnection/extension_test.go b/pkg/enterprise/clusterconnection/extension_test.go new file mode 100644 index 0000000000..e3d85604ab --- /dev/null +++ b/pkg/enterprise/clusterconnection/extension_test.go @@ -0,0 +1,116 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package clusterconnection_test + +import ( + "context" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + + v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" + + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/apis" + "github.com/tigera/operator/pkg/common" + "github.com/tigera/operator/pkg/controller/contexts" + ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" + "github.com/tigera/operator/pkg/render" +) + +var _ = Describe("clusterconnection enterprise controller extension", func() { + var cli client.Client + + // controllerContext builds a ControllerContext selecting the enterprise + // clusterconnection hook against the given client. + controllerContext := func() contexts.ControllerContext { + return contexts.ControllerContext{ + RenderContext: render.RenderContext{ + Installation: &operatorv1.InstallationSpec{Variant: operatorv1.CalicoEnterprise}, + }, + Controller: contexts.ClusterConnectionController, + Ctx: context.Background(), + Client: cli, + } + } + + clusterInformation := func() *v3.ClusterInformation { + return &v3.ClusterInformation{ + ObjectMeta: metav1.ObjectMeta{Name: "default"}, + Spec: v3.ClusterInformationSpec{CNXVersion: "v3.99.0", CalicoVersion: "v3.99.0-calico"}, + } + } + + newClient := func(objs ...client.Object) client.Client { + scheme := runtime.NewScheme() + Expect(apis.AddToScheme(scheme, false)).NotTo(HaveOccurred()) + return ctrlrfake.DefaultFakeClientBuilder(scheme).WithObjects(objs...).Build() + } + + Describe("Validate", func() { + It("passes when no ManagementCluster exists", func() { + cli = newClient() + Expect(ext.Validate(controllerContext())).NotTo(HaveOccurred()) + }) + + It("rejects a cluster that is both a management and a managed cluster", func() { + cli = newClient(&operatorv1.ManagementCluster{ObjectMeta: metav1.ObjectMeta{Name: "tigera-secure"}}) + err := ext.Validate(controllerContext()) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("not supported")) + }) + }) + + Describe("ExtendContext", func() { + It("reports the managed cluster CNX version", func() { + cli = newClient(clusterInformation()) + ecc, managed, err := ext.ExtendContext(controllerContext()) + rc := ecc.RenderContext + Expect(err).NotTo(HaveOccurred()) + Expect(managed).To(BeEmpty()) + + data, ok := render.GuardianRenderDataFromContext(rc) + Expect(ok).To(BeTrue()) + Expect(data.Version).To(Equal("v3.99.0")) + Expect(data.IncludeEgressNetworkPolicy).To(BeFalse()) + }) + + It("enables the egress network policy when the license has the feature", func() { + license := &v3.LicenseKey{ + ObjectMeta: metav1.ObjectMeta{Name: "default"}, + Status: v3.LicenseKeyStatus{Features: []string{common.EgressAccessControlFeature}}, + } + cli = newClient(clusterInformation(), license) + ecc, _, err := ext.ExtendContext(controllerContext()) + rc := ecc.RenderContext + Expect(err).NotTo(HaveOccurred()) + + data, ok := render.GuardianRenderDataFromContext(rc) + Expect(ok).To(BeTrue()) + Expect(data.IncludeEgressNetworkPolicy).To(BeTrue()) + }) + + It("errors when ClusterInformation is missing", func() { + cli = newClient() + _, _, err := ext.ExtendContext(controllerContext()) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("ClusterInformation")) + }) + }) +}) diff --git a/pkg/enterprise/clusterconnection/suite_test.go b/pkg/enterprise/clusterconnection/suite_test.go new file mode 100644 index 0000000000..abd8adc498 --- /dev/null +++ b/pkg/enterprise/clusterconnection/suite_test.go @@ -0,0 +1,34 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package clusterconnection_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/tigera/operator/pkg/enterprise" + "github.com/tigera/operator/pkg/extensions" +) + +// ext is the enterprise extension Set under test, shared across the suite. It is +// immutable once built and the specs only read it, so a single instance is safe. +var ext *extensions.Set = enterprise.New() + +func TestClusterConnection(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "pkg/enterprise/clusterconnection Suite") +} diff --git a/pkg/enterprise/guardian/extension.go b/pkg/enterprise/guardian/extension.go new file mode 100644 index 0000000000..6dc1a95410 --- /dev/null +++ b/pkg/enterprise/guardian/extension.go @@ -0,0 +1,649 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package guardian + +import ( + "net" + "net/url" + + "github.com/sirupsen/logrus" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + "sigs.k8s.io/controller-runtime/pkg/client" + + v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" + "github.com/tigera/api/pkg/lib/numorstring" + + "github.com/tigera/operator/pkg/common" + "github.com/tigera/operator/pkg/extensions" + "github.com/tigera/operator/pkg/render" + "github.com/tigera/operator/pkg/render/common/networkpolicy" + "github.com/tigera/operator/pkg/render/common/securitycontextconstraints" + operatorurl "github.com/tigera/operator/pkg/url" +) + +// Register wires the guardian extensions into the variant. +func Register(v *extensions.Variant) { + extensions.RegisterModifier(v, render.GuardianName, modifyGuardian) + extensions.RegisterModifier(v, render.ComponentNameGuardianPolicy, modifyGuardianPolicy) +} + +// modifyGuardianPolicy replaces the core OSS guardian network policy with the +// enterprise management-cluster policy. Building the enterprise egress rules can +// fail (proxy URL parsing); on failure we drop the policy entirely, matching the +// core behavior of omitting it rather than installing a partial policy. +func modifyGuardianPolicy(rc render.RenderContext, gpc render.GuardianPolicyExtensionContext, objs, del []client.Object) ([]client.Object, []client.Object) { + policy, ok := extensions.FindObject[*v3.NetworkPolicy](objs, render.GuardianPolicyName) + if !ok { + return objs, del + } + + spec, err := enterpriseGuardianPolicySpec(gpc) + if err != nil { + logrus.WithError(err).Error("Failed to build guardian network policy, policy will be omitted") + return removeObject(objs, policy), del + } + policy.Spec = spec + return objs, del +} + +func removeObject(objs []client.Object, drop client.Object) []client.Object { + out := objs[:0] + for _, o := range objs { + if o != drop { + out = append(out, o) + } + } + return out +} + +// enterpriseGuardianPolicySpec builds the network policy spec for guardian in a +// managed cluster: egress to the management cluster components and the tunnel +// destination(s), and ingress from the management-cluster components that reach +// back over the tunnel. +func enterpriseGuardianPolicySpec(gpc render.GuardianPolicyExtensionContext) (v3.NetworkPolicySpec, error) { + egressRules := []v3.Rule{ + { + Action: v3.Allow, + Protocol: &networkpolicy.TCPProtocol, + Destination: render.PacketCaptureEntityRule, + }, + } + egressRules = networkpolicy.AppendDNSEgressRules(egressRules, gpc.OpenShift) + egressRules = append(egressRules, []v3.Rule{ + { + Action: v3.Allow, + Protocol: &networkpolicy.TCPProtocol, + Destination: networkpolicy.KubeAPIServerEntityRule, + }, + { + Action: v3.Allow, + Protocol: &networkpolicy.TCPProtocol, + Destination: networkpolicy.PrometheusEntityRule, + }, + { + Action: v3.Allow, + Protocol: &networkpolicy.TCPProtocol, + Destination: render.TigeraAPIServerEntityRule, + }, + }...) + + // Create an egress rule for each unique destination the guardian pods connect + // to. With multiple pods whose proxy settings differ, there are multiple + // destinations that must be allowed. + allowedDestinations := map[string]bool{} + for _, podProxyConfig := range render.ProcessPodProxies(gpc.PodProxies) { + var proxyURL *url.URL + var err error + if podProxyConfig != nil && podProxyConfig.HTTPSProxy != "" { + // The scheme is HTTPS, as we establish an mTLS session with the target. + // We expect the URL to be of the form host:port. + targetURL := &url.URL{Scheme: "https", Host: gpc.URL} + proxyURL, err = podProxyConfig.ProxyFunc()(targetURL) + if err != nil { + return v3.NetworkPolicySpec{}, err + } + } + + var tunnelDestinationHostPort string + if proxyURL != nil { + proxyHostPort, err := operatorurl.ParseHostPortFromHTTPProxyURL(proxyURL) + if err != nil { + return v3.NetworkPolicySpec{}, err + } + tunnelDestinationHostPort = proxyHostPort + } else { + // gpc.URL has host:port form. + tunnelDestinationHostPort = gpc.URL + } + + if allowedDestinations[tunnelDestinationHostPort] { + continue + } + + host, port, err := net.SplitHostPort(tunnelDestinationHostPort) + if err != nil { + return v3.NetworkPolicySpec{}, err + } + parsedPort, err := numorstring.PortFromString(port) + if err != nil { + return v3.NetworkPolicySpec{}, err + } + parsedIP := net.ParseIP(host) + if parsedIP == nil { + // Domain-based egress rules require the EgressAccessControl license feature. + if !gpc.IncludeEgressNetworkPolicy { + continue + } + egressRules = append(egressRules, v3.Rule{ + Action: v3.Allow, + Protocol: &networkpolicy.TCPProtocol, + Destination: v3.EntityRule{ + Domains: []string{host}, + Ports: []numorstring.Port{parsedPort}, + }, + }) + allowedDestinations[tunnelDestinationHostPort] = true + } else { + netSuffix := "/32" + if parsedIP.To4() == nil { + netSuffix = "/128" + } + egressRules = append(egressRules, v3.Rule{ + Action: v3.Allow, + Protocol: &networkpolicy.TCPProtocol, + Destination: v3.EntityRule{ + Nets: []string{parsedIP.String() + netSuffix}, + Ports: []numorstring.Port{parsedPort}, + }, + }) + allowedDestinations[tunnelDestinationHostPort] = true + } + } + + egressRules = append(egressRules, v3.Rule{Action: v3.Pass}) + + dest := v3.EntityRule{Ports: networkpolicy.Ports(render.GuardianTargetPort)} + helper := networkpolicy.DefaultHelper() + ingressRules := []v3.Rule{ + {Action: v3.Allow, Protocol: &networkpolicy.TCPProtocol, Source: render.FluentdSourceEntityRule, Destination: dest}, + {Action: v3.Allow, Protocol: &networkpolicy.TCPProtocol, Source: helper.ComplianceBenchmarkerSourceEntityRule(), Destination: dest}, + {Action: v3.Allow, Protocol: &networkpolicy.TCPProtocol, Source: helper.ComplianceReporterSourceEntityRule(), Destination: dest}, + {Action: v3.Allow, Protocol: &networkpolicy.TCPProtocol, Source: helper.ComplianceSnapshotterSourceEntityRule(), Destination: dest}, + {Action: v3.Allow, Protocol: &networkpolicy.TCPProtocol, Source: helper.ComplianceControllerSourceEntityRule(), Destination: dest}, + {Action: v3.Allow, Protocol: &networkpolicy.TCPProtocol, Source: render.IntrusionDetectionSourceEntityRule, Destination: dest}, + {Action: v3.Allow, Protocol: &networkpolicy.TCPProtocol, Source: render.IntrusionDetectionInstallerSourceEntityRule, Destination: dest}, + {Action: v3.Allow, Protocol: &networkpolicy.TCPProtocol, Destination: dest}, + } + + return v3.NetworkPolicySpec{ + Order: &networkpolicy.HighPrecedenceOrder, + Tier: networkpolicy.CalicoTierName, + Selector: networkpolicy.KubernetesAppSelector(render.GuardianName), + Types: []v3.PolicyType{v3.PolicyTypeIngress, v3.PolicyTypeEgress}, + Ingress: ingressRules, + Egress: egressRules, + }, nil +} + +// modifyGuardian layers Calico Enterprise behavior onto the rendered guardian +// objects: the secrets Role/RoleBinding and default UI settings, the +// elasticsearch/kibana service ports, the management-cluster-request cluster +// role rules (which replace the OSS rules), and the CA bundle env vars. +func modifyGuardian(rc render.RenderContext, gc render.GuardianExtensionContext, objs, del []client.Object) ([]client.Object, []client.Object) { + if role, ok := extensions.FindObject[*rbacv1.ClusterRole](objs, render.GuardianClusterRoleName); ok { + role.Rules = guardianEnterpriseRules(gc) + } + + if svc, ok := extensions.FindObject[*corev1.Service](objs, render.GuardianServiceName); ok { + svc.Spec.Ports = append(svc.Spec.Ports, guardianEnterpriseServicePorts()...) + } + + if dep, ok := extensions.FindObject[*appsv1.Deployment](objs, render.GuardianDeploymentName); ok { + addGuardianEnterpriseEnv(gc, dep) + } + + return append(objs, + guardianSecretsRole(), + guardianSecretRoleBinding(), + // Default UI settings for this managed cluster. + render.ManagerClusterWideSettingsGroup(), + render.ManagerUserSpecificSettingsGroup(), + render.ManagerClusterWideTigeraLayer(), + render.ManagerClusterWideDefaultView(), + ), del +} + +// guardianEnterpriseRules are the cluster role rules guardian needs in Calico +// Enterprise. They wholly replace the OSS rules: the management cluster drives +// guardian over the tunnel, so it needs the union of the rules its components +// require, plus any configured impersonation and the OpenShift SCC. +func guardianEnterpriseRules(gc render.GuardianExtensionContext) []rbacv1.PolicyRule { + var rules []rbacv1.PolicyRule + + if imp := gc.Impersonation; imp != nil { + if imp.Users != nil { + rules = append(rules, rbacv1.PolicyRule{ + APIGroups: []string{""}, + Resources: []string{"users"}, + ResourceNames: imp.Users, + Verbs: []string{"impersonate"}, + }) + } + if imp.Groups != nil { + rules = append(rules, rbacv1.PolicyRule{ + APIGroups: []string{""}, + Resources: []string{"groups"}, + ResourceNames: imp.Groups, + Verbs: []string{"impersonate"}, + }) + } + if imp.ServiceAccounts != nil { + rules = append(rules, rbacv1.PolicyRule{ + APIGroups: []string{""}, + Resources: []string{"serviceaccounts"}, + ResourceNames: imp.ServiceAccounts, + Verbs: []string{"impersonate"}, + }) + } + } + + rules = append(rules, rulesForManagementClusterRequests(gc.OpenShift)...) + + if gc.OpenShift { + rules = append(rules, rbacv1.PolicyRule{ + APIGroups: []string{"security.openshift.io"}, + Resources: []string{"securitycontextconstraints"}, + Verbs: []string{"use"}, + ResourceNames: []string{securitycontextconstraints.NonRootV2}, + }) + } + + return rules +} + +func guardianEnterpriseServicePorts() []corev1.ServicePort { + return []corev1.ServicePort{ + { + Name: "elasticsearch", + Port: 9200, + TargetPort: intstr.IntOrString{Type: intstr.Int, IntVal: 8080}, + Protocol: corev1.ProtocolTCP, + }, + { + Name: "kibana", + Port: 5601, + TargetPort: intstr.IntOrString{Type: intstr.Int, IntVal: 8080}, + Protocol: corev1.ProtocolTCP, + }, + } +} + +func addGuardianEnterpriseEnv(gc render.GuardianExtensionContext, dep *appsv1.Deployment) { + for i := range dep.Spec.Template.Spec.Containers { + c := &dep.Spec.Template.Spec.Containers[i] + if c.Name != render.GuardianContainerName { + continue + } + c.Env = append(c.Env, + corev1.EnvVar{Name: "GUARDIAN_PACKET_CAPTURE_CA_BUNDLE_PATH", Value: gc.TrustedBundleMountPath}, + corev1.EnvVar{Name: "GUARDIAN_PROMETHEUS_CA_BUNDLE_PATH", Value: gc.TrustedBundleMountPath}, + corev1.EnvVar{Name: "GUARDIAN_QUERYSERVER_CA_BUNDLE_PATH", Value: gc.TrustedBundleMountPath}, + ) + } +} + +// guardianSecretsRole creates a Role that allows the management cluster to +// provision secrets to the tigera-operator Namespace, used to push secrets the +// managed cluster needs to access / authenticate with the management cluster. +func guardianSecretsRole() *rbacv1.Role { + return &rbacv1.Role{ + TypeMeta: metav1.TypeMeta{Kind: "Role", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: render.GuardianSecretsRole, + Namespace: common.OperatorNamespace(), + }, + Rules: []rbacv1.PolicyRule{ + { + APIGroups: []string{""}, + Resources: []string{"secrets"}, + Verbs: []string{"create", "delete", "deletecollection", "update"}, + }, + }, + } +} + +func guardianSecretRoleBinding() *rbacv1.RoleBinding { + return &rbacv1.RoleBinding{ + TypeMeta: metav1.TypeMeta{Kind: "RoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: render.GuardianSecretsRoleBindingName, + Namespace: common.OperatorNamespace(), + }, + RoleRef: rbacv1.RoleRef{ + APIGroup: "rbac.authorization.k8s.io", + Kind: "Role", + Name: render.GuardianSecretsRole, + }, + Subjects: []rbacv1.Subject{ + { + Kind: "ServiceAccount", + Name: render.GuardianServiceAccountName, + Namespace: render.GuardianNamespace, + }, + }, + } +} + +// rulesForManagementClusterRequests returns the set of RBAC rules guardian needs +// to satisfy requests from the management cluster over the tunnel. +func rulesForManagementClusterRequests(isOpenShift bool) []rbacv1.PolicyRule { + rules := []rbacv1.PolicyRule{ + // Common rules required to handle requests from multiple components in the management cluster. + { + // ID uses read-only permissions and kube-controllers uses both read and write verbs. + APIGroups: []string{""}, + Resources: []string{"configmaps"}, + Verbs: []string{"create", "delete", "get", "list", "update", "watch"}, + }, + { + // Allows Linseed to watch namespaces before copying its token. + // Also enables PolicyRecommendation to watch namespaces, + // and Manager/kube-controllers to list them. + APIGroups: []string{""}, + Resources: []string{"namespaces"}, + Verbs: []string{"get", "list", "watch"}, + }, + { + // kube-controllers watches Nodes to monitor for deletions. + // Manager performs a list operation on Nodes. + APIGroups: []string{""}, + Resources: []string{"nodes"}, + Verbs: []string{"get", "list", "watch"}, + }, + { + // kube-controllers watches Pods to verify existence for IPAM garbage collection. + // Manager performs get operations on Pods. + APIGroups: []string{""}, + Resources: []string{"pods"}, + Verbs: []string{"get", "list", "watch"}, + }, + { + // The Federated Services Controller needs access to the remote kubeconfig secret + // in order to create a remote syncer. + APIGroups: []string{""}, + Resources: []string{"secrets"}, + Verbs: []string{"get", "list", "watch"}, + }, + { + // Manager uses list; kube-controllers uses 'get', 'list', 'watch', 'update'. + APIGroups: []string{""}, + Resources: []string{"services"}, + Verbs: []string{"get", "list", "update", "watch"}, + }, + { + // Needed by kube-controllers to validate licenses; also used by ID. + APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, + Resources: []string{"licensekeys"}, + Verbs: []string{"get", "watch"}, + }, + { + // Manager uses list; PolicyRecommendation & ID uses all verbs. + APIGroups: []string{"projectcalico.org"}, + Resources: []string{ + "globalnetworksets", + "networkpolicies", + "tier.networkpolicies", + "stagednetworkpolicies", + "tier.stagednetworkpolicies", + }, + Verbs: []string{"create", "delete", "get", "list", "patch", "update", "watch"}, + }, + { + // Manager uses list; PolicyRecommendation uses all verbs. + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"tiers"}, + Verbs: []string{"create", "delete", "get", "list", "patch", "update", "watch"}, + }, + // Rules needed by guardian to handle manager authorization reviews. + { + APIGroups: []string{"rbac.authorization.k8s.io"}, + Resources: []string{"clusterroles", "clusterrolebindings", "roles", "rolebindings"}, + Verbs: []string{"list", "get"}, + }, + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"uisettings", "uisettingsgroups"}, + Verbs: []string{"list", "get"}, + }, + + // Rules needed by guardian to handle other manager requests. + { + APIGroups: []string{""}, + Resources: []string{"events"}, + Verbs: []string{"list"}, + }, + { + APIGroups: []string{""}, + Resources: []string{"serviceaccounts"}, + Verbs: []string{"list"}, + }, + { + // Allow query server talk to Prometheus via the manager user. + APIGroups: []string{""}, + Resources: []string{"services/proxy"}, + ResourceNames: []string{ + "calico-node-prometheus:9090", + "https:calico-api:8080", + }, + Verbs: []string{"create", "get"}, + }, + { + APIGroups: []string{"apps"}, + Resources: []string{"daemonsets", "replicasets", "statefulsets"}, + Verbs: []string{"list"}, + }, + { + APIGroups: []string{"authentication.k8s.io"}, + Resources: []string{"tokenreviews"}, + Verbs: []string{"create"}, + }, + { + APIGroups: []string{"authorization.k8s.io"}, + Resources: []string{"subjectaccessreviews"}, + Verbs: []string{"create"}, + }, + { + APIGroups: []string{"networking.k8s.io"}, + Resources: []string{"networkpolicies"}, + Verbs: []string{"get", "list"}, + }, + { + APIGroups: []string{"policy.networking.k8s.io"}, + Resources: []string{ + "clusternetworkpolicies", + "adminnetworkpolicies", + "baselineadminnetworkpolicies", + }, + Verbs: []string{"list"}, + }, + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"alertexceptions"}, + Verbs: []string{"get", "list", "update"}, + }, + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"felixconfigurations"}, + ResourceNames: []string{"default"}, + Verbs: []string{"get"}, + }, + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{ + "globalnetworkpolicies", + "networksets", + "stagedglobalnetworkpolicies", + "stagedkubernetesnetworkpolicies", + "tier.globalnetworkpolicies", + "tier.stagedglobalnetworkpolicies", + }, + Verbs: []string{"list"}, + }, + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"hostendpoints"}, + Verbs: []string{"list"}, + }, + + // Rules needed by guardian to handle policy recommendation requests. + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{ + "policyrecommendationscopes", + "policyrecommendationscopes/status", + }, + Verbs: []string{"create", "delete", "get", "list", "patch", "update", "watch"}, + }, + + // Rules needed by guardian to handle calico-kube-controller requests. + { + // Nodes are watched to monitor for deletions. + APIGroups: []string{""}, + Resources: []string{"endpoints"}, + Verbs: []string{"create", "delete", "get", "list", "update", "watch"}, + }, + { + APIGroups: []string{""}, + Resources: []string{"services/status"}, + Verbs: []string{"get", "list", "update", "watch"}, + }, + { + // Needs to manage hostendpoints. + APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, + Resources: []string{"hostendpoints"}, + Verbs: []string{"create", "delete", "get", "list", "update", "watch"}, + }, + { + // Needs access to update clusterinformations. + APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, + Resources: []string{"clusterinformations"}, + Verbs: []string{"create", "get", "list", "update", "watch"}, + }, + { + // Needs to manipulate kubecontrollersconfiguration, which contains its config. + // It creates a default if none exists, and updates status as well. + APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, + Resources: []string{"kubecontrollersconfigurations"}, + Verbs: []string{"create", "get", "list", "update", "watch"}, + }, + { + APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, + Resources: []string{"tiers"}, + Verbs: []string{"create"}, + }, + { + APIGroups: []string{"crd.projectcalico.org", "projectcalico.org"}, + Resources: []string{"deeppacketinspections"}, + Verbs: []string{"get", "list", "watch"}, + }, + { + APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, + Resources: []string{"deeppacketinspections/status"}, + Verbs: []string{"update"}, + }, + { + APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, + Resources: []string{"packetcaptures"}, + Verbs: []string{"get", "list", "update"}, + }, + { + APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, + Resources: []string{"remoteclusterconfigurations"}, + Verbs: []string{"get", "list", "watch"}, + }, + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"licensekeys"}, + Verbs: []string{"create", "get", "list", "update", "watch"}, + }, + { + // Grant permissions to access ClusterInformation resources in managed clusters. + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"clusterinformations"}, + Verbs: []string{"get", "list", "watch"}, + }, + { + APIGroups: []string{"usage.tigera.io"}, + Resources: []string{"licenseusagereports"}, + Verbs: []string{"create", "delete", "get", "list", "update", "watch"}, + }, + + // Rules needed by guardian to handle Intrusion detection requests. + { + APIGroups: []string{""}, + Resources: []string{"podtemplates"}, + Verbs: []string{"get"}, + }, + { + APIGroups: []string{"apps"}, + Resources: []string{"deployments"}, + Verbs: []string{"get"}, + }, + { + APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, + Resources: []string{"alertexceptions"}, + Verbs: []string{"get", "list"}, + }, + { + APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, + Resources: []string{"securityeventwebhooks"}, + Verbs: []string{"get", "list", "update", "watch"}, + }, + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{ + "globalalerts", + "globalalerts/status", + "globalthreatfeeds", + "globalthreatfeeds/status", + }, + Verbs: []string{"create", "delete", "get", "list", "patch", "update", "watch"}, + }, + // Rules needed to fetch the compliance reports + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"globalreporttypes", "globalreports"}, + Verbs: []string{"get", "list", "watch"}, + }, + } + + // Rules needed by policy recommendation in openshift. + if isOpenShift { + rules = append(rules, + rbacv1.PolicyRule{ + APIGroups: []string{"security.openshift.io"}, + Resources: []string{"securitycontextconstraints"}, + Verbs: []string{"use"}, + ResourceNames: []string{securitycontextconstraints.HostNetworkV2}, + }, + ) + } + + return rules +} diff --git a/pkg/enterprise/guardian/extension_test.go b/pkg/enterprise/guardian/extension_test.go new file mode 100644 index 0000000000..e38918d924 --- /dev/null +++ b/pkg/enterprise/guardian/extension_test.go @@ -0,0 +1,102 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package guardian_test + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + client "sigs.k8s.io/controller-runtime/pkg/client" + + v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/extensions" + "github.com/tigera/operator/pkg/extensions/extensionstest" + "github.com/tigera/operator/pkg/render" +) + +var _ = Describe("guardian enterprise modifier", func() { + + // newObjs returns the subset of rendered guardian objects the modifier touches. + newObjs := func() []client.Object { + return []client.Object{ + &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: render.GuardianClusterRoleName}, Rules: []rbacv1.PolicyRule{{Verbs: []string{"get"}}}}, + &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{Name: render.GuardianServiceName, Namespace: render.GuardianNamespace}, + Spec: corev1.ServiceSpec{Ports: []corev1.ServicePort{{Name: "https", Port: 443}}}, + }, + &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{Name: render.GuardianDeploymentName, Namespace: render.GuardianNamespace}, + Spec: appsv1.DeploymentSpec{Template: corev1.PodTemplateSpec{Spec: corev1.PodSpec{ + Containers: []corev1.Container{{Name: render.GuardianContainerName}}, + }}}, + }, + } + } + + entCtx := render.RenderContext{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.CalicoEnterprise}} + + It("appends the secrets RBAC and UI settings", func() { + out, _ := extensionstest.ApplyExtensionsWithContext(ext, render.GuardianName, entCtx, render.GuardianExtensionContext{}, newObjs(), nil) + _, ok := extensions.FindObject[*rbacv1.Role](out, render.GuardianSecretsRole) + Expect(ok).To(BeTrue()) + _, ok = extensions.FindObject[*rbacv1.RoleBinding](out, render.GuardianSecretsRoleBindingName) + Expect(ok).To(BeTrue()) + _, ok = extensions.FindObject[*v3.UISettingsGroup](out, render.ManagerClusterSettings) + Expect(ok).To(BeTrue()) + }) + + It("adds the elasticsearch and kibana service ports", func() { + out, _ := extensionstest.ApplyExtensionsWithContext(ext, render.GuardianName, entCtx, render.GuardianExtensionContext{}, newObjs(), nil) + svc, _ := extensions.FindObject[*corev1.Service](out, render.GuardianServiceName) + names := []string{} + for _, p := range svc.Spec.Ports { + names = append(names, p.Name) + } + Expect(names).To(ContainElements("https", "elasticsearch", "kibana")) + }) + + It("replaces the cluster role rules and adds impersonation", func() { + gc := render.GuardianExtensionContext{ + Impersonation: &operatorv1.Impersonation{Users: []string{"foo"}, Groups: []string{"bar"}}, + } + out, _ := extensionstest.ApplyExtensionsWithContext(ext, render.GuardianName, entCtx, gc, newObjs(), nil) + role, _ := extensions.FindObject[*rbacv1.ClusterRole](out, render.GuardianClusterRoleName) + + // The single OSS placeholder rule is gone, replaced by the enterprise set. + Expect(role.Rules).NotTo(ContainElement(rbacv1.PolicyRule{Verbs: []string{"get"}})) + Expect(role.Rules).To(ContainElement(HaveField("ResourceNames", Equal([]string{"foo"})))) + Expect(role.Rules).To(ContainElement(HaveField("ResourceNames", Equal([]string{"bar"})))) + }) + + It("adds the CA bundle env to the guardian container", func() { + gc := render.GuardianExtensionContext{TrustedBundleMountPath: "/ca/bundle"} + out, _ := extensionstest.ApplyExtensionsWithContext(ext, render.GuardianName, entCtx, gc, newObjs(), nil) + dep, _ := extensions.FindObject[*appsv1.Deployment](out, render.GuardianDeploymentName) + Expect(dep.Spec.Template.Spec.Containers[0].Env).To(ContainElement(corev1.EnvVar{Name: "GUARDIAN_PROMETHEUS_CA_BUNDLE_PATH", Value: "/ca/bundle"})) + }) + + It("does nothing for the Calico variant", func() { + ctx := render.RenderContext{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.Calico}} + out, _ := extensionstest.ApplyExtensions(ext, render.GuardianName, ctx, newObjs(), nil) + Expect(out).To(HaveLen(len(newObjs()))) + role, _ := extensions.FindObject[*rbacv1.ClusterRole](out, render.GuardianClusterRoleName) + Expect(role.Rules).To(Equal([]rbacv1.PolicyRule{{Verbs: []string{"get"}}})) + }) +}) diff --git a/pkg/enterprise/guardian/guardian_render_test.go b/pkg/enterprise/guardian/guardian_render_test.go new file mode 100644 index 0000000000..fdb8ab382a --- /dev/null +++ b/pkg/enterprise/guardian/guardian_render_test.go @@ -0,0 +1,445 @@ +// Copyright (c) 2020-2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package guardian_test + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + netv1 "k8s.io/api/networking/v1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + + v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" + + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/apis" + "github.com/tigera/operator/pkg/common" + "github.com/tigera/operator/pkg/components" + "github.com/tigera/operator/pkg/controller/certificatemanager" + ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" + "github.com/tigera/operator/pkg/extensions/extensionstest" + "github.com/tigera/operator/pkg/render" + rmeta "github.com/tigera/operator/pkg/render/common/meta" + "github.com/tigera/operator/pkg/render/common/networkpolicy" + rtest "github.com/tigera/operator/pkg/render/common/test" + "github.com/tigera/operator/pkg/render/testutils" +) + +// The guardian policy fixtures live next to the render package's testutils, so +// reference them relative to this enterprise subpackage. +const ( + clusterDomain = "cluster.local" + guardianPolicyJSON = "../../render/testutils/expected_policies/guardian.json" + guardianPolicyOCPJSON = "../../render/testutils/expected_policies/guardian_ocp.json" +) + +// guardianObjects renders the guardian component and applies the registered +// enterprise modifier the way the componentHandler does. +func guardianObjects(cfg *render.GuardianConfiguration) []client.Object { + g := render.Guardian(cfg) + ExpectWithOffset(1, g.ResolveImages(nil)).To(BeNil()) + objs, _ := g.Objects() + rc := render.RenderContext{Installation: cfg.Installation} + var extCtx any + if p, ok := g.(render.ExtensionContextProvider); ok { + extCtx = p.ExtensionContext() + } + out, _ := extensionstest.ApplyExtensionsWithContext(ext, render.GuardianName, rc, extCtx, objs, nil) + return out +} + +var _ = Describe("Guardian enterprise rendering tests", func() { + var cfg *render.GuardianConfiguration + var g render.Component + var resources []client.Object + var deleteResources []client.Object + + createGuardianConfig := func(i operatorv1.InstallationSpec, addr string, openshift bool) *render.GuardianConfiguration { + i.Variant = operatorv1.CalicoEnterprise + secret := &corev1.Secret{ + TypeMeta: metav1.TypeMeta{Kind: "Secret", APIVersion: "v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: render.GuardianSecretName, + Namespace: common.OperatorNamespace(), + }, + Data: map[string][]byte{ + "cert": []byte("foo"), + "key": []byte("bar"), + }, + } + scheme := runtime.NewScheme() + Expect(apis.AddToScheme(scheme, false)).NotTo(HaveOccurred()) + cli := ctrlrfake.DefaultFakeClientBuilder(scheme).Build() + + certificateManager, err := certificatemanager.Create(cli, nil, clusterDomain, common.OperatorNamespace(), certificatemanager.AllowCACreation()) + Expect(err).NotTo(HaveOccurred()) + + bundle := certificateManager.CreateTrustedBundle() + + return &render.GuardianConfiguration{ + URL: addr, + PullSecrets: []*corev1.Secret{{ + TypeMeta: metav1.TypeMeta{Kind: "Secret", APIVersion: "v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "pull-secret", + Namespace: common.OperatorNamespace(), + }, + }}, + Installation: &i, + TunnelSecret: secret, + TrustedCertBundle: bundle, + OpenShift: openshift, + ManagementClusterConnection: &operatorv1.ManagementClusterConnection{}, + IncludeEgressNetworkPolicy: true, + } + } + + Context("Guardian component", func() { + renderGuardian := func(i operatorv1.InstallationSpec) { + cfg = createGuardianConfig(i, "127.0.0.1:1234", false) + g = render.Guardian(cfg) + Expect(g.ResolveImages(nil)).To(BeNil()) + resources, deleteResources = g.Objects() + // Apply the registered enterprise modifier the way the componentHandler + // does, so these enterprise tests exercise the integrated output. + rc := render.RenderContext{Installation: cfg.Installation} + var extCtx any + if p, ok := g.(render.ExtensionContextProvider); ok { + extCtx = p.ExtensionContext() + } + resources, _ = extensionstest.ApplyExtensionsWithContext(ext, render.GuardianName, rc, extCtx, resources, nil) + } + + BeforeEach(func() { + renderGuardian(operatorv1.InstallationSpec{Registry: "my-reg/"}) + }) + + It("should render all resources for a managed cluster", func() { + expectedResources := []client.Object{ + &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: render.GuardianServiceAccountName, Namespace: render.GuardianNamespace}, TypeMeta: metav1.TypeMeta{Kind: "ServiceAccount", APIVersion: "v1"}}, + &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: render.GuardianClusterRoleName}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, + &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: render.GuardianClusterRoleBindingName}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, + &rbacv1.Role{ObjectMeta: metav1.ObjectMeta{Name: render.GuardianSecretsRole, Namespace: "tigera-operator"}, TypeMeta: metav1.TypeMeta{Kind: "Role", APIVersion: "rbac.authorization.k8s.io/v1"}}, + &rbacv1.RoleBinding{ObjectMeta: metav1.ObjectMeta{Name: render.GuardianSecretsRoleBindingName, Namespace: "tigera-operator"}, TypeMeta: metav1.TypeMeta{Kind: "RoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, + &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: render.GuardianDeploymentName, Namespace: render.GuardianNamespace}, TypeMeta: metav1.TypeMeta{Kind: "Deployment", APIVersion: "apps/v1"}}, + &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: render.GuardianServiceName, Namespace: render.GuardianNamespace}, TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: ""}}, + &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: render.GuardianSecretName, Namespace: render.GuardianNamespace}, TypeMeta: metav1.TypeMeta{Kind: "Secret", APIVersion: "v1"}}, + &v3.UISettingsGroup{ObjectMeta: metav1.ObjectMeta{Name: render.ManagerClusterSettings}, TypeMeta: metav1.TypeMeta{Kind: "UISettingsGroup", APIVersion: "projectcalico.org/v3"}}, + &v3.UISettingsGroup{ObjectMeta: metav1.ObjectMeta{Name: render.ManagerUserSettings}, TypeMeta: metav1.TypeMeta{Kind: "UISettingsGroup", APIVersion: "projectcalico.org/v3"}}, + &v3.UISettings{ObjectMeta: metav1.ObjectMeta{Name: render.ManagerClusterSettingsLayerTigera}, TypeMeta: metav1.TypeMeta{Kind: "UISettings", APIVersion: "projectcalico.org/v3"}}, + &v3.UISettings{ObjectMeta: metav1.ObjectMeta{Name: render.ManagerClusterSettingsViewDefault}, TypeMeta: metav1.TypeMeta{Kind: "UISettings", APIVersion: "projectcalico.org/v3"}}, + } + + expectedDeleteResources := []client.Object{ + &corev1.Namespace{TypeMeta: metav1.TypeMeta{Kind: "Namespace", APIVersion: "v1"}, ObjectMeta: metav1.ObjectMeta{Name: "tigera-guardian"}}, + &rbacv1.ClusterRole{TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, ObjectMeta: metav1.ObjectMeta{Name: "tigera-guardian"}}, + &rbacv1.ClusterRoleBinding{TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, ObjectMeta: metav1.ObjectMeta{Name: "tigera-guardian"}}, + &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "tigera-manager"}, TypeMeta: metav1.TypeMeta{Kind: "Namespace", APIVersion: "v1"}}, + &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: "tigera-manager", Namespace: "tigera-manager"}, TypeMeta: metav1.TypeMeta{Kind: "ServiceAccount", APIVersion: "v1"}}, + &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "tigera-manager-role"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, + &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "tigera-manager-binding"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, + &netv1.NetworkPolicy{ObjectMeta: metav1.ObjectMeta{Name: "guardian", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "NetworkPolicy", APIVersion: "networking.k8s.io/v1"}}, + } + + rtest.ExpectResources(resources, expectedResources) + rtest.ExpectResources(deleteResources, expectedDeleteResources) + + deployment := rtest.GetResource(resources, render.GuardianDeploymentName, render.GuardianNamespace, "apps", "v1", "Deployment").(*appsv1.Deployment) + Expect(deployment.Spec.Template.Spec.Containers).To(HaveLen(1)) + Expect(deployment.Spec.Template.Spec.Containers[0].Image).Should(Equal("my-reg/tigera/calico:" + components.ComponentTigeraCalico.Version)) + + Expect(*deployment.Spec.Template.Spec.Containers[0].SecurityContext.AllowPrivilegeEscalation).To(BeFalse()) + Expect(*deployment.Spec.Template.Spec.Containers[0].SecurityContext.Privileged).To(BeFalse()) + Expect(*deployment.Spec.Template.Spec.Containers[0].SecurityContext.RunAsGroup).To(BeEquivalentTo(10001)) + Expect(*deployment.Spec.Template.Spec.Containers[0].SecurityContext.RunAsNonRoot).To(BeTrue()) + Expect(*deployment.Spec.Template.Spec.Containers[0].SecurityContext.RunAsUser).To(BeEquivalentTo(10001)) + Expect(deployment.Spec.Template.Spec.Containers[0].SecurityContext.SeccompProfile).To(Equal( + &corev1.SeccompProfile{ + Type: corev1.SeccompProfileTypeRuntimeDefault, + })) + Expect(deployment.Spec.Template.Spec.Containers[0].SecurityContext.Capabilities).To(Equal( + &corev1.Capabilities{ + Drop: []corev1.Capability{"ALL"}, + }, + )) + }) + + It("should render controlPlaneTolerations", func() { + t := corev1.Toleration{ + Key: "foo", + Operator: corev1.TolerationOpEqual, + Value: "bar", + } + renderGuardian(operatorv1.InstallationSpec{ + ControlPlaneTolerations: []corev1.Toleration{t}, + }) + deployment := rtest.GetResource(resources, render.GuardianDeploymentName, render.GuardianNamespace, "apps", "v1", "Deployment").(*appsv1.Deployment) + Expect(deployment.Spec.Template.Spec.Tolerations).Should(ContainElements(append(rmeta.TolerateCriticalAddonsAndControlPlane, t))) + }) + + It("should render toleration on GKE", func() { + renderGuardian(operatorv1.InstallationSpec{ + KubernetesProvider: operatorv1.ProviderGKE, + }) + deployment := rtest.GetResource(resources, render.GuardianDeploymentName, render.GuardianNamespace, "apps", "v1", "Deployment").(*appsv1.Deployment) + Expect(deployment).NotTo(BeNil()) + Expect(deployment.Spec.Template.Spec.Tolerations).To(ContainElements(corev1.Toleration{ + Key: "kubernetes.io/arch", + Operator: corev1.TolerationOpEqual, + Value: "arm64", + Effect: corev1.TaintEffectNoSchedule, + })) + }) + + It("should render guardian with unlimited impersonation", func() { + cfg.ManagementClusterConnection = &operatorv1.ManagementClusterConnection{ + Spec: operatorv1.ManagementClusterConnectionSpec{ + Impersonation: &operatorv1.Impersonation{ + Users: []string{}, + Groups: []string{}, + ServiceAccounts: []string{}, + }, + }, + } + + resources := guardianObjects(cfg) + Expect(resources).ToNot(BeNil()) + + clusterRole, ok := rtest.GetResource(resources, render.GuardianClusterRoleName, "", "rbac.authorization.k8s.io", "v1", "ClusterRole").(*rbacv1.ClusterRole) + Expect(ok).To(BeTrue()) + + foundUserImp, foundGroupImp, foundSaImp := false, false, false + for _, rule := range clusterRole.Rules { + if rule.Verbs[0] == "impersonate" { + if rule.Resources[0] == "users" { + Expect(rule.ResourceNames).To(Equal([]string{})) + foundUserImp = true + } + if rule.Resources[0] == "groups" { + Expect(rule.ResourceNames).To(Equal([]string{})) + foundGroupImp = true + } + if rule.Resources[0] == "serviceaccounts" { + Expect(rule.ResourceNames).To(Equal([]string{})) + foundSaImp = true + } + } + } + + Expect(foundUserImp).To(BeTrue()) + Expect(foundGroupImp).To(BeTrue()) + Expect(foundSaImp).To(BeTrue()) + }) + + It("should render guardian with specific impersonation", func() { + cfg.ManagementClusterConnection = &operatorv1.ManagementClusterConnection{ + Spec: operatorv1.ManagementClusterConnectionSpec{ + Impersonation: &operatorv1.Impersonation{ + Users: []string{"foo"}, + Groups: []string{"bar"}, + ServiceAccounts: []string{"zaz"}, + }, + }, + } + + resources := guardianObjects(cfg) + Expect(resources).ToNot(BeNil()) + + clusterRole, ok := rtest.GetResource(resources, render.GuardianClusterRoleName, "", "rbac.authorization.k8s.io", "v1", "ClusterRole").(*rbacv1.ClusterRole) + Expect(ok).To(BeTrue()) + + foundUserImp, foundGroupImp, foundSaImp := false, false, false + for _, rule := range clusterRole.Rules { + if rule.Verbs[0] == "impersonate" { + if rule.Resources[0] == "users" { + Expect(rule.ResourceNames).To(Equal([]string{"foo"})) + foundUserImp = true + } + if rule.Resources[0] == "groups" { + Expect(rule.ResourceNames).To(Equal([]string{"bar"})) + foundGroupImp = true + } + if rule.Resources[0] == "serviceaccounts" { + Expect(rule.ResourceNames).To(Equal([]string{"zaz"})) + foundSaImp = true + } + } + } + + Expect(foundUserImp).To(BeTrue()) + Expect(foundGroupImp).To(BeTrue()) + Expect(foundSaImp).To(BeTrue()) + }) + + It("should render guardian with specific no sa permissions but with user and group", func() { + cfg.ManagementClusterConnection = &operatorv1.ManagementClusterConnection{ + Spec: operatorv1.ManagementClusterConnectionSpec{ + Impersonation: &operatorv1.Impersonation{ + Users: []string{}, + Groups: []string{}, + }, + }, + } + + resources := guardianObjects(cfg) + Expect(resources).ToNot(BeNil()) + + clusterRole, ok := rtest.GetResource(resources, render.GuardianClusterRoleName, "", "rbac.authorization.k8s.io", "v1", "ClusterRole").(*rbacv1.ClusterRole) + Expect(ok).To(BeTrue()) + + foundUserImp, foundGroupImp, foundSaImp := false, false, false + for _, rule := range clusterRole.Rules { + if rule.Verbs[0] == "impersonate" { + if rule.Resources[0] == "users" { + foundUserImp = true + } + if rule.Resources[0] == "groups" { + foundGroupImp = true + } + if rule.Resources[0] == "serviceaccounts" { + foundSaImp = true + } + } + } + + Expect(foundUserImp).To(BeTrue()) + Expect(foundGroupImp).To(BeTrue()) + Expect(foundSaImp).To(BeFalse()) + }) + }) + + It("should render SecurityContextConstrains properly when provider is OpenShift", func() { + cfg = createGuardianConfig(operatorv1.InstallationSpec{Registry: "my-reg/"}, "127.0.0.1:1234", false) + cfg.Installation.KubernetesProvider = operatorv1.ProviderOpenShift + cfg.OpenShift = true + resources := guardianObjects(cfg) + + role := rtest.GetResource(resources, render.GuardianClusterRoleName, "", "rbac.authorization.k8s.io", "v1", "ClusterRole").(*rbacv1.ClusterRole) + Expect(role.Rules).To(ContainElement(rbacv1.PolicyRule{ + APIGroups: []string{"security.openshift.io"}, + Resources: []string{"securitycontextconstraints"}, + Verbs: []string{"use"}, + ResourceNames: []string{"nonroot-v2"}, + })) + }) + + Context("GuardianPolicy component", func() { + guardianPolicy := testutils.GetExpectedPolicyFromFile(guardianPolicyJSON) + guardianPolicyForOCP := testutils.GetExpectedPolicyFromFile(guardianPolicyOCPJSON) + + renderGuardianPolicy := func(addr string, openshift bool, variant operatorv1.ProductVariant, includeEgressNetworkPolicy bool) { + installation := operatorv1.InstallationSpec{ + Registry: "my-reg/", + } + cfg := createGuardianConfig(installation, addr, openshift) + cfg.Installation.Variant = variant + cfg.IncludeEgressNetworkPolicy = includeEgressNetworkPolicy + g, err := render.GuardianPolicy(cfg) + Expect(err).NotTo(HaveOccurred()) + objs, _ := g.Objects() + // Apply the registered enterprise modifier the way the componentHandler + // does, so the enterprise policy is exercised. + rc := render.RenderContext{Installation: cfg.Installation} + var extCtx any + if p, ok := g.(render.ExtensionContextProvider); ok { + extCtx = p.ExtensionContext() + } + resources, _ = extensionstest.ApplyExtensionsWithContext(ext, render.ComponentNameGuardianPolicy, rc, extCtx, objs, nil) + } + + Context("policy rendering based on variant and IncludeEgressNetworkPolicy", func() { + It("should render Enterprise network policy without domain-based egress when IncludeEgressNetworkPolicy is false", func() { + // Enterprise variant with IncludeEgressNetworkPolicy=false should render a policy but skip domain-based egress rules + renderGuardianPolicy("my-management.example.com:1234", false, operatorv1.CalicoEnterprise, false) + + policyName := types.NamespacedName{Name: "calico-system.guardian-access", Namespace: "calico-system"} + policy := testutils.GetCalicoSystemPolicyFromResources(policyName, resources) + Expect(policy).NotTo(BeNil(), "Enterprise variant should always render a network policy when tier exists") + + // Verify it's the Enterprise policy (should have both Ingress and Egress types) + Expect(policy.Spec.Types).To(ConsistOf(v3.PolicyTypeIngress, v3.PolicyTypeEgress)) + Expect(policy.Spec.Egress).NotTo(BeEmpty()) + + // Verify no domain-based egress rules are present + for _, rule := range policy.Spec.Egress { + Expect(rule.Destination.Domains).To(BeEmpty(), + "Domain-based egress rules should not be present when IncludeEgressNetworkPolicy is false") + } + }) + + It("should render Enterprise network policy with domain-based egress when IncludeEgressNetworkPolicy is true", func() { + // Enterprise variant with IncludeEgressNetworkPolicy=true should render the full policy including domain-based egress + renderGuardianPolicy("my-management.example.com:1234", false, operatorv1.CalicoEnterprise, true) + + policyName := types.NamespacedName{Name: "calico-system.guardian-access", Namespace: "calico-system"} + policy := testutils.GetCalicoSystemPolicyFromResources(policyName, resources) + Expect(policy).NotTo(BeNil(), "Enterprise variant with IncludeEgressNetworkPolicy=true should render a network policy") + + // Verify it's the Enterprise policy (should have both Ingress and Egress types) + Expect(policy.Spec.Types).To(ConsistOf(v3.PolicyTypeIngress, v3.PolicyTypeEgress)) + Expect(policy.Spec.Egress).NotTo(BeEmpty()) + + // Verify domain-based egress rule is present + hasDomainRule := false + for _, rule := range policy.Spec.Egress { + if len(rule.Destination.Domains) > 0 { + hasDomainRule = true + break + } + } + Expect(hasDomainRule).To(BeTrue(), "Domain-based egress rule should be present when IncludeEgressNetworkPolicy is true") + }) + }) + + Context("calico-system rendering", func() { + policyName := types.NamespacedName{Name: "calico-system.guardian-access", Namespace: "calico-system"} + + getExpectedPolicy := func(name types.NamespacedName, scenario testutils.CalicoSystemScenario) *v3.NetworkPolicy { + if name.Name == "calico-system.guardian-access" && scenario.ManagedCluster { + return testutils.SelectPolicyByProvider(scenario, guardianPolicy, guardianPolicyForOCP) + } + + return nil + } + + DescribeTable("should render calico-system policy", + func(scenario testutils.CalicoSystemScenario) { + renderGuardianPolicy("127.0.0.1:1234", scenario.OpenShift, operatorv1.CalicoEnterprise, true) + policy := testutils.GetCalicoSystemPolicyFromResources(policyName, resources) + expectedPolicy := getExpectedPolicy(policyName, scenario) + Expect(policy).To(Equal(expectedPolicy)) + }, + Entry("for managed, kube-dns", testutils.CalicoSystemScenario{ManagedCluster: true, OpenShift: false}), + Entry("for managed, openshift-dns", testutils.CalicoSystemScenario{ManagedCluster: true, OpenShift: true}), + ) + + // The test matrix above validates against an IP-based management cluster address. + // Validate policy adaptation for domain-based management cluster address here. + It("should adapt Guardian policy if ManagementClusterAddr is domain-based", func() { + renderGuardianPolicy("mydomain.io:8080", false, operatorv1.CalicoEnterprise, true) + policy := testutils.GetCalicoSystemPolicyFromResources(policyName, resources) + managementClusterEgressRule := policy.Spec.Egress[5] + Expect(managementClusterEgressRule.Destination.Domains).To(Equal([]string{"mydomain.io"})) + Expect(managementClusterEgressRule.Destination.Ports).To(Equal(networkpolicy.Ports(8080))) + }) + }) + }) +}) diff --git a/pkg/enterprise/guardian/suite_test.go b/pkg/enterprise/guardian/suite_test.go new file mode 100644 index 0000000000..b83fbf63e8 --- /dev/null +++ b/pkg/enterprise/guardian/suite_test.go @@ -0,0 +1,34 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package guardian_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/tigera/operator/pkg/enterprise" + "github.com/tigera/operator/pkg/extensions" +) + +// ext is the enterprise extension Set under test, shared across the suite. It is +// immutable once built and the specs only read it, so a single instance is safe. +var ext *extensions.Set = enterprise.New() + +func TestGuardian(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "pkg/enterprise/guardian Suite") +} diff --git a/pkg/enterprise/installation/core.go b/pkg/enterprise/installation/core.go new file mode 100644 index 0000000000..d983c2d665 --- /dev/null +++ b/pkg/enterprise/installation/core.go @@ -0,0 +1,243 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package installation + +import ( + "fmt" + "strings" + + v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" + rbacv1 "k8s.io/api/rbac/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/handler" + + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/common" + "github.com/tigera/operator/pkg/controller/contexts" + "github.com/tigera/operator/pkg/controller/utils" + "github.com/tigera/operator/pkg/ctrlruntime" + "github.com/tigera/operator/pkg/dns" + "github.com/tigera/operator/pkg/extensions" + "github.com/tigera/operator/pkg/render" + relasticsearch "github.com/tigera/operator/pkg/render/common/elasticsearch" + "github.com/tigera/operator/pkg/render/kubecontrollers" + "github.com/tigera/operator/pkg/render/monitor" + "github.com/tigera/operator/pkg/tls/certificatemanagement" +) + +// Register wires the installation controller hook and the node and +// calico-kube-controllers modifiers into the variant. +func Register(v *extensions.Variant) { + v.Controller(contexts.InstallationController, coreControllerExtension{}) + registerNode(v) + registerKubeControllers(v) +} + +// coreControllerExtension is the Calico Enterprise controller-side hook for the +// installation controller. +type coreControllerExtension struct{} + +// installationRenderData is the controller-produced data the installation +// extension hands to its modifiers through RenderContext.Extension. The node +// modifier type-asserts it back out. +type installationRenderData struct { + nodePrometheusTLS certificatemanagement.KeyPairInterface + + // kubeControllerTLS is the calico-kube-controllers metrics serving keypair; the + // kube-controllers modifier mounts it onto the deployment. + kubeControllerTLS certificatemanagement.KeyPairInterface + + // collectProcessPath mirrors LogCollector.Spec.CollectProcessPath being + // enabled; the node modifier uses it to set HostPID and the felix env. + collectProcessPath bool + + // calico-kube-controllers enterprise additions the kube-controllers modifier + // applies: the enterprise cluster role rules, the enterprise enabled controllers, + // and the WAF v3 (Gateway API add-on) surface. + kubeControllerRules []rbacv1.PolicyRule + kubeControllerControllers []string + waf wafRenderData +} + +// installationData pulls the installation extension's render data back out of the +// render context, returning the zero value when none is set. +func installationData(rc render.RenderContext) installationRenderData { + return render.ExtractExtensionData[installationRenderData](rc) +} + +func collectProcessPathEnabled(lc *operatorv1.LogCollector) bool { + return lc != nil && + lc.Spec.CollectProcessPath != nil && + *lc.Spec.CollectProcessPath == operatorv1.CollectProcessPathEnable +} + +// Validate rejects installation config Calico Enterprise does not support. +func (coreControllerExtension) Validate(cc contexts.ControllerContext) error { + return ValidateReporterPort(cc.FelixConfiguration) +} + +// DefaultFelixConfiguration sets the Enterprise-only FelixConfiguration defaults. +// Some platforms run a DNS service that isn't named "kube-dns", so dnsTrustedServers +// needs a provider-specific default for Enterprise DNS logging to work. Returns +// whether it changed fc. +func (coreControllerExtension) DefaultFelixConfiguration(install *operatorv1.InstallationSpec, fc *v3.FelixConfiguration) (bool, error) { + dnsService := "" + switch install.KubernetesProvider { + case operatorv1.ProviderOpenShift: + dnsService = "k8s-service:openshift-dns/dns-default" + case operatorv1.ProviderRKE2: + dnsService = "k8s-service:kube-system/rke2-coredns-rke2-coredns" + } + if dnsService == "" { + return false, nil + } + + felixDefault := "k8s-service:kube-dns" + trustedServers := []string{dnsService} + // Keep any other values that are already configured, excepting the value we are + // setting and the kube-dns default. + existingSetting := "" + if fc.Spec.DNSTrustedServers != nil { + existingSetting = strings.Join(*fc.Spec.DNSTrustedServers, ",") + for _, server := range *fc.Spec.DNSTrustedServers { + if server != felixDefault && server != dnsService { + trustedServers = append(trustedServers, server) + } + } + } + if strings.Join(trustedServers, ",") == existingSetting { + return false, nil + } + fc.Spec.DNSTrustedServers = &trustedServers + return true, nil +} + +// Watches registers the enterprise resources the installation controller +// reconciles on. +func (coreControllerExtension) Watches(c ctrlruntime.Controller) error { + for _, obj := range []client.Object{ + &operatorv1.ManagementCluster{}, + &operatorv1.ManagementClusterConnection{}, + &operatorv1.LogCollector{}, + // GatewayAPI.spec.extensions.waf.state gates the WAF v3 surface on calico-kube-controllers. + &operatorv1.GatewayAPI{}, + } { + if err := c.WatchObject(obj, &handler.EnqueueRequestForObject{}); err != nil { + return err + } + } + // es-kube-controllers includes the manager internal TLS secret in its bundle. + return utils.AddSecretsWatch(c, render.ManagerInternalTLSSecretName, common.OperatorNamespace()) +} + +// ExtendContext does the controller-side work the modifiers can't: creating and +// fetching the certificates that feed the trusted bundle. It returns the render +// context carrying the produced node prometheus keypair, and that keypair as one +// the controller should manage. +func (coreControllerExtension) ExtendContext(cc contexts.ControllerContext) (contexts.ControllerContext, []certificatemanagement.KeyPairInterface, error) { + + nodePrometheusTLS, err := cc.CertificateManager.GetOrCreateKeyPair( + cc.Client, + render.NodePrometheusTLSServerSecret, + common.OperatorNamespace(), + dns.GetServiceDNSNames(render.CalicoNodeMetricsService, common.CalicoNamespace, cc.ClusterDomain), + ) + if err != nil { + return cc, nil, fmt.Errorf("error creating node prometheus TLS certificate: %w", err) + } + if nodePrometheusTLS != nil { + cc.TrustedBundle.AddCertificates(nodePrometheusTLS) + } + + // The calico-kube-controllers metrics endpoint is served with mTLS in + // Enterprise; the keypair is created here (cluster side effect) and mounted by + // the kube-controllers modifier. + kubeControllerTLS, err := cc.CertificateManager.GetOrCreateKeyPair( + cc.Client, + kubecontrollers.KubeControllerPrometheusTLSSecret, + common.OperatorNamespace(), + dns.GetServiceDNSNames(kubecontrollers.KubeControllerMetrics, common.CalicoNamespace, cc.ClusterDomain), + ) + if err != nil { + return cc, nil, fmt.Errorf("error creating kube-controllers metrics TLS certificate: %w", err) + } + if kubeControllerTLS != nil { + cc.TrustedBundle.AddCertificates(kubeControllerTLS) + } + + logCollector, err := utils.GetLogCollector(cc.Ctx, cc.Client) + if err != nil { + return cc, nil, fmt.Errorf("error reading LogCollector: %w", err) + } + + // calico-kube-controllers enterprise additions: the WAF surface, the enterprise + // cluster role rules, and the enterprise enabled controllers. A managed cluster's + // kube-controllers needs an extra license-push rule. + managementClusterConnection, err := utils.GetManagementClusterConnection(cc.Ctx, cc.Client) + if err != nil { + return cc, nil, fmt.Errorf("error reading ManagementClusterConnection: %w", err) + } + waf, wafWebhookTLS, err := buildWAFData(cc) + if err != nil { + return cc, nil, fmt.Errorf("error preparing WAF configuration: %w", err) + } + + cc.Extension = installationRenderData{ + nodePrometheusTLS: nodePrometheusTLS, + kubeControllerTLS: kubeControllerTLS, + collectProcessPath: collectProcessPathEnabled(logCollector), + kubeControllerRules: calicoKubeControllersEnterpriseRules(waf.enabled, managementClusterConnection != nil), + kubeControllerControllers: calicoKubeControllersEnterpriseControllers(waf.enabled), + waf: waf, + } + + prometheusClientCert, err := cc.CertificateManager.GetCertificate(cc.Client, monitor.PrometheusClientTLSSecretName, common.OperatorNamespace()) + if err != nil { + return cc, nil, fmt.Errorf("unable to fetch prometheus certificate: %w", err) + } + if prometheusClientCert != nil { + cc.TrustedBundle.AddCertificates(prometheusClientCert) + } + + esgwCertificate, err := cc.CertificateManager.GetCertificate(cc.Client, relasticsearch.PublicCertSecret, common.OperatorNamespace()) + if err != nil { + return cc, nil, fmt.Errorf("failed to retrieve / validate %s: %w", relasticsearch.PublicCertSecret, err) + } + if esgwCertificate != nil { + cc.TrustedBundle.AddCertificates(esgwCertificate) + } + + // es-kube-controllers talks to Voltron, so the shared bundle must trust the + // manager internal cert. + managerInternalTLS, err := cc.CertificateManager.GetCertificate(cc.Client, render.ManagerInternalTLSSecretName, common.OperatorNamespace()) + if err != nil { + return cc, nil, fmt.Errorf("failed to retrieve %s: %w", render.ManagerInternalTLSSecretName, err) + } + if managerInternalTLS != nil { + cc.TrustedBundle.AddCertificates(managerInternalTLS) + } + + var managed []certificatemanagement.KeyPairInterface + if nodePrometheusTLS != nil { + managed = append(managed, nodePrometheusTLS) + } + if kubeControllerTLS != nil { + managed = append(managed, kubeControllerTLS) + } + if wafWebhookTLS != nil { + managed = append(managed, wafWebhookTLS) + } + return cc, managed, nil +} diff --git a/pkg/enterprise/installation/core_test.go b/pkg/enterprise/installation/core_test.go new file mode 100644 index 0000000000..0723e7532a --- /dev/null +++ b/pkg/enterprise/installation/core_test.go @@ -0,0 +1,111 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package installation_test + +import ( + "context" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" + "k8s.io/apimachinery/pkg/runtime" + + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/apis" + "github.com/tigera/operator/pkg/common" + "github.com/tigera/operator/pkg/controller/certificatemanager" + "github.com/tigera/operator/pkg/controller/contexts" + ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" + "github.com/tigera/operator/pkg/render" + "github.com/tigera/operator/pkg/render/kubecontrollers" +) + +var _ = Describe("installation controller extension", func() { + It("rejects a zero prometheus reporter port", func() { + port := 0 + cc := newControllerContext(operatorv1.CalicoEnterprise) + cc.FelixConfiguration = &v3.FelixConfiguration{ + Spec: v3.FelixConfigurationSpec{PrometheusReporterPort: &port}, + } + Expect(ext.Validate(cc)).To(HaveOccurred()) + }) + + DescribeTable("defaults dnsTrustedServers for providers whose DNS service isn't kube-dns", + func(provider operatorv1.Provider, expected []string) { + fc := &v3.FelixConfiguration{} + install := &operatorv1.InstallationSpec{Variant: operatorv1.CalicoEnterprise, KubernetesProvider: provider} + updated, err := ext.DefaultFelixConfiguration(contexts.InstallationController, install, fc) + Expect(err).NotTo(HaveOccurred()) + if expected == nil { + Expect(updated).To(BeFalse()) + Expect(fc.Spec.DNSTrustedServers).To(BeNil()) + return + } + Expect(updated).To(BeTrue()) + Expect(*fc.Spec.DNSTrustedServers).To(ConsistOf(expected)) + }, + Entry("OpenShift", operatorv1.ProviderOpenShift, []string{"k8s-service:openshift-dns/dns-default"}), + Entry("RKE2", operatorv1.ProviderRKE2, []string{"k8s-service:kube-system/rke2-coredns-rke2-coredns"}), + Entry("other providers keep the felix default", operatorv1.ProviderNone, nil), + ) + + It("does no felix defaulting for the Calico variant", func() { + fc := &v3.FelixConfiguration{} + updated, err := ext.DefaultFelixConfiguration(contexts.InstallationController, &operatorv1.InstallationSpec{Variant: operatorv1.Calico, KubernetesProvider: operatorv1.ProviderOpenShift}, fc) + Expect(err).NotTo(HaveOccurred()) + Expect(updated).To(BeFalse()) + Expect(fc.Spec.DNSTrustedServers).To(BeNil()) + }) + + It("manages the node prometheus and kube-controllers metrics keypairs for the enterprise variant", func() { + _, managed, err := ext.ExtendContext(newControllerContext(operatorv1.CalicoEnterprise)) + Expect(err).NotTo(HaveOccurred()) + names := []string{} + for _, kp := range managed { + names = append(names, kp.GetName()) + } + Expect(names).To(ConsistOf(render.NodePrometheusTLSServerSecret, kubecontrollers.KubeControllerPrometheusTLSSecret)) + }) + + It("is a no-op for the Calico variant", func() { + _, managed, err := ext.ExtendContext(newControllerContext(operatorv1.Calico)) + Expect(err).NotTo(HaveOccurred()) + Expect(managed).To(BeEmpty()) + }) +}) + +func newControllerContext(variant operatorv1.ProductVariant) contexts.ControllerContext { + scheme := runtime.NewScheme() + Expect(apis.AddToScheme(scheme, false)).NotTo(HaveOccurred()) + c := ctrlrfake.DefaultFakeClientBuilder(scheme).Build() + + certManager, err := certificatemanager.Create(c, nil, "", common.OperatorNamespace(), certificatemanager.AllowCACreation()) + Expect(err).NotTo(HaveOccurred()) + trustedBundle := certManager.CreateTrustedBundle() + + return contexts.ControllerContext{ + RenderContext: render.RenderContext{ + Installation: &operatorv1.InstallationSpec{Variant: variant}, + FelixConfiguration: &v3.FelixConfiguration{}, + TrustedBundle: trustedBundle, + ClusterDomain: "cluster.local", + }, + Controller: contexts.InstallationController, + Ctx: context.Background(), + Client: c, + CertificateManager: certManager, + } +} diff --git a/pkg/enterprise/installation/kubecontrollers.go b/pkg/enterprise/installation/kubecontrollers.go new file mode 100644 index 0000000000..082e1f3fc2 --- /dev/null +++ b/pkg/enterprise/installation/kubecontrollers.go @@ -0,0 +1,415 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package installation + +import ( + "encoding/json" + "fmt" + "path/filepath" + "strings" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" + + "github.com/tigera/operator/pkg/common" + "github.com/tigera/operator/pkg/components" + "github.com/tigera/operator/pkg/controller/contexts" + "github.com/tigera/operator/pkg/controller/gatewayapi" + "github.com/tigera/operator/pkg/controller/utils" + "github.com/tigera/operator/pkg/controller/utils/imageset" + "github.com/tigera/operator/pkg/dns" + entkubecontrollers "github.com/tigera/operator/pkg/enterprise/kubecontrollers" + "github.com/tigera/operator/pkg/extensions" + "github.com/tigera/operator/pkg/render" + "github.com/tigera/operator/pkg/render/applicationlayer" + rmeta "github.com/tigera/operator/pkg/render/common/meta" + "github.com/tigera/operator/pkg/render/common/networkpolicy" + "github.com/tigera/operator/pkg/render/common/secret" + "github.com/tigera/operator/pkg/render/kubecontrollers" + "github.com/tigera/operator/pkg/render/monitor" + "github.com/tigera/operator/pkg/tls/certificatemanagement" +) + +// registerKubeControllers registers the calico-kube-controllers modifiers. There is +// no image override: kube-controllers runs from the combined calico image, which +// resolves by variant in the base render. +func registerKubeControllers(v *extensions.Variant) { + v.Modify(render.ComponentNameKubeControllers, modifyKubeControllers) + v.Modify(render.ComponentNameKubeControllersPolicy, modifyKubeControllersPolicy) +} + +// modifyKubeControllersPolicy adds the WAF admission webhook ingress rule to the +// calico-kube-controllers calico-system network policy, so the kube-apiserver can +// reach the in-process webhook on :9443 (EV-6386). Without it the calico-system +// default-deny drops the apiserver->:9443 call and WAF admission times out. +func modifyKubeControllersPolicy(rc render.RenderContext, objs, del []client.Object) ([]client.Object, []client.Object) { + if !installationData(rc).waf.enabled { + return objs, del + } + policy, ok := extensions.FindObject[*v3.NetworkPolicy](objs, kubecontrollers.KubeControllerNetworkPolicyName) + if !ok { + return objs, del + } + policy.Spec.Ingress = append(policy.Spec.Ingress, v3.Rule{ + Action: v3.Allow, + Protocol: &networkpolicy.TCPProtocol, + Destination: v3.EntityRule{ + Ports: networkpolicy.Ports(uint16(applicationlayer.WAFWebhookContainerPort)), + }, + }) + return objs, del +} + +// modifyKubeControllers layers the full Calico Enterprise surface onto the rendered +// calico-kube-controllers objects: the enterprise cluster role rules, the enterprise +// enabled controllers, the metrics serving TLS, and the WAF v3 (Gateway API add-on) +// surface. The modifier only runs for the enterprise variant, so everything it adds +// is enterprise-only by construction - the base render carries none of it. The +// controller-side inputs (keypairs, the resolved wasm image, the pull secret) are +// produced by the installation hook and handed in through rc. +func modifyKubeControllers(rc render.RenderContext, objs, del []client.Object) ([]client.Object, []client.Object) { + data := installationData(rc) + + if role, ok := extensions.FindObject[*rbacv1.ClusterRole](objs, kubecontrollers.KubeControllerRole); ok { + role.Rules = append(role.Rules, data.kubeControllerRules...) + } + + if dp, ok := extensions.FindObject[*appsv1.Deployment](objs, kubecontrollers.KubeController); ok { + modifyKubeControllersDeployment(rc, dp, data) + } + + // The WAF admission webhook surface (Service + ValidatingWebhookConfiguration), + // the wasm pull secret, and the wasm CA bundle. Created when WAF is enabled, + // deleted otherwise so toggling the extension off cleans them up. + webhookObjs := applicationlayer.WAFAdmissionWebhookComponents(data.waf.caBundle) + if data.waf.enabled { + objs = append(objs, webhookObjs...) + if data.waf.pullSecret != nil { + objs = append(objs, secret.ToRuntimeObjects(secret.CopyToNamespace(common.CalicoNamespace, data.waf.pullSecret)...)...) + } + if data.waf.caCert != nil { + objs = append(objs, data.waf.caCert) + } + } else { + del = append(del, webhookObjs...) + } + + return objs, del +} + +func modifyKubeControllersDeployment(rc render.RenderContext, dp *appsv1.Deployment, data installationRenderData) { + spec := &dp.Spec.Template.Spec + if dp.Spec.Template.Annotations == nil { + dp.Spec.Template.Annotations = map[string]string{} + } + + if tls := data.kubeControllerTLS; tls != nil { + spec.Volumes = append(spec.Volumes, tls.Volume()) + dp.Spec.Template.Annotations[tls.HashAnnotationKey()] = tls.HashAnnotationValue() + } + if waf := data.waf; waf.enabled && waf.webhookTLS != nil { + spec.Volumes = append(spec.Volumes, waf.webhookTLS.Volume()) + } + + for i := range spec.Containers { + c := &spec.Containers[i] + if c.Name != kubecontrollers.KubeController { + continue + } + + appendEnabledControllers(c, data.kubeControllerControllers) + c.Env = append(c.Env, enterpriseEnv(rc)...) + + if tls := data.kubeControllerTLS; tls != nil { + c.Env = append(c.Env, + corev1.EnvVar{Name: "TLS_KEY_PATH", Value: tls.VolumeMountKeyFilePath()}, + corev1.EnvVar{Name: "TLS_CRT_PATH", Value: tls.VolumeMountCertificateFilePath()}, + corev1.EnvVar{Name: "CLIENT_COMMON_NAME", Value: monitor.PrometheusClientTLSSecretName}, + ) + c.VolumeMounts = append(c.VolumeMounts, tls.VolumeMount(rmeta.OSTypeLinux)) + if tls.UseCertificateManagement() { + spec.InitContainers = append(spec.InitContainers, tls.InitContainer(common.CalicoNamespace, c.SecurityContext)) + } + } + + if waf := data.waf; waf.enabled { + c.Env = append(c.Env, wafEnv(waf)...) + c.Ports = append(c.Ports, corev1.ContainerPort{ + Name: "waf-webhook", + ContainerPort: applicationlayer.WAFWebhookContainerPort, + Protocol: corev1.ProtocolTCP, + }) + if waf.webhookTLS != nil { + c.VolumeMounts = append(c.VolumeMounts, waf.webhookTLS.VolumeMount(rmeta.OSTypeLinux)) + if waf.webhookTLS.UseCertificateManagement() { + spec.InitContainers = append(spec.InitContainers, waf.webhookTLS.InitContainer(common.CalicoNamespace, c.SecurityContext)) + } + } + } + } +} + +// appendEnabledControllers folds the enterprise controllers into the existing +// ENABLED_CONTROLLERS env the base render set (node,loadbalancer). +func appendEnabledControllers(c *corev1.Container, extra []string) { + if len(extra) == 0 { + return + } + for i := range c.Env { + if c.Env[i].Name == "ENABLED_CONTROLLERS" { + c.Env[i].Value = c.Env[i].Value + "," + strings.Join(extra, ",") + return + } + } +} + +// enterpriseEnv is the static enterprise env for calico-kube-controllers. The +// modifier runs only for the enterprise variant, so these are never rendered for core. +func enterpriseEnv(rc render.RenderContext) []corev1.EnvVar { + var env []corev1.EnvVar + if rc.TrustedBundle != nil { + env = append(env, corev1.EnvVar{Name: "MULTI_CLUSTER_FORWARDING_CA", Value: rc.TrustedBundle.MountPath()}) + } + if in := rc.Installation; in != nil && in.CalicoNetwork != nil && in.CalicoNetwork.MultiInterfaceMode != nil { + env = append(env, corev1.EnvVar{Name: "MULTI_INTERFACE_MODE", Value: in.CalicoNetwork.MultiInterfaceMode.Value()}) + } + return env +} + +// wafEnv is the WAF v3 env the kube-controllers binary consumes to program WAF policy +// attachments. WASM_IMAGE is the pre-resolved reference the hook produced. +func wafEnv(waf wafRenderData) []corev1.EnvVar { + var env []corev1.EnvVar + if waf.wasmImage != "" { + env = append(env, corev1.EnvVar{Name: "WASM_IMAGE", Value: waf.wasmImage}) + } + if waf.pullSecret != nil { + env = append(env, corev1.EnvVar{Name: "WASM_PULL_SECRET", Value: waf.pullSecret.Name}) + } + if waf.caCert != nil { + env = append(env, corev1.EnvVar{Name: "WASM_CA_CERT", Value: waf.caCert.Name}) + } + if waf.webhookTLS != nil { + env = append(env, corev1.EnvVar{Name: "WAF_WEBHOOK_CERT_DIR", Value: filepath.Dir(waf.webhookTLS.VolumeMountCertificateFilePath())}) + } + return env +} + +const ( + // WASMPullSecretName is the dedicated image-pull Secret (a merged copy of the + // install pull secrets) the WAF reconciler replicates into tenant namespaces for + // the Coraza wasm OCI pull. A dedicated name avoids clashing with the + // operator-managed tigera-pull-secret the GatewayAPI render also copies there (EV-6386). + WASMPullSecretName = "tigera-waf-pull-secret" + + // WASMCACertName is the dedicated CA-bundle ConfigMap the WAF reconciler + // replicates into tenant namespaces for the Coraza wasm OCI registry TLS check - + // a dedicated name avoids clashing with the operator-managed tigera-ca-bundle the + // GatewayAPI render also copies there (EV-6386). It is a renamed copy of the trusted bundle. + WASMCACertName = "tigera-waf-ca-bundle" +) + +// calicoKubeControllersEnterpriseRules are the enterprise cluster role rules layered +// onto calico-kube-controllers: the shared enterprise rules plus the calico-specific +// ones (federated endpoints, license usage reporting). +func calicoKubeControllersEnterpriseRules(wafEnabled, managedCluster bool) []rbacv1.PolicyRule { + rules := entkubecontrollers.KubeControllersEnterpriseCommonRules(wafEnabled, managedCluster) + return append(rules, + rbacv1.PolicyRule{ + APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, + Resources: []string{"remoteclusterconfigurations"}, + Verbs: []string{"watch", "list", "get"}, + }, + rbacv1.PolicyRule{ + APIGroups: []string{""}, + Resources: []string{"endpoints"}, + Verbs: []string{"create", "update", "delete"}, + }, + rbacv1.PolicyRule{ + APIGroups: []string{""}, + Resources: []string{"namespaces"}, + Verbs: []string{"get"}, + }, + rbacv1.PolicyRule{ + APIGroups: []string{"usage.tigera.io"}, + Resources: []string{"licenseusagereports"}, + Verbs: []string{"create", "update", "delete", "watch", "list", "get"}, + }, + ) +} + +// calicoKubeControllersEnterpriseControllers are the enterprise controllers added to +// the calico-kube-controllers ENABLED_CONTROLLERS list (on top of the base +// node,loadbalancer). applicationlayer is added only when the WAF extension is on. +func calicoKubeControllersEnterpriseControllers(wafEnabled bool) []string { + controllers := []string{"service", "federatedservices", "usage"} + if wafEnabled { + controllers = append(controllers, "applicationlayer") + } + return controllers +} + +// wafRenderData is the controller-produced WAF v3 (Gateway API add-on) state the +// installation hook hands the kube-controllers modifier through the render context. +// The zero value (enabled false) means the modifier deletes the webhook objects. +type wafRenderData struct { + enabled bool + wasmImage string + pullSecret *corev1.Secret + caCert *corev1.ConfigMap + webhookTLS certificatemanagement.KeyPairInterface + caBundle []byte +} + +// buildWAFData reads the GatewayAPI CR and, when the WAF extension is enabled, +// produces everything the modifier needs that it can't compute itself: the resolved +// wasm image, the webhook serving keypair (also returned as a managed keypair), the +// merged wasm pull secret, the wasm CA bundle ConfigMap, and the operator CA PEM. +func buildWAFData(cc contexts.ControllerContext) (wafRenderData, certificatemanagement.KeyPairInterface, error) { + gw, _, err := gatewayapi.GetGatewayAPI(cc.Ctx, cc.Client) + if err != nil && !apierrors.IsNotFound(err) { + return wafRenderData{}, nil, err + } + if gw == nil || !gw.Spec.IsWAFGatewayExtensionEnabled() { + return wafRenderData{}, nil, nil + } + + in := cc.Installation + // The wasm is baked into the gateway envoy-proxy image. Resolve it with the same + // GetReference the base render uses for every image; the hook has the ImageSet here. + imageSet, err := imageset.GetImageSet(cc.Ctx, cc.Client, in.Variant) + if err != nil { + return wafRenderData{}, nil, err + } + wasmImage, err := components.GetReference(components.ComponentGatewayAPIEnvoyProxy, in.Registry, in.ImagePath, in.ImagePrefix, imageSet) + if err != nil { + return wafRenderData{}, nil, err + } + + webhookTLS, err := cc.CertificateManager.GetOrCreateKeyPair( + cc.Client, + applicationlayer.WAFWebhookServerTLSSecretName, + common.OperatorNamespace(), + dns.GetServiceDNSNames(applicationlayer.WAFWebhookServiceName, common.CalicoNamespace, cc.ClusterDomain), + ) + if err != nil { + return wafRenderData{}, nil, err + } + + pullSecrets, err := utils.GetInstallationPullSecrets(in, cc.Client) + if err != nil { + return wafRenderData{}, nil, err + } + var pullSecret *corev1.Secret + if len(pullSecrets) > 0 { + pullSecret, _ = MergeWAFPullSecret(pullSecrets) + } + + var caCert *corev1.ConfigMap + if cc.TrustedBundle != nil { + caCert = cc.TrustedBundle.ConfigMap(common.CalicoNamespace) + caCert.Name = WASMCACertName + } + + return wafRenderData{ + enabled: true, + wasmImage: wasmImage, + pullSecret: pullSecret, + caCert: caCert, + webhookTLS: webhookTLS, + caBundle: cc.CertificateManager.KeyPair().GetCertificatePEM(), + }, webhookTLS, nil +} + +// MergeWAFPullSecret synthesizes the dedicated WAF wasm pull secret +// (tigera-waf-pull-secret) by merging the registry auths of every Installation pull +// secret. The EnvoyExtensionPolicy image source takes a single pullSecretRef, so a +// merged secret is the only way to honor multiple Installation pull secrets for the +// Coraza wasm OCI pull (e.g. the Tigera pull secret plus a private registry mirror). +// +// If the same registry appears in more than one secret, the first secret in +// Installation order wins. Secrets that cannot be parsed are skipped and their names +// returned, so the caller can log them without failing the reconcile. Returns a nil +// Secret when no registry auths could be collected. +func MergeWAFPullSecret(pullSecrets []*corev1.Secret) (*corev1.Secret, []string) { + merged := map[string]json.RawMessage{} + var skipped []string + for _, s := range pullSecrets { + auths, err := registryAuths(s) + if err != nil { + skipped = append(skipped, s.Name) + continue + } + for registry, auth := range auths { + if _, ok := merged[registry]; !ok { + merged[registry] = auth + } + } + } + if len(merged) == 0 { + return nil, skipped + } + + // Marshalling a map sorts its keys, so the rendered bytes are deterministic and + // do not churn the object on every reconcile. + data, err := json.Marshal(map[string]map[string]json.RawMessage{"auths": merged}) + if err != nil { + // Each auth entry round-trips from a successful Unmarshal above, so this + // cannot fail in practice; treat it as nothing to render. + return nil, skipped + } + + return &corev1.Secret{ + TypeMeta: metav1.TypeMeta{Kind: "Secret", APIVersion: "v1"}, + ObjectMeta: metav1.ObjectMeta{Name: WASMPullSecretName, Namespace: common.CalicoNamespace}, + Type: corev1.SecretTypeDockerConfigJson, + Data: map[string][]byte{corev1.DockerConfigJsonKey: data}, + }, skipped +} + +// registryAuths extracts the per-registry auth entries from a pull secret of either +// the dockerconfigjson type (auths nested under an "auths" key) or the legacy +// dockercfg type (a bare registry -> auth map). +func registryAuths(s *corev1.Secret) (map[string]json.RawMessage, error) { + if raw, ok := s.Data[corev1.DockerConfigJsonKey]; ok { + var cfg struct { + Auths map[string]json.RawMessage `json:"auths"` + } + if err := json.Unmarshal(raw, &cfg); err != nil { + return nil, err + } + if len(cfg.Auths) == 0 { + return nil, fmt.Errorf("secret %s has no auths entries", s.Name) + } + return cfg.Auths, nil + } + if raw, ok := s.Data[corev1.DockerConfigKey]; ok { + var auths map[string]json.RawMessage + if err := json.Unmarshal(raw, &auths); err != nil { + return nil, err + } + if len(auths) == 0 { + return nil, fmt.Errorf("secret %s has no auths entries", s.Name) + } + return auths, nil + } + return nil, fmt.Errorf("secret %s has neither a %s nor a %s key", s.Name, corev1.DockerConfigJsonKey, corev1.DockerConfigKey) +} diff --git a/pkg/enterprise/installation/kubecontrollers_test.go b/pkg/enterprise/installation/kubecontrollers_test.go new file mode 100644 index 0000000000..ad4867d610 --- /dev/null +++ b/pkg/enterprise/installation/kubecontrollers_test.go @@ -0,0 +1,300 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package installation_test + +import ( + "context" + "fmt" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/apis" + "github.com/tigera/operator/pkg/common" + "github.com/tigera/operator/pkg/components" + "github.com/tigera/operator/pkg/controller/certificatemanager" + "github.com/tigera/operator/pkg/controller/contexts" + ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" + "github.com/tigera/operator/pkg/enterprise/installation" + "github.com/tigera/operator/pkg/extensions" + "github.com/tigera/operator/pkg/extensions/extensionstest" + "github.com/tigera/operator/pkg/render" + "github.com/tigera/operator/pkg/render/applicationlayer" + rmeta "github.com/tigera/operator/pkg/render/common/meta" + "github.com/tigera/operator/pkg/render/common/networkpolicy" + "github.com/tigera/operator/pkg/render/kubecontrollers" + "github.com/tigera/operator/pkg/render/monitor" + "github.com/tigera/operator/pkg/tls" +) + +var _ = Describe("kube-controllers enterprise modifier", func() { + // kubeControllersDeployment is a minimal stand-in for the calico-kube-controllers + // deployment the base render produces, so the modifier has something to mount onto. + kubeControllersDeployment := func() *appsv1.Deployment { + return &appsv1.Deployment{ + TypeMeta: metav1.TypeMeta{Kind: "Deployment", APIVersion: "apps/v1"}, + ObjectMeta: metav1.ObjectMeta{Name: kubecontrollers.KubeController, Namespace: common.CalicoNamespace}, + Spec: appsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{{Name: kubecontrollers.KubeController}}, + }, + }, + }, + } + } + + It("mounts the metrics serving TLS keypair onto the deployment", func() { + ecc, _, err := ext.ExtendContext(newControllerContext(operatorv1.CalicoEnterprise)) + rc := ecc.RenderContext + Expect(err).NotTo(HaveOccurred()) + + objs, _ := extensionstest.ApplyExtensions(ext, render.ComponentNameKubeControllers, rc, []client.Object{kubeControllersDeployment()}, nil) + dp, ok := extensions.FindObject[*appsv1.Deployment](objs, kubecontrollers.KubeController) + Expect(ok).To(BeTrue()) + + c := dp.Spec.Template.Spec.Containers[0] + Expect(c.Env).To(ContainElements( + corev1.EnvVar{Name: "TLS_KEY_PATH", Value: "/calico-kube-controllers-metrics-tls/tls.key"}, + corev1.EnvVar{Name: "TLS_CRT_PATH", Value: "/calico-kube-controllers-metrics-tls/tls.crt"}, + corev1.EnvVar{Name: "CLIENT_COMMON_NAME", Value: monitor.PrometheusClientTLSSecretName}, + )) + Expect(c.VolumeMounts).To(ContainElement(HaveField("Name", kubecontrollers.KubeControllerPrometheusTLSSecret))) + Expect(dp.Spec.Template.Spec.Volumes).To(ContainElement(HaveField("Name", kubecontrollers.KubeControllerPrometheusTLSSecret))) + Expect(dp.Spec.Template.Annotations).NotTo(BeEmpty(), "expected the cert hash annotation") + }) + + It("adds the cert-management init container when certificate management is enabled", func() { + ecc, _, err := ext.ExtendContext(certManagementControllerContext()) + rc := ecc.RenderContext + Expect(err).NotTo(HaveOccurred()) + + objs, _ := extensionstest.ApplyExtensions(ext, render.ComponentNameKubeControllers, rc, []client.Object{kubeControllersDeployment()}, nil) + dp, ok := extensions.FindObject[*appsv1.Deployment](objs, kubecontrollers.KubeController) + Expect(ok).To(BeTrue()) + + Expect(dp.Spec.Template.Spec.InitContainers).To(HaveLen(1)) + Expect(dp.Spec.Template.Spec.InitContainers[0].Name).To(Equal(fmt.Sprintf("%s-key-cert-provisioner", kubecontrollers.KubeControllerPrometheusTLSSecret))) + }) +}) + +// certManagementControllerContext builds a controller context whose certificate +// manager issues cert-management (CSR-based) keypairs. +func certManagementControllerContext() contexts.ControllerContext { + scheme := runtime.NewScheme() + Expect(apis.AddToScheme(scheme, false)).NotTo(HaveOccurred()) + c := ctrlrfake.DefaultFakeClientBuilder(scheme).Build() + + ca, err := tls.MakeCA(rmeta.DefaultOperatorCASignerName()) + Expect(err).NotTo(HaveOccurred()) + caCert, _, err := ca.Config.GetPEMBytes() + Expect(err).NotTo(HaveOccurred()) + + installation := &operatorv1.InstallationSpec{ + Variant: operatorv1.CalicoEnterprise, + CertificateManagement: &operatorv1.CertificateManagement{CACert: caCert}, + } + certManager, err := certificatemanager.Create(c, installation, "", common.OperatorNamespace(), certificatemanager.AllowCACreation()) + Expect(err).NotTo(HaveOccurred()) + + return contexts.ControllerContext{ + RenderContext: render.RenderContext{ + Installation: installation, + FelixConfiguration: &v3.FelixConfiguration{}, + TrustedBundle: certManager.CreateTrustedBundle(), + ClusterDomain: "cluster.local", + }, + Controller: contexts.InstallationController, + Ctx: context.Background(), + Client: c, + CertificateManager: certManager, + } +} + +var _ = Describe("calico-kube-controllers enterprise surface", func() { + calicoKubeControllersCfg := func(cc contexts.ControllerContext) *kubecontrollers.KubeControllersConfiguration { + return &kubecontrollers.KubeControllersConfiguration{ + Installation: cc.Installation, + ClusterDomain: cc.ClusterDomain, + TrustedBundle: cc.TrustedBundle, + MetricsPort: 9094, + Namespace: common.CalicoNamespace, + BindingNamespaces: []string{common.CalicoNamespace}, + } + } + + // render builds the base calico-kube-controllers objects and applies the + // enterprise modifier, exactly as the component handler does. + renderKubeControllers := func(cc contexts.ControllerContext, rc render.RenderContext) []client.Object { + comp := kubecontrollers.NewCalicoKubeControllers(calicoKubeControllersCfg(cc)) + Expect(comp.ResolveImages(nil)).NotTo(HaveOccurred()) + create, del := comp.Objects() + out, _ := extensionstest.ApplyExtensions(ext, render.ComponentNameKubeControllers, rc, create, del) + return out + } + + kubeContainer := func(objs []client.Object) *corev1.Container { + dp, ok := extensions.FindObject[*appsv1.Deployment](objs, kubecontrollers.KubeController) + Expect(ok).To(BeTrue()) + return &dp.Spec.Template.Spec.Containers[0] + } + + It("layers the enterprise rules, controllers, and metrics TLS on (WAF off)", func() { + ecc, _, err := ext.ExtendContext(newControllerContext(operatorv1.CalicoEnterprise)) + rc := ecc.RenderContext + Expect(err).NotTo(HaveOccurred()) + objs := renderKubeControllers(newControllerContext(operatorv1.CalicoEnterprise), rc) + + role, ok := extensions.FindObject[*rbacv1.ClusterRole](objs, kubecontrollers.KubeControllerRole) + Expect(ok).To(BeTrue()) + Expect(role.Rules).To(ContainElement(HaveField("Resources", ContainElement("licensekeys")))) + + c := kubeContainer(objs) + Expect(c.Env).To(ContainElement(corev1.EnvVar{ + Name: "ENABLED_CONTROLLERS", Value: "node,loadbalancer,service,federatedservices,usage", + })) + // Metrics serving TLS wired from the keypair the hook created. + Expect(c.Env).To(ContainElement(HaveField("Name", "TLS_KEY_PATH"))) + // WAF is off, so no WASM env and no webhook objects. + Expect(c.Env).NotTo(ContainElement(HaveField("Name", "WASM_IMAGE"))) + _, ok = extensions.FindObject[*corev1.Service](objs, applicationlayer.WAFWebhookServiceName) + Expect(ok).To(BeFalse()) + }) + + It("layers the full WAF surface on when the GatewayAPI extension is enabled", func() { + cc := wafControllerContext() + ecc, managed, err := ext.ExtendContext(cc) + rc := ecc.RenderContext + Expect(err).NotTo(HaveOccurred()) + names := []string{} + for _, kp := range managed { + names = append(names, kp.GetName()) + } + Expect(names).To(ContainElement(applicationlayer.WAFWebhookServerTLSSecretName)) + + objs := renderKubeControllers(cc, rc) + + role, ok := extensions.FindObject[*rbacv1.ClusterRole](objs, kubecontrollers.KubeControllerRole) + Expect(ok).To(BeTrue()) + Expect(role.Rules).To(ContainElement(HaveField("Resources", ContainElement("wafpolicies")))) + + c := kubeContainer(objs) + Expect(c.Env).To(ContainElement(corev1.EnvVar{ + Name: "ENABLED_CONTROLLERS", Value: "node,loadbalancer,service,federatedservices,usage,applicationlayer", + })) + Expect(c.Env).To(ContainElement(corev1.EnvVar{ + Name: "WASM_IMAGE", Value: "test-reg/tigera/envoy-proxy:" + components.ComponentGatewayAPIEnvoyProxy.Version, + })) + Expect(c.Env).To(ContainElement(corev1.EnvVar{Name: "WASM_PULL_SECRET", Value: installation.WASMPullSecretName})) + Expect(c.Env).To(ContainElement(corev1.EnvVar{Name: "WASM_CA_CERT", Value: installation.WASMCACertName})) + Expect(c.Env).To(ContainElement(HaveField("Name", "WAF_WEBHOOK_CERT_DIR"))) + Expect(c.Ports).To(ContainElement(corev1.ContainerPort{Name: "waf-webhook", ContainerPort: int32(9443), Protocol: corev1.ProtocolTCP})) + + // The webhook surface, the wasm pull secret, and the wasm CA bundle are rendered. + _, ok = extensions.FindObject[*corev1.Service](objs, applicationlayer.WAFWebhookServiceName) + Expect(ok).To(BeTrue()) + _, ok = extensions.FindObject[*corev1.Secret](objs, installation.WASMPullSecretName) + Expect(ok).To(BeTrue()) + _, ok = extensions.FindObject[*corev1.ConfigMap](objs, installation.WASMCACertName) + Expect(ok).To(BeTrue()) + }) + + It("deletes the WAF webhook surface when the extension is disabled", func() { + cc := newControllerContext(operatorv1.CalicoEnterprise) + ecc, _, err := ext.ExtendContext(cc) + rc := ecc.RenderContext + Expect(err).NotTo(HaveOccurred()) + + comp := kubecontrollers.NewCalicoKubeControllers(calicoKubeControllersCfg(cc)) + Expect(comp.ResolveImages(nil)).NotTo(HaveOccurred()) + create, del := comp.Objects() + _, toDelete := extensionstest.ApplyExtensions(ext, render.ComponentNameKubeControllers, rc, create, del) + + _, ok := extensions.FindObject[*corev1.Service](toDelete, applicationlayer.WAFWebhookServiceName) + Expect(ok).To(BeTrue(), "the webhook Service should be queued for deletion") + }) + + It("adds the WAF webhook ingress rule to the network policy when enabled", func() { + cc := wafControllerContext() + ecc, _, err := ext.ExtendContext(cc) + rc := ecc.RenderContext + Expect(err).NotTo(HaveOccurred()) + + comp := kubecontrollers.NewCalicoKubeControllersPolicy(calicoKubeControllersCfg(cc), nil) + create, del := comp.Objects() + objs, _ := extensionstest.ApplyExtensions(ext, render.ComponentNameKubeControllersPolicy, rc, create, del) + + policy, ok := extensions.FindObject[*v3.NetworkPolicy](objs, kubecontrollers.KubeControllerNetworkPolicyName) + Expect(ok).To(BeTrue()) + Expect(policy.Spec.Ingress).To(ContainElement(v3.Rule{ + Action: v3.Allow, + Protocol: &networkpolicy.TCPProtocol, + Destination: v3.EntityRule{ + Ports: networkpolicy.Ports(uint16(applicationlayer.WAFWebhookContainerPort)), + }, + })) + }) +}) + +// wafControllerContext builds a controller context with a WAF-enabled GatewayAPI CR +// and an install pull secret, so the installation hook produces the full WAF data. +func wafControllerContext() contexts.ControllerContext { + scheme := runtime.NewScheme() + Expect(apis.AddToScheme(scheme, false)).NotTo(HaveOccurred()) + c := ctrlrfake.DefaultFakeClientBuilder(scheme).Build() + + Expect(c.Create(context.Background(), &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "pull", Namespace: common.OperatorNamespace()}, + Type: corev1.SecretTypeDockerConfigJson, + Data: map[string][]byte{corev1.DockerConfigJsonKey: []byte(`{"auths":{"reg.example.com":{"auth":"abc"}}}`)}, + })).NotTo(HaveOccurred()) + + enabled := operatorv1.WAFExtensionStateEnabled + Expect(c.Create(context.Background(), &operatorv1.GatewayAPI{ + ObjectMeta: metav1.ObjectMeta{Name: "default"}, + Spec: operatorv1.GatewayAPISpec{ + Extensions: &operatorv1.GatewayAPIExtensions{WAF: &operatorv1.WAFExtensionSpec{State: &enabled}}, + }, + })).NotTo(HaveOccurred()) + + certManager, err := certificatemanager.Create(c, nil, "", common.OperatorNamespace(), certificatemanager.AllowCACreation()) + Expect(err).NotTo(HaveOccurred()) + + return contexts.ControllerContext{ + RenderContext: render.RenderContext{ + Installation: &operatorv1.InstallationSpec{ + Variant: operatorv1.CalicoEnterprise, + Registry: "test-reg/", + ImagePullSecrets: []corev1.LocalObjectReference{{Name: "pull"}}, + }, + FelixConfiguration: &v3.FelixConfiguration{}, + TrustedBundle: certManager.CreateTrustedBundle(), + ClusterDomain: "cluster.local", + }, + Controller: contexts.InstallationController, + Ctx: context.Background(), + Client: c, + CertificateManager: certManager, + } +} diff --git a/pkg/enterprise/installation/node.go b/pkg/enterprise/installation/node.go new file mode 100644 index 0000000000..b862fe696d --- /dev/null +++ b/pkg/enterprise/installation/node.go @@ -0,0 +1,303 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package installation + +import ( + "errors" + "fmt" + "slices" + + v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + client "sigs.k8s.io/controller-runtime/pkg/client" + + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/common" + "github.com/tigera/operator/pkg/components" + "github.com/tigera/operator/pkg/controller/utils" + "github.com/tigera/operator/pkg/extensions" + "github.com/tigera/operator/pkg/render" + rmeta "github.com/tigera/operator/pkg/render/common/meta" +) + +const ( + // defaultNodeReporterPort is the port calico/node reports Enterprise internal + // metrics on when FelixConfiguration does not override prometheusReporterPort. + defaultNodeReporterPort = 9081 + + // defaultFelixMetricsPort is the Felix prometheus metrics port used when + // FelixConfiguration does not override prometheusMetricsPort. + defaultFelixMetricsPort = 9091 + + installCNIContainerName = "install-cni" +) + +func registerNode(v *extensions.Variant) { + v.Image(render.ComponentNameNode, components.ComponentTigeraNode) + v.Modify(render.ComponentNameNode, modifyNode) + + // The node component renders the cni-plugins init container; its image + // resolves through its own override key. + v.Image(render.ComponentNameCNIPlugins, components.ComponentTigeraCNIPlugins) +} + +// modifyNode layers Calico Enterprise behavior onto the rendered calico/node +// objects: the extra RBAC rules, the node-metrics Service, and the Enterprise +// daemonset configuration (flow/DNS log env, prometheus reporter, BGP metrics +// readiness check, multi-interface mode, and the calico log volume). +func modifyNode(rc render.RenderContext, objs, del []client.Object) ([]client.Object, []client.Object) { + if role, ok := extensions.FindObject[*rbacv1.ClusterRole](objs, render.CalicoNodeObjectName); ok { + role.Rules = append(role.Rules, nodeEnterpriseRules()...) + } + + // The Network resource is only available in Enterprise / Cloud at this time. + if role, ok := extensions.FindObject[*rbacv1.ClusterRole](objs, render.CalicoCNIPluginObjectName); ok { + role.Rules = append(role.Rules, rbacv1.PolicyRule{ + APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, + Resources: []string{"networks"}, + Verbs: []string{"get"}, + }) + } + + if ds, ok := extensions.FindObject[*appsv1.DaemonSet](objs, common.NodeDaemonSetName); ok { + modifyNodeDaemonSet(rc, ds) + } + + return append(objs, nodeMetricsService(rc)), del +} + +// nodeEnterpriseRules are the additional cluster role rules calico/node needs in +// Calico Enterprise. +func nodeEnterpriseRules() []rbacv1.PolicyRule { + return []rbacv1.PolicyRule{ + { + // Calico Enterprise needs to be able to read additional resources. + APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, + Resources: []string{ + "bfdconfigurations", + "egressgatewaypolicies", + "externalnetworks", + "licensekeys", + "networks", + "packetcaptures", + "remoteclusterconfigurations", + }, + Verbs: []string{"get", "list", "watch"}, + }, + { + // Tigera Secure updates status for packet captures. + APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, + Resources: []string{ + "packetcaptures", + "packetcaptures/status", + }, + Verbs: []string{"update"}, + }, + } +} + +// modifyNodeDaemonSet applies the Enterprise-specific daemonset changes that the +// base render leaves out: the Enterprise felix env, multi-interface mode, the +// BGP metrics readiness check, and the prometheus reporter keypair mount. The +// calico log volume is mounted by the base render for both variants, so it is +// not handled here. +func modifyNodeDaemonSet(rc render.RenderContext, ds *appsv1.DaemonSet) { + spec := &ds.Spec.Template.Spec + + // Collecting process info for flow logs reads from the host's process table. + if installationData(rc).collectProcessPath { + spec.HostPID = true + } + + multiInterfaceMode := multiInterfaceModeEnv(rc.Installation) + + for i := range spec.InitContainers { + if spec.InitContainers[i].Name == installCNIContainerName && multiInterfaceMode != nil { + spec.InitContainers[i].Env = append(spec.InitContainers[i].Env, *multiInterfaceMode) + } + } + + for i := range spec.Containers { + c := &spec.Containers[i] + if c.Name != render.CalicoNodeObjectName { + continue + } + + c.Env = append(c.Env, nodeEnterpriseEnv(rc)...) + + // Add the BGP metrics readiness check, but only when the base render kept + // the bird readiness check (i.e. BGP is in use and we're not on VPP). + if c.ReadinessProbe != nil && c.ReadinessProbe.Exec != nil && slices.Contains(c.ReadinessProbe.Exec.Command, "--bird-ready") { + c.ReadinessProbe.Exec.Command = append(c.ReadinessProbe.Exec.Command, "--bgp-metrics-ready") + } + } + + mountNodePrometheusTLS(rc, ds) +} + +// mountNodePrometheusTLS mounts the node prometheus reporter keypair onto the +// daemonset: the volume, the calico-node volume mount, the cert-management init +// container (when in use), and the pod hash annotation that rolls the pods on +// cert rotation. The keypair has cluster side effects, so the enterprise setup +// creates it and hands it in via rc rather than the modifier building it. In +// core (calico) the keypair is never created, so the base node render carries +// no prometheus mount at all. +func mountNodePrometheusTLS(rc render.RenderContext, ds *appsv1.DaemonSet) { + tls := installationData(rc).nodePrometheusTLS + if tls == nil { + return + } + spec := &ds.Spec.Template.Spec + + spec.Volumes = append(spec.Volumes, tls.Volume()) + + for i := range spec.Containers { + c := &spec.Containers[i] + if c.Name != render.CalicoNodeObjectName { + continue + } + c.VolumeMounts = append(c.VolumeMounts, tls.VolumeMount(rmeta.OSTypeLinux)) + if tls.UseCertificateManagement() { + spec.InitContainers = append(spec.InitContainers, tls.InitContainer(common.CalicoNamespace, c.SecurityContext)) + } + } + + if ds.Spec.Template.Annotations == nil { + ds.Spec.Template.Annotations = map[string]string{} + } + ds.Spec.Template.Annotations[tls.HashAnnotationKey()] = tls.HashAnnotationValue() +} + +// nodeEnterpriseEnv is the Enterprise felix configuration added to the +// calico/node container. +func nodeEnterpriseEnv(rc render.RenderContext) []corev1.EnvVar { + data := installationData(rc) + env := []corev1.EnvVar{ + {Name: "FELIX_PROMETHEUSREPORTERENABLED", Value: "true"}, + {Name: "FELIX_PROMETHEUSREPORTERPORT", Value: fmt.Sprintf("%d", NodeReporterPort(rc.FelixConfiguration))}, + {Name: "FELIX_FLOWLOGSFILEENABLED", Value: "true"}, + {Name: "FELIX_FLOWLOGSFILEINCLUDELABELS", Value: "true"}, + {Name: "FELIX_FLOWLOGSFILEINCLUDEPOLICIES", Value: "true"}, + {Name: "FELIX_FLOWLOGSFILEINCLUDESERVICE", Value: "true"}, + {Name: "FELIX_FLOWLOGSENABLENETWORKSETS", Value: "true"}, + {Name: "FELIX_FLOWLOGSCOLLECTPROCESSINFO", Value: "true"}, + {Name: "FELIX_DNSLOGSFILEENABLED", Value: "true"}, + {Name: "FELIX_DNSLOGSFILEPERNODELIMIT", Value: "1000"}, + } + + if data.collectProcessPath { + env = append(env, corev1.EnvVar{Name: "FELIX_FLOWLOGSCOLLECTPROCESSPATH", Value: "true"}) + } + + if mode := multiInterfaceModeEnv(rc.Installation); mode != nil { + env = append(env, *mode) + } + + tls := data.nodePrometheusTLS + if tls != nil && rc.TrustedBundle != nil { + env = append(env, + corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERCERTFILE", Value: tls.VolumeMountCertificateFilePath()}, + corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERKEYFILE", Value: tls.VolumeMountKeyFilePath()}, + corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERCAFILE", Value: rc.TrustedBundle.MountPath()}, + ) + } + + return env +} + +// multiInterfaceModeEnv returns the MULTI_INTERFACE_MODE env var when the +// installation configures it, or nil otherwise. +func multiInterfaceModeEnv(install *operatorv1.InstallationSpec) *corev1.EnvVar { + if install.CalicoNetwork != nil && install.CalicoNetwork.MultiInterfaceMode != nil { + return &corev1.EnvVar{Name: "MULTI_INTERFACE_MODE", Value: install.CalicoNetwork.MultiInterfaceMode.Value()} + } + return nil +} + +// nodeMetricsService builds the enterprise-only calico-node-metrics Service. +func nodeMetricsService(rc render.RenderContext) *corev1.Service { + reporterPort := NodeReporterPort(rc.FelixConfiguration) + felixPort := felixMetricsPort(rc.FelixConfiguration) + felixEnabled := rc.FelixConfiguration != nil && utils.IsFelixPrometheusMetricsEnabled(rc.FelixConfiguration) + + ports := []corev1.ServicePort{ + { + Name: "calico-metrics-port", + Port: int32(reporterPort), + TargetPort: intstr.FromInt(reporterPort), + Protocol: corev1.ProtocolTCP, + }, + { + Name: "calico-bgp-metrics-port", + Port: render.NodeBGPReporterPort, + TargetPort: intstr.FromInt(int(render.NodeBGPReporterPort)), + Protocol: corev1.ProtocolTCP, + }, + } + if felixEnabled { + ports = append(ports, corev1.ServicePort{ + Name: "felix-metrics-port", + Port: int32(felixPort), + TargetPort: intstr.FromInt(felixPort), + Protocol: corev1.ProtocolTCP, + }) + } + + return &corev1.Service{ + TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: render.CalicoNodeMetricsService, + Namespace: common.CalicoNamespace, + Labels: map[string]string{"k8s-app": render.CalicoNodeObjectName}, + }, + Spec: corev1.ServiceSpec{ + Selector: map[string]string{"k8s-app": render.CalicoNodeObjectName}, + ClusterIP: "None", + Ports: ports, + }, + } +} + +// ValidateReporterPort rejects the unsupported zero prometheus reporter port. +// The node and windows controller extensions share it. +func ValidateReporterPort(fc *v3.FelixConfiguration) error { + if fc != nil && fc.Spec.PrometheusReporterPort != nil && *fc.Spec.PrometheusReporterPort == 0 { + return errors.New("felixConfiguration prometheusReporterPort=0 not supported") + } + return nil +} + +// NodeReporterPort returns the reporter metrics port from the FelixConfiguration, +// falling back to the default. The node-metrics Service and the +// FELIX_PROMETHEUSREPORTERPORT env var both derive from here so they can't drift. +func NodeReporterPort(fc *v3.FelixConfiguration) int { + if fc != nil && fc.Spec.PrometheusReporterPort != nil { + return *fc.Spec.PrometheusReporterPort + } + return defaultNodeReporterPort +} + +// felixMetricsPort returns the Felix prometheus metrics port from the +// FelixConfiguration, falling back to the default. +func felixMetricsPort(fc *v3.FelixConfiguration) int { + if fc != nil && fc.Spec.PrometheusMetricsPort != nil { + return *fc.Spec.PrometheusMetricsPort + } + return defaultFelixMetricsPort +} diff --git a/pkg/enterprise/installation/node_enterprise_test.go b/pkg/enterprise/installation/node_enterprise_test.go new file mode 100644 index 0000000000..8e8af1099e --- /dev/null +++ b/pkg/enterprise/installation/node_enterprise_test.go @@ -0,0 +1,245 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package installation_test + +import ( + "context" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/apis" + "github.com/tigera/operator/pkg/common" + "github.com/tigera/operator/pkg/controller/certificatemanager" + "github.com/tigera/operator/pkg/controller/contexts" + "github.com/tigera/operator/pkg/controller/k8sapi" + ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" + "github.com/tigera/operator/pkg/dns" + "github.com/tigera/operator/pkg/extensions" + "github.com/tigera/operator/pkg/extensions/extensionstest" + "github.com/tigera/operator/pkg/render" + rmeta "github.com/tigera/operator/pkg/render/common/meta" + "github.com/tigera/operator/pkg/tls/certificatemanagement" +) + +// getTyphaNodeTLS builds the node/typha TLS bundle the node render expects. +func getTyphaNodeTLS(cli client.Client, certificateManager certificatemanager.CertificateManager) *render.TyphaNodeTLS { + nodeKeyPair, err := certificateManager.GetOrCreateKeyPair(cli, render.NodeTLSSecretName, common.OperatorNamespace(), []string{render.FelixCommonName}) + Expect(err).NotTo(HaveOccurred()) + + typhaKeyPair, err := certificateManager.GetOrCreateKeyPair(cli, render.TyphaTLSSecretName, common.OperatorNamespace(), []string{render.FelixCommonName}) + Expect(err).NotTo(HaveOccurred()) + + typhaNonClusterHostKeyPair, err := certificateManager.GetOrCreateKeyPair(cli, render.TyphaTLSSecretName+render.TyphaNonClusterHostSuffix, common.OperatorNamespace(), []string{render.FelixCommonName + render.TyphaNonClusterHostSuffix}) + Expect(err).NotTo(HaveOccurred()) + + trustedBundle := certificateManager.CreateTrustedBundle(nodeKeyPair, typhaKeyPair) + + return &render.TyphaNodeTLS{ + TrustedBundle: trustedBundle, + TyphaSecret: typhaKeyPair, + TyphaSecretNonClusterHost: typhaNonClusterHostKeyPair, + TyphaCommonName: render.TyphaCommonName, + NodeSecret: nodeKeyPair, + NodeCommonName: render.FelixCommonName, + } +} + +// These tests run the real node/typha render output through the registered +// enterprise modifiers. The render suite registers the enterprise extensions in +// its BeforeSuite, so this exercises the same integrated behavior the operator +// binary produces - and, importantly, catches a modifier whose FindObject stops +// matching because render renamed an object or container. +var _ = Describe("node enterprise modifier integration", func() { + var ( + cli client.Client + certManager certificatemanager.CertificateManager + typhaNodeTLS *render.TyphaNodeTLS + instance *operatorv1.InstallationSpec + renderCtx render.RenderContext + nodePrometheusTLS certificatemanagement.KeyPairInterface + ) + + nodeContainer := func(ds *appsv1.DaemonSet) *corev1.Container { + for i := range ds.Spec.Template.Spec.Containers { + if ds.Spec.Template.Spec.Containers[i].Name == render.CalicoNodeObjectName { + return &ds.Spec.Template.Spec.Containers[i] + } + } + return nil + } + + BeforeEach(func() { + scheme := runtime.NewScheme() + Expect(apis.AddToScheme(scheme, false)).NotTo(HaveOccurred()) + cli = ctrlrfake.DefaultFakeClientBuilder(scheme).Build() + + var err error + certManager, err = certificatemanager.Create(cli, nil, "", common.OperatorNamespace(), certificatemanager.AllowCACreation()) + Expect(err).NotTo(HaveOccurred()) + typhaNodeTLS = getTyphaNodeTLS(cli, certManager) + + nodePrometheusTLS, err = certManager.GetOrCreateKeyPair(cli, render.NodePrometheusTLSServerSecret, common.OperatorNamespace(), []string{"calico-node-metrics"}) + Expect(err).NotTo(HaveOccurred()) + typhaNodeTLS.TrustedBundle.AddCertificates(nodePrometheusTLS) + + confDir, binDir := render.DefaultCNIDirectories(operatorv1.ProviderNone) + bgp := operatorv1.BGPEnabled + instance = &operatorv1.InstallationSpec{ + Variant: operatorv1.CalicoEnterprise, + CNI: &operatorv1.CNISpec{ + Type: operatorv1.PluginCalico, + IPAM: &operatorv1.IPAMSpec{Type: operatorv1.IPAMPluginCalico}, + BinDir: &binDir, + ConfDir: &confDir, + }, + CalicoNetwork: &operatorv1.CalicoNetworkSpec{ + BGP: &bgp, + IPPools: []operatorv1.IPPool{{CIDR: "192.168.1.0/16"}}, + }, + } + + // Build the render context the way the controller does: run the enterprise + // controller extension, which stashes the node prometheus keypair in the + // context for the node modifier to read. + cc := contexts.ControllerContext{ + RenderContext: render.RenderContext{ + Installation: instance, + TrustedBundle: typhaNodeTLS.TrustedBundle, + ClusterDomain: dns.DefaultClusterDomain, + }, + Controller: contexts.InstallationController, + Ctx: context.Background(), + Client: cli, + CertificateManager: certManager, + } + ecc, _, err := ext.ExtendContext(cc) + Expect(err).NotTo(HaveOccurred()) + renderCtx = ecc.RenderContext + }) + + // renderNodeObjects renders the real node component and applies the registered + // modifier, exactly as the componentHandler does. + renderNodeObjects := func(rc render.RenderContext) []client.Object { + cfg := &render.NodeConfiguration{ + K8sServiceEp: k8sapi.ServiceEndpoint{}, + Installation: instance, + TLS: typhaNodeTLS, + ClusterDomain: dns.DefaultClusterDomain, + FelixHealthPort: 9099, + IPPools: instance.CalicoNetwork.IPPools, + } + comp := render.Node(cfg) + Expect(comp.ResolveImages(nil)).NotTo(HaveOccurred()) + objs, _ := comp.Objects() + out, _ := extensionstest.ApplyExtensions(ext, render.ComponentNameNode, rc, objs, nil) + return out + } + + It("appends the node metrics service to the real render output", func() { + objs := renderNodeObjects(renderCtx) + svc, ok := extensions.FindObject[*corev1.Service](objs, render.CalicoNodeMetricsService) + Expect(ok).To(BeTrue(), "expected the modifier to append %s", render.CalicoNodeMetricsService) + Expect(svc.Namespace).To(Equal(common.CalicoNamespace)) + }) + + It("adds the enterprise rules to the real cluster roles", func() { + objs := renderNodeObjects(renderCtx) + + nodeRole, ok := extensions.FindObject[*rbacv1.ClusterRole](objs, render.CalicoNodeObjectName) + Expect(ok).To(BeTrue()) + Expect(nodeRole.Rules).To(ContainElement(HaveField("Resources", ContainElement("licensekeys")))) + + cniRole, ok := extensions.FindObject[*rbacv1.ClusterRole](objs, render.CalicoCNIPluginObjectName) + Expect(ok).To(BeTrue()) + Expect(cniRole.Rules).To(ContainElement(HaveField("Resources", ContainElement("networks")))) + }) + + It("rewrites the real node daemonset for enterprise", func() { + objs := renderNodeObjects(renderCtx) + ds, ok := extensions.FindObject[*appsv1.DaemonSet](objs, common.NodeDaemonSetName) + Expect(ok).To(BeTrue()) + + c := nodeContainer(ds) + Expect(c).NotTo(BeNil()) + + Expect(c.Env).To(ContainElements( + corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERENABLED", Value: "true"}, + corev1.EnvVar{Name: "FELIX_FLOWLOGSFILEENABLED", Value: "true"}, + )) + // The reporter cert env is wired from the NodePrometheusTLS keypair the + // builder creates, and the modifier mounts that keypair onto the daemonset. + Expect(c.Env).To(ContainElement(HaveField("Name", "FELIX_PROMETHEUSREPORTERCERTFILE"))) + Expect(ds.Spec.Template.Spec.Volumes).To(ContainElement(nodePrometheusTLS.Volume())) + Expect(c.VolumeMounts).To(ContainElement(nodePrometheusTLS.VolumeMount(rmeta.OSTypeLinux))) + Expect(ds.Spec.Template.Annotations).To(HaveKey(nodePrometheusTLS.HashAnnotationKey())) + + // BGP is enabled, so the bird readiness check is present and the modifier + // adds the BGP metrics check. + Expect(c.ReadinessProbe.Exec.Command).To(ContainElement("--bgp-metrics-ready")) + }) + + It("enables process-path collection when the LogCollector requests it", func() { + enable := operatorv1.CollectProcessPathEnable + Expect(cli.Create(context.Background(), &operatorv1.LogCollector{ + ObjectMeta: metav1.ObjectMeta{Name: "tigera-secure"}, + Spec: operatorv1.LogCollectorSpec{CollectProcessPath: &enable}, + })).NotTo(HaveOccurred()) + + ecc, _, err := ext.ExtendContext(contexts.ControllerContext{ + RenderContext: render.RenderContext{ + Installation: instance, + TrustedBundle: typhaNodeTLS.TrustedBundle, + ClusterDomain: dns.DefaultClusterDomain, + }, + Controller: contexts.InstallationController, + Ctx: context.Background(), + Client: cli, + CertificateManager: certManager, + }) + Expect(err).NotTo(HaveOccurred()) + rc := ecc.RenderContext + + ds, ok := extensions.FindObject[*appsv1.DaemonSet](renderNodeObjects(rc), common.NodeDaemonSetName) + Expect(ok).To(BeTrue()) + Expect(ds.Spec.Template.Spec.HostPID).To(BeTrue()) + Expect(nodeContainer(ds).Env).To(ContainElement(corev1.EnvVar{Name: "FELIX_FLOWLOGSCOLLECTPROCESSPATH", Value: "true"})) + }) + + It("adds the enterprise rules to the real typha cluster role", func() { + comp := render.Typha(&render.TyphaConfiguration{ + K8sServiceEp: k8sapi.ServiceEndpoint{}, + Installation: instance, + TLS: typhaNodeTLS, + ClusterDomain: dns.DefaultClusterDomain, + FelixHealthPort: 9099, + }) + Expect(comp.ResolveImages(nil)).NotTo(HaveOccurred()) + objs, _ := comp.Objects() + objs, _ = extensionstest.ApplyExtensions(ext, render.ComponentNameTypha, renderCtx, objs, nil) + + role, ok := extensions.FindObject[*rbacv1.ClusterRole](objs, "calico-typha") + Expect(ok).To(BeTrue()) + Expect(role.Rules).To(ContainElement(HaveField("Resources", ContainElement("licensekeys")))) + }) +}) diff --git a/pkg/enterprise/installation/node_test.go b/pkg/enterprise/installation/node_test.go new file mode 100644 index 0000000000..e8f69a90ea --- /dev/null +++ b/pkg/enterprise/installation/node_test.go @@ -0,0 +1,191 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package installation_test + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + client "sigs.k8s.io/controller-runtime/pkg/client" + + v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/common" + "github.com/tigera/operator/pkg/components" + "github.com/tigera/operator/pkg/extensions" + "github.com/tigera/operator/pkg/extensions/extensionstest" + "github.com/tigera/operator/pkg/render" +) + +var _ = Describe("node enterprise image override", func() { + + It("selects the enterprise node image for the enterprise variant", func() { + ent := &operatorv1.InstallationSpec{Variant: operatorv1.CalicoEnterprise} + Expect(ext.ResolveImage("node", components.ComponentCalicoNode, ent)).To(Equal(components.ComponentTigeraNode)) + }) + + It("leaves the default in place for the Calico variant", func() { + calico := &operatorv1.InstallationSpec{Variant: operatorv1.Calico} + Expect(ext.ResolveImage("node", components.ComponentCalicoNode, calico)).To(Equal(components.ComponentCalicoNode)) + }) +}) + +var _ = Describe("node enterprise modifier", func() { + + // newObjs returns the subset of rendered node objects the modifier touches. + newObjs := func() []client.Object { + return []client.Object{ + &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: render.CalicoNodeObjectName}}, + &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: render.CalicoCNIPluginObjectName}}, + &appsv1.DaemonSet{ + ObjectMeta: metav1.ObjectMeta{Name: common.NodeDaemonSetName}, + Spec: appsv1.DaemonSetSpec{Template: corev1.PodTemplateSpec{Spec: corev1.PodSpec{ + InitContainers: []corev1.Container{{Name: "install-cni"}}, + Containers: []corev1.Container{{ + Name: render.CalicoNodeObjectName, + ReadinessProbe: &corev1.Probe{ProbeHandler: corev1.ProbeHandler{Exec: &corev1.ExecAction{ + Command: []string{"/bin/calico-node", "-bird-ready", "--bird-ready", "--felix-ready"}, + }}}, + }}, + }}}, + }, + } + } + + nodeContainer := func(ds *appsv1.DaemonSet) *corev1.Container { + for i := range ds.Spec.Template.Spec.Containers { + if ds.Spec.Template.Spec.Containers[i].Name == render.CalicoNodeObjectName { + return &ds.Spec.Template.Spec.Containers[i] + } + } + return nil + } + + entCtx := func() render.RenderContext { + return render.RenderContext{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.CalicoEnterprise}} + } + + It("adds the enterprise cluster role rules", func() { + out, _ := extensionstest.ApplyExtensions(ext, render.ComponentNameNode, entCtx(), newObjs(), nil) + + nodeRole, ok := extensions.FindObject[*rbacv1.ClusterRole](out, render.CalicoNodeObjectName) + Expect(ok).To(BeTrue()) + Expect(nodeRole.Rules).To(ContainElement(HaveField("Resources", ContainElement("licensekeys")))) + + cniRole, ok := extensions.FindObject[*rbacv1.ClusterRole](out, render.CalicoCNIPluginObjectName) + Expect(ok).To(BeTrue()) + Expect(cniRole.Rules).To(ContainElement(HaveField("Resources", ConsistOf("networks")))) + }) + + It("adds the enterprise felix env to the node container", func() { + out, _ := extensionstest.ApplyExtensions(ext, render.ComponentNameNode, entCtx(), newObjs(), nil) + ds, _ := extensions.FindObject[*appsv1.DaemonSet](out, common.NodeDaemonSetName) + c := nodeContainer(ds) + + Expect(c.Env).To(ContainElements( + corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERENABLED", Value: "true"}, + corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERPORT", Value: "9081"}, + corev1.EnvVar{Name: "FELIX_FLOWLOGSFILEENABLED", Value: "true"}, + corev1.EnvVar{Name: "FELIX_DNSLOGSFILEENABLED", Value: "true"}, + )) + }) + + It("derives the reporter port from FelixConfiguration", func() { + reporter := 7081 + ctx := entCtx() + ctx.FelixConfiguration = &v3.FelixConfiguration{Spec: v3.FelixConfigurationSpec{PrometheusReporterPort: &reporter}} + + out, _ := extensionstest.ApplyExtensions(ext, render.ComponentNameNode, ctx, newObjs(), nil) + ds, _ := extensions.FindObject[*appsv1.DaemonSet](out, common.NodeDaemonSetName) + Expect(nodeContainer(ds).Env).To(ContainElement(corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERPORT", Value: "7081"})) + }) + + It("appends the BGP metrics readiness check when the bird check is present", func() { + out, _ := extensionstest.ApplyExtensions(ext, render.ComponentNameNode, entCtx(), newObjs(), nil) + ds, _ := extensions.FindObject[*appsv1.DaemonSet](out, common.NodeDaemonSetName) + Expect(nodeContainer(ds).ReadinessProbe.Exec.Command).To(ContainElement("--bgp-metrics-ready")) + }) + + It("does not add the BGP metrics readiness check when the bird check is absent", func() { + objs := newObjs() + ds := objs[2].(*appsv1.DaemonSet) + ds.Spec.Template.Spec.Containers[0].ReadinessProbe.Exec.Command = []string{"/bin/calico-node", "--felix-ready"} + + out, _ := extensionstest.ApplyExtensions(ext, render.ComponentNameNode, entCtx(), objs, nil) + got, _ := extensions.FindObject[*appsv1.DaemonSet](out, common.NodeDaemonSetName) + Expect(nodeContainer(got).ReadinessProbe.Exec.Command).NotTo(ContainElement("--bgp-metrics-ready")) + }) + + It("adds MULTI_INTERFACE_MODE to the node and install-cni containers when configured", func() { + mode := operatorv1.MultiInterfaceModeMultus + ctx := entCtx() + ctx.Installation.CalicoNetwork = &operatorv1.CalicoNetworkSpec{MultiInterfaceMode: &mode} + + out, _ := extensionstest.ApplyExtensions(ext, render.ComponentNameNode, ctx, newObjs(), nil) + ds, _ := extensions.FindObject[*appsv1.DaemonSet](out, common.NodeDaemonSetName) + + want := corev1.EnvVar{Name: "MULTI_INTERFACE_MODE", Value: mode.Value()} + Expect(nodeContainer(ds).Env).To(ContainElement(want)) + Expect(ds.Spec.Template.Spec.InitContainers[0].Env).To(ContainElement(want)) + }) + + It("appends the node metrics service", func() { + out, _ := extensionstest.ApplyExtensions(ext, render.ComponentNameNode, entCtx(), newObjs(), nil) + svc, ok := extensions.FindObject[*corev1.Service](out, render.CalicoNodeMetricsService) + Expect(ok).To(BeTrue()) + Expect(svc.Spec.Ports).To(HaveLen(2)) + Expect(svc.Spec.Ports[0].Port).To(Equal(int32(9081))) + Expect(svc.Spec.Ports[1].Port).To(Equal(int32(9900))) + }) + + It("derives metrics service ports and felix-metrics-port from FelixConfiguration", func() { + reporter := 7081 + metrics := 7091 + enabled := true + ctx := entCtx() + ctx.FelixConfiguration = &v3.FelixConfiguration{Spec: v3.FelixConfigurationSpec{ + PrometheusReporterPort: &reporter, + PrometheusMetricsPort: &metrics, + PrometheusMetricsEnabled: &enabled, + }} + + out, _ := extensionstest.ApplyExtensions(ext, render.ComponentNameNode, ctx, newObjs(), nil) + svc, _ := extensions.FindObject[*corev1.Service](out, render.CalicoNodeMetricsService) + Expect(svc.Spec.Ports).To(HaveLen(3)) + Expect(svc.Spec.Ports[0].Port).To(Equal(int32(7081))) + Expect(svc.Spec.Ports[2].Name).To(Equal("felix-metrics-port")) + Expect(svc.Spec.Ports[2].Port).To(Equal(int32(7091))) + }) + + It("is a no-op for the Calico variant", func() { + ctx := render.RenderContext{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.Calico}} + out, _ := extensionstest.ApplyExtensions(ext, render.ComponentNameNode, ctx, newObjs(), nil) + + _, ok := extensions.FindObject[*corev1.Service](out, render.CalicoNodeMetricsService) + Expect(ok).To(BeFalse()) + nodeRole, _ := extensions.FindObject[*rbacv1.ClusterRole](out, render.CalicoNodeObjectName) + Expect(nodeRole.Rules).To(BeEmpty()) + }) + + It("does not panic on a zero RenderContext", func() { + out, _ := extensionstest.ApplyExtensions(ext, render.ComponentNameNode, render.RenderContext{}, newObjs(), nil) + _, ok := extensions.FindObject[*corev1.Service](out, render.CalicoNodeMetricsService) + Expect(ok).To(BeFalse()) + }) +}) diff --git a/pkg/enterprise/installation/suite_test.go b/pkg/enterprise/installation/suite_test.go new file mode 100644 index 0000000000..c3aa6ebbe7 --- /dev/null +++ b/pkg/enterprise/installation/suite_test.go @@ -0,0 +1,34 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package installation_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/tigera/operator/pkg/enterprise" + "github.com/tigera/operator/pkg/extensions" +) + +// ext is the enterprise extension Set under test, shared across the suite. It is +// immutable once built and the specs only read it, so a single instance is safe. +var ext *extensions.Set = enterprise.New() + +func TestInstallation(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "pkg/enterprise/installation Suite") +} diff --git a/pkg/render/kubecontrollers/waf_pull_secret_test.go b/pkg/enterprise/installation/waf_pull_secret_test.go similarity index 88% rename from pkg/render/kubecontrollers/waf_pull_secret_test.go rename to pkg/enterprise/installation/waf_pull_secret_test.go index 793374f169..6cb54f5f51 100644 --- a/pkg/render/kubecontrollers/waf_pull_secret_test.go +++ b/pkg/enterprise/installation/waf_pull_secret_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package kubecontrollers_test +package installation_test import ( "encoding/json" @@ -22,7 +22,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/tigera/operator/pkg/common" - "github.com/tigera/operator/pkg/render/kubecontrollers" + "github.com/tigera/operator/pkg/enterprise/installation" ) func dockerConfigJSONSecret(name string, auths map[string]any) *corev1.Secret { @@ -49,7 +49,7 @@ func mergedAuths(t *testing.T, s *corev1.Secret) map[string]map[string]string { } func TestMergeWAFPullSecret_MergesDisjointRegistries(t *testing.T) { - merged, skipped := kubecontrollers.MergeWAFPullSecret([]*corev1.Secret{ + merged, skipped := installation.MergeWAFPullSecret([]*corev1.Secret{ dockerConfigJSONSecret("tigera-pull-secret", map[string]any{"quay.io": map[string]string{"auth": "dGlnZXJh"}}), dockerConfigJSONSecret("mirror-pull-secret", map[string]any{"registry.example.com": map[string]string{"auth": "bWlycm9y"}}), }) @@ -59,7 +59,7 @@ func TestMergeWAFPullSecret_MergesDisjointRegistries(t *testing.T) { if merged == nil { t.Fatal("expected a merged secret") } - if merged.Name != kubecontrollers.WASMPullSecretName || merged.Namespace != common.CalicoNamespace { + if merged.Name != installation.WASMPullSecretName || merged.Namespace != common.CalicoNamespace { t.Fatalf("unexpected name/namespace: %s/%s", merged.Namespace, merged.Name) } if merged.Type != corev1.SecretTypeDockerConfigJson { @@ -72,7 +72,7 @@ func TestMergeWAFPullSecret_MergesDisjointRegistries(t *testing.T) { } func TestMergeWAFPullSecret_FirstSecretWinsOnDuplicateRegistry(t *testing.T) { - merged, _ := kubecontrollers.MergeWAFPullSecret([]*corev1.Secret{ + merged, _ := installation.MergeWAFPullSecret([]*corev1.Secret{ dockerConfigJSONSecret("first", map[string]any{"quay.io": map[string]string{"auth": "Zmlyc3Q="}}), dockerConfigJSONSecret("second", map[string]any{"quay.io": map[string]string{"auth": "c2Vjb25k"}}), }) @@ -88,7 +88,7 @@ func TestMergeWAFPullSecret_SkipsUnparseableSecrets(t *testing.T) { Type: corev1.SecretTypeDockerConfigJson, Data: map[string][]byte{corev1.DockerConfigJsonKey: []byte("not-json")}, } - merged, skipped := kubecontrollers.MergeWAFPullSecret([]*corev1.Secret{ + merged, skipped := installation.MergeWAFPullSecret([]*corev1.Secret{ bad, dockerConfigJSONSecret("good", map[string]any{"quay.io": map[string]string{"auth": "Z29vZA=="}}), }) @@ -111,7 +111,7 @@ func TestMergeWAFPullSecret_LegacyDockercfg(t *testing.T) { Type: corev1.SecretTypeDockercfg, Data: map[string][]byte{corev1.DockerConfigKey: cfg}, } - merged, skipped := kubecontrollers.MergeWAFPullSecret([]*corev1.Secret{legacy}) + merged, skipped := installation.MergeWAFPullSecret([]*corev1.Secret{legacy}) if len(skipped) != 0 { t.Fatalf("expected no skipped secrets, got %v", skipped) } @@ -127,7 +127,7 @@ func TestMergeWAFPullSecret_NothingUsableReturnsNil(t *testing.T) { Type: corev1.SecretTypeDockerConfigJson, Data: map[string][]byte{corev1.DockerConfigJsonKey: []byte("not-json")}, } - merged, skipped := kubecontrollers.MergeWAFPullSecret([]*corev1.Secret{bad}) + merged, skipped := installation.MergeWAFPullSecret([]*corev1.Secret{bad}) if merged != nil { t.Fatalf("expected nil secret, got %v", merged) } @@ -141,8 +141,8 @@ func TestMergeWAFPullSecret_DeterministicOutput(t *testing.T) { dockerConfigJSONSecret("a", map[string]any{"z.example.com": map[string]string{"auth": "eg=="}, "a.example.com": map[string]string{"auth": "YQ=="}}), dockerConfigJSONSecret("b", map[string]any{"m.example.com": map[string]string{"auth": "bQ=="}}), } - first, _ := kubecontrollers.MergeWAFPullSecret(in) - second, _ := kubecontrollers.MergeWAFPullSecret(in) + first, _ := installation.MergeWAFPullSecret(in) + second, _ := installation.MergeWAFPullSecret(in) if string(first.Data[corev1.DockerConfigJsonKey]) != string(second.Data[corev1.DockerConfigJsonKey]) { t.Fatal("merged secret bytes must be deterministic across reconciles") } diff --git a/pkg/enterprise/kubecontrollers/es_kube_controllers_test.go b/pkg/enterprise/kubecontrollers/es_kube_controllers_test.go new file mode 100644 index 0000000000..5c682fc99e --- /dev/null +++ b/pkg/enterprise/kubecontrollers/es_kube_controllers_test.go @@ -0,0 +1,369 @@ +// Copyright (c) 2020-2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package kubecontrollers + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + + v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" + + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/apis" + "github.com/tigera/operator/pkg/common" + "github.com/tigera/operator/pkg/components" + "github.com/tigera/operator/pkg/controller/certificatemanager" + "github.com/tigera/operator/pkg/controller/k8sapi" + ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" + "github.com/tigera/operator/pkg/dns" + "github.com/tigera/operator/pkg/render" + rtest "github.com/tigera/operator/pkg/render/common/test" + rkc "github.com/tigera/operator/pkg/render/kubecontrollers" + "github.com/tigera/operator/pkg/render/testutils" + "github.com/tigera/operator/pkg/tls/certificatemanagement" +) + +var _ = Describe("es-kube-controllers rendering tests", func() { + var ( + instance *operatorv1.InstallationSpec + k8sServiceEp k8sapi.ServiceEndpoint + cfg rkc.KubeControllersConfiguration + cli client.Client + ) + + esEnvs := []corev1.EnvVar{ + {Name: "ELASTIC_HOST", Value: "tigera-secure-es-gateway-http.tigera-elasticsearch.svc"}, + {Name: "ELASTIC_PORT", Value: "9200", ValueFrom: nil}, + { + Name: "ELASTIC_USERNAME", Value: "", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "tigera-ee-kube-controllers-elasticsearch-access", + }, + Key: "username", + }, + }, + }, + { + Name: "ELASTIC_PASSWORD", Value: "", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "tigera-ee-kube-controllers-elasticsearch-access", + }, + Key: "password", + }, + }, + }, + {Name: "ELASTIC_CA", Value: certificatemanagement.TrustedCertBundleMountPath}, + } + + // The es-kube-controllers policy fixtures live next to the render package's + // testutils, so reference them relative to this enterprise subpackage. + expectedESPolicy := testutils.GetExpectedPolicyFromFile("../../render/testutils/expected_policies/es-kubecontrollers.json") + expectedESPolicyForOpenshift := testutils.GetExpectedPolicyFromFile("../../render/testutils/expected_policies/es-kubecontrollers_ocp.json") + + BeforeEach(func() { + // Initialize a default instance to use. Each test can override this to its + // desired configuration. + + miMode := operatorv1.MultiInterfaceModeNone + instance = &operatorv1.InstallationSpec{ + CalicoNetwork: &operatorv1.CalicoNetworkSpec{ + IPPools: []operatorv1.IPPool{{CIDR: "192.168.1.0/16"}}, + MultiInterfaceMode: &miMode, + }, + Registry: "test-reg/", + } + k8sServiceEp = k8sapi.ServiceEndpoint{} + + scheme := runtime.NewScheme() + Expect(apis.AddToScheme(scheme, false)).NotTo(HaveOccurred()) + cli = ctrlrfake.DefaultFakeClientBuilder(scheme).Build() + + certificateManager, err := certificatemanager.Create(cli, nil, dns.DefaultClusterDomain, common.OperatorNamespace(), certificatemanager.AllowCACreation()) + Expect(err).NotTo(HaveOccurred()) + + cfg = rkc.KubeControllersConfiguration{ + K8sServiceEp: k8sServiceEp, + Installation: instance, + ClusterDomain: dns.DefaultClusterDomain, + MetricsPort: 9094, + TrustedBundle: certificateManager.CreateTrustedBundle(), + Namespace: common.CalicoNamespace, + BindingNamespaces: []string{common.CalicoNamespace}, + } + }) + + It("should render all es-calico-kube-controllers resources for a default configuration (standalone) using CalicoEnterprise when logstorage and secrets exist", func() { + expectedResources := []struct { + name string + ns string + group string + version string + kind string + }{ + {name: EsKubeControllerNetworkPolicyName, ns: common.CalicoNamespace, group: "projectcalico.org", version: "v3", kind: "NetworkPolicy"}, + {name: "calico-kube-controllers", ns: common.CalicoNamespace, group: "", version: "v1", kind: "ServiceAccount"}, + {name: EsKubeControllerRole, ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRole"}, + {name: EsKubeControllerRoleBinding, ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRoleBinding"}, + {name: EsKubeController, ns: common.CalicoNamespace, group: "apps", version: "v1", kind: "Deployment"}, + {name: ElasticsearchKubeControllersUserSecret, ns: common.CalicoNamespace, group: "", version: "v1", kind: "Secret"}, + {name: EsKubeControllerMetrics, ns: common.CalicoNamespace, group: "", version: "v1", kind: "Service"}, + } + + instance.Variant = operatorv1.CalicoEnterprise + cfg.KubeControllersGatewaySecret = &testutils.KubeControllersUserSecret + cfg.MetricsPort = 9094 + + component := NewElasticsearchKubeControllers(&cfg) + Expect(component.ResolveImages(nil)).To(BeNil()) + resources, _ := component.Objects() + Expect(len(resources)).To(Equal(len(expectedResources))) + + // Should render the correct resources. + i := 0 + for _, expectedRes := range expectedResources { + rtest.ExpectResourceTypeAndObjectMetadata(resources[i], expectedRes.name, expectedRes.ns, expectedRes.group, expectedRes.version, expectedRes.kind) + i++ + } + + // The Deployment should have the correct configuration. + dp := rtest.GetResource(resources, EsKubeController, common.CalicoNamespace, "apps", "v1", "Deployment").(*appsv1.Deployment) + + Expect(dp.Spec.Template.Spec.Containers[0].Image).To(Equal("test-reg/tigera/calico:" + components.ComponentTigeraCalico.Version)) + envs := dp.Spec.Template.Spec.Containers[0].Env + Expect(envs).To(ContainElement(corev1.EnvVar{ + Name: "ENABLED_CONTROLLERS", Value: "authorization,elasticsearchconfiguration", + })) + Expect(envs).To(ContainElements(esEnvs)) + + Expect(dp.Spec.Template.Spec.Containers[0].VolumeMounts).To(HaveLen(1)) + Expect(dp.Spec.Template.Spec.Containers[0].VolumeMounts[0].Name).To(Equal("tigera-ca-bundle")) + Expect(dp.Spec.Template.Spec.Containers[0].VolumeMounts[0].MountPath).To(Equal("/etc/pki/tls/certs")) + + Expect(dp.Spec.Template.Spec.Volumes).To(HaveLen(1)) + Expect(dp.Spec.Template.Spec.Volumes[0].Name).To(Equal("tigera-ca-bundle")) + Expect(dp.Spec.Template.Spec.Volumes[0].ConfigMap.Name).To(Equal("tigera-ca-bundle")) + + clusterRole := rtest.GetResource(resources, EsKubeControllerRole, "", "rbac.authorization.k8s.io", "v1", "ClusterRole").(*rbacv1.ClusterRole) + Expect(clusterRole.Rules).To(HaveLen(26), "cluster role should have 26 rules") + Expect(clusterRole.Rules).To(ContainElement( + rbacv1.PolicyRule{ + APIGroups: []string{""}, + Resources: []string{"configmaps"}, + Verbs: []string{"watch", "list", "get", "update", "create", "delete"}, + })) + Expect(clusterRole.Rules).To(ContainElement( + rbacv1.PolicyRule{ + APIGroups: []string{""}, + Resources: []string{"secrets"}, + Verbs: []string{"watch", "list", "get"}, + })) + }) + + It("should render all es-calico-kube-controllers resources for a default configuration using CalicoEnterprise and ClusterType is Management", func() { + expectedResources := []struct { + name string + ns string + group string + version string + kind string + }{ + {name: EsKubeControllerNetworkPolicyName, ns: common.CalicoNamespace, group: "projectcalico.org", version: "v3", kind: "NetworkPolicy"}, + {name: "calico-kube-controllers", ns: common.CalicoNamespace, group: "", version: "v1", kind: "ServiceAccount"}, + {name: EsKubeControllerRole, ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRole"}, + {name: EsKubeControllerRoleBinding, ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRoleBinding"}, + {name: rkc.ManagedClustersWatchRoleBindingName, ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRoleBinding"}, + {name: EsKubeController, ns: common.CalicoNamespace, group: "apps", version: "v1", kind: "Deployment"}, + {name: ElasticsearchKubeControllersUserSecret, ns: common.CalicoNamespace, group: "", version: "v1", kind: "Secret"}, + {name: EsKubeControllerMetrics, ns: common.CalicoNamespace, group: "", version: "v1", kind: "Service"}, + } + + // Override configuration to match expected Enterprise config. + instance.Variant = operatorv1.CalicoEnterprise + cfg.ManagementCluster = &operatorv1.ManagementCluster{} + cfg.KubeControllersGatewaySecret = &testutils.KubeControllersUserSecret + cfg.MetricsPort = 9094 + + component := NewElasticsearchKubeControllers(&cfg) + Expect(component.ResolveImages(nil)).To(BeNil()) + resources, _ := component.Objects() + Expect(len(resources)).To(Equal(len(expectedResources))) + + // Should render the correct resources. + i := 0 + for _, expectedRes := range expectedResources { + rtest.ExpectResourceTypeAndObjectMetadata(resources[i], expectedRes.name, expectedRes.ns, expectedRes.group, expectedRes.version, expectedRes.kind) + i++ + } + + // The Deployment should have the correct configuration. + dp := rtest.GetResource(resources, EsKubeController, common.CalicoNamespace, "apps", "v1", "Deployment").(*appsv1.Deployment) + + envs := dp.Spec.Template.Spec.Containers[0].Env + Expect(envs).To(ContainElement(corev1.EnvVar{ + Name: "ENABLED_CONTROLLERS", + Value: "authorization,elasticsearchconfiguration,managedcluster", + })) + + Expect(dp.Spec.Template.Spec.Containers[0].VolumeMounts).To(HaveLen(1)) + Expect(dp.Spec.Template.Spec.Containers[0].VolumeMounts[0].Name).To(Equal("tigera-ca-bundle")) + Expect(dp.Spec.Template.Spec.Containers[0].VolumeMounts[0].MountPath).To(Equal("/etc/pki/tls/certs")) + + Expect(dp.Spec.Template.Spec.Volumes).To(HaveLen(1)) + Expect(dp.Spec.Template.Spec.Volumes[0].Name).To(Equal("tigera-ca-bundle")) + Expect(dp.Spec.Template.Spec.Volumes[0].ConfigMap.Name).To(Equal("tigera-ca-bundle")) + + Expect(dp.Spec.Template.Spec.Containers[0].Image).To(Equal("test-reg/tigera/calico:" + components.ComponentTigeraCalico.Version)) + + clusterRole := rtest.GetResource(resources, EsKubeControllerRole, "", "rbac.authorization.k8s.io", "v1", "ClusterRole").(*rbacv1.ClusterRole) + Expect(clusterRole.Rules).To(HaveLen(26), "cluster role should have 26 rules") + Expect(clusterRole.Rules).To(ContainElement( + rbacv1.PolicyRule{ + APIGroups: []string{""}, + Resources: []string{"configmaps"}, + Verbs: []string{"watch", "list", "get", "update", "create", "delete"}, + })) + Expect(clusterRole.Rules).To(ContainElement( + rbacv1.PolicyRule{ + APIGroups: []string{""}, + Resources: []string{"secrets"}, + Verbs: []string{"watch", "list", "get"}, + })) + roleBindingWatch := rtest.GetResource(resources, rkc.ManagedClustersWatchRoleBindingName, "", "rbac.authorization.k8s.io", "v1", "ClusterRoleBinding").(*rbacv1.ClusterRoleBinding) + Expect(roleBindingWatch.RoleRef.Name).To(Equal(render.ManagedClustersWatchClusterRoleName)) + Expect(roleBindingWatch.Subjects).To(ConsistOf([]rbacv1.Subject{ + { + Kind: "ServiceAccount", + Name: rkc.KubeControllerServiceAccount, + Namespace: common.CalicoNamespace, + }, + })) + }) + + It("should add the OIDC prefix env variables", func() { + instance.Variant = operatorv1.CalicoEnterprise + cfg.ManagementCluster = &operatorv1.ManagementCluster{} + cfg.KubeControllersGatewaySecret = &testutils.KubeControllersUserSecret + cfg.MetricsPort = 9094 + cfg.Authentication = &operatorv1.Authentication{Spec: operatorv1.AuthenticationSpec{ + UsernamePrefix: "uOIDC:", + GroupsPrefix: "gOIDC:", + Openshift: &operatorv1.AuthenticationOpenshift{IssuerURL: "https://api.example.com"}, + }} + + component := NewElasticsearchKubeControllers(&cfg) + Expect(component.ResolveImages(nil)).To(BeNil()) + resources, _ := component.Objects() + + depResource := rtest.GetResource(resources, EsKubeController, common.CalicoNamespace, "apps", "v1", "Deployment") + Expect(depResource).ToNot(BeNil()) + deployment := depResource.(*appsv1.Deployment) + + var usernamePrefix, groupPrefix string + for _, container := range deployment.Spec.Template.Spec.Containers { + if container.Name == EsKubeController { + for _, env := range container.Env { + switch env.Name { + case "OIDC_AUTH_USERNAME_PREFIX": + usernamePrefix = env.Value + case "OIDC_AUTH_GROUP_PREFIX": + groupPrefix = env.Value + } + } + } + } + + Expect(usernamePrefix).To(Equal("uOIDC:")) + Expect(groupPrefix).To(Equal("gOIDC:")) + }) + + When("enableESOIDCWorkaround is true", func() { + It("should set the ENABLE_ELASTICSEARCH_OIDC_WORKAROUND env variable to true", func() { + instance.Variant = operatorv1.CalicoEnterprise + cfg.ManagementCluster = &operatorv1.ManagementCluster{} + cfg.KubeControllersGatewaySecret = &testutils.KubeControllersUserSecret + cfg.MetricsPort = 9094 + component := NewElasticsearchKubeControllers(&cfg) + resources, _ := component.Objects() + + depResource := rtest.GetResource(resources, EsKubeController, common.CalicoNamespace, "apps", "v1", "Deployment") + Expect(depResource).ToNot(BeNil()) + deployment := depResource.(*appsv1.Deployment) + + var esLicenseType string + for _, container := range deployment.Spec.Template.Spec.Containers { + if container.Name == EsKubeController { + for _, env := range container.Env { + if env.Name == "ENABLE_ELASTICSEARCH_OIDC_WORKAROUND" { + esLicenseType = env.Value + } + } + } + } + + Expect(esLicenseType).To(Equal("true")) + }) + }) + + Context("es-kube-controllers calico-system rendering", func() { + policyName := types.NamespacedName{Name: "calico-system.es-kube-controller-access", Namespace: common.CalicoNamespace} + + getExpectedPolicy := func(scenario testutils.CalicoSystemScenario) *v3.NetworkPolicy { + if scenario.ManagedCluster { + return nil + } + + return testutils.SelectPolicyByProvider(scenario, expectedESPolicy, expectedESPolicyForOpenshift) + } + + DescribeTable("should render calico-system policy", + func(scenario testutils.CalicoSystemScenario) { + if scenario.OpenShift { + cfg.Installation.KubernetesProvider = operatorv1.ProviderOpenShift + } else { + cfg.Installation.KubernetesProvider = operatorv1.ProviderNone + } + if scenario.ManagedCluster { + cfg.ManagementClusterConnection = &operatorv1.ManagementClusterConnection{} + } else { + cfg.ManagementClusterConnection = nil + } + instance.Variant = operatorv1.CalicoEnterprise + cfg.KubeControllersGatewaySecret = &testutils.KubeControllersUserSecret + + component := NewElasticsearchKubeControllers(&cfg) + resources, _ := component.Objects() + + policy := testutils.GetCalicoSystemPolicyFromResources(policyName, resources) + expectedPolicy := getExpectedPolicy(scenario) + Expect(policy).To(Equal(expectedPolicy)) + }, + Entry("for management/standalone, kube-dns", testutils.CalicoSystemScenario{ManagedCluster: false, OpenShift: false}), + Entry("for management/standalone, openshift-dns", testutils.CalicoSystemScenario{ManagedCluster: false, OpenShift: true}), + Entry("for managed, kube-dns", testutils.CalicoSystemScenario{ManagedCluster: true, OpenShift: false}), + Entry("for managed, openshift-dns", testutils.CalicoSystemScenario{ManagedCluster: true, OpenShift: true}), + ) + }) +}) diff --git a/pkg/enterprise/kubecontrollers/kubecontrollers.go b/pkg/enterprise/kubecontrollers/kubecontrollers.go new file mode 100644 index 0000000000..6141cd6b78 --- /dev/null +++ b/pkg/enterprise/kubecontrollers/kubecontrollers.go @@ -0,0 +1,335 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package kubecontrollers holds the enterprise es-calico-kube-controllers assembly +// (a distinct deployment the logstorage controller reconciles) and the enterprise +// kube-controllers cluster role rules shared with the calico-kube-controllers +// modifier in pkg/enterprise/installation. +package kubecontrollers + +import ( + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" + + "github.com/tigera/operator/pkg/render" + relasticsearch "github.com/tigera/operator/pkg/render/common/elasticsearch" + rmeta "github.com/tigera/operator/pkg/render/common/meta" + "github.com/tigera/operator/pkg/render/common/networkpolicy" + rkc "github.com/tigera/operator/pkg/render/kubecontrollers" + "github.com/tigera/operator/pkg/url" +) + +const ( + EsKubeController = "es-calico-kube-controllers" + EsKubeControllerRole = "es-calico-kube-controllers" + EsKubeControllerRoleBinding = "es-calico-kube-controllers" + EsKubeControllerMetrics = "es-calico-kube-controllers-metrics" + EsKubeControllerNetworkPolicyName = networkpolicy.CalicoComponentPolicyPrefix + "es-kube-controller-access" + + ElasticsearchKubeControllersUserSecret = "tigera-ee-kube-controllers-elasticsearch-access" + ElasticsearchKubeControllersUserName = "tigera-ee-kube-controllers" + ElasticsearchKubeControllersSecureUserSecret = "tigera-ee-kube-controllers-elasticsearch-access-gateway" + ElasticsearchKubeControllersVerificationUserSecret = "tigera-ee-kube-controllers-gateway-verification-credentials" +) + +// NewElasticsearchKubeControllers fills the generic kube-controllers configuration +// for the enterprise es-calico-kube-controllers deployment and returns the rendered +// component. es-kube-controllers is a distinct deployment (talks to Elasticsearch via +// es-gateway) reconciled by the logstorage kube-controllers controller, so it's +// assembled here rather than through the render-time modifier mechanism. +func NewElasticsearchKubeControllers(cfg *rkc.KubeControllersConfiguration) render.Component { + cfg.Name = EsKubeController + cfg.ConfigName = "elasticsearch" + cfg.RoleName = EsKubeControllerRole + cfg.RoleBindingName = EsKubeControllerRoleBinding + cfg.MetricsName = EsKubeControllerMetrics + cfg.DisableConfigAPI = cfg.Tenant.MultiTenant() + + cfg.Rules = rkc.KubeControllersRoleCommonRules(cfg) + cfg.Rules = append(cfg.Rules, KubeControllersEnterpriseCommonRules(false, cfg.ManagementClusterConnection != nil)...) + cfg.Rules = append(cfg.Rules, + rbacv1.PolicyRule{ + APIGroups: []string{"elasticsearch.k8s.elastic.co"}, + Resources: []string{"elasticsearches"}, + Verbs: []string{"watch", "get", "list"}, + }, + rbacv1.PolicyRule{ + APIGroups: []string{"rbac.authorization.k8s.io"}, + Resources: []string{"clusterroles", "clusterrolebindings"}, + Verbs: []string{"watch", "list", "get"}, + }, + ) + + if !cfg.Tenant.MultiTenant() { + // Zero and single tenant clusters need elasticsearch configuration. + cfg.EnabledControllers = append(cfg.EnabledControllers, "authorization", "elasticsearchconfiguration") + if cfg.ManagementCluster != nil && cfg.Tenant == nil { + // Enterprise requires the managedcluster controller to push licenses. + cfg.EnabledControllers = append(cfg.EnabledControllers, "managedcluster") + } + } + + cfg.NetworkPolicy = esKubeControllersCalicoSystemPolicy(cfg) + cfg.DeprecatedNetworkPolicyName = "es-kube-controller-access" + cfg.ExtraEnv = esKubeControllersEnv(cfg) + + return rkc.NewKubeControllers(cfg) +} + +// esKubeControllersEnv builds the enterprise env vars for es-calico-kube-controllers. +func esKubeControllersEnv(cfg *rkc.KubeControllersConfiguration) []corev1.EnvVar { + var env []corev1.EnvVar + + if cfg.Tenant != nil { + env = append(env, corev1.EnvVar{Name: "TENANT_ID", Value: cfg.Tenant.Spec.ID}) + } + + // What started as a workaround is now the default behaviour. This feature uses our backend in order to + // log into Kibana for users from external identity providers, rather than configuring an authn realm + // in the Elastic stack. + env = append(env, corev1.EnvVar{Name: "ENABLE_ELASTICSEARCH_OIDC_WORKAROUND", Value: "true"}) + if cfg.Authentication != nil { + env = append(env, + corev1.EnvVar{Name: "OIDC_AUTH_USERNAME_PREFIX", Value: cfg.Authentication.Spec.UsernamePrefix}, + corev1.EnvVar{Name: "OIDC_AUTH_GROUP_PREFIX", Value: cfg.Authentication.Spec.GroupsPrefix}, + ) + } + + if cfg.TrustedBundle != nil { + env = append(env, corev1.EnvVar{Name: "MULTI_CLUSTER_FORWARDING_CA", Value: cfg.TrustedBundle.MountPath()}) + } + if cfg.Installation.CalicoNetwork != nil && cfg.Installation.CalicoNetwork.MultiInterfaceMode != nil { + env = append(env, corev1.EnvVar{Name: "MULTI_INTERFACE_MODE", Value: cfg.Installation.CalicoNetwork.MultiInterfaceMode.Value()}) + } + + if !cfg.Tenant.MultiTenant() { + _, esHost, esPort, _ := url.ParseEndpoint(relasticsearch.GatewayEndpoint(rmeta.OSTypeLinux, cfg.ClusterDomain, render.ElasticsearchNamespace)) + env = append(env, + relasticsearch.ElasticHostEnvVar(esHost), + relasticsearch.ElasticPortEnvVar(esPort), + relasticsearch.ElasticUsernameEnvVar(ElasticsearchKubeControllersUserSecret), + relasticsearch.ElasticPasswordEnvVar(ElasticsearchKubeControllersUserSecret), + relasticsearch.ElasticCAEnvVar(rmeta.OSTypeLinux), + ) + } + + return env +} + +// KubeControllersEnterpriseCommonRules are the Calico Enterprise cluster role rules +// shared by calico-kube-controllers and es-calico-kube-controllers. wafEnabled adds +// the WAF v3 (Gateway API add-on) rules; managedCluster adds the license-push rule a +// managed cluster's kube-controllers needs. +func KubeControllersEnterpriseCommonRules(wafEnabled, managedCluster bool) []rbacv1.PolicyRule { + rules := []rbacv1.PolicyRule{ + { + APIGroups: []string{""}, + Resources: []string{"configmaps"}, + Verbs: []string{"watch", "list", "get", "update", "create", "delete"}, + }, + { + // The Federated Services Controller needs access to the remote kubeconfig secret + // in order to create a remote syncer. + APIGroups: []string{""}, + Resources: []string{"secrets"}, + Verbs: []string{"watch", "list", "get"}, + }, + { + // Needed to validate the license + APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, + Resources: []string{"licensekeys"}, + Verbs: []string{"get", "watch", "list"}, + }, + { + // Needed to update the status of the LicenseKey with the result of license validation. + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"licensekeys/status"}, + Verbs: []string{"update"}, + }, + { + APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, + Resources: []string{"deeppacketinspections"}, + Verbs: []string{"get", "watch", "list"}, + }, + { + APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, + Resources: []string{"deeppacketinspections/status"}, + Verbs: []string{"update"}, + }, + { + APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, + Resources: []string{"packetcaptures"}, + Verbs: []string{"get", "list", "update"}, + }, + { + APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, + Resources: []string{"packetcaptures/status"}, + Verbs: []string{"update"}, + }, + } + + if wafEnabled { + rules = append(rules, wafRules()...) + } + + if managedCluster { + rules = append(rules, + rbacv1.PolicyRule{ + APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, + Resources: []string{"licensekeys"}, + Verbs: []string{"get", "create", "update", "list", "watch"}, + }, + ) + } + + return rules +} + +// wafRules are the WAF v3 (Gateway API add-on) cluster role rules, gated by +// GatewayAPI.spec.extensions.waf.state == Enabled. +func wafRules() []rbacv1.PolicyRule { + return []rbacv1.PolicyRule{ + // Application-layer (gateway-addons) reconcilers reconcile WAF resources + // against Gateway API targetRefs and emit events on the policy objects. + { + APIGroups: []string{"applicationlayer.projectcalico.org"}, + Resources: []string{ + "wafpolicies", "globalwafpolicies", + "wafplugins", "globalwafplugins", + "wafvalidationpolicies", "globalwafvalidationpolicies", + }, + Verbs: []string{"get", "list", "watch", "create", "update", "patch", "delete"}, + }, + { + APIGroups: []string{"applicationlayer.projectcalico.org"}, + Resources: []string{ + "wafpolicies/status", "globalwafpolicies/status", + "wafplugins/status", "globalwafplugins/status", + "wafvalidationpolicies/status", "globalwafvalidationpolicies/status", + }, + Verbs: []string{"get", "update", "patch"}, + }, + { + APIGroups: []string{"applicationlayer.projectcalico.org"}, + Resources: []string{ + "wafpolicies/finalizers", "globalwafpolicies/finalizers", + "wafplugins/finalizers", "globalwafplugins/finalizers", + "wafvalidationpolicies/finalizers", "globalwafvalidationpolicies/finalizers", + }, + Verbs: []string{"update"}, + }, + { + // Validate Gateway API targetRefs and surface attachment status. + APIGroups: []string{"gateway.networking.k8s.io"}, + Resources: []string{"gateways", "httproutes", "tcproutes", "tlsroutes", "grpcroutes"}, + Verbs: []string{"get", "list", "watch", "update", "patch"}, + }, + { + APIGroups: []string{"gateway.networking.k8s.io"}, + Resources: []string{"gateways/status", "httproutes/status", "tcproutes/status", "tlsroutes/status", "grpcroutes/status"}, + Verbs: []string{"get", "update", "patch"}, + }, + // controller-runtime Reconcilers (e.g. the applicationlayer manager) record + // events on watched objects via Recorder.Eventf; both core and events.k8s.io + // API groups are emitted depending on the kubernetes version. + { + APIGroups: []string{""}, + Resources: []string{"events"}, + Verbs: []string{"create", "patch"}, + }, + { + APIGroups: []string{"events.k8s.io"}, + Resources: []string{"events"}, + Verbs: []string{"create", "patch"}, + }, + // Application-layer reconciler replicates the WAF wasm pull Secret from + // the controller namespace (calico-system) into each WAFPolicy's + // namespace so the rendered EnvoyExtensionPolicy can reference it. Also + // replicates CA-cert ConfigMaps when WASM_CA_CERT is set. + { + APIGroups: []string{""}, + Resources: []string{"secrets", "configmaps"}, + Verbs: []string{"get", "list", "watch", "create", "update", "patch", "delete"}, + }, + // Application-layer reconciler emits one EnvoyExtensionPolicy per WAF + // targetRef to bind the Coraza wasm filter at the gateway / route. + { + APIGroups: []string{"gateway.envoyproxy.io"}, + Resources: []string{"envoyextensionpolicies"}, + Verbs: []string{"get", "list", "watch", "create", "update", "patch", "delete"}, + }, + // Application-layer reconciler stamps each namespace with its allocated WAF + // rule-id range (applicationlayer.projectcalico.org/waf-id-range annotation) + // so application operators can author in-range rules. The base role already + // grants namespaces get/list/watch; the annotation write needs patch/update. + { + APIGroups: []string{""}, + Resources: []string{"namespaces"}, + Verbs: []string{"get", "patch", "update"}, + }, + } +} + +func esKubeControllersCalicoSystemPolicy(cfg *rkc.KubeControllersConfiguration) *v3.NetworkPolicy { + if cfg.ManagementClusterConnection != nil { + return nil + } + + egressRules := []v3.Rule{} + egressRules = networkpolicy.AppendDNSEgressRules(egressRules, cfg.Installation.KubernetesProvider.IsOpenShift()) + egressRules = append(egressRules, []v3.Rule{ + { + Action: v3.Allow, + Protocol: &networkpolicy.TCPProtocol, + Destination: v3.EntityRule{ + Ports: networkpolicy.Ports(443, 6443, 12388), + }, + }, + }...) + + egressRules = append(egressRules, []v3.Rule{ + { + Action: v3.Allow, + Protocol: &networkpolicy.TCPProtocol, + Destination: networkpolicy.DefaultHelper().ESGatewayEntityRule(), + }, + }...) + + networkpolicyHelper := networkpolicy.Helper(cfg.Tenant.MultiTenant(), cfg.Namespace) + egressRules = append(egressRules, []v3.Rule{ + { + Action: v3.Allow, + Protocol: &networkpolicy.TCPProtocol, + Destination: networkpolicyHelper.ManagerEntityRule(), + }, + }...) + + return &v3.NetworkPolicy{ + TypeMeta: metav1.TypeMeta{Kind: "NetworkPolicy", APIVersion: "projectcalico.org/v3"}, + ObjectMeta: metav1.ObjectMeta{ + Name: EsKubeControllerNetworkPolicyName, + Namespace: cfg.Namespace, + }, + Spec: v3.NetworkPolicySpec{ + Order: &networkpolicy.HighPrecedenceOrder, + Tier: networkpolicy.CalicoTierName, + Selector: networkpolicy.KubernetesAppSelector(EsKubeController), + Types: []v3.PolicyType{v3.PolicyTypeEgress}, + Egress: egressRules, + }, + } +} diff --git a/pkg/enterprise/kubecontrollers/suite_test.go b/pkg/enterprise/kubecontrollers/suite_test.go new file mode 100644 index 0000000000..a28166fba6 --- /dev/null +++ b/pkg/enterprise/kubecontrollers/suite_test.go @@ -0,0 +1,27 @@ +// Copyright (c) 2020-2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package kubecontrollers_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestKubeControllers(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "pkg/enterprise/kubecontrollers Suite") +} diff --git a/pkg/enterprise/options/options.go b/pkg/enterprise/options/options.go new file mode 100644 index 0000000000..9a621be5e5 --- /dev/null +++ b/pkg/enterprise/options/options.go @@ -0,0 +1,33 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package options + +import "github.com/tigera/operator/pkg/controller/contexts" + +// Options is the Calico Enterprise controller-phase options. The extension build +// computes it once at startup (see extensions.Set.ComputeOptions) and its +// controller hooks read it back off the ControllerContext via From. It lives in +// its own leaf package so the hooks and the operator's main can both reference it. +type Options struct { + // MultiTenant reports whether the operator runs in multi-tenant mode. + MultiTenant bool +} + +// From returns the enterprise options carried on the controller context, or the +// zero value when none are set. +func From(cc contexts.ControllerContext) Options { + o, _ := cc.Options.(Options) + return o +} diff --git a/pkg/enterprise/register.go b/pkg/enterprise/register.go new file mode 100644 index 0000000000..12f02fd64f --- /dev/null +++ b/pkg/enterprise/register.go @@ -0,0 +1,69 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package enterprise + +import ( + "context" + + "k8s.io/client-go/kubernetes" + + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/common/discovery" + "github.com/tigera/operator/pkg/enterprise/apiserver" + "github.com/tigera/operator/pkg/enterprise/clusterconnection" + "github.com/tigera/operator/pkg/enterprise/guardian" + "github.com/tigera/operator/pkg/enterprise/installation" + eoptions "github.com/tigera/operator/pkg/enterprise/options" + "github.com/tigera/operator/pkg/enterprise/typha" + "github.com/tigera/operator/pkg/enterprise/windows" + "github.com/tigera/operator/pkg/extensions" +) + +// New builds the extension Set for the in-repo Calico Enterprise variant: the +// controller extension, every component modifier, and the image overrides. The +// operator is handed this Set at startup (the core operator is handed none). +// After the monorepo split this is what calico-private's main will construct +// instead. Each per-component subpackage registers its own controller hook and +// modifiers through its Register func. +func New() *extensions.Set { + s := extensions.NewSet() + s.RegisterOptions(computeOptions) + + ent := s.Variant(operatorv1.CalicoEnterprise) + typha.Register(ent) + installation.Register(ent) + windows.Register(ent) + guardian.Register(ent) + apiserver.Register(ent) + clusterconnection.Register(ent) + + // When the enterprise operator manages a Calico installation, clean up the + // Enterprise objects left behind by a prior Enterprise installation. + cal := s.Variant(operatorv1.Calico) + apiserver.RegisterCalicoCleanup(cal) + + return s +} + +// computeOptions discovers the Calico Enterprise controller-phase options at +// startup. extensions.Set.ComputeOptions runs it from main; the result rides on +// each ControllerContext for the enterprise hooks to read. +func computeOptions(ctx context.Context, cli kubernetes.Interface) (any, error) { + multiTenant, err := discovery.MultiTenant(ctx, cli) + if err != nil { + return nil, err + } + return eoptions.Options{MultiTenant: multiTenant}, nil +} diff --git a/pkg/enterprise/typha/extension.go b/pkg/enterprise/typha/extension.go new file mode 100644 index 0000000000..203d8c6236 --- /dev/null +++ b/pkg/enterprise/typha/extension.go @@ -0,0 +1,63 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package typha + +import ( + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/tigera/operator/pkg/extensions" + "github.com/tigera/operator/pkg/render" +) + +// Register wires the typha extension into the variant. +func Register(v *extensions.Variant) { + v.Modify(render.ComponentNameTypha, modifyTypha) +} + +func modifyTypha(rc render.RenderContext, objs, del []client.Object) ([]client.Object, []client.Object) { + if role, ok := extensions.FindObject[*rbacv1.ClusterRole](objs, "calico-typha"); ok { + role.Rules = append(role.Rules, rbacv1.PolicyRule{ + APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, + Resources: []string{ + "bfdconfigurations", + "deeppacketinspections", + "egressgatewaypolicies", + "externalnetworks", + "licensekeys", + "networks", + "packetcaptures", + "remoteclusterconfigurations", + }, + Verbs: []string{"get", "list", "watch"}, + }) + } + + if dep, ok := extensions.FindObject[*appsv1.Deployment](objs, "calico-typha"); ok { + net := rc.Installation.CalicoNetwork + if net != nil && net.MultiInterfaceMode != nil { + for i := range dep.Spec.Template.Spec.Containers { + if dep.Spec.Template.Spec.Containers[i].Name == render.TyphaContainerName { + c := &dep.Spec.Template.Spec.Containers[i] + c.Env = append(c.Env, corev1.EnvVar{Name: "MULTI_INTERFACE_MODE", Value: net.MultiInterfaceMode.Value()}) + } + } + } + } + + return objs, del +} diff --git a/pkg/enterprise/typha/extension_test.go b/pkg/enterprise/typha/extension_test.go new file mode 100644 index 0000000000..c1eff29e53 --- /dev/null +++ b/pkg/enterprise/typha/extension_test.go @@ -0,0 +1,82 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package typha_test + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/extensions/extensionstest" + "github.com/tigera/operator/pkg/render" +) + +var _ = Describe("typha enterprise modifier", func() { + + multiMode := operatorv1.MultiInterfaceModeMultus + + newObjs := func() []client.Object { + return []client.Object{ + &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-typha"}}, + &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{Name: "calico-typha"}, + Spec: appsv1.DeploymentSpec{Template: corev1.PodTemplateSpec{Spec: corev1.PodSpec{ + Containers: []corev1.Container{{Name: render.TyphaContainerName}}, + }}}, + }, + } + } + + It("adds enterprise RBAC and MULTI_INTERFACE_MODE for the enterprise variant", func() { + ctx := render.RenderContext{Installation: &operatorv1.InstallationSpec{ + Variant: operatorv1.CalicoEnterprise, + CalicoNetwork: &operatorv1.CalicoNetworkSpec{MultiInterfaceMode: &multiMode}, + }} + out, _ := extensionstest.ApplyExtensions(ext, render.ComponentNameTypha, ctx, newObjs(), nil) + + role := out[0].(*rbacv1.ClusterRole) + Expect(role.Rules).To(ContainElement(HaveField("Resources", ContainElement("licensekeys")))) + + dep := out[1].(*appsv1.Deployment) + var c *corev1.Container + for i := range dep.Spec.Template.Spec.Containers { + if dep.Spec.Template.Spec.Containers[i].Name == render.TyphaContainerName { + c = &dep.Spec.Template.Spec.Containers[i] + } + } + Expect(c.Env).To(ContainElement(corev1.EnvVar{Name: "MULTI_INTERFACE_MODE", Value: multiMode.Value()})) + }) + + It("is a no-op for the Calico variant", func() { + ctx := render.RenderContext{Installation: &operatorv1.InstallationSpec{ + Variant: operatorv1.Calico, + CalicoNetwork: &operatorv1.CalicoNetworkSpec{MultiInterfaceMode: &multiMode}, + }} + out, _ := extensionstest.ApplyExtensions(ext, render.ComponentNameTypha, ctx, newObjs(), nil) + Expect(out[0].(*rbacv1.ClusterRole).Rules).To(BeEmpty()) + dep := out[1].(*appsv1.Deployment) + Expect(dep.Spec.Template.Spec.Containers[0].Env).To(BeEmpty()) + }) + + It("does not panic on a zero Context (nil Installation)", func() { + out, _ := extensionstest.ApplyExtensions(ext, render.ComponentNameTypha, render.RenderContext{}, newObjs(), nil) + Expect(out[0].(*rbacv1.ClusterRole).Rules).To(BeEmpty()) + }) +}) diff --git a/pkg/enterprise/typha/suite_test.go b/pkg/enterprise/typha/suite_test.go new file mode 100644 index 0000000000..960119299b --- /dev/null +++ b/pkg/enterprise/typha/suite_test.go @@ -0,0 +1,34 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package typha_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/tigera/operator/pkg/enterprise" + "github.com/tigera/operator/pkg/extensions" +) + +// ext is the enterprise extension Set under test, shared across the suite. It is +// immutable once built and the specs only read it, so a single instance is safe. +var ext *extensions.Set = enterprise.New() + +func TestTypha(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "pkg/enterprise/typha Suite") +} diff --git a/pkg/enterprise/windows/extension.go b/pkg/enterprise/windows/extension.go new file mode 100644 index 0000000000..d24b84083f --- /dev/null +++ b/pkg/enterprise/windows/extension.go @@ -0,0 +1,242 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package windows + +import ( + "fmt" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + "sigs.k8s.io/controller-runtime/pkg/client" + + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/common" + "github.com/tigera/operator/pkg/components" + "github.com/tigera/operator/pkg/controller/contexts" + "github.com/tigera/operator/pkg/controller/utils" + "github.com/tigera/operator/pkg/ctrlruntime" + "github.com/tigera/operator/pkg/dns" + "github.com/tigera/operator/pkg/enterprise/installation" + "github.com/tigera/operator/pkg/extensions" + "github.com/tigera/operator/pkg/render" + rmeta "github.com/tigera/operator/pkg/render/common/meta" + "github.com/tigera/operator/pkg/render/monitor" + "github.com/tigera/operator/pkg/tls/certificatemanagement" +) + +// windowsNodeContainers are the calico-node-windows containers that share the +// felix env and node volume mounts, so they receive the same enterprise layering. +var windowsNodeContainers = map[string]bool{"felix": true, "node": true, "confd": true} + +// Register wires the windows controller hook and modifiers into the variant. +func Register(v *extensions.Variant) { + v.Controller(contexts.WindowsController, windowsControllerExtension{}) + v.Image(render.ComponentNameWindowsNodeImg, components.ComponentTigeraNodeWindows) + v.Image(render.ComponentNameWindowsCNIImg, components.ComponentTigeraCNIWindows) + v.Modify(render.ComponentNameWindows, modifyWindows) +} + +// windowsControllerExtension is the Calico Enterprise controller-side hook for the +// windows controller. +type windowsControllerExtension struct{} + +// windowsRenderData is the controller-produced data the windows extension hands to +// its modifier through RenderContext.Extension. +type windowsRenderData struct { + prometheusServerTLS certificatemanagement.KeyPairInterface +} + +// windowsData pulls the windows extension's render data back out of the render +// context, returning the zero value when none is set. +func windowsData(rc render.RenderContext) windowsRenderData { + return render.ExtractExtensionData[windowsRenderData](rc) +} + +// Validate rejects windows installation config Calico Enterprise does not support. +func (windowsControllerExtension) Validate(cc contexts.ControllerContext) error { + return installation.ValidateReporterPort(cc.FelixConfiguration) +} + +// Watches registers the enterprise secrets the windows controller reconciles on. +func (windowsControllerExtension) Watches(c ctrlruntime.Controller) error { + for _, ns := range []string{common.CalicoNamespace, common.OperatorNamespace()} { + if err := utils.AddSecretsWatch(c, render.NodePrometheusTLSServerSecret, ns); err != nil { + return err + } + if err := utils.AddSecretsWatch(c, monitor.PrometheusClientTLSSecretName, ns); err != nil { + return err + } + } + return nil +} + +// ExtendContext fetches the node prometheus keypair the installation controller +// created and stashes it in the render context for the windows modifier. +func (windowsControllerExtension) ExtendContext(cc contexts.ControllerContext) (contexts.ControllerContext, []certificatemanagement.KeyPairInterface, error) { + tls, err := cc.CertificateManager.GetKeyPair( + cc.Client, + render.NodePrometheusTLSServerSecret, + common.OperatorNamespace(), + dns.GetServiceDNSNames(render.WindowsNodeMetricsService, common.CalicoNamespace, cc.ClusterDomain), + ) + if err != nil { + return cc, nil, fmt.Errorf("error getting node prometheus TLS certificate: %w", err) + } + cc.Extension = windowsRenderData{prometheusServerTLS: tls} + return cc, nil, nil +} + +// modifyWindows layers Calico Enterprise behavior onto the rendered +// calico-node-windows objects: the node-metrics Service and the Enterprise +// daemonset configuration (flow/DNS log env, prometheus reporter, trusted DNS +// servers, the calico log volume, and the prometheus reporter keypair mount). +func modifyWindows(rc render.RenderContext, objs, del []client.Object) ([]client.Object, []client.Object) { + if ds, ok := extensions.FindObject[*appsv1.DaemonSet](objs, common.WindowsDaemonSetName); ok { + modifyWindowsDaemonSet(rc, ds) + } + + return append(objs, windowsNodeMetricsService(rc)), del +} + +func modifyWindowsDaemonSet(rc render.RenderContext, ds *appsv1.DaemonSet) { + dirOrCreate := corev1.HostPathDirectoryOrCreate + spec := &ds.Spec.Template.Spec + + spec.Volumes = append(spec.Volumes, corev1.Volume{ + Name: "var-log-calico", + VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico", Type: &dirOrCreate}}, + }) + + for i := range spec.Containers { + c := &spec.Containers[i] + if !windowsNodeContainers[c.Name] { + continue + } + + c.Env = append(c.Env, windowsEnterpriseEnv(rc)...) + + // Enterprise mounts the calico log directory in place of the OSS CNI log + // directory, so drop the OSS mount before adding the enterprise one. + c.VolumeMounts = removeVolumeMount(c.VolumeMounts, "cni-log-dir") + c.VolumeMounts = append(c.VolumeMounts, corev1.VolumeMount{MountPath: "/var/log/calico", Name: "var-log-calico"}) + } + + mountWindowsPrometheusTLS(rc, ds) +} + +// windowsEnterpriseEnv is the Enterprise felix configuration added to the +// calico-node-windows containers. +func windowsEnterpriseEnv(rc render.RenderContext) []corev1.EnvVar { + tls := windowsData(rc).prometheusServerTLS + env := []corev1.EnvVar{ + {Name: "FELIX_PROMETHEUSREPORTERENABLED", Value: "true"}, + {Name: "FELIX_PROMETHEUSREPORTERPORT", Value: fmt.Sprintf("%d", installation.NodeReporterPort(rc.FelixConfiguration))}, + {Name: "FELIX_FLOWLOGSFILEENABLED", Value: "true"}, + {Name: "FELIX_FLOWLOGSFILEINCLUDELABELS", Value: "true"}, + {Name: "FELIX_FLOWLOGSFILEINCLUDEPOLICIES", Value: "true"}, + {Name: "FELIX_FLOWLOGSFILEINCLUDESERVICE", Value: "true"}, + {Name: "FELIX_FLOWLOGSENABLENETWORKSETS", Value: "true"}, + {Name: "FELIX_FLOWLOGSCOLLECTPROCESSINFO", Value: "true"}, + {Name: "FELIX_DNSLOGSFILEENABLED", Value: "true"}, + {Name: "FELIX_DNSLOGSFILEPERNODELIMIT", Value: "1000"}, + } + + if tls != nil && rc.TrustedBundle != nil { + env = append(env, + corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERCERTFILE", Value: tls.VolumeMountCertificateFilePath()}, + corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERKEYFILE", Value: tls.VolumeMountKeyFilePath()}, + corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERCAFILE", Value: rc.TrustedBundle.MountPath()}, + ) + } + + // Providers without a kube-dns service need a non-default trusted DNS server. + switch rc.Installation.KubernetesProvider { + case operatorv1.ProviderOpenShift: + env = append(env, corev1.EnvVar{Name: "FELIX_DNSTRUSTEDSERVERS", Value: "k8s-service:openshift-dns/dns-default"}) + case operatorv1.ProviderRKE2: + env = append(env, corev1.EnvVar{Name: "FELIX_DNSTRUSTEDSERVERS", Value: "k8s-service:kube-system/rke2-coredns-rke2-coredns"}) + } + + return env +} + +// mountWindowsPrometheusTLS mounts the node prometheus reporter keypair onto the +// windows daemonset: the volume, the volume mount on each node container, and +// the pod hash annotation that rolls the pods on cert rotation. +func mountWindowsPrometheusTLS(rc render.RenderContext, ds *appsv1.DaemonSet) { + tls := windowsData(rc).prometheusServerTLS + if tls == nil { + return + } + spec := &ds.Spec.Template.Spec + + spec.Volumes = append(spec.Volumes, tls.Volume()) + + for i := range spec.Containers { + c := &spec.Containers[i] + if windowsNodeContainers[c.Name] { + c.VolumeMounts = append(c.VolumeMounts, tls.VolumeMount(rmeta.OSTypeWindows)) + } + } + + if ds.Spec.Template.Annotations == nil { + ds.Spec.Template.Annotations = map[string]string{} + } + ds.Spec.Template.Annotations[tls.HashAnnotationKey()] = tls.HashAnnotationValue() +} + +// windowsNodeMetricsService builds the enterprise-only calico-node-metrics-windows +// Service. +func windowsNodeMetricsService(rc render.RenderContext) *corev1.Service { + reporterPort := installation.NodeReporterPort(rc.FelixConfiguration) + return &corev1.Service{ + TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: render.WindowsNodeMetricsService, + Namespace: common.CalicoNamespace, + Labels: map[string]string{"k8s-app": render.WindowsNodeObjectName}, + }, + Spec: corev1.ServiceSpec{ + Selector: map[string]string{"k8s-app": render.WindowsNodeObjectName}, + ClusterIP: "None", + Ports: []corev1.ServicePort{ + { + Name: "calico-metrics-port", + Port: int32(reporterPort), + TargetPort: intstr.FromInt(reporterPort), + Protocol: corev1.ProtocolTCP, + }, + { + Name: "calico-bgp-metrics-port", + Port: render.NodeBGPReporterPort, + TargetPort: intstr.FromInt(int(render.NodeBGPReporterPort)), + Protocol: corev1.ProtocolTCP, + }, + }, + }, + } +} + +func removeVolumeMount(mounts []corev1.VolumeMount, name string) []corev1.VolumeMount { + out := mounts[:0] + for _, m := range mounts { + if m.Name != name { + out = append(out, m) + } + } + return out +} diff --git a/pkg/enterprise/windows/extension_test.go b/pkg/enterprise/windows/extension_test.go new file mode 100644 index 0000000000..3709bbbcba --- /dev/null +++ b/pkg/enterprise/windows/extension_test.go @@ -0,0 +1,174 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package windows_test + +import ( + "context" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + client "sigs.k8s.io/controller-runtime/pkg/client" + + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/apis" + "github.com/tigera/operator/pkg/common" + "github.com/tigera/operator/pkg/components" + "github.com/tigera/operator/pkg/controller/certificatemanager" + "github.com/tigera/operator/pkg/controller/contexts" + ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" + "github.com/tigera/operator/pkg/dns" + "github.com/tigera/operator/pkg/extensions" + "github.com/tigera/operator/pkg/extensions/extensionstest" + "github.com/tigera/operator/pkg/render" +) + +var _ = Describe("windows enterprise image override", func() { + + ent := &operatorv1.InstallationSpec{Variant: operatorv1.CalicoEnterprise} + calico := &operatorv1.InstallationSpec{Variant: operatorv1.Calico} + + It("selects the enterprise windows images for the enterprise variant", func() { + Expect(ext.ResolveImage(render.ComponentNameWindowsNodeImg, components.ComponentCalicoNodeWindows, ent)).To(Equal(components.ComponentTigeraNodeWindows)) + Expect(ext.ResolveImage(render.ComponentNameWindowsCNIImg, components.ComponentCalicoCNIWindows, ent)).To(Equal(components.ComponentTigeraCNIWindows)) + }) + + It("leaves the defaults in place for the Calico variant", func() { + Expect(ext.ResolveImage(render.ComponentNameWindowsNodeImg, components.ComponentCalicoNodeWindows, calico)).To(Equal(components.ComponentCalicoNodeWindows)) + Expect(ext.ResolveImage(render.ComponentNameWindowsCNIImg, components.ComponentCalicoCNIWindows, calico)).To(Equal(components.ComponentCalicoCNIWindows)) + }) +}) + +var _ = Describe("windows enterprise modifier", func() { + + // newObjs returns a windows daemonset with the node containers and the OSS + // cni-log-dir mount the modifier swaps out. + newObjs := func() []client.Object { + nodeContainer := func(name string) corev1.Container { + return corev1.Container{ + Name: name, + VolumeMounts: []corev1.VolumeMount{{MountPath: "/var/log/calico/cni", Name: "cni-log-dir"}}, + } + } + return []client.Object{ + &appsv1.DaemonSet{ + ObjectMeta: metav1.ObjectMeta{Name: common.WindowsDaemonSetName}, + Spec: appsv1.DaemonSetSpec{Template: corev1.PodTemplateSpec{Spec: corev1.PodSpec{ + Containers: []corev1.Container{nodeContainer("felix"), nodeContainer("node"), nodeContainer("confd")}, + }}}, + }, + } + } + + ds := func(objs []client.Object) *appsv1.DaemonSet { + d, _ := extensions.FindObject[*appsv1.DaemonSet](objs, common.WindowsDaemonSetName) + return d + } + container := func(d *appsv1.DaemonSet, name string) *corev1.Container { + for i := range d.Spec.Template.Spec.Containers { + if d.Spec.Template.Spec.Containers[i].Name == name { + return &d.Spec.Template.Spec.Containers[i] + } + } + return nil + } + + ctxFor := func(provider operatorv1.Provider) render.RenderContext { + return render.RenderContext{ + Installation: &operatorv1.InstallationSpec{Variant: operatorv1.CalicoEnterprise, KubernetesProvider: provider}, + } + } + + It("appends the node-metrics service", func() { + out, _ := extensionstest.ApplyExtensions(ext, render.ComponentNameWindows, ctxFor(operatorv1.ProviderNone), newObjs(), nil) + svc, ok := extensions.FindObject[*corev1.Service](out, render.WindowsNodeMetricsService) + Expect(ok).To(BeTrue()) + Expect(svc.Namespace).To(Equal(common.CalicoNamespace)) + Expect(svc.Spec.Ports[0].Port).To(Equal(int32(9081))) + }) + + It("swaps the cni log mount for the calico log volume and adds enterprise env", func() { + out, _ := extensionstest.ApplyExtensions(ext, render.ComponentNameWindows, ctxFor(operatorv1.ProviderNone), newObjs(), nil) + d := ds(out) + + Expect(d.Spec.Template.Spec.Volumes).To(ContainElement(HaveField("Name", "var-log-calico"))) + for _, name := range []string{"felix", "node", "confd"} { + c := container(d, name) + Expect(c.VolumeMounts).To(ContainElement(HaveField("Name", "var-log-calico"))) + Expect(c.VolumeMounts).NotTo(ContainElement(HaveField("Name", "cni-log-dir"))) + Expect(c.Env).To(ContainElements( + corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERENABLED", Value: "true"}, + corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERPORT", Value: "9081"}, + corev1.EnvVar{Name: "FELIX_DNSLOGSFILEENABLED", Value: "true"}, + )) + } + }) + + It("sets the trusted DNS server on openshift", func() { + out, _ := extensionstest.ApplyExtensions(ext, render.ComponentNameWindows, ctxFor(operatorv1.ProviderOpenShift), newObjs(), nil) + Expect(container(ds(out), "node").Env).To(ContainElement(corev1.EnvVar{Name: "FELIX_DNSTRUSTEDSERVERS", Value: "k8s-service:openshift-dns/dns-default"})) + }) + + It("mounts the prometheus reporter keypair when present", func() { + scheme := runtime.NewScheme() + Expect(apis.AddToScheme(scheme, false)).NotTo(HaveOccurred()) + cli := ctrlrfake.DefaultFakeClientBuilder(scheme).Build() + cm, err := certificatemanager.Create(cli, nil, "", common.OperatorNamespace(), certificatemanager.AllowCACreation()) + Expect(err).NotTo(HaveOccurred()) + tls, err := cm.GetOrCreateKeyPair(cli, render.NodePrometheusTLSServerSecret, common.OperatorNamespace(), []string{"calico-node-metrics-windows"}) + Expect(err).NotTo(HaveOccurred()) + // The installation controller persists the secret; do the same here so the + // windows extension's GetKeyPair finds it. + Expect(cli.Create(context.Background(), tls.Secret(common.OperatorNamespace()))).NotTo(HaveOccurred()) + bundle := cm.CreateTrustedBundle() + + // Build the render context the way the windows controller does: run the + // windows extension, which fetches the keypair into the context. + cc := contexts.ControllerContext{ + RenderContext: render.RenderContext{ + Installation: ctxFor(operatorv1.ProviderNone).Installation, + TrustedBundle: bundle, + ClusterDomain: dns.DefaultClusterDomain, + }, + Controller: contexts.WindowsController, + Ctx: context.Background(), + Client: cli, + CertificateManager: cm, + } + ecc, _, err := ext.ExtendContext(cc) + rc := ecc.RenderContext + Expect(err).NotTo(HaveOccurred()) + + out, _ := extensionstest.ApplyExtensions(ext, render.ComponentNameWindows, rc, newObjs(), nil) + d := ds(out) + + Expect(d.Spec.Template.Spec.Volumes).To(ContainElement(tls.Volume())) + Expect(d.Spec.Template.Annotations).To(HaveKey(tls.HashAnnotationKey())) + Expect(container(d, "node").Env).To(ContainElement(HaveField("Name", "FELIX_PROMETHEUSREPORTERCERTFILE"))) + Expect(container(d, "node").VolumeMounts).To(ContainElement(tls.VolumeMount(render.Windows(&render.WindowsConfiguration{}).SupportedOSType()))) + }) + + It("does nothing for the Calico variant", func() { + ctx := render.RenderContext{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.Calico}} + out, _ := extensionstest.ApplyExtensions(ext, render.ComponentNameWindows, ctx, newObjs(), nil) + _, ok := extensions.FindObject[*corev1.Service](out, render.WindowsNodeMetricsService) + Expect(ok).To(BeFalse()) + Expect(ds(out).Spec.Template.Spec.Volumes).To(BeEmpty()) + }) +}) diff --git a/pkg/enterprise/windows/suite_test.go b/pkg/enterprise/windows/suite_test.go new file mode 100644 index 0000000000..3c96905e80 --- /dev/null +++ b/pkg/enterprise/windows/suite_test.go @@ -0,0 +1,34 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package windows_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/tigera/operator/pkg/enterprise" + "github.com/tigera/operator/pkg/extensions" +) + +// ext is the enterprise extension Set under test, shared across the suite. It is +// immutable once built and the specs only read it, so a single instance is safe. +var ext *extensions.Set = enterprise.New() + +func TestWindows(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "pkg/enterprise/windows Suite") +} diff --git a/pkg/enterprise/windows/windows_render_test.go b/pkg/enterprise/windows/windows_render_test.go new file mode 100644 index 0000000000..98b1cdd6b4 --- /dev/null +++ b/pkg/enterprise/windows/windows_render_test.go @@ -0,0 +1,937 @@ +// Copyright (c) 2021-2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package windows_test + +import ( + "fmt" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/onsi/gomega/gstruct" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/intstr" + "sigs.k8s.io/controller-runtime/pkg/client" + + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/apis" + "github.com/tigera/operator/pkg/common" + "github.com/tigera/operator/pkg/components" + "github.com/tigera/operator/pkg/controller/certificatemanager" + "github.com/tigera/operator/pkg/controller/k8sapi" + ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" + "github.com/tigera/operator/pkg/extensions/extensionstest" + "github.com/tigera/operator/pkg/render" + rmeta "github.com/tigera/operator/pkg/render/common/meta" + rtest "github.com/tigera/operator/pkg/render/common/test" + "github.com/tigera/operator/pkg/tls/certificatemanagement" +) + +var ( + bgpEnabled = operatorv1.BGPEnabled + bgpDisabled = operatorv1.BGPDisabled + logSeverity = operatorv1.LogLevelDebug + logFileMaxAgeDays = uint32(5) + logFileMaxCount = uint32(5) + logFileMaxSize = resource.MustParse("1Mi") +) + +// renderWindows renders the windows component and applies the registered +// enterprise modifier the way the componentHandler does, so enterprise tests +// exercise the integrated output (image overrides come from ResolveImages; the +// metrics service, env, volumes and mounts come from the modifier). +func renderWindows(cfg *render.WindowsConfiguration) []client.Object { + comp := render.Windows(cfg) + ExpectWithOffset(1, comp.ResolveImages(nil)).To(BeNil()) + objs, _ := comp.Objects() + rc := render.RenderContext{Installation: cfg.Installation} + out, _ := extensionstest.ApplyExtensions(ext, render.ComponentNameWindows, rc, objs, nil) + return out +} + +func getTyphaNodeTLS(cli client.Client, certificateManager certificatemanager.CertificateManager) *render.TyphaNodeTLS { + nodeKeyPair, err := certificateManager.GetOrCreateKeyPair(cli, render.NodeTLSSecretName, common.OperatorNamespace(), []string{render.FelixCommonName}) + Expect(err).NotTo(HaveOccurred()) + + typhaKeyPair, err := certificateManager.GetOrCreateKeyPair(cli, render.TyphaTLSSecretName, common.OperatorNamespace(), []string{render.FelixCommonName}) + Expect(err).NotTo(HaveOccurred()) + + typhaNonClusterHostKeyPair, err := certificateManager.GetOrCreateKeyPair(cli, render.TyphaTLSSecretName+render.TyphaNonClusterHostSuffix, common.OperatorNamespace(), []string{render.FelixCommonName + render.TyphaNonClusterHostSuffix}) + Expect(err).NotTo(HaveOccurred()) + + trustedBundle := certificateManager.CreateTrustedBundle(nodeKeyPair, typhaKeyPair) + + return &render.TyphaNodeTLS{ + TrustedBundle: trustedBundle, + TyphaSecret: typhaKeyPair, + TyphaSecretNonClusterHost: typhaNonClusterHostKeyPair, + TyphaCommonName: render.TyphaCommonName, + NodeSecret: nodeKeyPair, + NodeCommonName: render.FelixCommonName, + } +} + +// verifyWindowsProbesAndLifecycle asserts the expected node liveness and readiness probe plus pod lifecycle settings. +// The unused argument is kept temporarily so existing call sites compile while the OSS/Enterprise distinction +// is being phased out. +func verifyWindowsProbesAndLifecycle(ds *appsv1.DaemonSet, _ bool) { + livenessCmd := []string{"$env:CONTAINER_SANDBOX_MOUNT_POINT/CalicoWindows/calico.exe", "component", "node", "health", "--felix-live"} + readinessCmd := []string{"$env:CONTAINER_SANDBOX_MOUNT_POINT/CalicoWindows/calico.exe", "component", "node", "health", "--felix-ready"} + preStopCmd := []string{"$env:CONTAINER_SANDBOX_MOUNT_POINT/CalicoWindows/calico.exe", "component", "node", "shutdown"} + + expectedLiveness := &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + Exec: &corev1.ExecAction{Command: livenessCmd}, + }, + InitialDelaySeconds: 10, + FailureThreshold: 6, + TimeoutSeconds: 10, + PeriodSeconds: 10, + } + ExpectWithOffset(1, rtest.GetContainer(ds.Spec.Template.Spec.Containers, "felix").LivenessProbe).To(Equal(expectedLiveness)) + + expectedReadiness := &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + Exec: &corev1.ExecAction{Command: readinessCmd}, + }, + TimeoutSeconds: 10, + PeriodSeconds: 10, + } + ExpectWithOffset(1, rtest.GetContainer(ds.Spec.Template.Spec.Containers, "felix").ReadinessProbe).To(Equal(expectedReadiness)) + + expectedLifecycle := &corev1.Lifecycle{ + PreStop: &corev1.LifecycleHandler{ + Exec: &corev1.ExecAction{Command: preStopCmd}, + }, + } + ExpectWithOffset(1, rtest.GetContainer(ds.Spec.Template.Spec.Containers, "felix").Lifecycle).To(Equal(expectedLifecycle)) +} + +var _ = Describe("Windows enterprise rendering tests", func() { + var defaultInstance *operatorv1.InstallationSpec + var typhaNodeTLS *render.TyphaNodeTLS + var k8sServiceEp k8sapi.ServiceEndpoint + one := intstr.FromInt(1) + defaultNumExpectedResources := 2 + const defaultClusterDomain = "svc.cluster.local" + var defaultMode int32 = 420 + var cfg render.WindowsConfiguration + var cli client.Client + + BeforeEach(func() { + ff := true + hp := operatorv1.HostPortsEnabled + miMode := operatorv1.MultiInterfaceModeNone + defaultInstance = &operatorv1.InstallationSpec{ + CNI: &operatorv1.CNISpec{ + Type: "Calico", + IPAM: &operatorv1.IPAMSpec{Type: "Calico"}, + }, + CalicoNetwork: &operatorv1.CalicoNetworkSpec{ + BGP: &bgpEnabled, + IPPools: []operatorv1.IPPool{}, + NodeAddressAutodetectionV4: &operatorv1.NodeAddressAutodetection{}, + NodeAddressAutodetectionV6: &operatorv1.NodeAddressAutodetection{}, + HostPorts: &hp, + MultiInterfaceMode: &miMode, + }, + NodeUpdateStrategy: appsv1.DaemonSetUpdateStrategy{ + RollingUpdate: &appsv1.RollingUpdateDaemonSet{ + MaxUnavailable: &one, + }, + }, + Logging: &operatorv1.Logging{ + CNI: &operatorv1.CNILogging{ + LogSeverity: &logSeverity, + LogFileMaxSize: &logFileMaxSize, + LogFileMaxAgeDays: &logFileMaxAgeDays, + LogFileMaxCount: &logFileMaxCount, + }, + }, + WindowsNodes: &operatorv1.WindowsNodeSpec{ + CNIBinDir: "/opt/cni/bin", + CNIConfigDir: "/etc/cni/net.d", + CNILogDir: "/var/log/calico/cni", + }, + } + defaultInstance.CalicoNetwork.IPPools = append(defaultInstance.CalicoNetwork.IPPools, operatorv1.IPPool{CIDR: "192.168.1.0/16", Encapsulation: operatorv1.EncapsulationVXLAN}) + defaultInstance.CalicoNetwork.NodeAddressAutodetectionV4 = &operatorv1.NodeAddressAutodetection{FirstFound: &ff} + defaultInstance.ServiceCIDRs = []string{"10.96.0.0/12"} + scheme := runtime.NewScheme() + Expect(apis.AddToScheme(scheme, false)).NotTo(HaveOccurred()) + cli = ctrlrfake.DefaultFakeClientBuilder(scheme).Build() + certificateManager, err := certificatemanager.Create(cli, nil, defaultClusterDomain, common.OperatorNamespace(), certificatemanager.AllowCACreation()) + Expect(err).NotTo(HaveOccurred()) + // Create a dummy secret to pass as input. + typhaNodeTLS = getTyphaNodeTLS(cli, certificateManager) + + // Dummy service endpoint for k8s API. + k8sServiceEp = k8sapi.ServiceEndpoint{ + Host: "1.2.3.4", + Port: "6443", + } + + // Create a default configuration. + cfg = render.WindowsConfiguration{ + K8sServiceEp: k8sServiceEp, + K8sDNSServers: []string{"10.96.0.10"}, + Installation: defaultInstance, + ClusterDomain: defaultClusterDomain, + TLS: typhaNodeTLS, + VXLANVNI: 4096, + ImageOverrides: ext.Images(), + } + }) + + It("should render all resources for a default configuration using CalicoEnterprise", func() { + type testConf struct { + EnableBGP bool + EnableVXLAN bool + } + for _, testConfig := range []testConf{ + {true, false}, + {false, true}, + {true, true}, + } { + enableBGP := testConfig.EnableBGP + enableVXLAN := testConfig.EnableVXLAN + + if enableBGP { + defaultInstance.CalicoNetwork.BGP = &bgpEnabled + } else { + defaultInstance.CalicoNetwork.BGP = &bgpDisabled + } + + if enableVXLAN { + defaultInstance.CalicoNetwork.IPPools[0].Encapsulation = operatorv1.EncapsulationVXLAN + } else { + defaultInstance.CalicoNetwork.IPPools[0].Encapsulation = operatorv1.EncapsulationNone + } + By(fmt.Sprintf("BGP enabled: %v, VXLAN enabled: %v", enableBGP, enableVXLAN), func() { + expectedResources := []struct { + name string + ns string + group string + version string + kind string + }{ + {name: "cni-config-windows", ns: common.CalicoNamespace, group: "", version: "v1", kind: "ConfigMap"}, + {name: common.WindowsDaemonSetName, ns: common.CalicoNamespace, group: "apps", version: "v1", kind: "DaemonSet"}, + {name: "calico-node-metrics-windows", ns: "calico-system", group: "", version: "v1", kind: "Service"}, + } + defaultInstance.Variant = operatorv1.CalicoEnterprise + + resources := renderWindows(&cfg) + Expect(len(resources)).To(Equal(len(expectedResources))) + + // Should render the correct resources. + i := 0 + for _, expectedRes := range expectedResources { + rtest.ExpectResourceTypeAndObjectMetadata(resources[i], expectedRes.name, expectedRes.ns, expectedRes.group, expectedRes.version, expectedRes.kind) + i++ + } + + // The DaemonSet should have the correct configuration. + ds := rtest.GetResource(resources, "calico-node-windows", "calico-system", "apps", "v1", "DaemonSet").(*appsv1.DaemonSet) + + // The pod template should have node critical priority + Expect(ds.Spec.Template.Spec.PriorityClassName).To(Equal(render.NodePriorityClassName)) + + // The calico-node-windows daemonset has 3 containers (felix, node and confd). + // confd is only instantiated if using BGP. + numContainers := 3 + if !enableBGP { + numContainers = 2 + } + Expect(ds.Spec.Template.Spec.Containers).To(HaveLen(numContainers)) + for _, container := range ds.Spec.Template.Spec.Containers { + + // Windows node image override results in correct image. + Expect(container.Image).To(Equal(components.TigeraRegistry + "tigera/node-windows:" + components.ComponentTigeraNodeWindows.Version)) + Expect(container.SecurityContext.Capabilities).To(BeNil()) + Expect(container.SecurityContext.Privileged).To(BeNil()) + Expect(container.SecurityContext.SELinuxOptions).To(BeNil()) + Expect(container.SecurityContext.WindowsOptions).To(Not(BeNil())) + Expect(container.SecurityContext.WindowsOptions.GMSACredentialSpecName).To(BeNil()) + Expect(container.SecurityContext.WindowsOptions.GMSACredentialSpec).To(BeNil()) + Expect(*container.SecurityContext.WindowsOptions.RunAsUserName).To(Equal("NT AUTHORITY\\system")) + Expect(*container.SecurityContext.WindowsOptions.HostProcess).To(BeTrue()) + Expect(container.SecurityContext.RunAsUser).To(BeNil()) + Expect(container.SecurityContext.RunAsGroup).To(BeNil()) + Expect(container.SecurityContext.RunAsNonRoot).To(BeNil()) + Expect(container.SecurityContext.ReadOnlyRootFilesystem).To(BeNil()) + Expect(container.SecurityContext.AllowPrivilegeEscalation).To(BeNil()) + Expect(container.SecurityContext.ProcMount).To(BeNil()) + Expect(container.SecurityContext.SeccompProfile).To(BeNil()) + } + + felixContainer := rtest.GetContainer(ds.Spec.Template.Spec.Containers, "felix") + + // Windows node image override results in correct image. + Expect(felixContainer.Image).To(Equal(components.TigeraRegistry + "tigera/node-windows:" + components.ComponentTigeraNodeWindows.Version)) + Expect(felixContainer.SecurityContext.Capabilities).To(BeNil()) + Expect(felixContainer.SecurityContext.Privileged).To(BeNil()) + Expect(felixContainer.SecurityContext.SELinuxOptions).To(BeNil()) + Expect(felixContainer.SecurityContext.WindowsOptions).To(Not(BeNil())) + Expect(felixContainer.SecurityContext.WindowsOptions.GMSACredentialSpecName).To(BeNil()) + Expect(felixContainer.SecurityContext.WindowsOptions.GMSACredentialSpec).To(BeNil()) + Expect(*felixContainer.SecurityContext.WindowsOptions.RunAsUserName).To(Equal("NT AUTHORITY\\system")) + Expect(*felixContainer.SecurityContext.WindowsOptions.HostProcess).To(BeTrue()) + Expect(felixContainer.SecurityContext.RunAsUser).To(BeNil()) + Expect(felixContainer.SecurityContext.RunAsGroup).To(BeNil()) + Expect(felixContainer.SecurityContext.RunAsNonRoot).To(BeNil()) + Expect(felixContainer.SecurityContext.ReadOnlyRootFilesystem).To(BeNil()) + Expect(felixContainer.SecurityContext.AllowPrivilegeEscalation).To(BeNil()) + Expect(felixContainer.SecurityContext.ProcMount).To(BeNil()) + Expect(felixContainer.SecurityContext.SeccompProfile).To(BeNil()) + + nodeContainer := rtest.GetContainer(ds.Spec.Template.Spec.Containers, "node") + + // Windows node image override results in correct image. + Expect(nodeContainer.Image).To(Equal(components.TigeraRegistry + "tigera/node-windows:" + components.ComponentTigeraNodeWindows.Version)) + Expect(nodeContainer.SecurityContext.Capabilities).To(BeNil()) + Expect(nodeContainer.SecurityContext.Privileged).To(BeNil()) + Expect(nodeContainer.SecurityContext.SELinuxOptions).To(BeNil()) + Expect(nodeContainer.SecurityContext.WindowsOptions).To(Not(BeNil())) + Expect(nodeContainer.SecurityContext.WindowsOptions.GMSACredentialSpecName).To(BeNil()) + Expect(nodeContainer.SecurityContext.WindowsOptions.GMSACredentialSpec).To(BeNil()) + Expect(*nodeContainer.SecurityContext.WindowsOptions.RunAsUserName).To(Equal("NT AUTHORITY\\system")) + Expect(*nodeContainer.SecurityContext.WindowsOptions.HostProcess).To(BeTrue()) + Expect(nodeContainer.SecurityContext.RunAsUser).To(BeNil()) + Expect(nodeContainer.SecurityContext.RunAsGroup).To(BeNil()) + Expect(nodeContainer.SecurityContext.RunAsNonRoot).To(BeNil()) + Expect(nodeContainer.SecurityContext.ReadOnlyRootFilesystem).To(BeNil()) + Expect(nodeContainer.SecurityContext.AllowPrivilegeEscalation).To(BeNil()) + Expect(nodeContainer.SecurityContext.ProcMount).To(BeNil()) + Expect(nodeContainer.SecurityContext.SeccompProfile).To(BeNil()) + + if enableBGP { + confdContainer := rtest.GetContainer(ds.Spec.Template.Spec.Containers, "confd") + + // Windows node image override results in correct image. + Expect(confdContainer.Image).To(Equal(components.TigeraRegistry + "tigera/node-windows:" + components.ComponentTigeraNodeWindows.Version)) + Expect(confdContainer.SecurityContext.Capabilities).To(BeNil()) + Expect(confdContainer.SecurityContext.Privileged).To(BeNil()) + Expect(confdContainer.SecurityContext.SELinuxOptions).To(BeNil()) + Expect(confdContainer.SecurityContext.WindowsOptions).To(Not(BeNil())) + Expect(confdContainer.SecurityContext.WindowsOptions.GMSACredentialSpecName).To(BeNil()) + Expect(confdContainer.SecurityContext.WindowsOptions.GMSACredentialSpec).To(BeNil()) + Expect(*confdContainer.SecurityContext.WindowsOptions.RunAsUserName).To(Equal("NT AUTHORITY\\system")) + Expect(*confdContainer.SecurityContext.WindowsOptions.HostProcess).To(BeTrue()) + Expect(confdContainer.SecurityContext.RunAsUser).To(BeNil()) + Expect(confdContainer.SecurityContext.RunAsGroup).To(BeNil()) + Expect(confdContainer.SecurityContext.RunAsNonRoot).To(BeNil()) + Expect(confdContainer.SecurityContext.ReadOnlyRootFilesystem).To(BeNil()) + Expect(confdContainer.SecurityContext.AllowPrivilegeEscalation).To(BeNil()) + Expect(confdContainer.SecurityContext.ProcMount).To(BeNil()) + Expect(confdContainer.SecurityContext.SeccompProfile).To(BeNil()) + } + + // Validate correct number of init containers. + Expect(ds.Spec.Template.Spec.InitContainers).To(HaveLen(2)) + + // CNI container uses image override. + cniContainer := rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "install-cni") + rtest.ExpectEnv(cniContainer.Env, "CNI_NET_DIR", "/etc/cni/net.d") + Expect(cniContainer.Image).To(Equal(components.TigeraRegistry + "tigera/cni-windows:" + components.ComponentTigeraCNIWindows.Version)) + + Expect(cniContainer.SecurityContext.Capabilities).To(BeNil()) + Expect(cniContainer.SecurityContext.Privileged).To(BeNil()) + Expect(cniContainer.SecurityContext.SELinuxOptions).To(BeNil()) + Expect(cniContainer.SecurityContext.WindowsOptions).To(Not(BeNil())) + Expect(cniContainer.SecurityContext.WindowsOptions.GMSACredentialSpecName).To(BeNil()) + Expect(cniContainer.SecurityContext.WindowsOptions.GMSACredentialSpec).To(BeNil()) + Expect(*cniContainer.SecurityContext.WindowsOptions.RunAsUserName).To(Equal("NT AUTHORITY\\system")) + Expect(*cniContainer.SecurityContext.WindowsOptions.HostProcess).To(BeTrue()) + Expect(cniContainer.SecurityContext.RunAsUser).To(BeNil()) + Expect(cniContainer.SecurityContext.RunAsGroup).To(BeNil()) + Expect(cniContainer.SecurityContext.RunAsNonRoot).To(BeNil()) + Expect(cniContainer.SecurityContext.ReadOnlyRootFilesystem).To(BeNil()) + Expect(cniContainer.SecurityContext.AllowPrivilegeEscalation).To(BeNil()) + Expect(cniContainer.SecurityContext.ProcMount).To(BeNil()) + Expect(cniContainer.SecurityContext.SeccompProfile).To(BeNil()) + + // uninstall container uses image override. + uninstallContainer := rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "uninstall-calico") + Expect(uninstallContainer.Image).To(Equal(components.TigeraRegistry + "tigera/node-windows:" + components.ComponentTigeraNodeWindows.Version)) + + Expect(uninstallContainer.SecurityContext.Capabilities).To(BeNil()) + Expect(uninstallContainer.SecurityContext.Privileged).To(BeNil()) + Expect(uninstallContainer.SecurityContext.SELinuxOptions).To(BeNil()) + Expect(uninstallContainer.SecurityContext.WindowsOptions).To(Not(BeNil())) + Expect(uninstallContainer.SecurityContext.WindowsOptions.GMSACredentialSpecName).To(BeNil()) + Expect(uninstallContainer.SecurityContext.WindowsOptions.GMSACredentialSpec).To(BeNil()) + Expect(*uninstallContainer.SecurityContext.WindowsOptions.RunAsUserName).To(Equal("NT AUTHORITY\\system")) + Expect(*uninstallContainer.SecurityContext.WindowsOptions.HostProcess).To(BeTrue()) + Expect(uninstallContainer.SecurityContext.RunAsUser).To(BeNil()) + Expect(uninstallContainer.SecurityContext.RunAsGroup).To(BeNil()) + Expect(uninstallContainer.SecurityContext.RunAsNonRoot).To(BeNil()) + Expect(uninstallContainer.SecurityContext.ReadOnlyRootFilesystem).To(BeNil()) + Expect(uninstallContainer.SecurityContext.AllowPrivilegeEscalation).To(BeNil()) + Expect(uninstallContainer.SecurityContext.ProcMount).To(BeNil()) + Expect(uninstallContainer.SecurityContext.SeccompProfile).To(BeNil()) + + // Verify env + expectedNodeEnv := []corev1.EnvVar{ + {Name: "CNI_PLUGIN_TYPE", Value: "Calico"}, + {Name: "DATASTORE_TYPE", Value: "kubernetes"}, + {Name: "WAIT_FOR_DATASTORE", Value: "true"}, + {Name: "CALICO_MANAGE_CNI", Value: "true"}, + {Name: "CALICO_DISABLE_FILE_LOGGING", Value: "false"}, + {Name: "FELIX_DEFAULTENDPOINTTOHOSTACTION", Value: "ACCEPT"}, + {Name: "FELIX_HEALTHENABLED", Value: "true"}, + { + Name: "NODENAME", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{FieldPath: "spec.nodeName"}, + }, + }, + { + Name: "NAMESPACE", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.namespace"}, + }, + }, + {Name: "IP", Value: "autodetect"}, + {Name: "IP_AUTODETECTION_METHOD", Value: "first-found"}, + {Name: "IP6", Value: "none"}, + {Name: "FELIX_IPV6SUPPORT", Value: "false"}, + {Name: "FELIX_TYPHAK8SNAMESPACE", Value: "calico-system"}, + {Name: "FELIX_TYPHAK8SSERVICENAME", Value: "calico-typha"}, + {Name: "FELIX_TYPHACAFILE", Value: certificatemanagement.TrustedCertBundleMountPath}, + {Name: "FELIX_TYPHACERTFILE", Value: "/node-certs/tls.crt"}, + {Name: "FELIX_TYPHACN", Value: "typha-server"}, + {Name: "FELIX_TYPHAKEYFILE", Value: "/node-certs/tls.key"}, + + {Name: "VXLAN_VNI", Value: "4096"}, + {Name: "VXLAN_ADAPTER", Value: ""}, + {Name: "KUBE_NETWORK", Value: "Calico.*"}, + {Name: "KUBERNETES_SERVICE_HOST", Value: "1.2.3.4"}, + {Name: "KUBERNETES_SERVICE_PORT", Value: "6443"}, + + // Tigera-specific envvars + {Name: "FELIX_PROMETHEUSREPORTERENABLED", Value: "true"}, + {Name: "FELIX_PROMETHEUSREPORTERPORT", Value: "9081"}, + {Name: "FELIX_FLOWLOGSFILEENABLED", Value: "true"}, + {Name: "FELIX_FLOWLOGSFILEINCLUDELABELS", Value: "true"}, + {Name: "FELIX_FLOWLOGSFILEINCLUDEPOLICIES", Value: "true"}, + {Name: "FELIX_FLOWLOGSFILEINCLUDESERVICE", Value: "true"}, + {Name: "FELIX_FLOWLOGSENABLENETWORKSETS", Value: "true"}, + {Name: "FELIX_FLOWLOGSCOLLECTPROCESSINFO", Value: "true"}, + {Name: "FELIX_DNSLOGSFILEENABLED", Value: "true"}, + {Name: "FELIX_DNSLOGSFILEPERNODELIMIT", Value: "1000"}, + } + + // Set CALICO_NETWORKING_BACKEND + if enableBGP { + expectedNodeEnv = append(expectedNodeEnv, corev1.EnvVar{Name: "CALICO_NETWORKING_BACKEND", Value: "windows-bgp"}) + } else if enableVXLAN { + expectedNodeEnv = append(expectedNodeEnv, corev1.EnvVar{Name: "CALICO_NETWORKING_BACKEND", Value: "vxlan"}) + } else { + expectedNodeEnv = append(expectedNodeEnv, corev1.EnvVar{Name: "CALICO_NETWORKING_BACKEND", Value: "none"}) + } + + // Set CLUSTER_TYPE + if enableBGP { + expectedNodeEnv = append(expectedNodeEnv, corev1.EnvVar{Name: "CLUSTER_TYPE", Value: "k8s,operator,bgp,windows"}) + } else { + expectedNodeEnv = append(expectedNodeEnv, corev1.EnvVar{Name: "CLUSTER_TYPE", Value: "k8s,operator,windows"}) + } + + Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "node").Env).To(ConsistOf(expectedNodeEnv)) + Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "felix").Env).To(ConsistOf(expectedNodeEnv)) + if enableBGP { + Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "confd").Env).To(ConsistOf(expectedNodeEnv)) + } + + // Expect the SECURITY_GROUP env variables to not be set + Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "node").Env).NotTo(ContainElement(gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{"Name": Equal("TIGERA_DEFAULT_SECURITY_GROUPS")}))) + Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "node").Env).NotTo(ContainElement(gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{"Name": Equal("TIGERA_POD_SECURITY_GROUP")}))) + Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "felix").Env).NotTo(ContainElement(gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{"Name": Equal("TIGERA_DEFAULT_SECURITY_GROUPS")}))) + Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "felix").Env).NotTo(ContainElement(gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{"Name": Equal("TIGERA_POD_SECURITY_GROUP")}))) + if enableBGP { + Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "confd").Env).NotTo(ContainElement(gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{"Name": Equal("TIGERA_DEFAULT_SECURITY_GROUPS")}))) + Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "confd").Env).NotTo(ContainElement(gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{"Name": Equal("TIGERA_POD_SECURITY_GROUP")}))) + } + + expectedCNIEnv := []corev1.EnvVar{ + {Name: "SLEEP", Value: "false"}, + {Name: "CNI_PLUGIN_TYPE", Value: "Calico"}, + {Name: "CNI_BIN_DIR", Value: "/host/opt/cni/bin"}, + {Name: "CNI_CONF_NAME", Value: "10-calico.conflist"}, + {Name: "CNI_NET_DIR", Value: "/etc/cni/net.d"}, + {Name: "VXLAN_VNI", Value: "4096"}, + { + Name: "KUBERNETES_NODE_NAME", + Value: "", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{FieldPath: "spec.nodeName"}, + }, + }, + { + Name: "CNI_NETWORK_CONFIG", + ValueFrom: &corev1.EnvVarSource{ + ConfigMapKeyRef: &corev1.ConfigMapKeySelector{ + Key: "config", + LocalObjectReference: corev1.LocalObjectReference{ + Name: "cni-config-windows", + }, + }, + }, + }, + + {Name: "KUBERNETES_SERVICE_HOST", Value: "1.2.3.4"}, + {Name: "KUBERNETES_SERVICE_PORT", Value: "6443"}, + {Name: "KUBERNETES_SERVICE_CIDRS", Value: "10.96.0.0/12"}, + {Name: "KUBERNETES_DNS_SERVERS", Value: "10.96.0.10"}, + } + Expect(rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "install-cni").Env).To(ConsistOf(expectedCNIEnv)) + + expectedUninstallEnv := []corev1.EnvVar{ + {Name: "SLEEP", Value: "false"}, + {Name: "CNI_PLUGIN_TYPE", Value: "Calico"}, + {Name: "CNI_BIN_DIR", Value: "/host/opt/cni/bin"}, + {Name: "CNI_CONF_NAME", Value: "10-calico.conflist"}, + {Name: "CNI_NET_DIR", Value: "/host/etc/cni/net.d"}, + } + Expect(rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "uninstall-calico").Env).To(ConsistOf(expectedUninstallEnv)) + + // Verify volumes. + fileOrCreate := corev1.HostPathFileOrCreate + dirOrCreate := corev1.HostPathDirectoryOrCreate + expectedVols := []corev1.Volume{ + {Name: "lib-modules", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/lib/modules"}}}, + {Name: "var-run-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/run/calico", Type: &dirOrCreate}}}, + {Name: "var-lib-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/lib/calico", Type: &dirOrCreate}}}, + {Name: "xtables-lock", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/run/xtables.lock", Type: &fileOrCreate}}}, + {Name: "cni-bin-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/opt/cni/bin", Type: &dirOrCreate}}}, + {Name: "cni-net-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/etc/cni/net.d"}}}, + {Name: "cni-log-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico/cni", Type: &dirOrCreate}}}, + {Name: "policysync", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/run/nodeagent", Type: &dirOrCreate}}}, + { + Name: "tigera-ca-bundle", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "tigera-ca-bundle", + }, + }, + }, + }, + { + Name: render.NodeTLSSecretName, + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: render.NodeTLSSecretName, + DefaultMode: &defaultMode, + }, + }, + }, + {Name: "var-log-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico", Type: &dirOrCreate}}}, + } + Expect(ds.Spec.Template.Spec.Volumes).To(ConsistOf(expectedVols)) + + // Verify volume mounts. + expectedNodeVolumeMounts := []corev1.VolumeMount{ + {MountPath: "/host/etc/cni/net.d", Name: "cni-net-dir"}, + {MountPath: "/var/run/calico", Name: "var-run-calico"}, + {MountPath: "/var/lib/calico", Name: "var-lib-calico"}, + {MountPath: "c:/etc/pki/tls/certs", Name: "tigera-ca-bundle", ReadOnly: true}, + {MountPath: "c:/node-certs", Name: render.NodeTLSSecretName, ReadOnly: true}, + {MountPath: "/var/log/calico", Name: "var-log-calico", ReadOnly: false}, + } + Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "node").VolumeMounts).To(ConsistOf(expectedNodeVolumeMounts)) + Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "felix").VolumeMounts).To(ConsistOf(expectedNodeVolumeMounts)) + if enableBGP { + Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "confd").VolumeMounts).To(ConsistOf(expectedNodeVolumeMounts)) + } + + expectedCNIVolumeMounts := []corev1.VolumeMount{ + {MountPath: "/host/opt/cni/bin", Name: "cni-bin-dir"}, + {MountPath: "/host/etc/cni/net.d", Name: "cni-net-dir"}, + } + Expect(rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "install-cni").VolumeMounts).To(ConsistOf(expectedCNIVolumeMounts)) + + expectedUninstallVolumeMounts := []corev1.VolumeMount{ + {MountPath: "/host/opt/cni/bin", Name: "cni-bin-dir"}, + {MountPath: "/host/etc/cni/net.d", Name: "cni-net-dir"}, + } + Expect(rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "uninstall-calico").VolumeMounts).To(ConsistOf(expectedUninstallVolumeMounts)) + + // Verify tolerations. + Expect(ds.Spec.Template.Spec.Tolerations).To(ConsistOf(rmeta.TolerateAll)) + + // Verify readiness and liveness probes. + verifyWindowsProbesAndLifecycle(ds, false) + }) + } + }) + + It("should render all resources when variant is CalicoEnterprise and running on openshift", func() { + expectedResources := []struct { + name string + ns string + group string + version string + kind string + }{ + {name: "cni-config-windows", ns: common.CalicoNamespace, group: "", version: "v1", kind: "ConfigMap"}, + {name: common.WindowsDaemonSetName, ns: common.CalicoNamespace, group: "apps", version: "v1", kind: "DaemonSet"}, + {name: "calico-node-metrics-windows", ns: "calico-system", group: "", version: "v1", kind: "Service"}, + } + + defaultInstance.Variant = operatorv1.CalicoEnterprise + defaultInstance.KubernetesProvider = operatorv1.ProviderOpenShift + + resources := renderWindows(&cfg) + Expect(len(resources)).To(Equal(len(expectedResources))) + + // Should render the correct resources. + i := 0 + for _, expectedRes := range expectedResources { + rtest.ExpectResourceTypeAndObjectMetadata(resources[i], expectedRes.name, expectedRes.ns, expectedRes.group, expectedRes.version, expectedRes.kind) + i++ + } + + // The DaemonSet should have the correct configuration. + ds := rtest.GetResource(resources, "calico-node-windows", "calico-system", "apps", "v1", "DaemonSet").(*appsv1.DaemonSet) + + // The pod template should have node critical priority + Expect(ds.Spec.Template.Spec.PriorityClassName).To(Equal(render.NodePriorityClassName)) + + felixContainer := rtest.GetContainer(ds.Spec.Template.Spec.Containers, "felix") + Expect(felixContainer.Image).To(Equal(fmt.Sprintf("%s%s%s:%s", components.TigeraRegistry, components.TigeraImagePath, components.ComponentTigeraNodeWindows.Image, components.ComponentTigeraNodeWindows.Version))) + nodeContainer := rtest.GetContainer(ds.Spec.Template.Spec.Containers, "node") + Expect(nodeContainer.Image).To(Equal(fmt.Sprintf("%s%s%s:%s", components.TigeraRegistry, components.TigeraImagePath, components.ComponentTigeraNodeWindows.Image, components.ComponentTigeraNodeWindows.Version))) + cniContainer := rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "install-cni") + Expect(cniContainer.Image).To(Equal(fmt.Sprintf("%s%s%s:%s", components.TigeraRegistry, components.TigeraImagePath, components.ComponentTigeraCNIWindows.Image, components.ComponentTigeraCNIWindows.Version))) + uninstallContainer := rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "uninstall-calico") + Expect(uninstallContainer.Image).To(Equal(fmt.Sprintf("%s%s%s:%s", components.TigeraRegistry, components.TigeraImagePath, components.ComponentTigeraNodeWindows.Image, components.ComponentTigeraNodeWindows.Version))) + + // FIXME: confirm openshift CNI path defaults + expectedCNIVolumeMounts := []corev1.VolumeMount{ + {MountPath: "/host/opt/cni/bin", Name: "cni-bin-dir"}, + {MountPath: "/host/etc/cni/net.d", Name: "cni-net-dir"}, + } + Expect(rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "install-cni").VolumeMounts).To(ConsistOf(expectedCNIVolumeMounts)) + + // FIXME: confirm openshift CNI path defaults + expectedUninstallVolumeMounts := []corev1.VolumeMount{ + {MountPath: "/host/opt/cni/bin", Name: "cni-bin-dir"}, + {MountPath: "/host/etc/cni/net.d", Name: "cni-net-dir"}, + } + Expect(rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "uninstall-calico").VolumeMounts).To(ConsistOf(expectedUninstallVolumeMounts)) + + // Verify volumes + // FIXME: confirm openshift CNI path defaults + fileOrCreate := corev1.HostPathFileOrCreate + dirOrCreate := corev1.HostPathDirectoryOrCreate + expectedVols := []corev1.Volume{ + {Name: "lib-modules", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/lib/modules"}}}, + {Name: "var-run-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/run/calico", Type: &dirOrCreate}}}, + {Name: "var-lib-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/lib/calico", Type: &dirOrCreate}}}, + {Name: "xtables-lock", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/run/xtables.lock", Type: &fileOrCreate}}}, + {Name: "cni-bin-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/opt/cni/bin", Type: &dirOrCreate}}}, + {Name: "cni-net-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/etc/cni/net.d"}}}, + {Name: "cni-log-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico/cni", Type: &dirOrCreate}}}, + {Name: "policysync", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/run/nodeagent", Type: &dirOrCreate}}}, + { + Name: "tigera-ca-bundle", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "tigera-ca-bundle", + }, + }, + }, + }, + { + Name: render.NodeTLSSecretName, + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: render.NodeTLSSecretName, + DefaultMode: &defaultMode, + }, + }, + }, + {Name: "var-log-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico", Type: &dirOrCreate}}}, + } + Expect(ds.Spec.Template.Spec.Volumes).To(ConsistOf(expectedVols)) + + expectedNodeEnv := []corev1.EnvVar{ + // Default envvars. + {Name: "CNI_PLUGIN_TYPE", Value: "Calico"}, + {Name: "DATASTORE_TYPE", Value: "kubernetes"}, + {Name: "WAIT_FOR_DATASTORE", Value: "true"}, + {Name: "CALICO_MANAGE_CNI", Value: "true"}, + {Name: "CALICO_NETWORKING_BACKEND", Value: "windows-bgp"}, + {Name: "CLUSTER_TYPE", Value: "k8s,operator,openshift,bgp,windows"}, + {Name: "CALICO_DISABLE_FILE_LOGGING", Value: "false"}, + {Name: "FELIX_DEFAULTENDPOINTTOHOSTACTION", Value: "ACCEPT"}, + {Name: "FELIX_HEALTHENABLED", Value: "true"}, + { + Name: "NODENAME", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{FieldPath: "spec.nodeName"}, + }, + }, + { + Name: "NAMESPACE", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.namespace"}, + }, + }, + {Name: "IP", Value: "autodetect"}, + {Name: "IP_AUTODETECTION_METHOD", Value: "first-found"}, + {Name: "IP6", Value: "none"}, + {Name: "FELIX_IPV6SUPPORT", Value: "false"}, + {Name: "FELIX_TYPHAK8SNAMESPACE", Value: "calico-system"}, + {Name: "FELIX_TYPHAK8SSERVICENAME", Value: "calico-typha"}, + {Name: "FELIX_TYPHACAFILE", Value: certificatemanagement.TrustedCertBundleMountPath}, + {Name: "FELIX_TYPHACERTFILE", Value: "/node-certs/tls.crt"}, + {Name: "FELIX_TYPHACN", Value: "typha-server"}, + {Name: "FELIX_TYPHAKEYFILE", Value: "/node-certs/tls.key"}, + + {Name: "FELIX_DNSTRUSTEDSERVERS", Value: "k8s-service:openshift-dns/dns-default"}, + + // Tigera-specific envvars + {Name: "FELIX_PROMETHEUSREPORTERENABLED", Value: "true"}, + {Name: "FELIX_PROMETHEUSREPORTERPORT", Value: "9081"}, + {Name: "FELIX_FLOWLOGSFILEENABLED", Value: "true"}, + {Name: "FELIX_FLOWLOGSFILEINCLUDELABELS", Value: "true"}, + {Name: "FELIX_FLOWLOGSFILEINCLUDEPOLICIES", Value: "true"}, + {Name: "FELIX_FLOWLOGSFILEINCLUDESERVICE", Value: "true"}, + {Name: "FELIX_FLOWLOGSENABLENETWORKSETS", Value: "true"}, + {Name: "FELIX_FLOWLOGSCOLLECTPROCESSINFO", Value: "true"}, + {Name: "FELIX_DNSLOGSFILEENABLED", Value: "true"}, + {Name: "FELIX_DNSLOGSFILEPERNODELIMIT", Value: "1000"}, + + // Calico Windows specific envvars + {Name: "VXLAN_VNI", Value: "4096"}, + {Name: "VXLAN_ADAPTER", Value: ""}, + {Name: "KUBE_NETWORK", Value: "Calico.*"}, + {Name: "KUBERNETES_SERVICE_HOST", Value: "1.2.3.4"}, + {Name: "KUBERNETES_SERVICE_PORT", Value: "6443"}, + } + Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "node").Env).To(ConsistOf(expectedNodeEnv)) + Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "felix").Env).To(ConsistOf(expectedNodeEnv)) + Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "confd").Env).To(ConsistOf(expectedNodeEnv)) + + verifyWindowsProbesAndLifecycle(ds, false) + }) + + It("should render all resources when variant is CalicoEnterprise and running on RKE2", func() { + expectedResources := []struct { + name string + ns string + group string + version string + kind string + }{ + {name: "cni-config-windows", ns: common.CalicoNamespace, group: "", version: "v1", kind: "ConfigMap"}, + {name: common.WindowsDaemonSetName, ns: common.CalicoNamespace, group: "apps", version: "v1", kind: "DaemonSet"}, + {name: "calico-node-metrics-windows", ns: "calico-system", group: "", version: "v1", kind: "Service"}, + } + + defaultInstance.Variant = operatorv1.CalicoEnterprise + defaultInstance.KubernetesProvider = operatorv1.ProviderRKE2 + + resources := renderWindows(&cfg) + Expect(len(resources)).To(Equal(len(expectedResources)), fmt.Sprintf("Actual resources: %#v", resources)) + + // Should render the correct resources. + i := 0 + for _, expectedRes := range expectedResources { + rtest.ExpectResourceTypeAndObjectMetadata(resources[i], expectedRes.name, expectedRes.ns, expectedRes.group, expectedRes.version, expectedRes.kind) + i++ + } + + // The DaemonSet should have the correct configuration. + ds := rtest.GetResource(resources, "calico-node-windows", "calico-system", "apps", "v1", "DaemonSet").(*appsv1.DaemonSet) + + // The pod template should have node critical priority + Expect(ds.Spec.Template.Spec.PriorityClassName).To(Equal(render.NodePriorityClassName)) + + felixContainer := rtest.GetContainer(ds.Spec.Template.Spec.Containers, "felix") + Expect(felixContainer.Image).To(Equal(fmt.Sprintf("%s%s%s:%s", components.TigeraRegistry, components.TigeraImagePath, components.ComponentTigeraNodeWindows.Image, components.ComponentTigeraNodeWindows.Version))) + nodeContainer := rtest.GetContainer(ds.Spec.Template.Spec.Containers, "node") + Expect(nodeContainer.Image).To(Equal(fmt.Sprintf("%s%s%s:%s", components.TigeraRegistry, components.TigeraImagePath, components.ComponentTigeraNodeWindows.Image, components.ComponentTigeraNodeWindows.Version))) + cniContainer := rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "install-cni") + Expect(cniContainer.Image).To(Equal(fmt.Sprintf("%s%s%s:%s", components.TigeraRegistry, components.TigeraImagePath, components.ComponentTigeraCNIWindows.Image, components.ComponentTigeraCNIWindows.Version))) + uninstallContainer := rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "uninstall-calico") + Expect(uninstallContainer.Image).To(Equal(fmt.Sprintf("%s%s%s:%s", components.TigeraRegistry, components.TigeraImagePath, components.ComponentTigeraNodeWindows.Image, components.ComponentTigeraNodeWindows.Version))) + + // FIXME: confirm RKE2 CNI path defaults + expectedCNIVolumeMounts := []corev1.VolumeMount{ + {MountPath: "/host/opt/cni/bin", Name: "cni-bin-dir"}, + {MountPath: "/host/etc/cni/net.d", Name: "cni-net-dir"}, + } + Expect(rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "install-cni").VolumeMounts).To(ConsistOf(expectedCNIVolumeMounts)) + + // FIXME: confirm RKE2 CNI path defaults + expectedUninstallVolumeMounts := []corev1.VolumeMount{ + {MountPath: "/host/opt/cni/bin", Name: "cni-bin-dir"}, + {MountPath: "/host/etc/cni/net.d", Name: "cni-net-dir"}, + } + Expect(rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "uninstall-calico").VolumeMounts).To(ConsistOf(expectedUninstallVolumeMounts)) + + // Verify volumes + // FIXME: confirm RKE2 CNI path defaults + fileOrCreate := corev1.HostPathFileOrCreate + dirOrCreate := corev1.HostPathDirectoryOrCreate + expectedVols := []corev1.Volume{ + {Name: "lib-modules", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/lib/modules"}}}, + {Name: "var-run-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/run/calico", Type: &dirOrCreate}}}, + {Name: "var-lib-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/lib/calico", Type: &dirOrCreate}}}, + {Name: "xtables-lock", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/run/xtables.lock", Type: &fileOrCreate}}}, + {Name: "cni-bin-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/opt/cni/bin", Type: &dirOrCreate}}}, + {Name: "cni-net-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/etc/cni/net.d"}}}, + {Name: "cni-log-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico/cni", Type: &dirOrCreate}}}, + {Name: "policysync", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/run/nodeagent", Type: &dirOrCreate}}}, + { + Name: "tigera-ca-bundle", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "tigera-ca-bundle", + }, + }, + }, + }, + { + Name: render.NodeTLSSecretName, + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: render.NodeTLSSecretName, + DefaultMode: &defaultMode, + }, + }, + }, + {Name: "var-log-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico", Type: &dirOrCreate}}}, + } + Expect(ds.Spec.Template.Spec.Volumes).To(ConsistOf(expectedVols)) + + expectedNodeEnv := []corev1.EnvVar{ + // Default envvars. + {Name: "CNI_PLUGIN_TYPE", Value: "Calico"}, + {Name: "DATASTORE_TYPE", Value: "kubernetes"}, + {Name: "WAIT_FOR_DATASTORE", Value: "true"}, + {Name: "CALICO_MANAGE_CNI", Value: "true"}, + {Name: "CALICO_NETWORKING_BACKEND", Value: "windows-bgp"}, + {Name: "CLUSTER_TYPE", Value: "k8s,operator,bgp,windows"}, + {Name: "CALICO_DISABLE_FILE_LOGGING", Value: "false"}, + {Name: "FELIX_DEFAULTENDPOINTTOHOSTACTION", Value: "ACCEPT"}, + {Name: "FELIX_HEALTHENABLED", Value: "true"}, + { + Name: "NODENAME", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{FieldPath: "spec.nodeName"}, + }, + }, + { + Name: "NAMESPACE", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.namespace"}, + }, + }, + {Name: "IP", Value: "autodetect"}, + {Name: "IP_AUTODETECTION_METHOD", Value: "first-found"}, + {Name: "IP6", Value: "none"}, + {Name: "FELIX_IPV6SUPPORT", Value: "false"}, + {Name: "FELIX_TYPHAK8SNAMESPACE", Value: "calico-system"}, + {Name: "FELIX_TYPHAK8SSERVICENAME", Value: "calico-typha"}, + {Name: "FELIX_TYPHACAFILE", Value: certificatemanagement.TrustedCertBundleMountPath}, + {Name: "FELIX_TYPHACERTFILE", Value: "/node-certs/tls.crt"}, + {Name: "FELIX_TYPHACN", Value: "typha-server"}, + {Name: "FELIX_TYPHAKEYFILE", Value: "/node-certs/tls.key"}, + + {Name: "FELIX_DNSTRUSTEDSERVERS", Value: "k8s-service:kube-system/rke2-coredns-rke2-coredns"}, + + // Tigera-specific envvars + {Name: "FELIX_PROMETHEUSREPORTERENABLED", Value: "true"}, + {Name: "FELIX_PROMETHEUSREPORTERPORT", Value: "9081"}, + {Name: "FELIX_FLOWLOGSFILEENABLED", Value: "true"}, + {Name: "FELIX_FLOWLOGSFILEINCLUDELABELS", Value: "true"}, + {Name: "FELIX_FLOWLOGSFILEINCLUDEPOLICIES", Value: "true"}, + {Name: "FELIX_FLOWLOGSFILEINCLUDESERVICE", Value: "true"}, + {Name: "FELIX_FLOWLOGSENABLENETWORKSETS", Value: "true"}, + {Name: "FELIX_FLOWLOGSCOLLECTPROCESSINFO", Value: "true"}, + {Name: "FELIX_DNSLOGSFILEENABLED", Value: "true"}, + {Name: "FELIX_DNSLOGSFILEPERNODELIMIT", Value: "1000"}, + + // Calico Windows specific envvars + {Name: "VXLAN_VNI", Value: "4096"}, + {Name: "VXLAN_ADAPTER", Value: ""}, + {Name: "KUBE_NETWORK", Value: "Calico.*"}, + {Name: "KUBERNETES_SERVICE_HOST", Value: "1.2.3.4"}, + {Name: "KUBERNETES_SERVICE_PORT", Value: "6443"}, + } + Expect(ds.Spec.Template.Spec.Containers[0].Env).To(ConsistOf(expectedNodeEnv)) + Expect(len(ds.Spec.Template.Spec.Containers[0].Env)).To(Equal(len(expectedNodeEnv))) + + verifyWindowsProbesAndLifecycle(ds, false) + + // The metrics service should have the correct configuration. + ms := rtest.GetResource(resources, "calico-node-metrics-windows", "calico-system", "", "v1", "Service").(*corev1.Service) + Expect(ms.Spec.ClusterIP).To(Equal("None"), "metrics service should be headless to prevent kube-proxy from rendering too many iptables rules") + }) + + It("should not enable prometheus metrics if NodeMetricsPort is nil", func() { + defaultInstance.Variant = operatorv1.CalicoEnterprise + defaultInstance.NodeMetricsPort = nil + + resources := renderWindows(&cfg) + Expect(len(resources)).To(Equal(defaultNumExpectedResources + 1)) + + dsResource := rtest.GetResource(resources, "calico-node-windows", "calico-system", "apps", "v1", "DaemonSet") + Expect(dsResource).ToNot(BeNil()) + + notExpectedEnvVar := corev1.EnvVar{Name: "FELIX_PROMETHEUSMETRICSPORT"} + ds := dsResource.(*appsv1.DaemonSet) + Expect(ds.Spec.Template.Spec.Containers[0].Env).ToNot(ContainElement(notExpectedEnvVar)) + + // It should have the reporter port, though. + expected := corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERPORT"} + Expect(ds.Spec.Template.Spec.Containers[0].Env).ToNot(ContainElement(expected)) + }) + + It("should set FELIX_PROMETHEUSMETRICSPORT with a custom value if NodeMetricsPort is set", func() { + var nodeMetricsPort int32 = 1234 + defaultInstance.Variant = operatorv1.CalicoEnterprise + defaultInstance.NodeMetricsPort = &nodeMetricsPort + resources := renderWindows(&cfg) + Expect(len(resources)).To(Equal(defaultNumExpectedResources + 1)) + + dsResource := rtest.GetResource(resources, "calico-node-windows", "calico-system", "apps", "v1", "DaemonSet") + Expect(dsResource).ToNot(BeNil()) + + // Assert on expected env vars. + expectedEnvVars := []corev1.EnvVar{ + {Name: "FELIX_PROMETHEUSMETRICSPORT", Value: "1234"}, + {Name: "FELIX_PROMETHEUSMETRICSENABLED", Value: "true"}, + } + ds := dsResource.(*appsv1.DaemonSet) + for _, v := range expectedEnvVars { + Expect(ds.Spec.Template.Spec.Containers[0].Env).To(ContainElement(v)) + } + + // Assert we set annotations properly. + Expect(ds.Spec.Template.Annotations["prometheus.io/scrape"]).To(Equal("true")) + Expect(ds.Spec.Template.Annotations["prometheus.io/port"]).To(Equal("1234")) + }) +}) diff --git a/pkg/extensions/controllerextension.go b/pkg/extensions/controllerextension.go new file mode 100644 index 0000000000..18957a524f --- /dev/null +++ b/pkg/extensions/controllerextension.go @@ -0,0 +1,60 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package extensions + +import ( + v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" + + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/controller/contexts" + "github.com/tigera/operator/pkg/ctrlruntime" + "github.com/tigera/operator/pkg/tls/certificatemanagement" +) + +// ControllerExtension extends a controller's reconcile: it validates the +// configuration and builds the RenderContext the render phase consumes. The core +// operator registers none and runs with the base behavior; an extension build +// registers one per controller it extends. +type ControllerExtension interface { + // Validate rejects configuration the extension does not support, before any + // rendering happens. + Validate(cc contexts.ControllerContext) error + + // ExtendContext does the controller-side reconcile work the render phase + // cannot, returning the updated ControllerContext (its embedded RenderContext is + // what the render phase consumes) plus any keypairs the extension created that the + // controller should manage (add to certificate management and BYO-expiry + // warnings), or an error that aborts the reconcile. + ExtendContext(cc contexts.ControllerContext) (contexts.ControllerContext, []certificatemanagement.KeyPairInterface, error) +} + +// Watcher is an optional companion to ControllerExtension. A controller's Add() +// calls Set.SetupWatches, which invokes Watches on any registered extension that +// implements this, so the extension registers the watches it needs (its CRs, its +// secrets) instead of the controller naming them. +type Watcher interface { + Watches(c ctrlruntime.Controller) error +} + +// FelixConfigDefaulter is an optional companion to ControllerExtension. A +// controller's FelixConfiguration defaulting calls Set.DefaultFelixConfiguration, +// which invokes this on a registered extension that implements it, so the variant's +// FelixConfiguration defaults (e.g. the provider-specific dnsTrustedServers) live in +// the extension instead of the controller. It returns whether it changed fc. Felix +// defaulting persists early in reconcile, before ExtendContext runs, so it can't fold +// into ExtendContext. +type FelixConfigDefaulter interface { + DefaultFelixConfiguration(install *operatorv1.InstallationSpec, fc *v3.FelixConfiguration) (bool, error) +} diff --git a/pkg/extensions/controllerextension_test.go b/pkg/extensions/controllerextension_test.go new file mode 100644 index 0000000000..538d23d294 --- /dev/null +++ b/pkg/extensions/controllerextension_test.go @@ -0,0 +1,131 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package extensions_test + +import ( + "errors" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/controller/contexts" + "github.com/tigera/operator/pkg/ctrlruntime" + "github.com/tigera/operator/pkg/extensions" + "github.com/tigera/operator/pkg/render" + "github.com/tigera/operator/pkg/tls/certificatemanagement" +) + +var _ = Describe("controller extension", func() { + var s *extensions.Set + BeforeEach(func() { + s = extensions.NewSet() + }) + + It("returns the base render context when the variant has no extension", func() { + install := &operatorv1.InstallationSpec{Variant: operatorv1.Calico} + rc, _, err := s.ExtendContext(contexts.ControllerContext{ + RenderContext: render.RenderContext{Installation: install, ClusterDomain: "cluster.local"}, + Controller: contexts.InstallationController, + }) + Expect(err).NotTo(HaveOccurred()) + Expect(rc.Installation).To(BeIdenticalTo(install)) + Expect(rc.ClusterDomain).To(Equal("cluster.local")) + Expect(rc.Extension).To(BeNil()) + }) + + It("runs the extension registered for the installation variant", func() { + s.Variant(operatorv1.CalicoEnterprise).Controller(contexts.InstallationController, fakeController{}) + rc, _, err := s.ExtendContext(enterpriseContext()) + Expect(err).NotTo(HaveOccurred()) + Expect(rc.ClusterDomain).To(Equal("from-fake")) + }) + + It("ignores an extension registered for a different variant", func() { + s.Variant(operatorv1.CalicoEnterprise).Controller(contexts.InstallationController, fakeController{}) + rc, _, err := s.ExtendContext(contexts.ControllerContext{ + RenderContext: render.RenderContext{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.Calico}, ClusterDomain: "real"}, + Controller: contexts.InstallationController, + }) + Expect(err).NotTo(HaveOccurred()) + Expect(rc.ClusterDomain).To(Equal("real")) + }) + + It("surfaces the extension error", func() { + s.Variant(operatorv1.CalicoEnterprise).Controller(contexts.InstallationController, fakeController{err: errors.New("boom")}) + _, _, err := s.ExtendContext(enterpriseContext()) + Expect(err).To(MatchError("boom")) + }) + + It("runs the extension's validation", func() { + s.Variant(operatorv1.CalicoEnterprise).Controller(contexts.InstallationController, fakeController{validateErr: errors.New("invalid")}) + Expect(s.Validate(enterpriseContext())).To(MatchError("invalid")) + }) + + It("runs the watch hook of an extension that implements Watcher", func() { + called := false + s.Variant(operatorv1.CalicoEnterprise).Controller(contexts.InstallationController, watchingController{called: &called}) + Expect(s.SetupWatches(contexts.InstallationController, nil)).NotTo(HaveOccurred()) + Expect(called).To(BeTrue()) + }) + + It("returns the base context and no validation error for a nil Set", func() { + var nilSet *extensions.Set + cc := enterpriseContext() + cc.ClusterDomain = "real" + rc, _, err := nilSet.ExtendContext(cc) + Expect(err).NotTo(HaveOccurred()) + Expect(rc.ClusterDomain).To(Equal("real")) + Expect(nilSet.Validate(cc)).NotTo(HaveOccurred()) + }) +}) + +func enterpriseContext() contexts.ControllerContext { + return contexts.ControllerContext{ + RenderContext: render.RenderContext{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.CalicoEnterprise}}, + Controller: contexts.InstallationController, + } +} + +// fakeController is a ControllerExtension whose Validate and ExtendContext return +// configurable results. +type fakeController struct { + err error + validateErr error +} + +func (f fakeController) Validate(_ contexts.ControllerContext) error { + return f.validateErr +} + +func (f fakeController) ExtendContext(cc contexts.ControllerContext) (contexts.ControllerContext, []certificatemanagement.KeyPairInterface, error) { + if f.err != nil { + return cc, nil, f.err + } + cc.ClusterDomain = "from-fake" + return cc, nil, nil +} + +// watchingController is a fakeController that also implements the Watcher +// companion, recording that its watch hook ran. +type watchingController struct { + fakeController + called *bool +} + +func (w watchingController) Watches(ctrlruntime.Controller) error { + *w.called = true + return nil +} diff --git a/pkg/extensions/doc.go b/pkg/extensions/doc.go new file mode 100644 index 0000000000..ec06d9597b --- /dev/null +++ b/pkg/extensions/doc.go @@ -0,0 +1,44 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package extensions is the seam other product variants (today just Calico +// Enterprise) use to layer variant-specific behavior onto the core operator's +// render output, so core code never branches on variant. +// +// A Set holds the extensions for every variant. Per reconcile the controller +// selects one Variant from the installation's variant, so a registered hook only +// ever runs for its own variant and never re-checks it. A Variant bundles two +// kinds of extension: +// +// A ControllerExtension is the controller-side hook. It runs once per reconcile +// in the installation controller, has cluster access (Client, +// CertificateManager) via the ControllerContext, and does the side-effecting +// work a pure render hook can't: rejecting unsupported config (Validate) and +// creating certificates / extending the trusted bundle (ExtendContext). It +// returns the RenderContext, the read-only baton passed to the render phase. +// +// Per-component modifiers are the render phase: pure hooks that run after a +// component builds its objects. An image override swaps the component's image +// (resolved during ResolveImages); a Modifier post-processes the rendered +// objects (run at the componentHandler, which renders the decorated component). +// Register a modifier with Variant.Modify, or with RegisterModifier when it +// needs the component's own typed config. +// +// ControllerContext (controller phase) and RenderContext (render phase) are a +// pair: ControllerContext embeds RenderContext and adds the cluster-access deps, +// which is why modifiers, given only a RenderContext, can't do I/O. +// +// A variant wires up its controller extension and modifiers in one place at +// startup - see pkg/enterprise. +package extensions diff --git a/pkg/extensions/extension.go b/pkg/extensions/extension.go new file mode 100644 index 0000000000..625e371afe --- /dev/null +++ b/pkg/extensions/extension.go @@ -0,0 +1,40 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package extensions + +import ( + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/tigera/operator/pkg/render" +) + +// Modifier post-processes the objects a render component produced. It receives +// the component's create and delete lists and returns the (possibly extended) +// lists. A modifier may mutate matched objects, append objects to create, and +// append objects to delete (e.g. to clean up resources a prior variant left +// behind). It runs only for the variant it is registered under, so it need not +// re-check the variant. +type Modifier func(rc render.RenderContext, create, delete []client.Object) (newCreate, newDelete []client.Object) + +// FindObject returns the first object of type T with the given name. +func FindObject[T client.Object](objs []client.Object, name string) (T, bool) { + var zero T + for _, o := range objs { + if t, ok := o.(T); ok && o.GetName() == name { + return t, true + } + } + return zero, false +} diff --git a/pkg/extensions/extension_test.go b/pkg/extensions/extension_test.go new file mode 100644 index 0000000000..69ae36aa6a --- /dev/null +++ b/pkg/extensions/extension_test.go @@ -0,0 +1,103 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package extensions_test + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/extensions" + "github.com/tigera/operator/pkg/extensions/extensionstest" + "github.com/tigera/operator/pkg/render" +) + +var _ = Describe("extension registry", func() { + var s *extensions.Set + BeforeEach(func() { + s = extensions.NewSet() + }) + + entCtx := render.RenderContext{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.CalicoEnterprise}} + + It("applies a registered modifier to the matching component and variant", func() { + s.Variant(operatorv1.CalicoEnterprise).Modify("test", func(ctx render.RenderContext, objs, del []client.Object) ([]client.Object, []client.Object) { + cm, ok := extensions.FindObject[*corev1.ConfigMap](objs, "cm") + Expect(ok).To(BeTrue()) + cm.Data = map[string]string{"k": "v"} + return append(objs, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "extra"}}), del + }) + + in := []client.Object{&corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "cm"}}} + out, _ := extensionstest.ApplyExtensions(s, "test", entCtx, in, nil) + + Expect(out).To(HaveLen(2)) + cm := out[0].(*corev1.ConfigMap) + Expect(cm.Data).To(HaveKeyWithValue("k", "v")) + Expect(out[1].GetName()).To(Equal("extra")) + }) + + It("lets a modifier append to the delete list", func() { + s.Variant(operatorv1.CalicoEnterprise).Modify("test", func(_ render.RenderContext, objs, del []client.Object) ([]client.Object, []client.Object) { + return objs, append(del, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "stale"}}) + }) + + in := []client.Object{&corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "cm"}}} + out, del := extensionstest.ApplyExtensions(s, "test", entCtx, in, nil) + Expect(out).To(Equal(in)) + Expect(del).To(HaveLen(1)) + Expect(del[0].GetName()).To(Equal("stale")) + }) + + It("returns objects unchanged when no modifier is registered", func() { + in := []client.Object{&corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "cm"}}} + out, _ := extensionstest.ApplyExtensions(s, "unregistered", entCtx, in, nil) + Expect(out).To(Equal(in)) + }) + + It("does not apply a modifier registered for a different variant", func() { + s.Variant(operatorv1.CalicoEnterprise).Modify("test", func(_ render.RenderContext, objs, del []client.Object) ([]client.Object, []client.Object) { + return append(objs, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "extra"}}), del + }) + + calicoCtx := render.RenderContext{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.Calico}} + in := []client.Object{&corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "cm"}}} + out, _ := extensionstest.ApplyExtensions(s, "test", calicoCtx, in, nil) + Expect(out).To(Equal(in)) + }) + + It("returns objects unchanged when no installation is set", func() { + in := []client.Object{&corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "cm"}}} + out, _ := extensionstest.ApplyExtensions(s, "test", render.RenderContext{}, in, nil) + Expect(out).To(Equal(in)) + }) + + It("replaces rather than stacks when a component modifier is registered twice", func() { + add := func(name string) { + s.Variant(operatorv1.CalicoEnterprise).Modify("test", func(_ render.RenderContext, objs, del []client.Object) ([]client.Object, []client.Object) { + return append(objs, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: name}}), del + }) + } + add("first") + add("second") + + out, _ := extensionstest.ApplyExtensions(s, "test", entCtx, nil, nil) + Expect(out).To(HaveLen(1)) + Expect(out[0].GetName()).To(Equal("second")) + }) +}) diff --git a/pkg/extensions/extensions_suite_test.go b/pkg/extensions/extensions_suite_test.go new file mode 100644 index 0000000000..791eedb737 --- /dev/null +++ b/pkg/extensions/extensions_suite_test.go @@ -0,0 +1,27 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package extensions_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestExtensions(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "pkg/extensions Suite") +} diff --git a/pkg/extensions/extensionstest/decorate.go b/pkg/extensions/extensionstest/decorate.go new file mode 100644 index 0000000000..05d621e14f --- /dev/null +++ b/pkg/extensions/extensionstest/decorate.go @@ -0,0 +1,77 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package extensionstest holds shared helpers for exercising a registered +// extension through the same Set.Decorate seam the component handler uses. It is +// test support imported by the extension test suites (extensions, render, and the +// per-component enterprise packages), so the helper lives once instead of being +// copied into each test package. +package extensionstest + +import ( + client "sigs.k8s.io/controller-runtime/pkg/client" + + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/extensions" + "github.com/tigera/operator/pkg/render" + rmeta "github.com/tigera/operator/pkg/render/common/meta" +) + +// StubComponent adapts raw object lists to a render.Component so a registered +// extension can be exercised through Set.Decorate. Key selects the extension; +// ExtCtx is delivered as the component's ExtensionContext (the typed config a +// RegisterModifier modifier reads). +type StubComponent struct { + Key string + ExtCtx any + Create, Delete []client.Object +} + +func (s StubComponent) ResolveImages(*operatorv1.ImageSet) error { + return nil +} + +func (s StubComponent) Objects() ([]client.Object, []client.Object) { + return s.Create, s.Delete +} + +func (s StubComponent) Ready() bool { + return true +} + +func (s StubComponent) SupportedOSType() rmeta.OSType { + return rmeta.OSTypeAny +} + +func (s StubComponent) ModifierKey() string { + return s.Key +} + +func (s StubComponent) ExtensionContext() any { + return s.ExtCtx +} + +// ApplyExtensions decorates a stub component holding the given objects with the +// extension registered under key, then renders it. For a modifier that needs the +// component's typed config, use ApplyExtensionsWithContext. +func ApplyExtensions(s *extensions.Set, key string, rc render.RenderContext, create, del []client.Object) ([]client.Object, []client.Object) { + return ApplyExtensionsWithContext(s, key, rc, nil, create, del) +} + +// ApplyExtensionsWithContext is ApplyExtensions for a modifier that reads the +// component's typed config: extCtx is delivered as the stub's ExtensionContext. +func ApplyExtensionsWithContext(s *extensions.Set, key string, rc render.RenderContext, extCtx any, create, del []client.Object) ([]client.Object, []client.Object) { + stub := StubComponent{Key: key, ExtCtx: extCtx, Create: create, Delete: del} + return s.Decorate(stub, rc).Objects() +} diff --git a/pkg/extensions/image_test.go b/pkg/extensions/image_test.go new file mode 100644 index 0000000000..736f110ca3 --- /dev/null +++ b/pkg/extensions/image_test.go @@ -0,0 +1,42 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package extensions_test + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/components" + "github.com/tigera/operator/pkg/extensions" +) + +var _ = Describe("image overrides", func() { + var s *extensions.Set + BeforeEach(func() { + s = extensions.NewSet() + s.Variant(operatorv1.CalicoEnterprise).Image("node", components.ComponentTigeraNode) + }) + + It("uses the override registered for the installation variant", func() { + ent := &operatorv1.InstallationSpec{Variant: operatorv1.CalicoEnterprise} + Expect(s.ResolveImage("node", components.ComponentCalicoNode, ent)).To(Equal(components.ComponentTigeraNode)) + }) + + It("falls back to the default for a variant with no override", func() { + calico := &operatorv1.InstallationSpec{Variant: operatorv1.Calico} + Expect(s.ResolveImage("node", components.ComponentCalicoNode, calico)).To(Equal(components.ComponentCalicoNode)) + }) +}) diff --git a/pkg/extensions/set.go b/pkg/extensions/set.go new file mode 100644 index 0000000000..4b8161cae1 --- /dev/null +++ b/pkg/extensions/set.go @@ -0,0 +1,200 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package extensions + +import ( + "context" + + v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" + "k8s.io/client-go/kubernetes" + + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/components" + "github.com/tigera/operator/pkg/controller/contexts" + "github.com/tigera/operator/pkg/ctrlruntime" + "github.com/tigera/operator/pkg/imageoverride" + "github.com/tigera/operator/pkg/render" + "github.com/tigera/operator/pkg/tls/certificatemanagement" +) + +// Set is all the variant extensions the operator runs with, indexed by product +// variant. The core operator runs with a nil Set; an extension build (e.g. +// Calico Enterprise) constructs a populated one and hands it in through +// options.ControllerOptions. This replaces what used to be package-level +// registries, so nothing is wired by import side effect. +// +// Per reconcile the controller selects one Variant from the installation's +// variant. The methods the controller calls (Decorate, Validate, ExtendContext, +// Images, ResolveImage) are nil-safe, so the core operator's nil Set yields base +// behavior. +type Set struct { + variants map[operatorv1.ProductVariant]*Variant + images *imageoverride.Overrides + + // computeOptions, registered by the extension build, discovers the variant's + // controller-phase options at startup. ComputeOptions runs it. The core operator + // registers none. + computeOptions func(context.Context, kubernetes.Interface) (any, error) + + // options is the result of ComputeOptions, carried onto every ControllerContext + // so a hook can read its variant's options. + options any +} + +// NewSet returns an empty Set ready to register variant extensions into. +func NewSet() *Set { + return &Set{ + variants: map[operatorv1.ProductVariant]*Variant{}, + images: imageoverride.New(), + } +} + +// RegisterOptions sets the function that computes the variant's controller-phase +// options. The extension build registers one; the core operator registers none. +func (s *Set) RegisterOptions(f func(context.Context, kubernetes.Interface) (any, error)) { + s.computeOptions = f +} + +// ComputeOptions runs the registered options computer (if any) and stores the +// result, which Validate and ExtendContext then carry onto the ControllerContext. +// It's meant to be called once at startup, before any reconcile. Nil-safe. +func (s *Set) ComputeOptions(ctx context.Context, cli kubernetes.Interface) error { + if s == nil || s.computeOptions == nil { + return nil + } + o, err := s.computeOptions(ctx, cli) + if err != nil { + return err + } + s.options = o + return nil +} + +// Variant returns the extension bundle for v, creating an empty one if needed. +// Used at registration time to build up a variant's extensions. +func (s *Set) Variant(v operatorv1.ProductVariant) *Variant { + if s.variants[v] == nil { + s.variants[v] = &Variant{ + variant: v, + controllers: map[contexts.ControllerName]ControllerExtension{}, + modifiers: map[string]decorator{}, + images: s.images, + } + } + return s.variants[v] +} + +// variant looks up the bundle for v, returning nil when none is registered. +// Nil-safe. +func (s *Set) variant(v operatorv1.ProductVariant) *Variant { + if s == nil { + return nil + } + return s.variants[v] +} + +// Decorate wraps component with the extension registered for it under the +// installation's variant, so that when the handler renders the component its +// objects are post-processed by that modifier. A decorated component is itself a +// render.Component, so it flows through the component handler like any other. +// Returns component unchanged when no extension applies. Nil-safe. +func (s *Set) Decorate(component render.Component, ctx render.RenderContext) render.Component { + if ctx.Installation == nil { + return component + } + return s.variant(ctx.Installation.Variant).decorate(component, ctx) +} + +// Validate runs the cc.Controller extension's validation for the installation's +// variant, or returns nil when no extension is registered. Nil-safe. +func (s *Set) Validate(cc contexts.ControllerContext) error { + if s != nil { + cc.Options = s.options + } + if cc.Installation == nil { + return nil + } + return s.variant(cc.Installation.Variant).validate(cc) +} + +// ExtendContext runs the cc.Controller extension for the installation's variant +// and returns the updated ControllerContext plus any keypairs the extension wants +// the controller to manage, or the context unchanged and no keypairs when no +// extension is registered. Nil-safe. +func (s *Set) ExtendContext(cc contexts.ControllerContext) (contexts.ControllerContext, []certificatemanagement.KeyPairInterface, error) { + if s != nil { + cc.Options = s.options + } + if cc.Installation == nil { + return cc, nil, nil + } + return s.variant(cc.Installation.Variant).extendContext(cc) +} + +// SetupWatches registers the watches every variant's extension declares for the +// named controller. It runs at controller startup, which is variant-agnostic, so +// it registers the union across variants (in practice the one active extension +// build's). Nil-safe. +func (s *Set) SetupWatches(controller contexts.ControllerName, c ctrlruntime.Controller) error { + if s == nil { + return nil + } + for _, v := range s.variants { + w, ok := v.controllers[controller].(Watcher) + if !ok { + continue + } + if err := w.Watches(c); err != nil { + return err + } + } + return nil +} + +// DefaultFelixConfiguration runs the named controller's extension FelixConfiguration +// defaulting for the installation's variant, returning whether it changed fc. The +// extension implements the optional FelixConfigDefaulter companion; when it doesn't +// (or no extension is registered) this is a no-op. Nil-safe. +func (s *Set) DefaultFelixConfiguration(controller contexts.ControllerName, install *operatorv1.InstallationSpec, fc *v3.FelixConfiguration) (bool, error) { + if install == nil { + return false, nil + } + v := s.variant(install.Variant) + if v == nil { + return false, nil + } + d, ok := v.controllers[controller].(FelixConfigDefaulter) + if !ok { + return false, nil + } + return d.DefaultFelixConfiguration(install, fc) +} + +// Images returns the shared image override table. The render package resolves a +// component's image through it directly (the imageoverride leaf, so render need +// not import extensions). Nil-safe, returning nil overrides that resolve to the +// default image. +func (s *Set) Images() *imageoverride.Overrides { + if s == nil { + return nil + } + return s.images +} + +// ResolveImage resolves key for the installation through the image overrides, +// returning def when no override applies. Nil-safe. +func (s *Set) ResolveImage(key string, def components.Component, in *operatorv1.InstallationSpec) components.Component { + return s.Images().Resolve(key, def, in) +} diff --git a/pkg/extensions/variant.go b/pkg/extensions/variant.go new file mode 100644 index 0000000000..a7438fdac4 --- /dev/null +++ b/pkg/extensions/variant.go @@ -0,0 +1,143 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package extensions + +import ( + "github.com/sirupsen/logrus" + "sigs.k8s.io/controller-runtime/pkg/client" + + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/components" + "github.com/tigera/operator/pkg/controller/contexts" + "github.com/tigera/operator/pkg/imageoverride" + "github.com/tigera/operator/pkg/render" + "github.com/tigera/operator/pkg/tls/certificatemanagement" +) + +// Variant bundles everything that extends the core operator for one product +// variant: the controller-side hook, the per-component modifiers, and the image +// overrides. The Set selects one Variant per reconcile from the installation's +// variant, so within a Variant there is at most one extension per component and +// nothing here is itself keyed by variant. +type Variant struct { + variant operatorv1.ProductVariant + controllers map[contexts.ControllerName]ControllerExtension + modifiers map[string]decorator + images *imageoverride.Overrides // shared with the owning Set +} + +// decorator wraps a base component, returning one whose Objects() are augmented +// by a registered modifier. +type decorator func(base render.Component, rc render.RenderContext) render.Component + +// Controller registers the variant's controller-side extension for the named +// controller. A controller has at most one; registering replaces any prior one. +func (v *Variant) Controller(name contexts.ControllerName, c ControllerExtension) { + v.controllers[name] = c +} + +// Image registers an image override for the named component. +func (v *Variant) Image(component string, image components.Component) { + v.images.Register(v.variant, component, image) +} + +// Modify registers a modifier for a component that needs no per-component +// config. For components whose modifier needs the component's own typed config, +// use RegisterModifier. +func (v *Variant) Modify(component string, m Modifier) { + v.modifiers[component] = func(base render.Component, rc render.RenderContext) render.Component { + return &decoratedComponent{Component: base, rc: rc, modify: m} + } +} + +// RegisterModifier registers a modifier for component whose modifier needs the +// component's own typed config. The component supplies it via +// render.ExtensionContextProvider; RegisterModifier asserts it to Cfg once, here, +// and hands the typed value to modify - so the modifier body needs no assertion. +// It is a free function because Go has no generic methods. +func RegisterModifier[Cfg any]( + v *Variant, + component string, + modify func(rc render.RenderContext, cfg Cfg, create, delete []client.Object) ([]client.Object, []client.Object), +) { + v.modifiers[component] = func(base render.Component, rc render.RenderContext) render.Component { + provider, ok := base.(render.ExtensionContextProvider) + if !ok { + logrus.Errorf("BUG: component %q has a registered modifier but provides no extension context; leaving it unmodified", component) + return base + } + cfg, ok := provider.ExtensionContext().(Cfg) + if !ok { + var want Cfg + logrus.Errorf("BUG: component %q extension context is %T, want %T; leaving it unmodified", component, provider.ExtensionContext(), want) + return base + } + bound := func(rc render.RenderContext, create, delete []client.Object) ([]client.Object, []client.Object) { + return modify(rc, cfg, create, delete) + } + return &decoratedComponent{Component: base, rc: rc, modify: bound} + } +} + +// decorate wraps component with the modifier registered for its extension key, +// or returns it unchanged when the component exposes no extension point or none +// is registered. Nil-safe. +func (v *Variant) decorate(component render.Component, rc render.RenderContext) render.Component { + if v == nil { + return component + } + ext, ok := component.(render.Extensible) + if !ok { + return component + } + build, ok := v.modifiers[ext.ModifierKey()] + if !ok { + return component + } + return build(component, rc) +} + +// validate runs the cc.Controller extension's validation, or nil when the +// variant has none for it. Nil-safe. +func (v *Variant) validate(cc contexts.ControllerContext) error { + if v == nil || v.controllers[cc.Controller] == nil { + return nil + } + return v.controllers[cc.Controller].Validate(cc) +} + +// extendContext runs the cc.Controller extension, or returns the context unchanged +// and no managed keypairs when the variant has none for it. Nil-safe. +func (v *Variant) extendContext(cc contexts.ControllerContext) (contexts.ControllerContext, []certificatemanagement.KeyPairInterface, error) { + if v == nil || v.controllers[cc.Controller] == nil { + return cc, nil, nil + } + return v.controllers[cc.Controller].ExtendContext(cc) +} + +// decoratedComponent is the render.Component produced by decorate: it renders +// its embedded base component and then runs the variant modifier over the +// result. It embeds the base render.Component, so ResolveImages, SupportedOSType, +// and Ready delegate to the base; only Objects is augmented. +type decoratedComponent struct { + render.Component + rc render.RenderContext + modify Modifier +} + +func (d *decoratedComponent) Objects() ([]client.Object, []client.Object) { + create, del := d.Component.Objects() + return d.modify(d.rc, create, del) +} diff --git a/pkg/imageoverride/imageoverride.go b/pkg/imageoverride/imageoverride.go new file mode 100644 index 0000000000..ee4c48d559 --- /dev/null +++ b/pkg/imageoverride/imageoverride.go @@ -0,0 +1,61 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package imageoverride is a leaf package (no render/operator dependencies) +// that holds the image override table. The render package imports it to resolve +// a component's image without depending on pkg/extensions, which would cycle. +package imageoverride + +import ( + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/components" +) + +type overrideKey struct { + variant operatorv1.ProductVariant + key string +} + +// Overrides maps a component (keyed by variant) to the image it should resolve +// to, letting a variant swap a component's image without the render package +// branching on variant. The render component holds one and resolves through it. +// Registry, image path, and FIPS handling are applied downstream in the render +// package, so an override only picks which component. +type Overrides struct { + m map[overrideKey]components.Component +} + +// New returns an empty Overrides. +func New() *Overrides { + return &Overrides{m: map[overrideKey]components.Component{}} +} + +// Register stores image under key for the given variant. The key is the render +// component's image identifier (e.g. "node"). +func (o *Overrides) Register(variant operatorv1.ProductVariant, key string, image components.Component) { + o.m[overrideKey{variant, key}] = image +} + +// Resolve returns the override registered for key under the installation's +// variant, otherwise def. It is safe to call on a nil *Overrides (the core +// operator hands render no overrides), which always returns def. +func (o *Overrides) Resolve(key string, def components.Component, in *operatorv1.InstallationSpec) components.Component { + if o == nil || in == nil { + return def + } + if image, ok := o.m[overrideKey{in.Variant, key}]; ok { + return image + } + return def +} diff --git a/pkg/render/apiserver.go b/pkg/render/apiserver.go index 5be75e7ebd..90a64ca0b8 100644 --- a/pkg/render/apiserver.go +++ b/pkg/render/apiserver.go @@ -16,7 +16,6 @@ package render import ( "fmt" - "net/url" "strings" admregv1 "k8s.io/api/admissionregistration/v1" @@ -28,19 +27,15 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" apiregv1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1" - "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" - "github.com/tigera/api/pkg/lib/numorstring" operatorv1 "github.com/tigera/operator/api/v1" "github.com/tigera/operator/pkg/common" "github.com/tigera/operator/pkg/components" "github.com/tigera/operator/pkg/controller/k8sapi" - "github.com/tigera/operator/pkg/render/common/authentication" rcomp "github.com/tigera/operator/pkg/render/common/components" - relasticsearch "github.com/tigera/operator/pkg/render/common/elasticsearch" rmeta "github.com/tigera/operator/pkg/render/common/meta" "github.com/tigera/operator/pkg/render/common/networkpolicy" "github.com/tigera/operator/pkg/render/common/podaffinity" @@ -57,8 +52,14 @@ const ( APIServerPortName = "apiserver" APIServerPolicyName = networkpolicy.CalicoComponentPolicyPrefix + "apiserver-access" - auditLogsVolumeName = "calico-audit-logs" - auditPolicyVolumeName = "calico-audit-policy" + // ComponentNameAPIServer is the extension key under which a variant registers + // its API server modifier and image override. + ComponentNameAPIServer = "apiserver" + + // ComponentNameAPIServerPolicy keys the API server network policy modifier (the + // OIDC egress rule and the L7 admission controller ingress port). The base policy + // carries neither. + ComponentNameAPIServerPolicy = "apiserver-policy" ) const ( @@ -114,14 +115,33 @@ func APIServer(cfg *APIServerConfiguration) (Component, error) { }, nil } +// apiServerPolicyComponent wraps the API server network policy passthrough so it is +// render.Extensible: the variant modifier adds the OIDC egress rule and the L7 +// admission controller ingress port. The base policy carries neither. +type apiServerPolicyComponent struct { + Component + cfg *APIServerConfiguration +} + +func (apiServerPolicyComponent) ModifierKey() string { return ComponentNameAPIServerPolicy } + +// ExtensionContext hands the policy modifier the config it needs to look up container +// ports. +func (c apiServerPolicyComponent) ExtensionContext() any { + return APIServerExtensionContext{Config: c.cfg} +} + func APIServerPolicy(cfg *APIServerConfiguration) Component { - return NewPassthrough( - []client.Object{calicoSystemAPIServerPolicy(cfg)}, - []client.Object{ - // allow-tigera Tier was renamed to calico-system - networkpolicy.DeprecatedAllowTigeraNetworkPolicyObject("apiserver-access", APIServerNamespace), - }, - ) + return apiServerPolicyComponent{ + Component: NewPassthrough( + []client.Object{calicoSystemAPIServerPolicy(cfg)}, + []client.Object{ + // allow-tigera Tier was renamed to calico-system + networkpolicy.DeprecatedAllowTigeraNetworkPolicyObject("apiserver-access", APIServerNamespace), + }, + ), + cfg: cfg, + } } // APIServerConfiguration contains all the config information needed to render the component. @@ -131,62 +151,55 @@ type APIServerConfiguration struct { Installation *operatorv1.InstallationSpec APIServer *operatorv1.APIServerSpec ForceHostNetwork bool - ApplicationLayer *operatorv1.ApplicationLayer - ManagementCluster *operatorv1.ManagementCluster - ManagementClusterConnection *operatorv1.ManagementClusterConnection TLSKeyPair certificatemanagement.KeyPairInterface PullSecrets []*corev1.Secret OpenShift bool TrustedBundle certificatemanagement.TrustedBundle MultiTenant bool - KeyValidatorConfig authentication.KeyValidatorConfig KubernetesVersion *common.VersionInfo ClusterDomain string // Whether or not we should run the aggregation API server for projectcalico.org/v3 APIs // as part of this component. RequiresAggregationServer bool - - // When certificate management is enabled, we need a separate init container to create a cert, running - // with the same permissions as query server. - QueryServerTLSKeyPairCertificateManagementOnly certificatemanagement.KeyPairInterface } type apiServerComponent struct { - cfg *APIServerConfiguration - calicoImage string - l7AdmissionControllerEnvoyImage string - dikastesImage string + cfg *APIServerConfiguration + calicoImage string +} + +// APIServerExtensionContext carries the API server's render configuration and resolved +// image to a variant modifier. The modifier uses these to build variant-specific objects +// and to layer additional containers, volumes, and configuration onto the rendered +// deployment. +type APIServerExtensionContext struct { + Config *APIServerConfiguration + CalicoImage string +} + +// ModifierKey implements render.Extensible: the API server's variant-specific objects are +// applied by the modifier registered under this key. +func (c *apiServerComponent) ModifierKey() string { return ComponentNameAPIServer } + +// ExtensionContext implements render.ExtensionContextProvider, handing the modifier the +// config and resolved image it needs. +func (c *apiServerComponent) ExtensionContext() any { + return APIServerExtensionContext{Config: c.cfg, CalicoImage: c.calicoImage} } func (c *apiServerComponent) ResolveImages(is *operatorv1.ImageSet) error { reg := c.cfg.Installation.Registry path := c.cfg.Installation.ImagePath prefix := c.cfg.Installation.ImagePrefix - var err error - errMsgs := []string{} - enterprise := c.cfg.Installation.Variant.IsEnterprise() - if enterprise || c.cfg.RequiresAggregationServer { - c.calicoImage, err = components.GetReference(components.CombinedCalicoImage(c.cfg.Installation), reg, path, prefix, is) - if err != nil { - errMsgs = append(errMsgs, err.Error()) - } - } - - if enterprise && c.cfg.IsSidecarInjectionEnabled() { - c.l7AdmissionControllerEnvoyImage, err = components.GetReference(components.ComponentEnvoyProxy, reg, path, prefix, is) - if err != nil { - errMsgs = append(errMsgs, err.Error()) - } - c.dikastesImage, err = components.GetReference(components.ComponentDikastes, reg, path, prefix, is) - if err != nil { - errMsgs = append(errMsgs, err.Error()) - } - } - - if len(errMsgs) != 0 { - return fmt.Errorf("%s", strings.Join(errMsgs, ",")) + // Resolve the calico image unconditionally: the base uses it for the aggregation + // API server container, and a variant modifier needs it for the query server + // container and the deployment skeleton it may render itself. + var err error + c.calicoImage, err = components.GetReference(components.CombinedCalicoImage(c.cfg.Installation), reg, path, prefix, is) + if err != nil { + return err } return nil } @@ -196,8 +209,8 @@ func (c *apiServerComponent) SupportedOSType() rmeta.OSType { } func (c *apiServerComponent) Objects() ([]client.Object, []client.Object) { - // Start with all of the cluster-scoped resources that are used for both Calico and Calico Enterprise. - // When switching between Calico / Enterprise, these objects are simply updated in-place. + // Cluster-scoped resources used by the API server, independent of variant. Any + // variant-specific objects are layered on by the variant's modifier. globalObjects := []client.Object{ c.calicoCustomResourcesClusterRole(), c.calicoCustomResourcesClusterRoleBinding(), @@ -209,31 +222,20 @@ func (c *apiServerComponent) Objects() ([]client.Object, []client.Object) { } objsToDelete := []client.Object{} - - // Namespaced objects common to both Calico and Calico Enterprise. - // These objects will be updated when switching between the variants. namespacedObjects := []client.Object{} // Add in image pull secrets. secrets := secret.CopyToNamespace(APIServerNamespace, c.cfg.PullSecrets...) namespacedObjects = append(namespacedObjects, secret.ToRuntimeObjects(secrets...)...) - // The deployment and its supporting objects are needed when running the aggregation API server - // or when running Enterprise (which always needs the queryserver). - if c.cfg.RequiresAggregationServer || c.cfg.Installation.Variant.IsEnterprise() { - namespacedObjects = append(namespacedObjects, - c.apiServerServiceAccount(), - c.apiServerDeployment(), - c.apiServerService(), - c.apiServerPodDisruptionBudget(), - ) + // The deployment and its supporting objects are needed when running the aggregation + // API server. A variant that needs the deployment without an aggregation server (e.g. + // to run only a query server) renders the skeleton itself in its modifier and pulls + // these back out of the delete list. + if c.cfg.RequiresAggregationServer { + namespacedObjects = append(namespacedObjects, c.deploymentObjects()...) } else { - objsToDelete = append(objsToDelete, - &corev1.ServiceAccount{TypeMeta: metav1.TypeMeta{Kind: "ServiceAccount", APIVersion: "v1"}, ObjectMeta: metav1.ObjectMeta{Name: APIServerServiceAccountName, Namespace: APIServerNamespace}}, - &appsv1.Deployment{TypeMeta: metav1.TypeMeta{Kind: "Deployment", APIVersion: "apps/v1"}, ObjectMeta: metav1.ObjectMeta{Name: APIServerName, Namespace: APIServerNamespace}}, - &corev1.Service{TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}, ObjectMeta: metav1.ObjectMeta{Name: APIServerServiceName, Namespace: APIServerNamespace}}, - &policyv1.PodDisruptionBudget{TypeMeta: metav1.TypeMeta{Kind: "PodDisruptionBudget", APIVersion: "policy/v1"}, ObjectMeta: metav1.ObjectMeta{Name: APIServerName, Namespace: APIServerNamespace}}, - ) + objsToDelete = append(objsToDelete, APIServerDeploymentObjectMeta()...) } // These are objects that only need to exist when we are running an aggregation API server to @@ -246,16 +248,6 @@ func (c *apiServerComponent) Objects() ([]client.Object, []client.Object) { c.authReaderRoleBinding(), } - if c.cfg.Installation.Variant.IsEnterprise() { - aggregationAPIServerObjects = append(aggregationAPIServerObjects, - c.uiSettingsGroupGetterClusterRole(), - c.kubeControllerManagerUISettingsGroupGetterClusterRoleBinding(), - c.uiSettingsPassthruClusterRole(), - c.uiSettingsPassthruClusterRolebinding(), - c.auditPolicyConfigMap(), - ) - } - // Add in certificates for API server TLS. if !c.cfg.TLSKeyPair.UseCertificateManagement() { aggregationAPIServerObjects = append(aggregationAPIServerObjects, c.apiServiceRegistration(c.cfg.TLSKeyPair.GetCertificatePEM())) @@ -263,73 +255,11 @@ func (c *apiServerComponent) Objects() ([]client.Object, []client.Object) { aggregationAPIServerObjects = append(aggregationAPIServerObjects, c.apiServiceRegistration(c.cfg.Installation.CertificateManagement.CACert)) } - // Global enterprise-only objects. - globalEnterpriseObjects := []client.Object{ - c.tigeraAPIServerClusterRole(), - c.tigeraAPIServerClusterRoleBinding(), - } - - if !c.cfg.MultiTenant { - // These resources are only installed in zero-tenant clusters. Multi-tenant clusters don't use the default - // RBAC resources. - globalEnterpriseObjects = append(globalEnterpriseObjects, - c.tigeraUserClusterRole(), - c.tigeraNetworkAdminClusterRole(), - ) - } - - if c.cfg.ManagementCluster != nil { - globalEnterpriseObjects = append(globalEnterpriseObjects, c.managedClusterWatchClusterRole()) - if c.cfg.MultiTenant { - // Multi-tenant management cluster API servers need access to per-tenant CA secrets in order to sign - // per-tenant guardian certificates when creating ManagedClusters. - globalEnterpriseObjects = append(globalEnterpriseObjects, c.multiTenantSecretsRBAC()...) - // Multi-tenant management cluster components impersonate the single-tenant canonical service account - // in order to retrieve informations from the managed cluster. A cluster role will be created and each - // component will create a role binding in the tenant namespace - globalEnterpriseObjects = append(globalEnterpriseObjects, c.multiTenantManagedClusterAccessClusterRoles()...) - } else { - globalEnterpriseObjects = append(globalEnterpriseObjects, c.secretsRBAC()...) - } - } else { - // If we're not a management cluster, the API server doesn't need permissions to access secrets. - objsToDelete = append(objsToDelete, c.multiTenantSecretsRBAC()...) - objsToDelete = append(objsToDelete, c.secretsRBAC()...) - objsToDelete = append(objsToDelete, c.multiTenantManagedClusterAccessClusterRoles()...) - objsToDelete = append(objsToDelete, c.managedClusterWatchClusterRole()) - } - - // Namespaced enterprise-only objects. - namespacedEnterpriseObjects := []client.Object{} - - if c.cfg.TrustedBundle != nil { - namespacedEnterpriseObjects = append(namespacedEnterpriseObjects, c.cfg.TrustedBundle.ConfigMap(QueryserverNamespace)) - } - if c.cfg.IsSidecarInjectionEnabled() { - namespacedEnterpriseObjects = append(namespacedEnterpriseObjects, c.sidecarMutatingWebhookConfig()) - } else { - objsToDelete = append(objsToDelete, &admregv1.MutatingWebhookConfiguration{ObjectMeta: metav1.ObjectMeta{Name: common.SidecarMutatingWebhookConfigName}}) - } - if c.cfg.ManagementClusterConnection != nil { - namespacedEnterpriseObjects = append(namespacedEnterpriseObjects, - c.externalLinseedRoleBinding(), - ) - } - - // Compile the final arrays based on the variant. - if c.cfg.Installation.Variant.IsEnterprise() { - // Create any enterprise specific objects. - globalObjects = append(globalObjects, globalEnterpriseObjects...) - namespacedObjects = append(namespacedObjects, namespacedEnterpriseObjects...) - - // Clean up cluster-scoped resources that were created with the 'tigera' prefix. - // The apiserver now uses consistent resource names with 'calico' prefix across both EE and OSS variants. - objsToDelete = append(objsToDelete, c.deprecatedResources()...) - } else { - // Explicitly delete any global enterprise objects. - // Namespaced objects will be handled by namespace deletion. - objsToDelete = append(objsToDelete, globalEnterpriseObjects...) - } + // The L7 sidecar mutating webhook is an enterprise (ApplicationLayer) concern added + // by the variant modifier. The base always queues it for deletion so a cluster + // without sidecar injection (including any OSS install) never retains a stale one; + // the modifier pulls it back out of the delete list when sidecar injection is on. + objsToDelete = append(objsToDelete, &admregv1.MutatingWebhookConfiguration{ObjectMeta: metav1.ObjectMeta{Name: common.SidecarMutatingWebhookConfigName}}) // Clean up deprecated k8s NetworkPolicy, regardless of variant, // avoiding leftovers in the case of switching between variants. @@ -358,6 +288,40 @@ func (c *apiServerComponent) Ready() bool { return true } +// deploymentObjects returns the API server Deployment and its supporting objects. +func (c *apiServerComponent) deploymentObjects() []client.Object { + return []client.Object{ + c.apiServerServiceAccount(), + c.apiServerDeployment(), + c.apiServerService(), + c.apiServerPodDisruptionBudget(), + } +} + +// APIServerDeploymentObjects returns the API server Deployment and its supporting +// objects (ServiceAccount, Service, PodDisruptionBudget). The base renders these when +// running an aggregation API server; a variant modifier renders them itself when it +// needs the deployment but the base did not (e.g. a query-server-only deployment in +// v3-CRD mode). calicoImage is the resolved image for any base containers. +func APIServerDeploymentObjects(cfg *APIServerConfiguration, calicoImage string) []client.Object { + c := &apiServerComponent{cfg: cfg, calicoImage: calicoImage} + return c.deploymentObjects() +} + +// APIServerDeploymentObjectMeta returns empty-shell copies of the API server Deployment +// and its supporting objects, identifying them by name/kind/namespace. The base queues +// these for deletion when it isn't running an aggregation API server; a variant modifier +// matches against them to pull them back out of the delete list when it renders the +// deployment skeleton itself. +func APIServerDeploymentObjectMeta() []client.Object { + return []client.Object{ + &corev1.ServiceAccount{TypeMeta: metav1.TypeMeta{Kind: "ServiceAccount", APIVersion: "v1"}, ObjectMeta: metav1.ObjectMeta{Name: APIServerServiceAccountName, Namespace: APIServerNamespace}}, + &appsv1.Deployment{TypeMeta: metav1.TypeMeta{Kind: "Deployment", APIVersion: "apps/v1"}, ObjectMeta: metav1.ObjectMeta{Name: APIServerName, Namespace: APIServerNamespace}}, + &corev1.Service{TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}, ObjectMeta: metav1.ObjectMeta{Name: APIServerServiceName, Namespace: APIServerNamespace}}, + &policyv1.PodDisruptionBudget{TypeMeta: metav1.TypeMeta{Kind: "PodDisruptionBudget", APIVersion: "policy/v1"}, ObjectMeta: metav1.ObjectMeta{Name: APIServerName, Namespace: APIServerNamespace}}, + } +} + // For legacy reasons we use apiserver: true here instead of the k8s-app: name label, // so we need to set it explicitly rather than use the common labeling logic. func (c *apiServerComponent) deploymentSelector() *metav1.LabelSelector { @@ -502,13 +466,6 @@ func calicoSystemAPIServerPolicy(cfg *APIServerConfiguration) *v3.NetworkPolicy }, }...) - if cfg.KeyValidatorConfig != nil { - if parsedURL, err := url.Parse(cfg.KeyValidatorConfig.Issuer()); err == nil { - oidcEgressRule := networkpolicy.GetOIDCEgressRule(parsedURL) - egressRules = append(egressRules, oidcEgressRule) - } - } - if r, err := cfg.K8SServiceEndpoint.DestinationEntityRule(); r != nil && err == nil { egressRules = append(egressRules, v3.Rule{ Action: v3.Allow, @@ -523,15 +480,12 @@ func calicoSystemAPIServerPolicy(cfg *APIServerConfiguration) *v3.NetworkPolicy Action: v3.Pass, }) - apiServerContainerPort := getContainerPort(cfg, APIServerContainerName).ContainerPort - queryServerContainerPort := getContainerPort(cfg, TigeraAPIServerQueryServerContainerName).ContainerPort - l7AdmCtrlContainerPort := getContainerPort(cfg, L7AdmissionControllerContainerName).ContainerPort + apiServerContainerPort := GetContainerPort(cfg, APIServerContainerName).ContainerPort + queryServerContainerPort := GetContainerPort(cfg, TigeraAPIServerQueryServerContainerName).ContainerPort - // The ports Calico Enterprise API Server and Calico Enterprise Query Server are configured to listen on. + // The ports the API server and query server listen on. The L7 admission controller + // ingress port is added by the variant modifier when sidecar injection is enabled. ingressPorts := networkpolicy.Ports(443, uint16(apiServerContainerPort), uint16(queryServerContainerPort), 10443) - if cfg.IsSidecarInjectionEnabled() { - ingressPorts = append(ingressPorts, numorstring.Port{MinPort: uint16(l7AdmCtrlContainerPort), MaxPort: uint16(l7AdmCtrlContainerPort)}) - } return &v3.NetworkPolicy{ TypeMeta: metav1.TypeMeta{Kind: "NetworkPolicy", APIVersion: "projectcalico.org/v3"}, @@ -768,17 +722,6 @@ func (c *apiServerComponent) authClusterRole() client.Object { } } -// multiTenantSecretsRBAC provides the tigera API server with the ability to read secrets on the cluster. -// This is needed in multi-tenant management clusters only, in order to read tenant secrets for signing managed cluster certificates. -func (c *apiServerComponent) multiTenantSecretsRBAC() []client.Object { - return TunnelSecretRBAC(APIServerSecretsRBACName, APIServerServiceAccountName, c.cfg.ManagementCluster, true) -} - -// secretsRBAC provides the tigera API server with the ability to read secrets from the API server's namespace. -func (c *apiServerComponent) secretsRBAC() []client.Object { - return TunnelSecretRBAC(APIServerSecretsRBACName, APIServerServiceAccountName, c.cfg.ManagementCluster, false) -} - // authClusterRoleBinding returns a clusterrolebinding to create, and a clusterrolebinding to delete. // // Both Calico and Calico Enterprise, with different names. @@ -854,7 +797,7 @@ func (c *apiServerComponent) webhookReaderClusterRoleBinding() client.Object { } } -func getContainerPort(cfg *APIServerConfiguration, containerName ContainerName) *operatorv1.APIServerDeploymentContainerPort { +func GetContainerPort(cfg *APIServerConfiguration, containerName ContainerName) *operatorv1.APIServerDeploymentContainerPort { // Try to get the override port if cfg != nil && cfg.APIServer != nil && @@ -889,11 +832,10 @@ func getContainerPort(cfg *APIServerConfiguration, containerName ContainerName) return nil } -// apiServerService creates a service backed by the API server and - for enterprise - query server. +// apiServerService creates a service backed by the API server. A variant modifier may add +// additional ports (e.g. the query server port). func (c *apiServerComponent) apiServerService() *corev1.Service { - apiServerTargetPort := getContainerPort(c.cfg, APIServerContainerName) - queryServerTargetPort := getContainerPort(c.cfg, TigeraAPIServerQueryServerContainerName) - l7AdmissionControllerTargetPort := getContainerPort(c.cfg, L7AdmissionControllerContainerName) + apiServerTargetPort := GetContainerPort(c.cfg, APIServerContainerName) s := &corev1.Service{ TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}, @@ -917,29 +859,6 @@ func (c *apiServerComponent) apiServerService() *corev1.Service { }, } - if c.cfg.Installation.Variant.IsEnterprise() { - // Add port for queryserver if enterprise. - s.Spec.Ports = append(s.Spec.Ports, - corev1.ServicePort{ - Name: QueryServerPortName, - Port: QueryServerPort, - Protocol: corev1.ProtocolTCP, - TargetPort: intstr.FromInt32(queryServerTargetPort.ContainerPort), - }, - ) - } - - if c.cfg.IsSidecarInjectionEnabled() { - s.Spec.Ports = append(s.Spec.Ports, - corev1.ServicePort{ - Name: L7AdmissionControllerPortName, - Port: L7AdmissionControllerPort, - Protocol: corev1.ProtocolTCP, - TargetPort: intstr.FromInt32(l7AdmissionControllerTargetPort.ContainerPort), - }, - ) - } - return s } @@ -958,31 +877,22 @@ func (c *apiServerComponent) apiServerDeployment() *appsv1.Deployment { c.cfg.TLSKeyPair.HashAnnotationKey(): c.cfg.TLSKeyPair.HashAnnotationValue(), } + // The API server cert init container is only needed when running the aggregation API + // server under certificate management. A variant modifier adds any further init + // containers it needs (e.g. the query server cert). var initContainers []corev1.Container - if c.cfg.TLSKeyPair.UseCertificateManagement() { - if c.cfg.RequiresAggregationServer { - // Only include the API server init container if we're running the aggregation API server! - initContainerAPIServer := c.cfg.TLSKeyPair.InitContainer(APIServerNamespace, c.apiServerContainer().SecurityContext) - initContainerAPIServer.Name = fmt.Sprintf("%s-%s", CalicoAPIServerTLSSecretName, certificatemanagement.CSRInitContainerName) - initContainers = append(initContainers, initContainerAPIServer) - } - - initContainerQueryServer := c.cfg.QueryServerTLSKeyPairCertificateManagementOnly.InitContainer(APIServerNamespace, c.queryServerContainer().SecurityContext) - annotations[c.cfg.QueryServerTLSKeyPairCertificateManagementOnly.HashAnnotationKey()] = c.cfg.QueryServerTLSKeyPairCertificateManagementOnly.HashAnnotationValue() - initContainers = append(initContainers, initContainerQueryServer) + if c.cfg.TLSKeyPair.UseCertificateManagement() && c.cfg.RequiresAggregationServer { + initContainerAPIServer := c.cfg.TLSKeyPair.InitContainer(APIServerNamespace, c.apiServerContainer().SecurityContext) + initContainerAPIServer.Name = fmt.Sprintf("%s-%s", CalicoAPIServerTLSSecretName, certificatemanagement.CSRInitContainerName) + initContainers = append(initContainers, initContainerAPIServer) } - // Determine which containers to run. + // Determine which containers to run. A variant modifier may add additional + // containers (e.g. the query server and the L7 admission controller). containers := []corev1.Container{} if c.cfg.RequiresAggregationServer { containers = append(containers, c.apiServerContainer()) } - if c.cfg.IsSidecarInjectionEnabled() { - containers = append(containers, c.l7AdmissionControllerContainer()) - } - if c.cfg.Installation.Variant.IsEnterprise() { - containers = append(containers, c.queryServerContainer()) - } d := &appsv1.Deployment{ TypeMeta: metav1.TypeMeta{Kind: "Deployment", APIVersion: "apps/v1"}, @@ -1027,15 +937,6 @@ func (c *apiServerComponent) apiServerDeployment() *appsv1.Deployment { d.Spec.Template.Spec.Affinity = podaffinity.NewPodAntiAffinity(APIServerName, []string{APIServerNamespace, "tigera-system", "calico-apiserver"}) } - if c.cfg.Installation.Variant.IsEnterprise() { - if c.cfg.TrustedBundle != nil { - trustedBundleHashAnnotations := c.cfg.TrustedBundle.HashAnnotations() - for k, v := range trustedBundleHashAnnotations { - d.Spec.Template.Annotations[k] = v - } - } - } - if overrides := c.cfg.APIServer.APIServerDeployment; overrides != nil { rcomp.ApplyDeploymentOverrides(d, overrides) } @@ -1043,70 +944,17 @@ func (c *apiServerComponent) apiServerDeployment() *appsv1.Deployment { return d } -// apiServer creates a MutatingWebhookConfiguration for sidecars. -func (c *apiServerComponent) sidecarMutatingWebhookConfig() *admregv1.MutatingWebhookConfiguration { - var cacert []byte - svcPort := getContainerPort(c.cfg, L7AdmissionControllerContainerName).ContainerPort - - svcpath := "/sidecar-webhook" - svcref := admregv1.ServiceReference{ - Name: QueryserverServiceName, - Namespace: QueryserverNamespace, - Path: &svcpath, - Port: &svcPort, - } - failpol := admregv1.Fail - labelsel := metav1.LabelSelector{ - MatchLabels: map[string]string{ - "applicationlayer.projectcalico.org/sidecar": "true", - }, - } - rules := []admregv1.RuleWithOperations{ - { - Rule: admregv1.Rule{ - APIGroups: []string{""}, - APIVersions: []string{"v1"}, - Resources: []string{"pods"}, - }, - Operations: []admregv1.OperationType{admregv1.Create}, - }, - } - sidefx := admregv1.SideEffectClassNone - if !c.cfg.TLSKeyPair.UseCertificateManagement() { - cacert = c.cfg.TLSKeyPair.GetIssuer().GetCertificatePEM() - } else { - cacert = c.cfg.Installation.CertificateManagement.CACert - } - mwc := admregv1.MutatingWebhookConfiguration{ - TypeMeta: metav1.TypeMeta{ - Kind: "MutatingWebhookConfiguration", - APIVersion: "admissionregistration.k8s.io/v1", - }, - ObjectMeta: metav1.ObjectMeta{Name: common.SidecarMutatingWebhookConfigName}, - Webhooks: []admregv1.MutatingWebhook{ - { - AdmissionReviewVersions: []string{"v1"}, - ClientConfig: admregv1.WebhookClientConfig{ - Service: &svcref, - CABundle: cacert, - }, - Name: "sidecar.projectcalico.org", - FailurePolicy: &failpol, - ObjectSelector: &labelsel, - Rules: rules, - SideEffects: &sidefx, - }, - }, - } - - return &mwc +func (c *apiServerComponent) hostNetwork() bool { + return HostNetwork(c.cfg) } -func (c *apiServerComponent) hostNetwork() bool { - if c.cfg.ForceHostNetwork { +// HostNetwork reports whether the API server deployment runs on the host network, +// accounting for both the forced setting and the provider-driven requirement. +func HostNetwork(cfg *APIServerConfiguration) bool { + if cfg.ForceHostNetwork { return true } - return HostNetworkRequired(c.cfg.Installation) + return HostNetworkRequired(cfg.Installation) } func HostNetworkRequired(installation *operatorv1.InstallationSpec) bool { @@ -1126,12 +974,6 @@ func (c *apiServerComponent) apiServerContainer() corev1.Container { volumeMounts := []corev1.VolumeMount{ c.cfg.TLSKeyPair.VolumeMount(c.SupportedOSType()), } - if c.cfg.Installation.Variant.IsEnterprise() { - volumeMounts = append(volumeMounts, - corev1.VolumeMount{Name: auditLogsVolumeName, MountPath: "/var/log/calico/audit"}, - corev1.VolumeMount{Name: auditPolicyVolumeName, MountPath: "/etc/tigera/audit"}, - ) - } env := []corev1.EnvVar{ {Name: "DATASTORE_TYPE", Value: "kubernetes"}, @@ -1161,7 +1003,7 @@ func (c *apiServerComponent) apiServerContainer() corev1.Container { env = append(env, corev1.EnvVar{Name: "MULTI_INTERFACE_MODE", Value: c.cfg.Installation.CalicoNetwork.MultiInterfaceMode.Value()}) } - apiServerTargetPort := getContainerPort(c.cfg, APIServerContainerName).ContainerPort + apiServerTargetPort := GetContainerPort(c.cfg, APIServerContainerName).ContainerPort apiServer := corev1.Container{ Name: string(APIServerContainerName), @@ -1183,19 +1025,13 @@ func (c *apiServerComponent) apiServerContainer() corev1.Container { PeriodSeconds: 60, }, } - // In case of OpenShift, apiserver needs privileged access to write audit logs to host path volume. - // Audit logs are owned by root on hosts so we need to be root user and group. Audit logs are supported only in Enterprise version. - if c.cfg.Installation.Variant.IsEnterprise() { - apiServer.SecurityContext = securitycontext.NewRootContext(c.cfg.OpenShift) - } else { - apiServer.SecurityContext = securitycontext.NewNonRootContext() - } + apiServer.SecurityContext = securitycontext.NewNonRootContext() return apiServer } func (c *apiServerComponent) startUpArgs() []string { - apiServerTargetPort := getContainerPort(c.cfg, APIServerContainerName).ContainerPort + apiServerTargetPort := GetContainerPort(c.cfg, APIServerContainerName).ContainerPort args := []string{ fmt.Sprintf("--secure-port=%d", apiServerTargetPort), @@ -1203,25 +1039,9 @@ func (c *apiServerComponent) startUpArgs() []string { fmt.Sprintf("--tls-cert-file=%s", c.cfg.TLSKeyPair.VolumeMountCertificateFilePath()), } - if c.cfg.Installation.Variant.IsEnterprise() { - args = append(args, - "--audit-policy-file=/etc/tigera/audit/policy.conf", - "--audit-log-path=/var/log/calico/audit/tsee-audit.log", - ) - } - - if c.cfg.ManagementCluster != nil { - args = append(args, "--enable-managed-clusters-create-api=true") - if c.cfg.ManagementCluster.Spec.Address != "" { - args = append(args, fmt.Sprintf("--managementClusterAddr=%s", c.cfg.ManagementCluster.Spec.Address)) - } - if c.cfg.ManagementCluster.Spec.TLS != nil && c.cfg.ManagementCluster.Spec.TLS.SecretName != "" { - if c.cfg.ManagementCluster.Spec.TLS.SecretName == ManagerTLSSecretName { - args = append(args, "--managementClusterCAType=Public") - } - args = append(args, fmt.Sprintf("--tunnelSecretName=%s", c.cfg.ManagementCluster.Spec.TLS.SecretName)) - } - } + // The management-cluster tunnel args (--enable-managed-clusters-create-api, + // --managementClusterAddr, --tunnelSecretName, --managementClusterCAType) are an + // enterprise concern, appended to this container by the variant modifier. if c.cfg.KubernetesVersion != nil && c.cfg.KubernetesVersion.Major < 2 && c.cfg.KubernetesVersion.Minor < 30 { // Disable this API as it is not available by default. If we don't, the server fails to start, due to trying to // establish watches for unavailable APIs. @@ -1230,181 +1050,13 @@ func (c *apiServerComponent) startUpArgs() []string { return args } -// queryServerContainer creates the query server container. -func (c *apiServerComponent) queryServerContainer() corev1.Container { - queryServerTargetPort := getContainerPort(c.cfg, TigeraAPIServerQueryServerContainerName).ContainerPort - - var tlsSecret certificatemanagement.KeyPairInterface - if c.cfg.QueryServerTLSKeyPairCertificateManagementOnly != nil { - tlsSecret = c.cfg.QueryServerTLSKeyPairCertificateManagementOnly - } else { - tlsSecret = c.cfg.TLSKeyPair - } - env := []corev1.EnvVar{ - {Name: "DATASTORE_TYPE", Value: "kubernetes"}, - {Name: "LISTEN_ADDR", Value: fmt.Sprintf(":%d", queryServerTargetPort)}, - {Name: "TLS_CERT", Value: fmt.Sprintf("/%s/tls.crt", tlsSecret.GetName())}, - {Name: "TLS_KEY", Value: fmt.Sprintf("/%s/tls.key", tlsSecret.GetName())}, - } - if c.cfg.TrustedBundle != nil { - env = append(env, corev1.EnvVar{Name: "TRUSTED_BUNDLE_PATH", Value: c.cfg.TrustedBundle.MountPath()}) - } - - if c.hostNetwork() { - env = append(env, c.cfg.K8SServiceEndpoint.EnvVars()...) - } else { - env = append(env, c.cfg.K8SServiceEndpointPodNetwork.EnvVars()...) - } - - if c.cfg.Installation.CalicoNetwork != nil && c.cfg.Installation.CalicoNetwork.MultiInterfaceMode != nil { - env = append(env, corev1.EnvVar{Name: "MULTI_INTERFACE_MODE", Value: c.cfg.Installation.CalicoNetwork.MultiInterfaceMode.Value()}) - } - - if c.cfg.KeyValidatorConfig != nil { - env = append(env, c.cfg.KeyValidatorConfig.RequiredEnv("")...) - } - - linseedURL := relasticsearch.LinseedEndpoint(c.SupportedOSType(), c.cfg.ClusterDomain, ElasticsearchNamespace, c.cfg.ManagementClusterConnection != nil, false) - env = append(env, - corev1.EnvVar{Name: "LINSEED_URL", Value: linseedURL}, - corev1.EnvVar{Name: "LINSEED_CLIENT_CERT", Value: fmt.Sprintf("/%s/tls.crt", tlsSecret.GetName())}, - corev1.EnvVar{Name: "LINSEED_CLIENT_KEY", Value: fmt.Sprintf("/%s/tls.key", tlsSecret.GetName())}, - ) - if c.cfg.ManagementClusterConnection != nil { - env = append(env, - corev1.EnvVar{Name: "CLUSTER_ID", Value: ""}, - corev1.EnvVar{Name: "LINSEED_TOKEN", Value: GetLinseedTokenPath(true)}, - ) - } - if c.cfg.TrustedBundle != nil { - env = append(env, corev1.EnvVar{Name: "LINSEED_CA", Value: c.cfg.TrustedBundle.MountPath()}) - } - - // set LogLEVEL for queryserver container - if logging := c.cfg.APIServer.Logging; logging != nil && - logging.QueryServerLogging != nil && logging.QueryServerLogging.LogSeverity != nil { - env = append(env, - corev1.EnvVar{Name: "LOGLEVEL", Value: strings.ToLower(string(*logging.QueryServerLogging.LogSeverity))}) - } else { - // set default LOGLEVEL to info when not set by the user - env = append(env, corev1.EnvVar{Name: "LOGLEVEL", Value: "info"}) - } - - volumeMounts := []corev1.VolumeMount{ - tlsSecret.VolumeMount(c.SupportedOSType()), - } - if c.cfg.TrustedBundle != nil { - volumeMounts = append(volumeMounts, c.cfg.TrustedBundle.VolumeMounts(c.SupportedOSType())...) - } - if c.cfg.ManagementClusterConnection != nil { - volumeMounts = append(volumeMounts, corev1.VolumeMount{ - Name: LinseedTokenVolumeName, - MountPath: LinseedVolumeMountPath, - }) - } - - container := corev1.Container{ - Name: string(TigeraAPIServerQueryServerContainerName), - Image: c.calicoImage, - Command: []string{components.CalicoBinaryPath, "component", "queryserver"}, - Env: env, - LivenessProbe: &corev1.Probe{ - ProbeHandler: corev1.ProbeHandler{ - HTTPGet: &corev1.HTTPGetAction{ - Path: "/version", - Port: intstr.FromInt32(queryServerTargetPort), - Scheme: corev1.URISchemeHTTPS, - }, - }, - InitialDelaySeconds: 90, - }, - SecurityContext: securitycontext.NewNonRootContext(), - VolumeMounts: volumeMounts, - } - return container -} - -func (c *apiServerComponent) externalLinseedRoleBinding() *rbacv1.RoleBinding { - return &rbacv1.RoleBinding{ - TypeMeta: metav1.TypeMeta{Kind: "RoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{ - Name: "tigera-linseed", - Namespace: APIServerNamespace, - }, - RoleRef: rbacv1.RoleRef{ - APIGroup: "rbac.authorization.k8s.io", - Kind: "ClusterRole", - Name: TigeraLinseedSecretsClusterRole, - }, - Subjects: []rbacv1.Subject{ - { - Kind: "ServiceAccount", - Name: GuardianServiceAccountName, - Namespace: GuardianNamespace, - }, - }, - } -} - -// apiServerVolumes creates the volumes used by the API server deployment. +// apiServerVolumes creates the volumes used by the API server deployment. A variant +// modifier adds any further volumes it needs (the query server cert and the Linseed +// token). func (c *apiServerComponent) apiServerVolumes() []corev1.Volume { - volumes := []corev1.Volume{ + return []corev1.Volume{ c.cfg.TLSKeyPair.Volume(), } - if c.cfg.QueryServerTLSKeyPairCertificateManagementOnly != nil { - volumes = append(volumes, c.cfg.QueryServerTLSKeyPairCertificateManagementOnly.Volume()) - } - - if c.cfg.Installation.Variant.IsEnterprise() && c.cfg.RequiresAggregationServer { - // Only include these volumes if we're running the aggregation API server, since audit logging is done through the - // main API server otherwise. - volumes = append(volumes, - corev1.Volume{ - Name: auditLogsVolumeName, - VolumeSource: corev1.VolumeSource{ - HostPath: &corev1.HostPathVolumeSource{ - Path: "/var/log/calico/audit", - Type: ptr.To(corev1.HostPathDirectoryOrCreate), - }, - }, - }, - corev1.Volume{ - Name: auditPolicyVolumeName, - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{Name: auditPolicyVolumeName}, - Items: []corev1.KeyToPath{ - { - Key: "config", - Path: "policy.conf", - }, - }, - }, - }, - }, - ) - } - - if c.cfg.Installation.Variant.IsEnterprise() && c.cfg.TrustedBundle != nil { - volumes = append(volumes, c.cfg.TrustedBundle.Volume()) - } - - if c.cfg.ManagementClusterConnection != nil { - // Optional: the Secret is delivered over the Guardian tunnel, which can't be - // established until calico-apiserver is Ready. - volumes = append(volumes, corev1.Volume{ - Name: LinseedTokenVolumeName, - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: fmt.Sprintf(LinseedTokenSecret, "calico-apiserver"), - Items: []corev1.KeyToPath{{Key: LinseedTokenKey, Path: LinseedTokenSubPath}}, - Optional: ptr.To(true), - }, - }, - }) - } - - return volumes } // tolerations creates the tolerations used by the API server deployment. @@ -1419,119 +1071,6 @@ func (c *apiServerComponent) tolerations() []corev1.Toleration { return tolerations } -// tigeraAPIServerClusterRole creates a clusterrole that gives permissions to access backing CRDs -// -// Calico Enterprise only -func (c *apiServerComponent) tigeraAPIServerClusterRole() *rbacv1.ClusterRole { - rules := []rbacv1.PolicyRule{ - { - // Read access to Linseed policy activity data for queryserver enrichment. - APIGroups: []string{"linseed.tigera.io"}, - Resources: []string{"policyactivity"}, - Verbs: []string{"get"}, - }, - { - // Calico Enterprise backing storage. - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{ - "alertexceptions", - "bfdconfigurations", - "deeppacketinspections", - "deeppacketinspections/status", - "egressgatewaypolicies", - "externalnetworks", - "globalalerts", - "globalalerts/status", - "globalalerttemplates", - "globalreports", - "globalreports/status", - "globalreporttypes", - "globalthreatfeeds", - "globalthreatfeeds/status", - "licensekeys", - "managedclusters", - "managedclusters/status", - "networks", - "packetcaptures", - "packetcaptures/status", - "policyrecommendationscopes", - "policyrecommendationscopes/status", - "remoteclusterconfigurations", - "securityeventwebhooks", - "securityeventwebhooks/status", - "uisettings", - "uisettingsgroups", - }, - Verbs: []string{ - "get", - "list", - "watch", - "create", - "update", - "delete", - "patch", - }, - }, - { - // The queryserver's RBAC calculator needs to list tiers, - // uisettingsgroups, and managedclusters via the aggregated - // API to evaluate user permissions for the /policies endpoint. - APIGroups: []string{"projectcalico.org"}, - Resources: []string{ - "tiers", - "uisettingsgroups", - "managedclusters", - }, - Verbs: []string{"get", "list", "watch"}, - }, - { - // Required by the AuthorizationReview calculator in queryserver to evaluate - // RBAC permissions for users. - APIGroups: []string{"rbac.authorization.k8s.io"}, - Resources: []string{ - "clusterroles", - "clusterrolebindings", - "roles", - "rolebindings", - }, - Verbs: []string{"get", "list", "watch"}, - }, - } - - return &rbacv1.ClusterRole{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{ - Name: APIServerName, - }, - Rules: rules, - } -} - -// tigeraAPIServerClusterRoleBinding creates a clusterrolebinding that applies tigeraAPIServerClusterRole to -// the calico-apiserver service account. -// -// Calico Enterprise only -func (c *apiServerComponent) tigeraAPIServerClusterRoleBinding() *rbacv1.ClusterRoleBinding { - return &rbacv1.ClusterRoleBinding{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{ - Name: APIServerName, - }, - Subjects: []rbacv1.Subject{ - { - Kind: "ServiceAccount", - Name: APIServerServiceAccountName, - Namespace: APIServerNamespace, - }, - }, - RoleRef: rbacv1.RoleRef{ - Kind: "ClusterRole", - Name: APIServerName, - APIGroup: "rbac.authorization.k8s.io", - }, - } -} - // tierGetterClusterRole creates a clusterrole that gives permissions to get tiers. func (c *apiServerComponent) tierGetterClusterRole() *rbacv1.ClusterRole { return &rbacv1.ClusterRole{ @@ -1575,534 +1114,11 @@ func (c *apiServerComponent) kubeControllerMgrTierGetterClusterRoleBinding() *rb } } -// uiSettingsGroupGetterClusterRole creates a clusterrole that gives permissions to get uisettingsgroups. -// -// Calico Enterprise only -func (c *apiServerComponent) uiSettingsGroupGetterClusterRole() *rbacv1.ClusterRole { - return &rbacv1.ClusterRole{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{ - Name: "calico-uisettingsgroup-getter", - }, - Rules: []rbacv1.PolicyRule{ - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{ - "uisettingsgroups", - }, - Verbs: []string{"get"}, - }, - }, - } -} - -// kubeControllerManagerUISettingsGroupGetterClusterRoleBinding creates a rolebinding that allows the k8s kube-controller -// manager to get uisettingsgroups. -// -// In k8s 1.15+, cascading resource deletions (for instance pods for a replicaset) failed due to k8s kube-controller -// not having permissions to get tiers. UISettings and UISettingsGroups RBAC works in a similar way to tiered policy -// and so we need similar RBAC for UISettingsGroups. -// -// Calico Enterprise only -func (c *apiServerComponent) kubeControllerManagerUISettingsGroupGetterClusterRoleBinding() *rbacv1.ClusterRoleBinding { - return &rbacv1.ClusterRoleBinding{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{ - Name: "calico-uisettingsgroup-getter", - }, - RoleRef: rbacv1.RoleRef{ - Kind: "ClusterRole", - Name: "calico-uisettingsgroup-getter", - APIGroup: "rbac.authorization.k8s.io", - }, - Subjects: []rbacv1.Subject{ - { - Kind: "User", - Name: "system:kube-controller-manager", - APIGroup: "rbac.authorization.k8s.io", - }, - }, - } -} - -// tigeraUserClusterRole returns a cluster role for a default Calico Enterprise user. -// -// Calico Enterprise only -func (c *apiServerComponent) tigeraUserClusterRole() *rbacv1.ClusterRole { - rules := []rbacv1.PolicyRule{ - // List requests that the Tigera manager needs. - { - APIGroups: []string{ - "projectcalico.org", - "networking.k8s.io", - "extensions", - "", - }, - // Use both the networkpolicies and tier.networkpolicies resource types to ensure identical behavior - // irrespective of the Calico RBAC scheme (see the ClusterRole "calico-tiered-policy-passthrough" for - // more details). Similar for all tiered policy resource types. - Resources: []string{ - "tiers", - "networkpolicies", - "tier.networkpolicies", - "globalnetworkpolicies", - "tier.globalnetworkpolicies", - "namespaces", - "globalnetworksets", - "networksets", - "managedclusters", - "stagedglobalnetworkpolicies", - "tier.stagedglobalnetworkpolicies", - "stagednetworkpolicies", - "tier.stagednetworkpolicies", - "stagedkubernetesnetworkpolicies", - "policyrecommendationscopes", - }, - Verbs: []string{"watch", "list"}, - }, - { - APIGroups: []string{"policy.networking.k8s.io"}, - Resources: []string{ - "clusternetworkpolicies", - "adminnetworkpolicies", - "baselineadminnetworkpolicies", - }, - Verbs: []string{"watch", "list"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"packetcaptures/files"}, - Verbs: []string{"get"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"packetcaptures"}, - Verbs: []string{"get", "list", "watch"}, - }, - // Allow the user to view Networks. - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"networks"}, - Verbs: []string{"get", "list", "watch"}, - }, - // Additional "list" requests required to view flows. - { - APIGroups: []string{""}, - Resources: []string{"pods"}, - Verbs: []string{"list"}, - }, - // Additional "list" requests required to view serviceaccount labels. - { - APIGroups: []string{""}, - Resources: []string{"serviceaccounts"}, - Verbs: []string{"list"}, - }, - // Access for WAF API to read in coreruleset configmap - { - APIGroups: []string{""}, - Resources: []string{"configmaps"}, - ResourceNames: []string{"coreruleset-default"}, - Verbs: []string{"get"}, - }, - // Access to statistics. - { - APIGroups: []string{""}, - Resources: []string{"services/proxy"}, - ResourceNames: []string{ - "https:calico-api:8080", "calico-node-prometheus:9090", - }, - Verbs: []string{"get", "create"}, - }, - // Access to policies in all tiers - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"tiers"}, - Verbs: []string{"get"}, - }, - // List and download the reports in the Tigera Secure manager. - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"globalreports"}, - Verbs: []string{"get", "list"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"globalreporttypes"}, - Verbs: []string{"get"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"clusterinformations"}, - Verbs: []string{"get", "list"}, - }, - // Access to hostendpoints from the UI ServiceGraph. - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"hostendpoints"}, - Verbs: []string{"get", "list"}, - }, - // List and view the threat defense configuration - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{ - "alertexceptions", - "globalalerts", - "globalalerts/status", - "globalalerttemplates", - "globalthreatfeeds", - "globalthreatfeeds/status", - "securityeventwebhooks", - }, - Verbs: []string{"get", "watch", "list"}, - }, - // User can: - // - read UISettings in the cluster-settings group - // - read and write UISettings in the user-settings group - // Default settings group and settings are created in manager.go. - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"uisettingsgroups"}, - Verbs: []string{"get"}, - ResourceNames: []string{"cluster-settings", "user-settings"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"uisettingsgroups/data"}, - Verbs: []string{"get", "list", "watch"}, - ResourceNames: []string{"cluster-settings"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"uisettingsgroups/data"}, - Verbs: []string{"*"}, - ResourceNames: []string{"user-settings"}, - }, - // Allow the user to read applicationlayers to detect if WAF is enabled/disabled. - { - APIGroups: []string{"operator.tigera.io"}, - Resources: []string{"applicationlayers", "packetcaptureapis", "compliances", "intrusiondetections"}, - Verbs: []string{"get"}, - }, - // Allow the user to view WAF policies, plugins, and validation policies. - { - APIGroups: []string{"applicationlayer.projectcalico.org"}, - Resources: []string{ - "globalwafpolicies", - "globalwafplugins", - "globalwafvalidationpolicies", - "wafpolicies", - "wafplugins", - "wafvalidationpolicies", - }, - Verbs: []string{"get", "watch", "list"}, - }, - { - APIGroups: []string{"apps"}, - Resources: []string{"deployments"}, - Verbs: []string{"get", "list", "watch"}, - }, - // Allow the user to read services to view WAF configuration. - { - APIGroups: []string{""}, - Resources: []string{"services"}, - Verbs: []string{"get", "list", "watch"}, - }, - // Allow the user to read felixconfigurations to detect if wireguard and/or other features are enabled. - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"felixconfigurations"}, - Verbs: []string{"get", "list"}, - }, - // Allow the user to only view securityeventwebhooks. - { - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{"securityeventwebhooks"}, - Verbs: []string{"get", "list"}, - }, - } - - // Privileges for lma.tigera.io have no effect on managed clusters. - if c.cfg.ManagementClusterConnection == nil { - // Access to flow logs, audit logs, and statistics. - // Access to log into Kibana for oidc users. - rules = append(rules, rbacv1.PolicyRule{ - APIGroups: []string{"lma.tigera.io"}, - Resources: []string{"*"}, - ResourceNames: []string{ - "flows", "audit*", "l7", "events", "dns", "waf", "kibana_login", "recommendations", - }, - Verbs: []string{"get"}, - }) - } - - return &rbacv1.ClusterRole{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{ - Name: "tigera-ui-user", - }, - Rules: rules, - } -} - -// tigeraNetworkAdminClusterRole returns a cluster role for a Tigera Secure manager network admin. -// -// Calico Enterprise only -func (c *apiServerComponent) tigeraNetworkAdminClusterRole() *rbacv1.ClusterRole { - rules := []rbacv1.PolicyRule{ - // Full access to all network policies - { - APIGroups: []string{ - "projectcalico.org", - "networking.k8s.io", - "extensions", - }, - // Use both the networkpolicies and tier.networkpolicies resource types to ensure identical behavior - // irrespective of the Calico RBAC scheme (see the ClusterRole "calico-tiered-policy-passthrough" for - // more details). Similar for all tiered policy resource types. - Resources: []string{ - "tiers", - "networkpolicies", - "tier.networkpolicies", - "globalnetworkpolicies", - "tier.globalnetworkpolicies", - "stagedglobalnetworkpolicies", - "tier.stagedglobalnetworkpolicies", - "stagednetworkpolicies", - "tier.stagednetworkpolicies", - "stagedkubernetesnetworkpolicies", - "globalnetworksets", - "networksets", - "managedclusters", - "packetcaptures", - "policyrecommendationscopes", - }, - Verbs: []string{"create", "update", "delete", "patch", "get", "watch", "list"}, - }, - { - APIGroups: []string{ - "policy.networking.k8s.io", - }, - Resources: []string{ - "clusternetworkpolicies", - "adminnetworkpolicies", - "baselineadminnetworkpolicies", - }, - Verbs: []string{"create", "update", "delete", "patch", "get", "watch", "list"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"packetcaptures/files"}, - Verbs: []string{"get", "delete"}, - }, - // Allow the user to CRUD Networks. - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"networks"}, - Verbs: []string{"create", "update", "delete", "patch", "get", "watch", "list"}, - }, - // Additional "list" requests that the Tigera Secure manager needs - { - APIGroups: []string{""}, - Resources: []string{"namespaces"}, - Verbs: []string{"watch", "list"}, - }, - // Additional "list" requests required to view flows. - { - APIGroups: []string{""}, - Resources: []string{"pods"}, - Verbs: []string{"list"}, - }, - // Additional "list" requests required to view serviceaccount labels. - { - APIGroups: []string{""}, - Resources: []string{"serviceaccounts"}, - Verbs: []string{"list"}, - }, - // Access for WAF API to read in coreruleset configmap - { - APIGroups: []string{""}, - Resources: []string{"configmaps"}, - ResourceNames: []string{"coreruleset-default"}, - Verbs: []string{"get"}, - }, - // Access to statistics. - { - APIGroups: []string{""}, - Resources: []string{"services/proxy"}, - ResourceNames: []string{ - "https:calico-api:8080", "calico-node-prometheus:9090", - }, - Verbs: []string{"get", "create"}, - }, - // Manage globalreport configuration, view report generation status, and list reports in the Tigera Secure manager. - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"globalreports"}, - Verbs: []string{"*"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"globalreports/status"}, - Verbs: []string{"get", "list", "watch"}, - }, - // List and download the reports in the Tigera Secure manager. - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"globalreporttypes"}, - Verbs: []string{"get"}, - }, - // Access to cluster information containing Calico and EE versions from the UI. - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"clusterinformations"}, - Verbs: []string{"get", "list"}, - }, - // Access to hostendpoints from the UI ServiceGraph. - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"hostendpoints"}, - Verbs: []string{"get", "list"}, - }, - // Manage the threat defense configuration - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{ - "alertexceptions", - "globalalerts", - "globalalerts/status", - "globalalerttemplates", - "globalthreatfeeds", - "globalthreatfeeds/status", - "securityeventwebhooks", - }, - Verbs: []string{"create", "update", "delete", "patch", "get", "watch", "list"}, - }, - // User can: - // - read and write UISettings in the cluster-settings group, and rename the group - // - read and write UISettings in the user-settings group, and rename the group - // Default settings group and settings are created in manager.go. - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"uisettingsgroups"}, - Verbs: []string{"get", "patch", "update"}, - ResourceNames: []string{"cluster-settings", "user-settings"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"uisettingsgroups/data"}, - Verbs: []string{"*"}, - ResourceNames: []string{"cluster-settings", "user-settings"}, - }, - // Allow the user to read and write applicationlayers to enable/disable WAF. - { - APIGroups: []string{"operator.tigera.io"}, - Resources: []string{"applicationlayers", "packetcaptureapis", "compliances", "intrusiondetections"}, - Verbs: []string{"get", "update", "patch", "create", "delete"}, - }, - // Allow the user to manage WAF policies, plugins, and validation policies. - { - APIGroups: []string{"applicationlayer.projectcalico.org"}, - Resources: []string{ - "globalwafpolicies", - "globalwafplugins", - "globalwafvalidationpolicies", - "wafpolicies", - "wafplugins", - "wafvalidationpolicies", - }, - Verbs: []string{"create", "update", "delete", "patch", "get", "watch", "list"}, - }, - // Allow the user to read deployments to view WAF configuration. - { - APIGroups: []string{"apps"}, - Resources: []string{"deployments"}, - Verbs: []string{"get", "list", "watch", "patch"}, - }, - { - APIGroups: []string{""}, - Resources: []string{"services"}, - Verbs: []string{"get", "list", "watch", "patch"}, - }, - // Allow the user to read felixconfigurations to detect if wireguard and/or other features are enabled. - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"felixconfigurations"}, - Verbs: []string{"get", "list"}, - }, - // Allow the user to perform CRUD operations on securityeventwebhooks. - { - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{"securityeventwebhooks"}, - Verbs: []string{"get", "list", "update", "patch", "create", "delete"}, - }, - // Allow the user to create secrets. - { - APIGroups: []string{""}, - Resources: []string{ - "secrets", - }, - Verbs: []string{"create"}, - }, - // Allow the user to patch webhooks-secret secret. - { - APIGroups: []string{""}, - Resources: []string{ - "secrets", - }, - ResourceNames: []string{ - "webhooks-secret", - }, - Verbs: []string{"patch"}, - }, - } - - // Privileges for lma.tigera.io have no effect on managed clusters. - if c.cfg.ManagementClusterConnection == nil { - // Access to flow logs, audit logs, and statistics. - // Elasticsearch superuser access once logged into Kibana. - rules = append(rules, rbacv1.PolicyRule{ - APIGroups: []string{"lma.tigera.io"}, - Resources: []string{"*"}, - ResourceNames: []string{ - "flows", "audit*", "l7", "events", "dns", "waf", "kibana_login", "elasticsearch_superuser", "recommendations", - }, - Verbs: []string{"get"}, - }) - } - - // In v3 CRD / webhooks mode there is no aggregated apiserver, and the - // calico-uisettings-passthrough ClusterRole that normally grants the broad - // uisettings permission isn't deployed. Grant write verbs here so the - // calico-webhooks UISettings handler (which narrows access via a SAR on - // uisettingsgroups/data) gets invoked instead of being short-circuited by - // kube-apiserver RBAC. - if !c.cfg.RequiresAggregationServer { - rules = append(rules, rbacv1.PolicyRule{ - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"uisettings"}, - Verbs: []string{"create", "update", "delete", "patch"}, - }) - } - - return &rbacv1.ClusterRole{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{ - Name: "tigera-network-admin", - }, - Rules: rules, - } -} - // calicoPolicyPassthruClusterRole creates a clusterrole that is used to control the RBAC // mechanism for Calico tiered policy. func (c *apiServerComponent) calicoPolicyPassthruClusterRole() *rbacv1.ClusterRole { resources := []string{"networkpolicies", "globalnetworkpolicies"} - // Append additional resources for enterprise Variant. - if c.cfg.Installation.Variant.IsEnterprise() { - resources = append(resources, "stagednetworkpolicies", "stagedglobalnetworkpolicies") - } - return &rbacv1.ClusterRole{ TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, ObjectMeta: metav1.ObjectMeta{ @@ -2143,132 +1159,6 @@ func (c *apiServerComponent) calicoPolicyPassthruClusterRolebinding() *rbacv1.Cl } } -// uiSettingsPassthruClusterRole creates a clusterrole that is used to control the RBAC mechanism for Tigera UI Settings. -// RBAC for these is handled within the Tigera API Server which checks uisettingsgroups/data permissions for the user. -// -// Calico Enterprise only -func (c *apiServerComponent) uiSettingsPassthruClusterRole() *rbacv1.ClusterRole { - return &rbacv1.ClusterRole{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{ - Name: "calico-uisettings-passthrough", - }, - Rules: []rbacv1.PolicyRule{ - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"uisettings"}, - Verbs: []string{"*"}, - }, - }, - } -} - -// uiSettingsPassthruClusterRolebinding creates a clusterrolebinding that applies uiSettingsPassthruClusterRole to all -// users. -// -// Calico Enterprise only. -func (c *apiServerComponent) uiSettingsPassthruClusterRolebinding() *rbacv1.ClusterRoleBinding { - return &rbacv1.ClusterRoleBinding{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{ - Name: "calico-uisettings-passthrough", - }, - Subjects: []rbacv1.Subject{ - { - Kind: "Group", - Name: "system:authenticated", - APIGroup: "rbac.authorization.k8s.io", - }, - }, - RoleRef: rbacv1.RoleRef{ - Kind: "ClusterRole", - Name: "calico-uisettings-passthrough", - APIGroup: "rbac.authorization.k8s.io", - }, - } -} - -// auditPolicyConfigMap returns a configmap with contents to configure audit logging for -// projectcalico.org/v3 APIs. -// -// Calico Enterprise only -func (c *apiServerComponent) auditPolicyConfigMap() *corev1.ConfigMap { - const defaultAuditPolicy = `apiVersion: audit.k8s.io/v1 -kind: Policy -rules: -- level: RequestResponse - omitStages: - - RequestReceived - verbs: - - create - - patch - - update - - delete - resources: - - group: projectcalico.org - resources: - - globalnetworkpolicies - - networkpolicies - - stagedglobalnetworkpolicies - - stagednetworkpolicies - - stagedkubernetesnetworkpolicies - - globalnetworksets - - networksets - - tiers - - hostendpoints` - - return &corev1.ConfigMap{ - TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"}, - ObjectMeta: metav1.ObjectMeta{ - // This object is for Enterprise only, so pass it explicitly. - Namespace: APIServerNamespace, - Name: auditPolicyVolumeName, - }, - Data: map[string]string{ - "config": defaultAuditPolicy, - }, - } -} - -func (c *apiServerComponent) multiTenantManagedClusterAccessClusterRoles() []client.Object { - var objects []client.Object - objects = append(objects, &rbacv1.ClusterRole{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{Name: MultiTenantManagedClustersAccessClusterRoleName}, - Rules: []rbacv1.PolicyRule{ - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"managedclusters"}, - Verbs: []string{ - // The Authentication Proxy in Voltron checks if Enterprise Components (using impersonation headers for - // the service in the canonical namespace) can get a managed clusters before sending the request down the tunnel. - // This ClusterRole will be assigned to each component using a RoleBinding in the canonical or tenant namespace. - "get", - }, - }, - }, - }) - - return objects -} - -// managedClusterWatchClusterRole creates a ClusterRole for watching the ManagedCluster API -func (c *apiServerComponent) managedClusterWatchClusterRole() client.Object { - return &rbacv1.ClusterRole{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{Name: ManagedClustersWatchClusterRoleName}, - Rules: []rbacv1.PolicyRule{ - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"managedclusters"}, - Verbs: []string{ - "get", "list", "watch", - }, - }, - }, - } -} - func (c *apiServerComponent) getDeprecatedResources() []client.Object { var renamedRscList []client.Object @@ -2288,23 +1178,21 @@ func (c *apiServerComponent) getDeprecatedResources() []client.Object { }, }) - // The following resources were not present in Calico OSS, so there is no need to clean up in OSS. - if c.cfg.Installation.Variant.IsEnterprise() { - // Renamed ClusterRoleBinging tigera-tier-getter to calico-tier-getter since Tier is available in OSS - renamedRscList = append(renamedRscList, &rbacv1.ClusterRoleBinding{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{ - Name: "tigera-tier-getter", - }, - }) - // Renamed ClusterRole tigera-tier-getter to calico-tier-getter since Tier is available in OSS - renamedRscList = append(renamedRscList, &rbacv1.ClusterRole{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{ - Name: "tigera-tier-getter", - }, - }) - } + // Renamed ClusterRoleBinding tigera-tier-getter to calico-tier-getter since Tier is available in OSS. + // Deleting an object that was never created (e.g. in a fresh OSS install) is a no-op. + renamedRscList = append(renamedRscList, &rbacv1.ClusterRoleBinding{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "tigera-tier-getter", + }, + }) + // Renamed ClusterRole tigera-tier-getter to calico-tier-getter since Tier is available in OSS + renamedRscList = append(renamedRscList, &rbacv1.ClusterRole{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "tigera-tier-getter", + }, + }) renamedRscList = append(renamedRscList, &corev1.Namespace{ TypeMeta: metav1.TypeMeta{Kind: "Namespace", APIVersion: "v1"}, @@ -2328,162 +1216,3 @@ func (c *apiServerComponent) getDeprecatedResources() []client.Object { return renamedRscList } - -func (cfg *APIServerConfiguration) IsSidecarInjectionEnabled() bool { - return cfg.ApplicationLayer != nil && - cfg.ApplicationLayer.Spec.SidecarInjection != nil && - *cfg.ApplicationLayer.Spec.SidecarInjection == operatorv1.SidecarEnabled -} - -func (c *apiServerComponent) l7AdmissionControllerContainer() corev1.Container { - volumeMounts := []corev1.VolumeMount{ - c.cfg.TLSKeyPair.VolumeMount(c.SupportedOSType()), - } - - l7AdmissionControllerTargetPort := getContainerPort(c.cfg, L7AdmissionControllerContainerName).ContainerPort - - dataplane := "iptables" - if c.cfg.Installation.IsNftables() { - dataplane = "nftables" - } - - l7AdmssCtrl := corev1.Container{ - Name: string(L7AdmissionControllerContainerName), - Image: c.calicoImage, - Command: []string{components.CalicoBinaryPath, "component", "l7-admission-controller"}, - Env: []corev1.EnvVar{ - { - Name: "L7ADMCTRL_TLSCERTPATH", - Value: c.cfg.TLSKeyPair.VolumeMountCertificateFilePath(), - }, - { - Name: "L7ADMCTRL_TLSKEYPATH", - Value: c.cfg.TLSKeyPair.VolumeMountKeyFilePath(), - }, - { - Name: "L7ADMCTRL_ENVOYIMAGE", - Value: c.l7AdmissionControllerEnvoyImage, - }, - { - Name: "L7ADMCTRL_DIKASTESIMAGE", - Value: c.dikastesImage, - }, - { - Name: "L7ADMCTRL_LISTENADDR", - Value: fmt.Sprintf(":%d", l7AdmissionControllerTargetPort), - }, - { - Name: "DATAPLANE", - Value: dataplane, - }, - }, - VolumeMounts: volumeMounts, - LivenessProbe: &corev1.Probe{ - ProbeHandler: corev1.ProbeHandler{ - HTTPGet: &corev1.HTTPGetAction{ - Path: "/live", - Port: intstr.FromInt32(l7AdmissionControllerTargetPort), - Scheme: corev1.URISchemeHTTPS, - }, - }, - }, - } - - return l7AdmssCtrl -} - -// deprecatedResources removes legacy cluster-scoped resources created with the 'tigera' prefix (EE-only). -// Moving forward, both EE and OSS variants standardize on the 'calico' prefix for all shared resources. -// TODO to clean up the below deprecated logic with 14 resources in 3.25+ -func (c *apiServerComponent) deprecatedResources() []client.Object { - return []client.Object{ - &rbacv1.ClusterRole{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{Name: "tigera-extension-apiserver-secrets-access"}, - }, - &rbacv1.ClusterRoleBinding{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{Name: "tigera-extension-apiserver-secrets-access"}, - }, - - // delegateAuthClusterRoleBinding - &rbacv1.ClusterRoleBinding{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{Name: "tigera-apiserver-delegate-auth"}, - }, - - // authClusterRole - &rbacv1.ClusterRole{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{Name: "tigera-extension-apiserver-auth-access"}, - }, - - // authClusterRoleBinding - &rbacv1.ClusterRoleBinding{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{Name: "tigera-extension-apiserver-auth-access"}, - }, - // authReaderRoleBinding - need clean up in diff namespace kube-system - &rbacv1.RoleBinding{ - TypeMeta: metav1.TypeMeta{Kind: "RoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{ - Name: "tigera-auth-reader", - Namespace: "kube-system", - }, - }, - // webhookReaderClusterRole - &rbacv1.ClusterRole{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{Name: "tigera-webhook-reader"}, - }, - - // webhookReaderClusterRoleBinding - &rbacv1.ClusterRoleBinding{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{Name: "tigera-apiserver-webhook-reader"}, - }, - - // calico-apiserver CR and CRB - &rbacv1.ClusterRole{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{Name: "tigera-apiserver"}, - }, - &rbacv1.ClusterRoleBinding{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{Name: "tigera-apiserver"}, - }, - - &rbacv1.ClusterRole{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{Name: "tigera-uisettingsgroup-getter"}, - }, - &rbacv1.ClusterRoleBinding{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{Name: "tigera-uisettingsgroup-getter"}, - }, - - &rbacv1.ClusterRole{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{Name: "tigera-tiered-policy-passthrough"}, - }, - &rbacv1.ClusterRoleBinding{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{Name: "tigera-tiered-policy-passthrough"}, - }, - - &rbacv1.ClusterRole{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{Name: "tigera-uisettings-passthrough"}, - }, - &rbacv1.ClusterRoleBinding{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{Name: "tigera-uisettings-passthrough"}, - }, - - // Clean up legacy secrets in the tigera-operator namespace - &corev1.Secret{ - TypeMeta: metav1.TypeMeta{Kind: "Secret", APIVersion: "v1"}, - ObjectMeta: metav1.ObjectMeta{Name: "tigera-api-cert", Namespace: common.OperatorNamespace()}, - }, - } -} diff --git a/pkg/render/apiserver_test.go b/pkg/render/apiserver_test.go index 9878c7ed1b..25c3f85ad5 100644 --- a/pkg/render/apiserver_test.go +++ b/pkg/render/apiserver_test.go @@ -15,1487 +15,44 @@ package render_test import ( - "crypto/tls" - "crypto/x509" - "encoding/pem" "fmt" - "strings" - "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "github.com/onsi/gomega/gstruct" - "github.com/openshift/library-go/pkg/crypto" - - calicov3 "github.com/tigera/api/pkg/apis/projectcalico/v3" - - operatorv1 "github.com/tigera/operator/api/v1" - "github.com/tigera/operator/pkg/apis" - "github.com/tigera/operator/pkg/common" - "github.com/tigera/operator/pkg/components" - "github.com/tigera/operator/pkg/controller/certificatemanager" - "github.com/tigera/operator/pkg/controller/k8sapi" - ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" - "github.com/tigera/operator/pkg/dns" - "github.com/tigera/operator/pkg/render" - rmeta "github.com/tigera/operator/pkg/render/common/meta" - "github.com/tigera/operator/pkg/render/common/networkpolicy" - "github.com/tigera/operator/pkg/render/common/podaffinity" - rtest "github.com/tigera/operator/pkg/render/common/test" - "github.com/tigera/operator/pkg/render/testutils" - "github.com/tigera/operator/pkg/tls/certificatemanagement" - "github.com/tigera/operator/test" - - appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" - policyv1 "k8s.io/api/policy/v1" - rbacv1 "k8s.io/api/rbac/v1" - "k8s.io/apimachinery/pkg/api/resource" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/types" - apiregv1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1" - "k8s.io/utils/ptr" - "sigs.k8s.io/controller-runtime/pkg/client" -) - -var _ = Describe("API server rendering tests (Calico Enterprise)", func() { - apiServerPolicy := testutils.GetExpectedPolicyFromFile("./testutils/expected_policies/apiserver.json") - apiServerPolicyForOCP := testutils.GetExpectedPolicyFromFile("./testutils/expected_policies/apiserver_ocp.json") - var ( - instance *operatorv1.InstallationSpec - apiserver *operatorv1.APIServerSpec - managementCluster = &operatorv1.ManagementCluster{Spec: operatorv1.ManagementClusterSpec{Address: "example.com:1234"}} - cfg *render.APIServerConfiguration - trustedBundle certificatemanagement.TrustedBundle - dnsNames []string - cli client.Client - certificateManager certificatemanager.CertificateManager - err error - ) - - BeforeEach(func() { - instance = &operatorv1.InstallationSpec{ - ControlPlaneReplicas: ptr.To(int32(2)), - Registry: "testregistry.com/", - Variant: operatorv1.CalicoEnterprise, - } - apiserver = &operatorv1.APIServerSpec{} - dnsNames = dns.GetServiceDNSNames(render.APIServerServiceName, render.APIServerNamespace, clusterDomain) - scheme := runtime.NewScheme() - Expect(apis.AddToScheme(scheme, false)).NotTo(HaveOccurred()) - - cli = ctrlrfake.DefaultFakeClientBuilder(scheme).Build() - certificateManager, err = certificatemanager.Create(cli, nil, clusterDomain, common.OperatorNamespace(), certificatemanager.AllowCACreation()) - Expect(err).NotTo(HaveOccurred()) - - kp, err := certificateManager.GetOrCreateKeyPair(cli, render.CalicoAPIServerTLSSecretName, common.OperatorNamespace(), dnsNames) - Expect(err).NotTo(HaveOccurred()) - - trustedBundle = certificatemanagement.CreateTrustedBundle(nil) - - cfg = &render.APIServerConfiguration{ - RequiresAggregationServer: true, - K8SServiceEndpoint: k8sapi.ServiceEndpoint{}, - Installation: instance, - APIServer: apiserver, - OpenShift: true, - TLSKeyPair: kp, - TrustedBundle: trustedBundle, - KubernetesVersion: &common.VersionInfo{ - Major: 1, - Minor: 31, - }, - } - }) - - DescribeTable("should render an API server with default configuration", func(clusterDomain string) { - expectedResources := []client.Object{ - &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "calico-audit-policy", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"}}, - &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "tigera-ca-bundle", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"}}, - &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "ServiceAccount", APIVersion: "v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-crds"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver-access-calico-crds"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-tier-getter"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-tier-getter"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-tiered-policy-passthrough"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-tiered-policy-passthrough"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-uisettings-passthrough"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-uisettings-passthrough"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-extension-apiserver-auth-access"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-extension-apiserver-auth-access"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver-delegate-auth"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.RoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver-auth-reader", Namespace: "kube-system"}, TypeMeta: metav1.TypeMeta{Kind: "RoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &apiregv1.APIService{ObjectMeta: metav1.ObjectMeta{Name: "v3.projectcalico.org"}, TypeMeta: metav1.TypeMeta{Kind: "APIService", APIVersion: "apiregistration.k8s.io/v1"}}, - &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "Deployment", APIVersion: "apps/v1"}}, - &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: "calico-api", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}}, - &policyv1.PodDisruptionBudget{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "PodDisruptionBudget", APIVersion: "policy/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-uisettingsgroup-getter"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-uisettingsgroup-getter"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "tigera-ui-user"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "tigera-network-admin"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-webhook-reader"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver-webhook-reader"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - } - - dnsNames := dns.GetServiceDNSNames(render.APIServerServiceName, render.APIServerNamespace, clusterDomain) - kp, err := certificateManager.GetOrCreateKeyPair(cli, render.CalicoAPIServerTLSSecretName, common.OperatorNamespace(), dnsNames) - Expect(err).NotTo(HaveOccurred()) - cfg.TLSKeyPair = kp - component, err := render.APIServer(cfg) - Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - Expect(component.ResolveImages(nil)).To(BeNil()) - - resources, _ := component.Objects() - - // Should render the correct resources. - // - 1 namespace - // - 1 ConfigMap audit Policy - // - 1 ConfigMap Tigera CA bundle - // - 1 Service account - // - 2 ServiceAccount ClusterRole and binding for calico CRDs - // - 2 ServiceAccount ClusterRole and binding for tigera CRDs - // - 2 ClusterRole and binding for auth configmap - // - 2 calico policy passthru ClusterRole and binding - // - 2 tiered policy passthru ClusterRole and binding - // - 1 Role binding for tigera-operator to manage secrets - // - 1 delegate auth binding - // - 1 auth reader binding - // - 2 webhook reader ClusterRole and binding - // - 2 cert secrets - // - 1 api server - // - 1 service registration - // - 1 Server service - rtest.ExpectResources(resources, expectedResources) - - apiService, ok := rtest.GetResource(resources, "v3.projectcalico.org", "", "apiregistration.k8s.io", "v1", "APIService").(*apiregv1.APIService) - Expect(ok).To(BeTrue(), "Expected v1.APIService") - verifyAPIService(apiService, true, clusterDomain) - - d := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment").(*appsv1.Deployment) - - Expect(d.Name).To(Equal("calico-apiserver")) - Expect(len(d.Labels)).To(Equal(1)) - Expect(d.Labels).To(HaveKeyWithValue("apiserver", "true")) - - Expect(*d.Spec.Replicas).To(BeEquivalentTo(2)) - Expect(d.Spec.Strategy.Type).To(Equal(appsv1.RollingUpdateDeploymentStrategyType)) - Expect(len(d.Spec.Selector.MatchLabels)).To(Equal(1)) - Expect(d.Spec.Selector.MatchLabels).To(HaveKeyWithValue("apiserver", "true")) - - Expect(d.Spec.Template.Name).To(Equal("calico-apiserver")) - Expect(d.Spec.Template.Namespace).To(Equal("calico-system")) - Expect(len(d.Spec.Template.Labels)).To(Equal(1)) - Expect(d.Spec.Template.Labels).To(HaveKeyWithValue("apiserver", "true")) - - Expect(d.Spec.Template.Spec.ServiceAccountName).To(Equal("calico-apiserver")) - - Expect(d.Spec.Template.Spec.Tolerations).To(ConsistOf(rmeta.TolerateControlPlane)) - - Expect(d.Spec.Template.Spec.ImagePullSecrets).To(BeEmpty()) - Expect(d.Spec.Template.Spec.Containers).To(HaveLen(2)) - Expect(d.Spec.Template.Spec.Containers[0].Name).To(Equal("calico-apiserver")) - Expect(d.Spec.Template.Spec.Containers[0].Image).To(Equal( - fmt.Sprintf("testregistry.com/%s%s:%s", components.TigeraImagePath, components.ComponentTigeraCalico.Image, components.ComponentTigeraCalico.Version), - )) - - expectedArgs := []string{ - "--secure-port=5443", - "--tls-private-key-file=/calico-apiserver-certs/tls.key", - "--tls-cert-file=/calico-apiserver-certs/tls.crt", - "--audit-policy-file=/etc/tigera/audit/policy.conf", - "--audit-log-path=/var/log/calico/audit/tsee-audit.log", - } - Expect(d.Spec.Template.Spec.Containers[0].Args).To(ConsistOf(expectedArgs)) - Expect(len(d.Spec.Template.Spec.Containers[0].Env)).To(Equal(2)) - Expect(d.Spec.Template.Spec.Containers[0].Env[0].Name).To(Equal("DATASTORE_TYPE")) - Expect(d.Spec.Template.Spec.Containers[0].Env[0].Value).To(Equal("kubernetes")) - Expect(d.Spec.Template.Spec.Containers[0].Env[0].ValueFrom).To(BeNil()) - Expect(d.Spec.Template.Spec.Containers[0].Env[1].Name).To(Equal("LOG_LEVEL")) - Expect(d.Spec.Template.Spec.Containers[0].Env[1].Value).To(Equal("info")) - Expect(d.Spec.Template.Spec.Containers[0].Env[1].ValueFrom).To(BeNil()) - - Expect(len(d.Spec.Template.Spec.Containers[0].VolumeMounts)).To(Equal(3)) - Expect(d.Spec.Template.Spec.Containers[0].VolumeMounts[0].Name).To(Equal("calico-apiserver-certs")) - Expect(d.Spec.Template.Spec.Containers[0].VolumeMounts[1].MountPath).To(Equal("/var/log/calico/audit")) - Expect(d.Spec.Template.Spec.Containers[0].VolumeMounts[1].Name).To(Equal("calico-audit-logs")) - - Expect(d.Spec.Template.Spec.Containers[0].ReadinessProbe.HTTPGet.Path).To(Equal("/readyz")) - Expect(d.Spec.Template.Spec.Containers[0].ReadinessProbe.HTTPGet.Port.String()).To(BeEquivalentTo("5443")) - Expect(d.Spec.Template.Spec.Containers[0].ReadinessProbe.HTTPGet.Scheme).To(BeEquivalentTo("HTTPS")) - Expect(d.Spec.Template.Spec.Containers[0].ReadinessProbe.PeriodSeconds).To(BeEquivalentTo(60)) - - Expect(*d.Spec.Template.Spec.Containers[0].SecurityContext.AllowPrivilegeEscalation).To(BeTrue()) - Expect(*d.Spec.Template.Spec.Containers[0].SecurityContext.Privileged).To(BeTrue()) - Expect(*d.Spec.Template.Spec.Containers[0].SecurityContext.RunAsGroup).To(BeEquivalentTo(0)) - Expect(*d.Spec.Template.Spec.Containers[0].SecurityContext.RunAsNonRoot).To(BeFalse()) - Expect(*d.Spec.Template.Spec.Containers[0].SecurityContext.RunAsUser).To(BeEquivalentTo(0)) - Expect(d.Spec.Template.Spec.Containers[0].SecurityContext.Capabilities).To(Equal( - &corev1.Capabilities{ - Drop: []corev1.Capability{"ALL"}, - }, - )) - Expect(d.Spec.Template.Spec.Containers[0].SecurityContext.SeccompProfile).To(Equal( - &corev1.SeccompProfile{ - Type: corev1.SeccompProfileTypeRuntimeDefault, - })) - - Expect(d.Spec.Template.Spec.Containers[1].Name).To(Equal("tigera-queryserver")) - Expect(d.Spec.Template.Spec.Containers[1].Image).To(Equal( - fmt.Sprintf("testregistry.com/%s%s:%s", components.TigeraImagePath, components.ComponentTigeraCalico.Image, components.ComponentTigeraCalico.Version), - )) - Expect(d.Spec.Template.Spec.Containers[1].Args).To(BeEmpty()) - - Expect(d.Spec.Template.Spec.Containers[1].Env).To(HaveLen(10)) - - Expect(d.Spec.Template.Spec.Containers[1].Env[0].Name).To(Equal("DATASTORE_TYPE")) - Expect(d.Spec.Template.Spec.Containers[1].Env[0].Value).To(Equal("kubernetes")) - Expect(d.Spec.Template.Spec.Containers[1].Env[0].ValueFrom).To(BeNil()) - Expect(d.Spec.Template.Spec.Containers[1].Env[1].Name).To(Equal("LISTEN_ADDR")) - Expect(d.Spec.Template.Spec.Containers[1].Env[1].Value).To(Equal(":8080")) - Expect(d.Spec.Template.Spec.Containers[1].Env[1].ValueFrom).To(BeNil()) - Expect(d.Spec.Template.Spec.Containers[1].Env[2].Name).To(Equal("TLS_CERT")) - Expect(d.Spec.Template.Spec.Containers[1].Env[2].Value).To(Equal("/calico-apiserver-certs/tls.crt")) - Expect(d.Spec.Template.Spec.Containers[1].Env[2].ValueFrom).To(BeNil()) - Expect(d.Spec.Template.Spec.Containers[1].Env[3].Name).To(Equal("TLS_KEY")) - Expect(d.Spec.Template.Spec.Containers[1].Env[3].Value).To(Equal("/calico-apiserver-certs/tls.key")) - Expect(d.Spec.Template.Spec.Containers[1].Env[3].ValueFrom).To(BeNil()) - Expect(d.Spec.Template.Spec.Containers[1].Env[4].Name).To(Equal("TRUSTED_BUNDLE_PATH")) - Expect(d.Spec.Template.Spec.Containers[1].Env[4].Value).To(Equal("/etc/pki/tls/certs/tigera-ca-bundle.crt")) - Expect(d.Spec.Template.Spec.Containers[1].Env[5].Name).To(Equal("LINSEED_URL")) - Expect(d.Spec.Template.Spec.Containers[1].Env[5].Value).To(Equal("https://tigera-linseed.tigera-elasticsearch.svc")) - Expect(d.Spec.Template.Spec.Containers[1].Env[6].Name).To(Equal("LINSEED_CLIENT_CERT")) - Expect(d.Spec.Template.Spec.Containers[1].Env[6].Value).To(Equal("/calico-apiserver-certs/tls.crt")) - Expect(d.Spec.Template.Spec.Containers[1].Env[7].Name).To(Equal("LINSEED_CLIENT_KEY")) - Expect(d.Spec.Template.Spec.Containers[1].Env[7].Value).To(Equal("/calico-apiserver-certs/tls.key")) - Expect(d.Spec.Template.Spec.Containers[1].Env[8].Name).To(Equal("LINSEED_CA")) - Expect(d.Spec.Template.Spec.Containers[1].Env[8].Value).To(Equal("/etc/pki/tls/certs/tigera-ca-bundle.crt")) - Expect(d.Spec.Template.Spec.Containers[1].Env[9].Name).To(Equal("LOGLEVEL")) - Expect(d.Spec.Template.Spec.Containers[1].Env[9].Value).To(Equal("info")) - Expect(d.Spec.Template.Spec.Containers[1].Env[9].ValueFrom).To(BeNil()) - - // Expect the SECURITY_GROUP env variables to not be set - Expect(d.Spec.Template.Spec.Containers[1].Env).NotTo(ContainElement(gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{"Name": Equal("TIGERA_DEFAULT_SECURITY_GROUPS")}))) - Expect(d.Spec.Template.Spec.Containers[1].Env).NotTo(ContainElement(gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{"Name": Equal("TIGERA_POD_SECURITY_GROUP")}))) - - Expect(d.Spec.Template.Spec.Containers[1].VolumeMounts).To(HaveLen(2)) - Expect(d.Spec.Template.Spec.Containers[1].VolumeMounts[0].Name).To(Equal("calico-apiserver-certs")) - Expect(d.Spec.Template.Spec.Containers[1].VolumeMounts[0].MountPath).To(Equal("/calico-apiserver-certs")) - Expect(d.Spec.Template.Spec.Containers[1].VolumeMounts[0].ReadOnly).To(BeTrue()) - Expect(d.Spec.Template.Spec.Containers[1].VolumeMounts[0].SubPath).To(Equal("")) - Expect(d.Spec.Template.Spec.Containers[1].VolumeMounts[0].MountPropagation).To(BeNil()) - Expect(d.Spec.Template.Spec.Containers[1].VolumeMounts[0].SubPathExpr).To(Equal("")) - Expect(d.Spec.Template.Spec.Containers[1].VolumeMounts[1].Name).To(Equal("tigera-ca-bundle")) - Expect(d.Spec.Template.Spec.Containers[1].VolumeMounts[1].MountPath).To(Equal("/etc/pki/tls/certs")) - Expect(d.Spec.Template.Spec.Containers[1].VolumeMounts[1].ReadOnly).To(BeTrue()) - Expect(d.Spec.Template.Spec.Containers[1].VolumeMounts[1].SubPath).To(Equal("")) - Expect(d.Spec.Template.Spec.Containers[1].VolumeMounts[1].MountPropagation).To(BeNil()) - Expect(d.Spec.Template.Spec.Containers[1].VolumeMounts[1].SubPathExpr).To(Equal("")) - - Expect(d.Spec.Template.Spec.Containers[1].LivenessProbe.HTTPGet.Path).To(Equal("/version")) - Expect(d.Spec.Template.Spec.Containers[1].LivenessProbe.HTTPGet.Port.String()).To(BeEquivalentTo("8080")) - Expect(d.Spec.Template.Spec.Containers[1].LivenessProbe.HTTPGet.Scheme).To(BeEquivalentTo("HTTPS")) - Expect(d.Spec.Template.Spec.Containers[1].LivenessProbe.InitialDelaySeconds).To(BeEquivalentTo(90)) - - Expect(*d.Spec.Template.Spec.Containers[1].SecurityContext.AllowPrivilegeEscalation).To(BeFalse()) - Expect(*d.Spec.Template.Spec.Containers[1].SecurityContext.Privileged).To(BeFalse()) - Expect(*d.Spec.Template.Spec.Containers[1].SecurityContext.RunAsGroup).To(BeEquivalentTo(10001)) - Expect(*d.Spec.Template.Spec.Containers[1].SecurityContext.RunAsNonRoot).To(BeTrue()) - Expect(*d.Spec.Template.Spec.Containers[1].SecurityContext.RunAsUser).To(BeEquivalentTo(10001)) - Expect(d.Spec.Template.Spec.Containers[1].SecurityContext.Capabilities).To(Equal( - &corev1.Capabilities{ - Drop: []corev1.Capability{"ALL"}, - }, - )) - Expect(d.Spec.Template.Spec.Containers[1].SecurityContext.SeccompProfile).To(Equal( - &corev1.SeccompProfile{ - Type: corev1.SeccompProfileTypeRuntimeDefault, - })) - - Expect(d.Spec.Template.Spec.Volumes).To(HaveLen(4)) - Expect(d.Spec.Template.Spec.Volumes[0].Name).To(Equal("calico-apiserver-certs")) - Expect(d.Spec.Template.Spec.Volumes[0].Secret.SecretName).To(Equal("calico-apiserver-certs")) - Expect(d.Spec.Template.Spec.Volumes[1].Name).To(Equal("calico-audit-logs")) - Expect(d.Spec.Template.Spec.Volumes[1].HostPath.Path).To(Equal("/var/log/calico/audit")) - Expect(*d.Spec.Template.Spec.Volumes[1].HostPath.Type).To(BeEquivalentTo("DirectoryOrCreate")) - Expect(d.Spec.Template.Spec.Volumes[2].Name).To(Equal("calico-audit-policy")) - Expect(d.Spec.Template.Spec.Volumes[2].ConfigMap.Name).To(Equal("calico-audit-policy")) - Expect(d.Spec.Template.Spec.Volumes[2].ConfigMap.Items).To(HaveLen(1)) - Expect(d.Spec.Template.Spec.Volumes[2].ConfigMap.Items[0].Key).To(Equal("config")) - Expect(d.Spec.Template.Spec.Volumes[2].ConfigMap.Items[0].Path).To(Equal("policy.conf")) - Expect(d.Spec.Template.Spec.Volumes[3].Name).To(Equal("tigera-ca-bundle")) - Expect(d.Spec.Template.Spec.Volumes[3].ConfigMap.Name).To(Equal("tigera-ca-bundle")) - - clusterRole := rtest.GetResource(resources, "tigera-network-admin", "", "rbac.authorization.k8s.io", "v1", "ClusterRole").(*rbacv1.ClusterRole) - Expect(clusterRole.Rules).To(ConsistOf(networkAdminPolicyRules)) - - clusterRole = rtest.GetResource(resources, "tigera-ui-user", "", "rbac.authorization.k8s.io", "v1", "ClusterRole").(*rbacv1.ClusterRole) - Expect(clusterRole.Rules).To(ConsistOf(uiUserPolicyRules)) - - clusterRoleBinding := rtest.GetResource(resources, "calico-extension-apiserver-auth-access", "", "rbac.authorization.k8s.io", "v1", "ClusterRoleBinding").(*rbacv1.ClusterRoleBinding) - Expect(clusterRoleBinding.RoleRef.Name).To(Equal("calico-extension-apiserver-auth-access")) - - svc := rtest.GetResource(resources, "calico-api", "calico-system", "", "v1", "Service").(*corev1.Service) - Expect(svc.GetObjectMeta().GetLabels()).To(HaveLen(1)) - Expect(svc.GetObjectMeta().GetLabels()).To(HaveKeyWithValue("k8s-app", "calico-api")) - - Expect(svc.Spec.Ports).To(HaveLen(2)) - serviceFound := 0 - for _, p := range svc.Spec.Ports { - switch p.Name { - case render.APIServerPortName: - Expect(p.Port).To(Equal(int32(443))) - Expect(p.TargetPort.IntValue()).To(Equal(5443)) - serviceFound++ - case render.QueryServerPortName: - Expect(p.Port).To(Equal(int32(8080))) - Expect(p.TargetPort.IntValue()).To(Equal(8080)) - serviceFound++ - } - } - Expect(serviceFound).To(Equal(2)) - - cr := rtest.GetResource(resources, "calico-tiered-policy-passthrough", "", "rbac.authorization.k8s.io", "v1", "ClusterRole").(*rbacv1.ClusterRole) - var tieredPolicyRules []string - for _, rule := range cr.Rules { - tieredPolicyRules = append(tieredPolicyRules, rule.Resources...) - } - Expect(tieredPolicyRules).To(ContainElements("networkpolicies", "globalnetworkpolicies", "stagednetworkpolicies", "stagedglobalnetworkpolicies")) - - apiserverClusterRole := rtest.GetResource(resources, - "calico-crds", "", rbacv1.GroupName, "v1", "ClusterRole").(*rbacv1.ClusterRole) - Expect(apiserverClusterRole.Rules).To(ContainElement(rbacv1.PolicyRule{ - APIGroups: []string{"admissionregistration.k8s.io"}, - Resources: []string{ - "validatingadmissionpolicies", - "validatingadmissionpolicybindings", - }, - Verbs: []string{ - "get", - "list", - "watch", - }, - })) - }, - Entry("default cluster domain", dns.DefaultClusterDomain), - Entry("custom cluster domain", "custom-domain.internal"), - ) - - It("should render resources without an aggregation server", func() { - cfg.RequiresAggregationServer = false - - component, err := render.APIServer(cfg) - Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - resources, _ := component.Objects() - - expectedResources := []client.Object{ - &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "tigera-ca-bundle", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"}}, - &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "ServiceAccount", APIVersion: "v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-crds"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver-access-calico-crds"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-tier-getter"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-tier-getter"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver-delegate-auth"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "Deployment", APIVersion: "apps/v1"}}, - &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: "calico-api", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}}, - &policyv1.PodDisruptionBudget{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "PodDisruptionBudget", APIVersion: "policy/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "tigera-ui-user"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "tigera-network-admin"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-webhook-reader"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver-webhook-reader"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - } - - rtest.ExpectResources(resources, expectedResources) - - // In CRD/webhooks mode the calico-uisettings-passthrough ClusterRole is not deployed, so - // tigera-network-admin needs to grant write access to uisettings itself for the - // calico-webhooks UISettings handler to do the narrowing instead of being short-circuited - // by kube-apiserver RBAC. - networkAdmin := rtest.GetResource(resources, "tigera-network-admin", "", "rbac.authorization.k8s.io", "v1", "ClusterRole").(*rbacv1.ClusterRole) - Expect(networkAdmin.Rules).To(ContainElement(rbacv1.PolicyRule{ - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"uisettings"}, - Verbs: []string{"create", "update", "delete", "patch"}, - })) - }) - - It("should grant the calico-apiserver SA write access to globalreports/status", func() { - component, err := render.APIServer(cfg) - Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - resources, _ := component.Objects() - - cr := rtest.GetResource(resources, "calico-apiserver", "", "rbac.authorization.k8s.io", "v1", "ClusterRole").(*rbacv1.ClusterRole) - - // Find the backing-storage rule and assert it covers globalreports/status. - // Without this, compliance-controller can't update status.lastScheduledReportJob - // and no compliance jobs ever run. - var found bool - for _, rule := range cr.Rules { - hasGlobalReports := false - hasGlobalReportsStatus := false - for _, r := range rule.Resources { - if r == "globalreports" { - hasGlobalReports = true - } - if r == "globalreports/status" { - hasGlobalReportsStatus = true - } - } - if hasGlobalReports { - found = true - Expect(hasGlobalReportsStatus).To(BeTrue(), "calico-apiserver ClusterRole rule covering globalreports must also cover globalreports/status") - Expect(rule.Verbs).To(ContainElement("update")) - } - } - Expect(found).To(BeTrue(), "calico-apiserver ClusterRole should have a rule covering globalreports") - }) - - It("should render L7 Admission Controller with default config when SidecarInjection is Enabled", func() { - sidecarEnabled := operatorv1.SidecarEnabled - cfg.ApplicationLayer = &operatorv1.ApplicationLayer{ - Spec: operatorv1.ApplicationLayerSpec{ - SidecarInjection: &sidecarEnabled, - }, - } - - component, err := render.APIServer(cfg) - Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - resources, _ := component.Objects() - - d, ok := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment").(*appsv1.Deployment) - Expect(ok).To(BeTrue()) - var container corev1.Container - for _, c := range d.Spec.Template.Spec.Containers { - if c.Name == "calico-l7-admission-controller" { - container = c - } - } - Expect(container.Env[4].Name).To(Equal("L7ADMCTRL_LISTENADDR")) - Expect(container.Env[4].Value).To(Equal(":6443")) - - // Check the Service configuration - svc := rtest.GetResource(resources, "calico-api", "calico-system", "", "v1", "Service").(*corev1.Service) - var servicePort corev1.ServicePort - for _, p := range svc.Spec.Ports { - if p.Name == render.L7AdmissionControllerPortName { - servicePort = p - } - } - Expect(servicePort.Port).To(Equal(int32(6443))) - Expect(servicePort.TargetPort.IntValue()).To(Equal(6443)) - }) - - It("should render log severity when provided", func() { - errorLog := operatorv1.LogSeverityError - debugLog := operatorv1.LogSeverityDebug - cfg.APIServer.Logging = &operatorv1.APIServerPodLogging{ - APIServerLogging: &operatorv1.APIServerLogging{ - LogSeverity: &errorLog, - }, - QueryServerLogging: &operatorv1.QueryServerLogging{ - LogSeverity: &debugLog, - }, - } - component, err := render.APIServer(cfg) - Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - resources, _ := component.Objects() - - deploy, ok := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment").(*appsv1.Deployment) - Expect(ok).To(BeTrue()) - - containers := deploy.Spec.Template.Spec.Containers - for _, container := range containers { - envs := container.Env - if strings.Contains(container.Name, "apiserver") { - for _, env := range envs { - if env.Name == "LOG_LEVEL" { - Expect(env.Value).To(Equal("error")) - } - } - } else if strings.Contains(container.Name, "queryserver") { - for _, env := range envs { - if env.Name == "LOGLEVEL" { - Expect(env.Value).To(Equal("debug")) - } - } - } - } - Expect(deploy.Spec.Template.Spec.Containers).NotTo(BeNil()) - Expect(deploy.Spec.Template.Spec.Affinity).To(Equal(podaffinity.NewPodAntiAffinity("calico-apiserver", []string{"calico-system", "tigera-system", "calico-apiserver"}))) - }) - - It("should render SecurityContextConstrains properly when provider is OpenShift", func() { - cfg.Installation.KubernetesProvider = operatorv1.ProviderOpenShift - cfg.Installation.Variant = operatorv1.CalicoEnterprise - component, err := render.APIServer(cfg) - Expect(err).NotTo(HaveOccurred()) - Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() - - role := rtest.GetResource(resources, "calico-extension-apiserver-auth-access", "", "rbac.authorization.k8s.io", "v1", "ClusterRole").(*rbacv1.ClusterRole) - Expect(role.Rules).To(ContainElement(rbacv1.PolicyRule{ - APIGroups: []string{"security.openshift.io"}, - Resources: []string{"securitycontextconstraints"}, - Verbs: []string{"use"}, - ResourceNames: []string{"privileged"}, - })) - Expect(role.Rules).To(ContainElement(rbacv1.PolicyRule{ - APIGroups: []string{"config.openshift.io"}, - Resources: []string{"infrastructures"}, - Verbs: []string{"get", "list", "watch"}, - })) - }) - - It("should render an API server with custom configuration", func() { - expectedResources := []client.Object{ - &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "calico-audit-policy", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"}}, - &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "tigera-ca-bundle", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"}}, - &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "ServiceAccount", APIVersion: "v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-crds"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver-access-calico-crds"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-tier-getter"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-tier-getter"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-tiered-policy-passthrough"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-tiered-policy-passthrough"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-uisettings-passthrough"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-uisettings-passthrough"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-extension-apiserver-auth-access"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-extension-apiserver-auth-access"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver-delegate-auth"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.RoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver-auth-reader", Namespace: "kube-system"}, TypeMeta: metav1.TypeMeta{Kind: "RoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &apiregv1.APIService{ObjectMeta: metav1.ObjectMeta{Name: "v3.projectcalico.org"}, TypeMeta: metav1.TypeMeta{Kind: "APIService", APIVersion: "apiregistration.k8s.io/v1"}}, - &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "Deployment", APIVersion: "apps/v1"}}, - &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: "calico-api", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}}, - &policyv1.PodDisruptionBudget{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "PodDisruptionBudget", APIVersion: "policy/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-uisettingsgroup-getter"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-uisettingsgroup-getter"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "tigera-ui-user"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "tigera-network-admin"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-webhook-reader"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver-webhook-reader"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - } - - component, err := render.APIServer(cfg) - Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() - - rtest.ExpectResources(resources, expectedResources) - - dep := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment") - rtest.ExpectResourceTypeAndObjectMetadata(dep, "calico-apiserver", "calico-system", "apps", "v1", "Deployment") - d := dep.(*appsv1.Deployment) - - Expect(d.Spec.Template.Spec.Volumes).To(HaveLen(4)) - }) - - It("should render needed resources for k8s kube-controller", func() { - expectedResources := []client.Object{ - &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "calico-audit-policy", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"}}, - &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "tigera-ca-bundle", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"}}, - &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "ServiceAccount", APIVersion: "v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-crds"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver-access-calico-crds"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-tier-getter"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-tier-getter"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-tiered-policy-passthrough"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-tiered-policy-passthrough"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-uisettings-passthrough"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-uisettings-passthrough"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-extension-apiserver-auth-access"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-extension-apiserver-auth-access"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver-delegate-auth"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.RoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver-auth-reader", Namespace: "kube-system"}, TypeMeta: metav1.TypeMeta{Kind: "RoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &apiregv1.APIService{ObjectMeta: metav1.ObjectMeta{Name: "v3.projectcalico.org"}, TypeMeta: metav1.TypeMeta{Kind: "APIService", APIVersion: "apiregistration.k8s.io/v1"}}, - &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "Deployment", APIVersion: "apps/v1"}}, - &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: "calico-api", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}}, - &policyv1.PodDisruptionBudget{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "PodDisruptionBudget", APIVersion: "policy/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-uisettingsgroup-getter"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-uisettingsgroup-getter"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "tigera-ui-user"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "tigera-network-admin"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-webhook-reader"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver-webhook-reader"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - } - - component, err := render.APIServer(cfg) - Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() - - rtest.ExpectResources(resources, expectedResources) - - // Should render the correct resources. - cr := rtest.GetResource(resources, "calico-tier-getter", "", "rbac.authorization.k8s.io", "v1", "ClusterRole").(*rbacv1.ClusterRole) - Expect(len(cr.Rules)).To(Equal(1)) - Expect(len(cr.Rules[0].Resources)).To(Equal(1)) - Expect(cr.Rules[0].Resources[0]).To(Equal("tiers")) - Expect(len(cr.Rules[0].Verbs)).To(Equal(1)) - Expect(cr.Rules[0].Verbs[0]).To(Equal("get")) - - crb := rtest.GetResource(resources, "calico-tier-getter", "", "rbac.authorization.k8s.io", "v1", "ClusterRoleBinding").(*rbacv1.ClusterRoleBinding) - Expect(crb.RoleRef.Kind).To(Equal("ClusterRole")) - Expect(crb.RoleRef.Name).To(Equal("calico-tier-getter")) - Expect(len(crb.Subjects)).To(Equal(1)) - Expect(crb.Subjects[0].Kind).To(Equal("User")) - Expect(crb.Subjects[0].Name).To(Equal("system:kube-controller-manager")) - - cr = rtest.GetResource(resources, "calico-uisettingsgroup-getter", "", "rbac.authorization.k8s.io", "v1", "ClusterRole").(*rbacv1.ClusterRole) - Expect(len(cr.Rules)).To(Equal(1)) - Expect(len(cr.Rules[0].Resources)).To(Equal(1)) - Expect(cr.Rules[0].Resources[0]).To(Equal("uisettingsgroups")) - Expect(len(cr.Rules[0].Verbs)).To(Equal(1)) - Expect(cr.Rules[0].Verbs[0]).To(Equal("get")) - - crb = rtest.GetResource(resources, "calico-uisettingsgroup-getter", "", "rbac.authorization.k8s.io", "v1", "ClusterRoleBinding").(*rbacv1.ClusterRoleBinding) - Expect(crb.RoleRef.Kind).To(Equal("ClusterRole")) - Expect(crb.RoleRef.Name).To(Equal("calico-uisettingsgroup-getter")) - Expect(len(crb.Subjects)).To(Equal(1)) - Expect(crb.Subjects[0].Kind).To(Equal("User")) - Expect(crb.Subjects[0].Name).To(Equal("system:kube-controller-manager")) - }) - - It("should include a ControlPlaneNodeSelector when specified", func() { - expectedResources := []client.Object{ - &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "calico-audit-policy", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"}}, - &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "tigera-ca-bundle", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"}}, - &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "ServiceAccount", APIVersion: "v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-crds"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver-access-calico-crds"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-tier-getter"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-tier-getter"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-tiered-policy-passthrough"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-tiered-policy-passthrough"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-uisettings-passthrough"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-uisettings-passthrough"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-extension-apiserver-auth-access"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-extension-apiserver-auth-access"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver-delegate-auth"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.RoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver-auth-reader", Namespace: "kube-system"}, TypeMeta: metav1.TypeMeta{Kind: "RoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &apiregv1.APIService{ObjectMeta: metav1.ObjectMeta{Name: "v3.projectcalico.org"}, TypeMeta: metav1.TypeMeta{Kind: "APIService", APIVersion: "apiregistration.k8s.io/v1"}}, - &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "Deployment", APIVersion: "apps/v1"}}, - &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: "calico-api", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}}, - &policyv1.PodDisruptionBudget{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "PodDisruptionBudget", APIVersion: "policy/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-uisettingsgroup-getter"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-uisettingsgroup-getter"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "tigera-ui-user"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "tigera-network-admin"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-webhook-reader"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver-webhook-reader"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - } - - cfg.Installation.ControlPlaneNodeSelector = map[string]string{"nodeName": "control01"} - component, err := render.APIServer(cfg) - Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() - - rtest.ExpectResources(resources, expectedResources) - - d := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment").(*appsv1.Deployment) - - Expect(d.Spec.Template.Spec.NodeSelector).To(HaveLen(1)) - Expect(d.Spec.Template.Spec.NodeSelector).To(HaveKeyWithValue("nodeName", "control01")) - }) - - It("should include a ControlPlaneToleration when specified", func() { - tol := corev1.Toleration{ - Key: "foo", - Operator: corev1.TolerationOpEqual, - Value: "bar", - Effect: corev1.TaintEffectNoExecute, - } - cfg.Installation.ControlPlaneTolerations = []corev1.Toleration{tol} - - component, err := render.APIServer(cfg) - Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - resources, _ := component.Objects() - d := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment").(*appsv1.Deployment) - Expect(d.Spec.Template.Spec.Tolerations).To(ContainElements(append(rmeta.TolerateControlPlane, tol))) - }) - - It("should include a ClusterRole and ClusterRoleBindings for reading webhook configuration", func() { - expectedResources := []client.Object{ - &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "calico-audit-policy", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"}}, - &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "tigera-ca-bundle", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"}}, - &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "ServiceAccount", APIVersion: "v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-crds"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver-access-calico-crds"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-tier-getter"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-tier-getter"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-tiered-policy-passthrough"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-tiered-policy-passthrough"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-uisettings-passthrough"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-uisettings-passthrough"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-extension-apiserver-auth-access"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-extension-apiserver-auth-access"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver-delegate-auth"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.RoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver-auth-reader", Namespace: "kube-system"}, TypeMeta: metav1.TypeMeta{Kind: "RoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &apiregv1.APIService{ObjectMeta: metav1.ObjectMeta{Name: "v3.projectcalico.org"}, TypeMeta: metav1.TypeMeta{Kind: "APIService", APIVersion: "apiregistration.k8s.io/v1"}}, - &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "Deployment", APIVersion: "apps/v1"}}, - &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: "calico-api", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}}, - &policyv1.PodDisruptionBudget{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "PodDisruptionBudget", APIVersion: "policy/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-uisettingsgroup-getter"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-uisettingsgroup-getter"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "tigera-ui-user"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "tigera-network-admin"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-webhook-reader"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver-webhook-reader"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - } - - component, err := render.APIServer(cfg) - Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() - - rtest.ExpectResources(resources, expectedResources) - - // Should render the correct resources. - cr := rtest.GetResource(resources, "calico-webhook-reader", "", "rbac.authorization.k8s.io", "v1", "ClusterRole").(*rbacv1.ClusterRole) - Expect(len(cr.Rules)).To(Equal(1)) - Expect(len(cr.Rules[0].Resources)).To(Equal(2)) - Expect(cr.Rules[0].Resources[0]).To(Equal("mutatingwebhookconfigurations")) - Expect(cr.Rules[0].Resources[1]).To(Equal("validatingwebhookconfigurations")) - Expect(len(cr.Rules[0].Verbs)).To(Equal(3)) - Expect(cr.Rules[0].Verbs[0]).To(Equal("get")) - Expect(cr.Rules[0].Verbs[1]).To(Equal("list")) - Expect(cr.Rules[0].Verbs[2]).To(Equal("watch")) - - crb := rtest.GetResource(resources, "calico-apiserver-webhook-reader", "", "rbac.authorization.k8s.io", "v1", "ClusterRoleBinding").(*rbacv1.ClusterRoleBinding) - Expect(crb.RoleRef.Kind).To(Equal("ClusterRole")) - Expect(crb.RoleRef.Name).To(Equal("calico-webhook-reader")) - Expect(len(crb.Subjects)).To(Equal(1)) - Expect(crb.Subjects[0].Kind).To(Equal("ServiceAccount")) - Expect(crb.Subjects[0].Name).To(Equal("calico-apiserver")) - Expect(crb.Subjects[0].Namespace).To(Equal("calico-system")) - }) - - It("should set KUBERENETES_SERVICE_... variables if host networked", func() { - cfg.K8SServiceEndpoint.Host = "k8shost" - cfg.K8SServiceEndpoint.Port = "1234" - cfg.ForceHostNetwork = true - component, err := render.APIServer(cfg) - Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() - - deploymentResource := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment") - Expect(deploymentResource).ToNot(BeNil()) - - deployment := deploymentResource.(*appsv1.Deployment) - rtest.ExpectK8sServiceEpEnvVars(deployment.Spec.Template.Spec, "k8shost", "1234") - }) - - It("should set RecreateDeploymentStrategyType if host networked", func() { - cfg.ForceHostNetwork = true - component, err := render.APIServer(cfg) - Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - resources, _ := component.Objects() - d := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment").(*appsv1.Deployment) - Expect(d.Spec.Strategy.Type).To(Equal(appsv1.RecreateDeploymentStrategyType)) - }) - - It("should add egress policy with Enterprise variant and K8SServiceEndpoint defined", func() { - cfg.K8SServiceEndpoint.Host = "k8shost" - cfg.K8SServiceEndpoint.Port = "1234" - cfg.ForceHostNetwork = true - - component := render.APIServerPolicy(cfg) - resources, _ := component.Objects() - policyName := types.NamespacedName{Name: "calico-system.apiserver-access", Namespace: "calico-system"} - policy := testutils.GetCalicoSystemPolicyFromResources(policyName, resources) - Expect(policy).ToNot(BeNil()) - Expect(policy.Spec).ToNot(BeNil()) - Expect(policy.Spec.Egress).ToNot(BeNil()) - Expect(policy.Spec.Egress).To(ContainElement(calicov3.Rule{ - Action: calicov3.Allow, - Protocol: &networkpolicy.TCPProtocol, - Destination: calicov3.EntityRule{ - Ports: networkpolicy.Ports(1234), - Domains: []string{"k8shost"}, - }, - })) - }) - - It("should add egress policy with Enterprise variant and K8SServiceEndpoint as IP defined", func() { - cfg.K8SServiceEndpoint.Host = "169.169.169.169" - cfg.K8SServiceEndpoint.Port = "4321" - cfg.ForceHostNetwork = false - - component := render.APIServerPolicy(cfg) - resources, _ := component.Objects() - policyName := types.NamespacedName{Name: "calico-system.apiserver-access", Namespace: "calico-system"} - policy := testutils.GetCalicoSystemPolicyFromResources(policyName, resources) - Expect(policy).ToNot(BeNil()) - Expect(policy.Spec).ToNot(BeNil()) - Expect(policy.Spec.Egress).ToNot(BeNil()) - Expect(policy.Spec.Egress).To(ContainElement(calicov3.Rule{ - Action: calicov3.Allow, - Protocol: &networkpolicy.TCPProtocol, - Destination: calicov3.EntityRule{ - Ports: networkpolicy.Ports(4321), - Nets: []string{"169.169.169.169/32"}, - }, - })) - }) - - It("should not set KUBERENETES_SERVICE_... variables if not host networked on Docker EE with proxy.local", func() { - cfg.K8SServiceEndpoint.Host = "proxy.local" - cfg.K8SServiceEndpoint.Port = "1234" - cfg.Installation.KubernetesProvider = operatorv1.ProviderDockerEE - - component, err := render.APIServer(cfg) - Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() - - deploymentResource := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment") - Expect(deploymentResource).ToNot(BeNil()) - - deployment := deploymentResource.(*appsv1.Deployment) - rtest.ExpectNoK8sServiceEpEnvVars(deployment.Spec.Template.Spec) - }) - - It("should set KUBERENETES_SERVICE_... variables if not host networked on Docker EE with non-proxy address", func() { - cfg.K8SServiceEndpointPodNetwork.Host = "k8shost" - cfg.K8SServiceEndpointPodNetwork.Port = "1234" - cfg.Installation.KubernetesProvider = operatorv1.ProviderDockerEE - - component, err := render.APIServer(cfg) - Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() - - deploymentResource := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment") - Expect(deploymentResource).ToNot(BeNil()) - - deployment := deploymentResource.(*appsv1.Deployment) - rtest.ExpectK8sServiceEpEnvVars(deployment.Spec.Template.Spec, "k8shost", "1234") - }) - - It("should render an API server with custom configuration with MCM enabled at startup", func() { - cfg.ManagementCluster = managementCluster - component, err := render.APIServer(cfg) - Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - Expect(component.ResolveImages(nil)).To(BeNil()) - - resources, _ := component.Objects() - - expectedResources := []client.Object{ - &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "calico-audit-policy", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"}}, - &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "tigera-ca-bundle", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"}}, - &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "ServiceAccount", APIVersion: "v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-crds"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver-access-calico-crds"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-tier-getter"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-tier-getter"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-tiered-policy-passthrough"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-tiered-policy-passthrough"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-uisettings-passthrough"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-uisettings-passthrough"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-extension-apiserver-auth-access"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-extension-apiserver-auth-access"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver-delegate-auth"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.RoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver-auth-reader", Namespace: "kube-system"}, TypeMeta: metav1.TypeMeta{Kind: "RoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &apiregv1.APIService{ObjectMeta: metav1.ObjectMeta{Name: "v3.projectcalico.org"}, TypeMeta: metav1.TypeMeta{Kind: "APIService", APIVersion: "apiregistration.k8s.io/v1"}}, - &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "Deployment", APIVersion: "apps/v1"}}, - &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: "calico-api", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}}, - &policyv1.PodDisruptionBudget{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "PodDisruptionBudget", APIVersion: "policy/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-uisettingsgroup-getter"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-uisettingsgroup-getter"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "tigera-ui-user"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "tigera-network-admin"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: render.ManagedClustersWatchClusterRoleName}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-webhook-reader"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver-webhook-reader"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.Role{ObjectMeta: metav1.ObjectMeta{Name: render.APIServerSecretsRBACName, Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "Role", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.RoleBinding{ObjectMeta: metav1.ObjectMeta{Name: render.APIServerSecretsRBACName, Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "RoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - } - - rtest.ExpectResources(resources, expectedResources) - - By("Validating the newly created tunnel secret") - tunnelSecret, err := certificatemanagement.CreateSelfSignedSecret(render.VoltronTunnelSecretName, common.OperatorNamespace(), "tigera-voltron", []string{"voltron"}) - Expect(err).ToNot(HaveOccurred()) - - // Use the x509 package to validate that the cert was signed with the privatekey - validateTunnelSecret(tunnelSecret) - - dep := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment") - Expect(dep).ToNot(BeNil()) - - By("Validating startup args") - expectedArgs := []string{ - "--secure-port=5443", - "--tls-private-key-file=/calico-apiserver-certs/tls.key", - "--tls-cert-file=/calico-apiserver-certs/tls.crt", - "--audit-policy-file=/etc/tigera/audit/policy.conf", - "--audit-log-path=/var/log/calico/audit/tsee-audit.log", - "--enable-managed-clusters-create-api=true", - "--managementClusterAddr=example.com:1234", - } - Expect((dep.(*appsv1.Deployment)).Spec.Template.Spec.Containers[0].Args).To(ConsistOf(expectedArgs)) - }) - - It("should render an API server with custom configuration with MCM enabled at restart", func() { - cfg.ManagementCluster = managementCluster - component, err := render.APIServer(cfg) - Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - Expect(component.ResolveImages(nil)).To(BeNil()) - - resources, _ := component.Objects() - - expected := []client.Object{ - &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "calico-audit-policy", Namespace: "calico-system"}}, - &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "tigera-ca-bundle", Namespace: "calico-system"}}, - &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver", Namespace: "calico-system"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-crds"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver-access-calico-crds"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-tier-getter"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-tier-getter"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-tiered-policy-passthrough"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-tiered-policy-passthrough"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-uisettings-passthrough"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-uisettings-passthrough"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-extension-apiserver-auth-access"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-extension-apiserver-auth-access"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver-delegate-auth"}}, - &rbacv1.RoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver-auth-reader", Namespace: "kube-system"}}, - &apiregv1.APIService{ObjectMeta: metav1.ObjectMeta{Name: "v3.projectcalico.org"}}, - &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver", Namespace: "calico-system"}}, - &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: "calico-api", Namespace: "calico-system"}}, - &policyv1.PodDisruptionBudget{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver", Namespace: "calico-system"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-uisettingsgroup-getter"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-uisettingsgroup-getter"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "tigera-ui-user"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "tigera-network-admin"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: render.ManagedClustersWatchClusterRoleName}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-webhook-reader"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver-webhook-reader"}}, - &rbacv1.Role{ObjectMeta: metav1.ObjectMeta{Name: render.APIServerSecretsRBACName, Namespace: "calico-system"}}, - &rbacv1.RoleBinding{ObjectMeta: metav1.ObjectMeta{Name: render.APIServerSecretsRBACName, Namespace: "calico-system"}}, - } - rtest.ExpectResources(resources, expected) - - dep := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment") - Expect(dep).ToNot(BeNil()) - - By("Validating startup args") - expectedArgs := []string{ - "--secure-port=5443", - "--tls-private-key-file=/calico-apiserver-certs/tls.key", - "--tls-cert-file=/calico-apiserver-certs/tls.crt", - "--audit-policy-file=/etc/tigera/audit/policy.conf", - "--audit-log-path=/var/log/calico/audit/tsee-audit.log", - "--enable-managed-clusters-create-api=true", - "--managementClusterAddr=example.com:1234", - } - Expect((dep.(*appsv1.Deployment)).Spec.Template.Spec.Containers[0].Args).To(ConsistOf(expectedArgs)) - }) - - It("should render an API server with signed ca bundles enabled", func() { - cfg.ManagementCluster = managementCluster - cfg.ManagementCluster.Spec.TLS = &operatorv1.TLS{ - SecretName: render.ManagerTLSSecretName, - } - component, err := render.APIServer(cfg) - Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - - resources, _ := component.Objects() - - dep := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment") - Expect(dep).ToNot(BeNil()) - - Expect((dep.(*appsv1.Deployment)).Spec.Template.Spec.Containers[0].Args).To(ContainElement("--managementClusterCAType=Public")) - Expect((dep.(*appsv1.Deployment)).Spec.Template.Spec.Containers[0].Args).To(ContainElement(fmt.Sprintf("--tunnelSecretName=%s", render.ManagerTLSSecretName))) - }) - - It("should pass tunnelSecretName when TLS secret is not manager-tls", func() { - cfg.ManagementCluster = managementCluster - cfg.ManagementCluster.Spec.TLS = &operatorv1.TLS{ - SecretName: render.VoltronTunnelSecretName, - } - component, err := render.APIServer(cfg) - Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - - resources, _ := component.Objects() - - dep := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment") - Expect(dep).ToNot(BeNil()) - - args := (dep.(*appsv1.Deployment)).Spec.Template.Spec.Containers[0].Args - Expect(args).To(ContainElement(fmt.Sprintf("--tunnelSecretName=%s", render.VoltronTunnelSecretName))) - Expect(args).ToNot(ContainElement("--managementClusterCAType=Public")) - }) - - It("should add an init container if certificate management is enabled", func() { - cfg.Installation.CertificateManagement = &operatorv1.CertificateManagement{SignerName: "a.b/c", CACert: cfg.TLSKeyPair.GetCertificatePEM()} - certificateManager, err := certificatemanager.Create(cli, cfg.Installation, clusterDomain, common.OperatorNamespace(), certificatemanager.AllowCACreation()) - Expect(err).NotTo(HaveOccurred()) - kp, err := certificateManager.GetOrCreateKeyPair(cli, render.CalicoAPIServerTLSSecretName, common.OperatorNamespace(), dnsNames) - Expect(err).NotTo(HaveOccurred()) - qskp, err := certificateManager.GetOrCreateKeyPair(cli, render.CalicoAPIServerTLSSecretName, common.OperatorNamespace(), dnsNames) - cfg.TLSKeyPair = kp - cfg.QueryServerTLSKeyPairCertificateManagementOnly = qskp - Expect(err).NotTo(HaveOccurred()) - component, err := render.APIServer(cfg) - Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - resources, _ := component.Objects() - expectedResources := []client.Object{ - &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "calico-audit-policy", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"}}, - &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "tigera-ca-bundle", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"}}, - &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "ServiceAccount", APIVersion: "v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-crds"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver-access-calico-crds"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-tier-getter"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-tier-getter"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-tiered-policy-passthrough"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-tiered-policy-passthrough"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-uisettings-passthrough"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-uisettings-passthrough"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-extension-apiserver-auth-access"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-extension-apiserver-auth-access"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver-delegate-auth"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.RoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver-auth-reader", Namespace: "kube-system"}, TypeMeta: metav1.TypeMeta{Kind: "RoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &apiregv1.APIService{ObjectMeta: metav1.ObjectMeta{Name: "v3.projectcalico.org"}, TypeMeta: metav1.TypeMeta{Kind: "APIService", APIVersion: "apiregistration.k8s.io/v1"}}, - &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "Deployment", APIVersion: "apps/v1"}}, - &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: "calico-api", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}}, - &policyv1.PodDisruptionBudget{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "PodDisruptionBudget", APIVersion: "policy/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-uisettingsgroup-getter"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-uisettingsgroup-getter"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "tigera-ui-user"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "tigera-network-admin"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-webhook-reader"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver-webhook-reader"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - } - rtest.ExpectResources(resources, expectedResources) - - dep := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment") - Expect(dep).ToNot(BeNil()) - deploy, ok := dep.(*appsv1.Deployment) - Expect(ok).To(BeTrue()) - Expect(deploy.Spec.Template.Spec.InitContainers).To(HaveLen(2)) - Expect(deploy.Spec.Template.Spec.InitContainers[0].Name).To(Equal("calico-apiserver-certs-key-cert-provisioner")) - rtest.ExpectEnv(deploy.Spec.Template.Spec.InitContainers[0].Env, "SIGNER", "a.b/c") - }) - - It("should not render PodAffinity when ControlPlaneReplicas is 1", func() { - cfg.Installation.ControlPlaneReplicas = ptr.To(int32(1)) - component, err := render.APIServer(cfg) - Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - resources, _ := component.Objects() - - deploy, ok := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment").(*appsv1.Deployment) - Expect(ok).To(BeTrue()) - Expect(deploy.Spec.Template.Spec.Affinity).To(BeNil()) - }) - - It("should render PodAffinity when ControlPlaneReplicas is greater than 1", func() { - cfg.Installation.ControlPlaneReplicas = ptr.To(int32(2)) - component, err := render.APIServer(cfg) - Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - resources, _ := component.Objects() - - deploy, ok := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment").(*appsv1.Deployment) - Expect(ok).To(BeTrue()) - Expect(deploy.Spec.Template.Spec.Affinity).NotTo(BeNil()) - Expect(deploy.Spec.Template.Spec.Affinity).To(Equal(podaffinity.NewPodAntiAffinity("calico-apiserver", []string{"calico-system", "tigera-system", "calico-apiserver"}))) - }) - - It("should render Linseed routing for the queryserver when ManagementClusterConnection is set", func() { - cfg.ManagementClusterConnection = &operatorv1.ManagementClusterConnection{} - cfg.ClusterDomain = "cluster.local" - - component, err := render.APIServer(cfg) - Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - resources, _ := component.Objects() - - rb, ok := rtest.GetResource(resources, "tigera-linseed", "calico-system", "rbac.authorization.k8s.io", "v1", "RoleBinding").(*rbacv1.RoleBinding) - Expect(ok).To(BeTrue(), "expected tigera-linseed RoleBinding in calico-system") - Expect(rb.RoleRef.Name).To(Equal("tigera-linseed-secrets")) - Expect(rb.Subjects).To(ConsistOf(rbacv1.Subject{ - Kind: "ServiceAccount", - Name: render.GuardianServiceAccountName, - Namespace: render.GuardianNamespace, - })) - - deploy, ok := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment").(*appsv1.Deployment) - Expect(ok).To(BeTrue()) - var qs *corev1.Container - for i := range deploy.Spec.Template.Spec.Containers { - if deploy.Spec.Template.Spec.Containers[i].Name == "tigera-queryserver" { - qs = &deploy.Spec.Template.Spec.Containers[i] - } - } - Expect(qs).NotTo(BeNil()) - Expect(qs.Env).To(ContainElement(corev1.EnvVar{Name: "LINSEED_URL", Value: "https://guardian.calico-system.svc"})) - Expect(qs.Env).To(ContainElement(corev1.EnvVar{Name: "CLUSTER_ID", Value: ""})) - Expect(qs.Env).To(ContainElement(corev1.EnvVar{Name: "LINSEED_TOKEN", Value: "/var/run/secrets/tigera.io/linseed/token"})) - Expect(qs.VolumeMounts).To(ContainElement(corev1.VolumeMount{ - Name: render.LinseedTokenVolumeName, - MountPath: render.LinseedVolumeMountPath, - })) - - var tokenVol *corev1.Volume - for i := range deploy.Spec.Template.Spec.Volumes { - if deploy.Spec.Template.Spec.Volumes[i].Name == render.LinseedTokenVolumeName { - tokenVol = &deploy.Spec.Template.Spec.Volumes[i] - } - } - Expect(tokenVol).NotTo(BeNil()) - Expect(tokenVol.Secret).NotTo(BeNil()) - Expect(tokenVol.Secret.SecretName).To(Equal("calico-apiserver-tigera-linseed-token")) - }) - - Context("calico-system rendering", func() { - policyName := types.NamespacedName{Name: "calico-system.apiserver-access", Namespace: "calico-system"} - - DescribeTable("should render calico-system policy", - func(scenario testutils.CalicoSystemScenario) { - cfg.OpenShift = scenario.OpenShift - if scenario.ManagedCluster { - cfg.ManagementClusterConnection = &operatorv1.ManagementClusterConnection{} - } else { - cfg.ManagementClusterConnection = nil - } - - component := render.APIServerPolicy(cfg) - resources, _ := component.Objects() - - policy := testutils.GetCalicoSystemPolicyFromResources(policyName, resources) - expectedPolicy := testutils.SelectPolicyByProvider(scenario, apiServerPolicy, apiServerPolicyForOCP) - Expect(policy).To(Equal(expectedPolicy)) - }, - Entry("for management/standalone, kube-dns", testutils.CalicoSystemScenario{ManagedCluster: false, OpenShift: false}), - Entry("for management/standalone, openshift-dns", testutils.CalicoSystemScenario{ManagedCluster: false, OpenShift: true}), - Entry("for managed, kube-dns", testutils.CalicoSystemScenario{ManagedCluster: true, OpenShift: false}), - Entry("for managed, openshift-dns", testutils.CalicoSystemScenario{ManagedCluster: true, OpenShift: true}), - ) - }) - - Context("With APIServer Deployment overrides", func() { - rr1 := corev1.ResourceRequirements{ - Limits: corev1.ResourceList{ - "cpu": resource.MustParse("2"), - "memory": resource.MustParse("300Mi"), - "storage": resource.MustParse("20Gi"), - }, - Requests: corev1.ResourceList{ - "cpu": resource.MustParse("1"), - "memory": resource.MustParse("150Mi"), - "storage": resource.MustParse("10Gi"), - }, - } - - rr2 := corev1.ResourceRequirements{ - Requests: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("250m"), - corev1.ResourceMemory: resource.MustParse("64Mi"), - }, - Limits: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("500m"), - corev1.ResourceMemory: resource.MustParse("500Mi"), - }, - } - - It("should handle APIServerDeployment overrides", func() { - var minReadySeconds int32 = 20 - - affinity := &corev1.Affinity{ - NodeAffinity: &corev1.NodeAffinity{ - RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{ - NodeSelectorTerms: []corev1.NodeSelectorTerm{{ - MatchExpressions: []corev1.NodeSelectorRequirement{{ - Key: "custom-affinity-key", - Operator: corev1.NodeSelectorOpExists, - }}, - }}, - }, - }, - } - toleration := corev1.Toleration{ - Key: "foo", - Operator: corev1.TolerationOpEqual, - Value: "bar", - } - - apiServerPort := operatorv1.APIServerDeploymentContainerPort{ - Name: render.APIServerPortName, - ContainerPort: 1111, - } - queryServerPort := operatorv1.APIServerDeploymentContainerPort{ - Name: render.QueryServerPortName, - ContainerPort: 2222, - } - l7AdmCtrlPort := operatorv1.APIServerDeploymentContainerPort{ - Name: render.L7AdmissionControllerPortName, - ContainerPort: 3333, - } - - sidecarEnabled := operatorv1.SidecarEnabled - cfg.ApplicationLayer = &operatorv1.ApplicationLayer{ - Spec: operatorv1.ApplicationLayerSpec{ - SidecarInjection: &sidecarEnabled, - }, - } - - cfg.APIServer.APIServerDeployment = &operatorv1.APIServerDeployment{ - Metadata: &operatorv1.Metadata{ - Labels: map[string]string{"top-level": "label1"}, - Annotations: map[string]string{"top-level": "annot1"}, - }, - Spec: &operatorv1.APIServerDeploymentSpec{ - MinReadySeconds: &minReadySeconds, - Template: &operatorv1.APIServerDeploymentPodTemplateSpec{ - Metadata: &operatorv1.Metadata{ - Labels: map[string]string{"template-level": "label2"}, - Annotations: map[string]string{"template-level": "annot2"}, - }, - Spec: &operatorv1.APIServerDeploymentPodSpec{ - Containers: []operatorv1.APIServerDeploymentContainer{ - { - Name: "calico-apiserver", - Resources: &rr1, - Ports: []operatorv1.APIServerDeploymentContainerPort{apiServerPort}, - }, - { - Name: "tigera-queryserver", - Resources: &rr2, - Ports: []operatorv1.APIServerDeploymentContainerPort{queryServerPort}, - }, - { - Name: "calico-l7-admission-controller", - Resources: &rr2, - Ports: []operatorv1.APIServerDeploymentContainerPort{l7AdmCtrlPort}, - }, - }, - InitContainers: []operatorv1.APIServerDeploymentInitContainer{ - { - Name: "calico-apiserver-certs-key-cert-provisioner", - Resources: &rr2, - }, - }, - NodeSelector: map[string]string{ - "custom-node-selector": "value", - }, - TopologySpreadConstraints: []corev1.TopologySpreadConstraint{ - { - MaxSkew: 1, - }, - }, - Affinity: affinity, - Tolerations: []corev1.Toleration{toleration}, - }, - }, - }, - } - // Enable certificate management. - cfg.Installation.CertificateManagement = &operatorv1.CertificateManagement{SignerName: "a.b/c", CACert: cfg.TLSKeyPair.GetCertificatePEM()} - certificateManager, err := certificatemanager.Create(cli, cfg.Installation, clusterDomain, common.OperatorNamespace(), certificatemanager.AllowCACreation()) - Expect(err).NotTo(HaveOccurred()) - - // Create and add the TLS keypair so the initContainer is rendered. - dnsNames := dns.GetServiceDNSNames(render.APIServerServiceName, render.APIServerNamespace, clusterDomain) - kp, err := certificateManager.GetOrCreateKeyPair(cli, render.CalicoAPIServerTLSSecretName, common.OperatorNamespace(), dnsNames) - Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - cfg.TLSKeyPair = kp - - qsKP, err := certificateManager.GetOrCreateKeyPair(cli, render.CalicoAPIServerTLSSecretName, common.OperatorNamespace(), dnsNames) - Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - cfg.QueryServerTLSKeyPairCertificateManagementOnly = qsKP - - component, err := render.APIServer(cfg) - Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - resources, _ := component.Objects() - - d, ok := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment").(*appsv1.Deployment) - Expect(ok).To(BeTrue()) - - // API server has apiserver: true label - Expect(d.Labels).To(HaveLen(2)) - Expect(d.Labels["apiserver"]).To(Equal("true")) - Expect(d.Labels["top-level"]).To(Equal("label1")) - Expect(d.Annotations).To(HaveLen(1)) - Expect(d.Annotations["top-level"]).To(Equal("annot1")) - - Expect(d.Spec.MinReadySeconds).To(Equal(minReadySeconds)) - - // At runtime, the operator will also add some standard labels to the - // deployment such as "k8s-app=calico-apiserver". But the APIServer - // deployment object produced by the render will have no labels so we expect just the one - // provided. - Expect(d.Spec.Template.Labels).To(HaveLen(2)) - Expect(d.Spec.Template.Labels["apiserver"]).To(Equal("true")) - Expect(d.Spec.Template.Labels["template-level"]).To(Equal("label2")) - - // With the default instance we expect 2 template-level annotations - // - 1 added by the operator by default - // - 1 added by the calicoNodeDaemonSet override - Expect(d.Spec.Template.Annotations).To(HaveLen(2)) - Expect(d.Spec.Template.Annotations).To(HaveKey("tigera-operator.hash.operator.tigera.io/calico-apiserver-certs")) - Expect(d.Spec.Template.Annotations["template-level"]).To(Equal("annot2")) - - Expect(d.Spec.Template.Spec.Containers).To(HaveLen(3)) - containersFound := 0 - for _, c := range d.Spec.Template.Spec.Containers { - switch c.Name { - case "calico-apiserver": - Expect(c.Resources).To(Equal(rr1)) - Expect(c.Ports[0].Name).To(Equal(apiServerPort.Name)) - Expect(c.Ports[0].ContainerPort).To(Equal(apiServerPort.ContainerPort)) - - Expect(c.Args[0]).To(ContainSubstring(fmt.Sprintf("--secure-port=%d", apiServerPort.ContainerPort))) - containersFound++ - case "tigera-queryserver": - Expect(c.Resources).To(Equal(rr2)) - Expect(c.Ports[0].Name).To(Equal(queryServerPort.Name)) - Expect(c.Ports[0].ContainerPort).To(Equal(queryServerPort.ContainerPort)) - - Expect(c.Env[1].Name).To(Equal("LISTEN_ADDR")) - Expect(c.Env[1].Value).To(Equal(fmt.Sprintf(":%d", queryServerPort.ContainerPort))) - containersFound++ - case "calico-l7-admission-controller": - Expect(c.Resources).To(Equal(rr2)) - Expect(c.Ports[0].Name).To(Equal(l7AdmCtrlPort.Name)) - Expect(c.Ports[0].ContainerPort).To(Equal(l7AdmCtrlPort.ContainerPort)) - - Expect(c.Env[4].Name).To(Equal("L7ADMCTRL_LISTENADDR")) - Expect(c.Env[4].Value).To(Equal(fmt.Sprintf(":%d", l7AdmCtrlPort.ContainerPort))) - containersFound++ - } - } - Expect(containersFound).To(Equal(3)) - - Expect(d.Spec.Template.Spec.InitContainers).To(HaveLen(2)) - Expect(d.Spec.Template.Spec.InitContainers[0].Name).To(Equal("calico-apiserver-certs-key-cert-provisioner")) - Expect(d.Spec.Template.Spec.InitContainers[0].Resources).To(Equal(rr2)) - - Expect(d.Spec.Template.Spec.NodeSelector).To(HaveLen(1)) - Expect(d.Spec.Template.Spec.NodeSelector).To(HaveKeyWithValue("custom-node-selector", "value")) - - Expect(d.Spec.Template.Spec.TopologySpreadConstraints).To(HaveLen(1)) - Expect(d.Spec.Template.Spec.TopologySpreadConstraints[0].MaxSkew).To(Equal(int32(1))) - - Expect(d.Spec.Template.Spec.Tolerations).To(HaveLen(1)) - Expect(d.Spec.Template.Spec.Tolerations[0]).To(Equal(toleration)) - - // Check the Service configuration - svc := rtest.GetResource(resources, "calico-api", "calico-system", "", "v1", "Service").(*corev1.Service) - Expect(svc.Spec.Ports).To(HaveLen(3)) - servicesFound := 0 - for _, p := range svc.Spec.Ports { - switch p.Name { - case render.APIServerPortName: - Expect(p.Port).To(Equal(int32(443))) - Expect(p.TargetPort.IntVal).To(Equal(apiServerPort.ContainerPort)) - servicesFound++ - case render.QueryServerPortName: - Expect(p.Port).To(Equal(int32(8080))) - Expect(p.TargetPort.IntVal).To(Equal(queryServerPort.ContainerPort)) - servicesFound++ - case render.L7AdmissionControllerPortName: - Expect(p.Port).To(Equal(int32(6443))) - Expect(p.TargetPort.IntVal).To(Equal(l7AdmCtrlPort.ContainerPort)) - servicesFound++ - } - } - Expect(servicesFound).To(Equal(3)) - }) - - It("should override a ControlPlaneNodeSelector when specified", func() { - cfg.Installation.ControlPlaneNodeSelector = map[string]string{"nodeName": "control01"} - - cfg.APIServer.APIServerDeployment = &operatorv1.APIServerDeployment{ - Spec: &operatorv1.APIServerDeploymentSpec{ - Template: &operatorv1.APIServerDeploymentPodTemplateSpec{ - Spec: &operatorv1.APIServerDeploymentPodSpec{ - NodeSelector: map[string]string{ - "custom-node-selector": "value", - }, - }, - }, - }, - } - component, err := render.APIServer(cfg) - Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() - d, ok := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment").(*appsv1.Deployment) - Expect(ok).To(BeTrue()) - // nodeSelectors are merged - Expect(d.Spec.Template.Spec.NodeSelector).To(HaveLen(2)) - Expect(d.Spec.Template.Spec.NodeSelector).To(HaveKeyWithValue("nodeName", "control01")) - Expect(d.Spec.Template.Spec.NodeSelector).To(HaveKeyWithValue("custom-node-selector", "value")) - }) - - It("should override ControlPlaneTolerations when specified", func() { - cfg.Installation.ControlPlaneTolerations = rmeta.TolerateControlPlane - - tol := corev1.Toleration{ - Key: "foo", - Operator: corev1.TolerationOpEqual, - Value: "bar", - Effect: corev1.TaintEffectNoExecute, - } + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/apis" + "github.com/tigera/operator/pkg/common" + "github.com/tigera/operator/pkg/components" + "github.com/tigera/operator/pkg/controller/certificatemanager" + "github.com/tigera/operator/pkg/controller/k8sapi" + ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" + "github.com/tigera/operator/pkg/dns" + "github.com/tigera/operator/pkg/render" + rmeta "github.com/tigera/operator/pkg/render/common/meta" + "github.com/tigera/operator/pkg/render/common/podaffinity" + rtest "github.com/tigera/operator/pkg/render/common/test" + "github.com/tigera/operator/test" - cfg.APIServer.APIServerDeployment = &operatorv1.APIServerDeployment{ - Spec: &operatorv1.APIServerDeploymentSpec{ - Template: &operatorv1.APIServerDeploymentPodTemplateSpec{ - Spec: &operatorv1.APIServerDeploymentPodSpec{ - Tolerations: []corev1.Toleration{tol}, - }, - }, - }, - } - component, err := render.APIServer(cfg) - Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() - d, ok := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment").(*appsv1.Deployment) - Expect(ok).To(BeTrue()) - Expect(d.Spec.Template.Spec.Tolerations).To(HaveLen(1)) - Expect(d.Spec.Template.Spec.Tolerations).To(ConsistOf(tol)) - }) + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + policyv1 "k8s.io/api/policy/v1" + rbacv1 "k8s.io/api/rbac/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + apiregv1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" +) - It("should disable ValidatingAdmissionPolicy on older k8s versions", func() { - cfg.KubernetesVersion = &common.VersionInfo{ - Major: 1, - Minor: 28, - } - component, err := render.APIServer(cfg) - Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - resources, _ := component.Objects() - d := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment").(*appsv1.Deployment) - Expect(d.Spec.Template.Spec.Containers[0].Args).To(ConsistOf([]string{ - "--secure-port=5443", - "--tls-private-key-file=/calico-apiserver-certs/tls.key", - "--tls-cert-file=/calico-apiserver-certs/tls.crt", - "--audit-policy-file=/etc/tigera/audit/policy.conf", - "--audit-log-path=/var/log/calico/audit/tsee-audit.log", - "--enable-validating-admission-policy=false", - })) - }) - }) -}) +// apiServerObjects renders the base (OSS) API server component. The enterprise +// modifier and the Calico-variant cleanup are exercised in pkg/enterprise/apiserver; +// these tests cover the OSS render path, which the base render handles on its own +// (including the deletes it queues for itself). +func apiServerObjects(c render.Component) ([]client.Object, []client.Object) { + return c.Objects() +} func verifyAPIService(service *apiregv1.APIService, enterprise bool, clusterDomain string) { Expect(service.Name).To(Equal("v3.projectcalico.org")) @@ -1517,405 +74,6 @@ func verifyAPIService(service *apiregv1.APIService, enterprise bool, clusterDoma test.VerifyCertSANs(ca, expectedDNSNames...) } -func validateTunnelSecret(voltronSecret *corev1.Secret) { - var newCert *x509.Certificate - - cert := voltronSecret.Data[corev1.TLSCertKey] - key := voltronSecret.Data[corev1.TLSPrivateKeyKey] - _, err := tls.X509KeyPair(cert, key) - Expect(err).ShouldNot(HaveOccurred()) - - roots := x509.NewCertPool() - ok := roots.AppendCertsFromPEM([]byte(cert)) - Expect(ok).To(BeTrue()) - - block, _ := pem.Decode([]byte(cert)) - Expect(err).ShouldNot(HaveOccurred()) - Expect(block).To(Not(BeNil())) - - newCert, err = x509.ParseCertificate(block.Bytes) - Expect(err).ShouldNot(HaveOccurred()) - - opts := x509.VerifyOptions{ - DNSName: "voltron", - Roots: roots, - } - - _, err = newCert.Verify(opts) - Expect(err).ShouldNot(HaveOccurred()) - - opts = x509.VerifyOptions{ - DNSName: "voltron", - Roots: x509.NewCertPool(), - CurrentTime: time.Now().Add(crypto.DefaultCACertificateLifetimeDuration + 24*time.Hour), - } - _, err = newCert.Verify(opts) - Expect(err).Should(HaveOccurred()) -} - -var ( - uiUserPolicyRules = []rbacv1.PolicyRule{ - { - APIGroups: []string{ - "projectcalico.org", - "networking.k8s.io", - "extensions", - "", - }, - Resources: []string{ - "tiers", - "networkpolicies", - "tier.networkpolicies", - "globalnetworkpolicies", - "tier.globalnetworkpolicies", - "namespaces", - "globalnetworksets", - "networksets", - "managedclusters", - "stagedglobalnetworkpolicies", - "tier.stagedglobalnetworkpolicies", - "stagednetworkpolicies", - "tier.stagednetworkpolicies", - "stagedkubernetesnetworkpolicies", - "policyrecommendationscopes", - }, - Verbs: []string{"watch", "list"}, - }, - { - APIGroups: []string{"policy.networking.k8s.io"}, - Resources: []string{ - "clusternetworkpolicies", - "adminnetworkpolicies", - "baselineadminnetworkpolicies", - }, - Verbs: []string{"watch", "list"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"packetcaptures/files"}, - Verbs: []string{"get"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"packetcaptures"}, - Verbs: []string{"get", "list", "watch"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"networks"}, - Verbs: []string{"get", "list", "watch"}, - }, - { - APIGroups: []string{""}, - Resources: []string{"pods"}, - Verbs: []string{"list"}, - }, - { - APIGroups: []string{""}, - Resources: []string{"serviceaccounts"}, - Verbs: []string{"list"}, - }, - { - APIGroups: []string{""}, - Resources: []string{"configmaps"}, - ResourceNames: []string{"coreruleset-default"}, - Verbs: []string{"get"}, - }, - { - APIGroups: []string{""}, - Resources: []string{"services/proxy"}, - ResourceNames: []string{ - "https:calico-api:8080", "calico-node-prometheus:9090", - }, - Verbs: []string{"get", "create"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"tiers"}, - Verbs: []string{"get"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"globalreports"}, - Verbs: []string{"get", "list"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"globalreporttypes"}, - Verbs: []string{"get"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"clusterinformations"}, - Verbs: []string{"get", "list"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"hostendpoints"}, - Verbs: []string{"get", "list"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{ - "alertexceptions", - "globalalerts", - "globalalerts/status", - "globalalerttemplates", - "globalthreatfeeds", - "globalthreatfeeds/status", - "securityeventwebhooks", - }, - Verbs: []string{"get", "watch", "list"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"uisettingsgroups"}, - Verbs: []string{"get"}, - ResourceNames: []string{"cluster-settings", "user-settings"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"uisettingsgroups/data"}, - Verbs: []string{"get", "list", "watch"}, - ResourceNames: []string{"cluster-settings"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"uisettingsgroups/data"}, - Verbs: []string{"*"}, - ResourceNames: []string{"user-settings"}, - }, - { - APIGroups: []string{"lma.tigera.io"}, - Resources: []string{"*"}, - ResourceNames: []string{ - "flows", "audit*", "l7", "events", "dns", "waf", "kibana_login", "recommendations", - }, - Verbs: []string{"get"}, - }, - { - APIGroups: []string{"operator.tigera.io"}, - Resources: []string{"applicationlayers", "packetcaptureapis", "compliances", "intrusiondetections"}, - Verbs: []string{"get"}, - }, - { - APIGroups: []string{"applicationlayer.projectcalico.org"}, - Resources: []string{ - "globalwafpolicies", - "globalwafplugins", - "globalwafvalidationpolicies", - "wafpolicies", - "wafplugins", - "wafvalidationpolicies", - }, - Verbs: []string{"get", "watch", "list"}, - }, - { - APIGroups: []string{"apps"}, - Resources: []string{"deployments"}, - Verbs: []string{"get", "list", "watch"}, - }, - { - APIGroups: []string{""}, - Resources: []string{"services"}, - Verbs: []string{"get", "list", "watch"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"felixconfigurations"}, - Verbs: []string{"get", "list"}, - }, - { - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{"securityeventwebhooks"}, - Verbs: []string{"get", "list"}, - }, - } - networkAdminPolicyRules = []rbacv1.PolicyRule{ - { - APIGroups: []string{ - "projectcalico.org", - "networking.k8s.io", - "extensions", - }, - Resources: []string{ - "tiers", - "networkpolicies", - "tier.networkpolicies", - "globalnetworkpolicies", - "tier.globalnetworkpolicies", - "stagedglobalnetworkpolicies", - "tier.stagedglobalnetworkpolicies", - "stagednetworkpolicies", - "tier.stagednetworkpolicies", - "stagedkubernetesnetworkpolicies", - "globalnetworksets", - "networksets", - "managedclusters", - "packetcaptures", - "policyrecommendationscopes", - }, - Verbs: []string{"create", "update", "delete", "patch", "get", "watch", "list"}, - }, - { - APIGroups: []string{ - "policy.networking.k8s.io", - }, - Resources: []string{ - "clusternetworkpolicies", - "adminnetworkpolicies", - "baselineadminnetworkpolicies", - }, - Verbs: []string{"create", "update", "delete", "patch", "get", "watch", "list"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"packetcaptures/files"}, - Verbs: []string{"get", "delete"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"networks"}, - Verbs: []string{"create", "update", "delete", "patch", "get", "watch", "list"}, - }, - { - APIGroups: []string{""}, - Resources: []string{"namespaces"}, - Verbs: []string{"watch", "list"}, - }, - { - APIGroups: []string{""}, - Resources: []string{"pods"}, - Verbs: []string{"list"}, - }, - { - APIGroups: []string{""}, - Resources: []string{"serviceaccounts"}, - Verbs: []string{"list"}, - }, - { - APIGroups: []string{""}, - Resources: []string{"configmaps"}, - ResourceNames: []string{"coreruleset-default"}, - Verbs: []string{"get"}, - }, - { - APIGroups: []string{""}, - Resources: []string{"services/proxy"}, - ResourceNames: []string{ - "https:calico-api:8080", "calico-node-prometheus:9090", - }, - Verbs: []string{"get", "create"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"globalreports"}, - Verbs: []string{"*"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"globalreports/status"}, - Verbs: []string{"get", "list", "watch"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"globalreporttypes"}, - Verbs: []string{"get"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"clusterinformations"}, - Verbs: []string{"get", "list"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"hostendpoints"}, - Verbs: []string{"get", "list"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{ - "alertexceptions", - "globalalerts", - "globalalerts/status", - "globalalerttemplates", - "globalthreatfeeds", - "globalthreatfeeds/status", - "securityeventwebhooks", - }, - Verbs: []string{"create", "update", "delete", "patch", "get", "watch", "list"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"uisettingsgroups"}, - Verbs: []string{"get", "patch", "update"}, - ResourceNames: []string{"cluster-settings", "user-settings"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"uisettingsgroups/data"}, - Verbs: []string{"*"}, - ResourceNames: []string{"cluster-settings", "user-settings"}, - }, - { - APIGroups: []string{"lma.tigera.io"}, - Resources: []string{"*"}, - ResourceNames: []string{ - "flows", "audit*", "l7", "events", "dns", "waf", "kibana_login", "elasticsearch_superuser", "recommendations", - }, - Verbs: []string{"get"}, - }, - { - APIGroups: []string{"operator.tigera.io"}, - Resources: []string{"applicationlayers", "packetcaptureapis", "compliances", "intrusiondetections"}, - Verbs: []string{"get", "update", "patch", "create", "delete"}, - }, - { - APIGroups: []string{"applicationlayer.projectcalico.org"}, - Resources: []string{ - "globalwafpolicies", - "globalwafplugins", - "globalwafvalidationpolicies", - "wafpolicies", - "wafplugins", - "wafvalidationpolicies", - }, - Verbs: []string{"create", "update", "delete", "patch", "get", "watch", "list"}, - }, - { - APIGroups: []string{"apps"}, - Resources: []string{"deployments"}, - Verbs: []string{"get", "list", "watch", "patch"}, - }, - { - APIGroups: []string{""}, - Resources: []string{"services"}, - Verbs: []string{"get", "list", "watch", "patch"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"felixconfigurations"}, - Verbs: []string{"get", "list"}, - }, - { - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{"securityeventwebhooks"}, - Verbs: []string{"get", "list", "update", "patch", "create", "delete"}, - }, - { - APIGroups: []string{""}, - Resources: []string{"secrets"}, - Verbs: []string{"create"}, - }, - { - APIGroups: []string{""}, - Resources: []string{"secrets"}, - ResourceNames: []string{"webhooks-secret"}, - Verbs: []string{"patch"}, - }, - } -) - var _ = Describe("API server rendering tests (Calico)", func() { var instance *operatorv1.InstallationSpec var apiserver *operatorv1.APIServerSpec @@ -1979,7 +137,7 @@ var _ = Describe("API server rendering tests (Calico)", func() { Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) Expect(component.ResolveImages(nil)).To(BeNil()) - resources, deleteResources := component.Objects() + resources, deleteResources := apiServerObjects(component) rtest.ExpectResources(resources, expectedResources) rtest.ExpectResourceInList(deleteResources, "allow-apiserver", "calico-system", "networking.k8s.io", "v1", "NetworkPolicy") @@ -2075,7 +233,7 @@ var _ = Describe("API server rendering tests (Calico)", func() { component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) Expect(component.ResolveImages(nil)).To(BeNil()) - resources, deleteResources := component.Objects() + resources, deleteResources := apiServerObjects(component) // Should not include deployment, service, SA, or PDB. Expect(rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment")).To(BeNil()) @@ -2118,7 +276,7 @@ var _ = Describe("API server rendering tests (Calico)", func() { component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) Expect(component.ResolveImages(nil)).To(BeNil()) - resources, deleteResources := component.Objects() + resources, deleteResources := apiServerObjects(component) // Should render the correct resources. By("Checking each expected resource is actually rendered") @@ -2155,7 +313,7 @@ var _ = Describe("API server rendering tests (Calico)", func() { component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) d := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment").(*appsv1.Deployment) Expect(d.Spec.Template.Spec.NodeSelector).To(HaveLen(1)) Expect(d.Spec.Template.Spec.NodeSelector).To(HaveKeyWithValue("nodeName", "control01")) @@ -2171,7 +329,7 @@ var _ = Describe("API server rendering tests (Calico)", func() { cfg.Installation.ControlPlaneTolerations = []corev1.Toleration{tol} component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) d := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment").(*appsv1.Deployment) Expect(d.Spec.Template.Spec.Tolerations).To(ContainElements(append(rmeta.TolerateControlPlane, tol))) }) @@ -2185,7 +343,7 @@ var _ = Describe("API server rendering tests (Calico)", func() { component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) deploymentResource := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment") Expect(deploymentResource).ToNot(BeNil()) @@ -2198,7 +356,7 @@ var _ = Describe("API server rendering tests (Calico)", func() { cfg.ForceHostNetwork = true component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) d := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment").(*appsv1.Deployment) Expect(d.Spec.Strategy.Type).To(Equal(appsv1.RecreateDeploymentStrategyType)) }) @@ -2211,7 +369,7 @@ var _ = Describe("API server rendering tests (Calico)", func() { component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) deploymentResource := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment") Expect(deploymentResource).ToNot(BeNil()) @@ -2228,7 +386,7 @@ var _ = Describe("API server rendering tests (Calico)", func() { component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) deploymentResource := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment") Expect(deploymentResource).ToNot(BeNil()) @@ -2243,7 +401,7 @@ var _ = Describe("API server rendering tests (Calico)", func() { component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) deploy, ok := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment").(*appsv1.Deployment) Expect(ok).To(BeTrue()) @@ -2256,7 +414,7 @@ var _ = Describe("API server rendering tests (Calico)", func() { component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) deploy, ok := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment").(*appsv1.Deployment) Expect(ok).To(BeTrue()) @@ -2270,7 +428,7 @@ var _ = Describe("API server rendering tests (Calico)", func() { component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) Expect(component.ResolveImages(nil)).To(BeNil()) - _, _ = component.Objects() + _, _ = apiServerObjects(component) }) It("should render host networked with TKG provider", func() { @@ -2281,7 +439,7 @@ var _ = Describe("API server rendering tests (Calico)", func() { component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) deploy, ok := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment").(*appsv1.Deployment) Expect(ok).To(BeTrue()) @@ -2387,13 +545,10 @@ var _ = Describe("API server rendering tests (Calico)", func() { kp, err := certificateManager.GetOrCreateKeyPair(cli, render.CalicoAPIServerTLSSecretName, common.OperatorNamespace(), dnsNames) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) cfg.TLSKeyPair = kp - qskp, err := certificateManager.GetOrCreateKeyPair(cli, render.CalicoAPIServerTLSSecretName, common.OperatorNamespace(), dnsNames) - Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - cfg.QueryServerTLSKeyPairCertificateManagementOnly = qskp component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) d, ok := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment").(*appsv1.Deployment) Expect(ok).To(BeTrue()) @@ -2429,7 +584,7 @@ var _ = Describe("API server rendering tests (Calico)", func() { Expect(d.Spec.Template.Spec.Containers[0].Ports[0].ContainerPort).To(Equal(apiServerPort.ContainerPort)) Expect(d.Spec.Template.Spec.Containers[0].Args[0]).To(ContainSubstring(fmt.Sprintf("--secure-port=%d", apiServerPort.ContainerPort))) - Expect(d.Spec.Template.Spec.InitContainers).To(HaveLen(2)) + Expect(d.Spec.Template.Spec.InitContainers).To(HaveLen(1)) Expect(d.Spec.Template.Spec.InitContainers[0].Name).To(Equal("calico-apiserver-certs-key-cert-provisioner")) Expect(d.Spec.Template.Spec.InitContainers[0].Resources).To(Equal(rr2)) @@ -2467,7 +622,7 @@ var _ = Describe("API server rendering tests (Calico)", func() { component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) d := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment").(*appsv1.Deployment) // nodeSelectors are merged Expect(d.Spec.Template.Spec.NodeSelector).To(HaveLen(2)) @@ -2497,7 +652,7 @@ var _ = Describe("API server rendering tests (Calico)", func() { component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) d := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment").(*appsv1.Deployment) Expect(d.Spec.Template.Spec.Tolerations).To(HaveLen(1)) Expect(d.Spec.Template.Spec.Tolerations).To(ConsistOf(tol)) @@ -2509,7 +664,7 @@ var _ = Describe("API server rendering tests (Calico)", func() { component, err := render.APIServer(cfg) Expect(err).NotTo(HaveOccurred(), "Expected APIServer to create successfully %s", err) Expect(component.ResolveImages(nil)).NotTo(HaveOccurred()) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) d := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment").(*appsv1.Deployment) Expect(d).NotTo(BeNil()) Expect(d.Spec.Template.Spec.Tolerations).To(ContainElement(corev1.Toleration{ @@ -2520,62 +675,4 @@ var _ = Describe("API server rendering tests (Calico)", func() { })) }) }) - - Context("multi-tenant", func() { - BeforeEach(func() { - cfg.MultiTenant = true - cfg.ManagementCluster = &operatorv1.ManagementCluster{Spec: operatorv1.ManagementClusterSpec{Address: "example.com:1234"}} - cfg.Installation = &operatorv1.InstallationSpec{ - ControlPlaneReplicas: ptr.To(int32(2)), - Registry: "testregistry.com/", - Variant: operatorv1.CalicoEnterprise, - } - }) - - It("should not install tigera-network-admin and tigera-ui-user", func() { - component, err := render.APIServer(cfg) - Expect(err).NotTo(HaveOccurred()) - - // Expect no UISettings / UISettingsGroups to be installed. - resources, _ := component.Objects() - obj := rtest.GetResource(resources, "tigera-network-admin", "", "rbac.authorization.k8s.io", "v1", "ClusterRole") - Expect(obj).To(BeNil()) - obj = rtest.GetResource(resources, "tigera-ui-user", "", "rbac.authorization.k8s.io", "v1", "ClusterRole") - Expect(obj).To(BeNil()) - }) - - It("should create a cluster role that get managed clusters", func() { - component, err := render.APIServer(cfg) - Expect(err).NotTo(HaveOccurred()) - - resources, _ := component.Objects() - managedClusterAccessRole := rtest.GetResource(resources, - render.MultiTenantManagedClustersAccessClusterRoleName, "", rbacv1.GroupName, "v1", "ClusterRole").(*rbacv1.ClusterRole) - expectedManagedClusterAccessRules := []rbacv1.PolicyRule{ - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"managedclusters"}, - Verbs: []string{"get"}, - }, - } - Expect(managedClusterAccessRole.Rules).To(ContainElements(expectedManagedClusterAccessRules)) - }) - - It("should create a cluster role for watching managed clusters", func() { - component, err := render.APIServer(cfg) - Expect(err).NotTo(HaveOccurred()) - - resources, _ := component.Objects() - managedClusterAccessRole := rtest.GetResource(resources, - render.ManagedClustersWatchClusterRoleName, "", rbacv1.GroupName, "v1", "ClusterRole").(*rbacv1.ClusterRole) - expectedManagedClusterAccessRules := []rbacv1.PolicyRule{ - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"managedclusters"}, - Verbs: []string{"get", "list", "watch"}, - }, - } - Expect(managedClusterAccessRole.Rules).To(ContainElements(expectedManagedClusterAccessRules)) - }) - }) }) diff --git a/pkg/render/component.go b/pkg/render/component.go index eea911fbe3..ad14722304 100644 --- a/pkg/render/component.go +++ b/pkg/render/component.go @@ -1,4 +1,4 @@ -// Copyright (c) 2021-2024 Tigera, Inc. All rights reserved. +// Copyright (c) 2021-2026 Tigera, Inc. All rights reserved. // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -39,3 +39,50 @@ type Component interface { // that create pods. Return OSTypeAny means that no node selector should be set for the "kubernetes.io/os" label. SupportedOSType() rmeta.OSType } + +// Extensible is implemented by components that expose extension points. The +// componentHandler uses ModifierKey() to look up registered modifiers. +// Components without extensions need not implement it. The method name is +// deliberately specific (not a generic Name()) so an unrelated method can't +// make a component modifier-eligible by accident. +type Extensible interface { + ModifierKey() string +} + +// ExtensionContextProvider is an optional companion to Extensible. A component +// implements it to hand its modifier component-specific context that can't be +// derived from the shared RenderContext - config only the component's controller +// has, such as a keypair the controller created. extensions.RegisterModifier +// asserts the returned value to the modifier's config type and passes it through, +// so the modifier body needs no assertion. +type ExtensionContextProvider interface { + ExtensionContext() any +} + +// Component names used as keys into the extension modifier registry. Keep these +// in sync with the ModifierKey() methods that return them. +const ( + ComponentNameTypha = "typha" + ComponentNameNode = "node" + + // ComponentNameCNIPlugins keys the upstream CNI plugins image. The node + // component renders the cni-plugins init container, so the image resolves + // through its own override key. + ComponentNameCNIPlugins = "cni-plugins" + + // ComponentNameWindows keys the windows daemonset modifier. The two windows + // images resolve through their own override keys, since one component renders + // both. + ComponentNameWindows = "windows" + ComponentNameWindowsNodeImg = "windows-node-image" + ComponentNameWindowsCNIImg = "windows-cni-image" + + // ComponentNameKubeControllers keys the calico-kube-controllers modifier. The + // es-calico-kube-controllers deployment shares the component type but leaves + // its modifier key empty, so it is not decorated. + ComponentNameKubeControllers = "kube-controllers" + + // ComponentNameKubeControllersPolicy keys the calico-kube-controllers network + // policy modifier (the WAF admission webhook ingress rule). + ComponentNameKubeControllersPolicy = "kube-controllers-policy" +) diff --git a/pkg/render/guardian.go b/pkg/render/guardian.go index f90bf3402a..6fe327a3d0 100644 --- a/pkg/render/guardian.go +++ b/pkg/render/guardian.go @@ -18,13 +18,9 @@ package render import ( "fmt" - "net" - "net/url" "golang.org/x/net/http/httpproxy" - operatorurl "github.com/tigera/operator/pkg/url" - appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" netv1 "k8s.io/api/networking/v1" @@ -35,8 +31,6 @@ import ( v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" - "github.com/tigera/api/pkg/lib/numorstring" - operatorv1 "github.com/tigera/operator/api/v1" "github.com/tigera/operator/pkg/common" "github.com/tigera/operator/pkg/components" @@ -45,7 +39,6 @@ import ( "github.com/tigera/operator/pkg/render/common/networkpolicy" "github.com/tigera/operator/pkg/render/common/secret" "github.com/tigera/operator/pkg/render/common/securitycontext" - "github.com/tigera/operator/pkg/render/common/securitycontextconstraints" "github.com/tigera/operator/pkg/tls/certificatemanagement" ) @@ -85,27 +78,51 @@ func Guardian(cfg *GuardianConfiguration) Component { } } +// GuardianPolicy renders the guardian network policy. The core operator renders +// the OSS policy; the enterprise modifier (keyed ComponentNameGuardianPolicy) +// replaces it with the management-cluster policy. The error return is retained +// for callers but is always nil now that the fallible enterprise computation +// lives in the modifier. func GuardianPolicy(cfg *GuardianConfiguration) (Component, error) { - var policies []client.Object + return &guardianPolicyComponent{cfg: cfg}, nil +} - guardianAccessPolicy, err := guardianCalicoSystemPolicy(cfg) - if err != nil { - return nil, err - } - if guardianAccessPolicy != nil { - policies = []client.Object{ - guardianAccessPolicy, - } +const ComponentNameGuardianPolicy = "guardian-policy" + +type guardianPolicyComponent struct { + cfg *GuardianConfiguration +} + +func (c *guardianPolicyComponent) ResolveImages(*operatorv1.ImageSet) error { return nil } +func (c *guardianPolicyComponent) SupportedOSType() rmeta.OSType { return rmeta.OSTypeAny } +func (c *guardianPolicyComponent) Ready() bool { return true } +func (c *guardianPolicyComponent) ModifierKey() string { return ComponentNameGuardianPolicy } + +// GuardianPolicyExtensionContext is the per-component context the guardian +// policy modifier reads (via RenderContext.Component). The enterprise guardian +// network policy is built entirely from these inputs. +type GuardianPolicyExtensionContext struct { + URL string + PodProxies []*httpproxy.Config + OpenShift bool + IncludeEgressNetworkPolicy bool +} + +func (c *guardianPolicyComponent) ExtensionContext() any { + return GuardianPolicyExtensionContext{ + URL: c.cfg.URL, + PodProxies: c.cfg.PodProxies, + OpenShift: c.cfg.OpenShift, + IncludeEgressNetworkPolicy: c.cfg.IncludeEgressNetworkPolicy, } +} - return NewPassthrough( - policies, - []client.Object{ - // allow-tigera Tier was renamed to calico-system - networkpolicy.DeprecatedAllowTigeraNetworkPolicyObject("guardian-access", GuardianNamespace), - networkpolicy.DeprecatedAllowTigeraNetworkPolicyObject("default-deny", GuardianNamespace), - }, - ), nil +func (c *guardianPolicyComponent) Objects() ([]client.Object, []client.Object) { + return []client.Object{ossNetworkPolicy(c.cfg)}, []client.Object{ + // allow-tigera Tier was renamed to calico-system + networkpolicy.DeprecatedAllowTigeraNetworkPolicyObject("guardian-access", GuardianNamespace), + networkpolicy.DeprecatedAllowTigeraNetworkPolicyObject("default-deny", GuardianNamespace), + } } // GuardianConfiguration contains all the config information needed to render the component. @@ -132,6 +149,31 @@ type GuardianConfiguration struct { Version string } +// GuardianRenderData is the variant-specific Guardian input a controller extension +// computes during reconcile and stashes in RenderContext.Extension. The +// clusterconnection controller reads it back to fill GuardianConfiguration without +// depending on the extension: when present it carries the enterprise values +// (the management-cluster version and the license-gated egress policy flag) and +// signals that the controller should not create the OSS Guardian client keypair. +// It lives in render so the controller can read it generically. +type GuardianRenderData struct { + // Version is the managed cluster version reported by ClusterInformation + // (CNXVersion for Enterprise, CalicoVersion for the OSS default). + Version string + + // IncludeEgressNetworkPolicy enables the domain-based egress rules in the + // Guardian policy, gated on an Enterprise license feature. + IncludeEgressNetworkPolicy bool +} + +// GuardianRenderDataFromContext returns the GuardianRenderData a controller +// extension stashed in the render context, and whether it was present. Absent +// means the OSS path: the controller applies its own defaults. +func GuardianRenderDataFromContext(rc RenderContext) (GuardianRenderData, bool) { + data, ok := rc.Extension.(GuardianRenderData) + return data, ok +} + type GuardianComponent struct { cfg *GuardianConfiguration calicoImage string @@ -150,32 +192,40 @@ func (c *GuardianComponent) SupportedOSType() rmeta.OSType { return rmeta.OSTypeLinux } +func (c *GuardianComponent) ModifierKey() string { return GuardianName } + +// GuardianExtensionContext is the per-component context the guardian modifier +// reads (via RenderContext.Component). It carries the inputs the enterprise +// guardian behavior needs that a modifier can't derive from the installation: +// the management cluster's impersonation config, whether we're on OpenShift, +// and the trusted bundle mount path the CA env vars reference. +type GuardianExtensionContext struct { + OpenShift bool + Impersonation *operatorv1.Impersonation + TrustedBundleMountPath string +} + +func (c *GuardianComponent) ExtensionContext() any { + var impersonation *operatorv1.Impersonation + if c.cfg.ManagementClusterConnection != nil { + impersonation = c.cfg.ManagementClusterConnection.Spec.Impersonation + } + return GuardianExtensionContext{ + OpenShift: c.cfg.OpenShift, + Impersonation: impersonation, + TrustedBundleMountPath: c.cfg.TrustedCertBundle.MountPath(), + } +} + func (c *GuardianComponent) Objects() ([]client.Object, []client.Object) { objs := []client.Object{ - // common RBAC for EE and OSS c.serviceAccount(), c.clusterRole(), c.clusterRoleBinding(), - } - - if c.cfg.Installation.Variant.IsEnterprise() { - // Enterprise-specific RBAC and settings - objs = append(objs, - c.secretsRole(), - c.secretRoleBinding(), - // Install default UI settings for this managed cluster. - managerClusterWideSettingsGroup(), - managerUserSpecificSettingsGroup(), - managerClusterWideTigeraLayer(), - managerClusterWideDefaultView(), - ) - } - - objs = append(objs, c.deployment(), c.service(), secret.CopyToNamespace(GuardianNamespace, c.cfg.TunnelSecret)[0], - ) + } return objs, deprecatedObjects() } @@ -197,28 +247,6 @@ func (c *GuardianComponent) service() *corev1.Service { }, } - if c.cfg.Installation.Variant.IsEnterprise() { - ports = append(ports, - corev1.ServicePort{ - Name: "elasticsearch", - Port: 9200, - TargetPort: intstr.IntOrString{ - Type: intstr.Int, - IntVal: 8080, - }, - Protocol: corev1.ProtocolTCP, - }, - corev1.ServicePort{ - Name: "kibana", - Port: 5601, - TargetPort: intstr.IntOrString{ - Type: intstr.Int, - IntVal: 8080, - }, - Protocol: corev1.ProtocolTCP, - }, - ) - } return &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: GuardianServiceName, @@ -241,87 +269,42 @@ func (c *GuardianComponent) serviceAccount() *corev1.ServiceAccount { } func (c *GuardianComponent) clusterRole() *rbacv1.ClusterRole { - var policyRules []rbacv1.PolicyRule - if c.cfg.Installation.Variant.IsEnterprise() { - impersonation := c.cfg.ManagementClusterConnection.Spec.Impersonation - if impersonation != nil { - if impersonation.Users != nil { - policyRules = append(policyRules, - rbacv1.PolicyRule{ - APIGroups: []string{""}, - Resources: []string{"users"}, - ResourceNames: impersonation.Users, - Verbs: []string{"impersonate"}, - }) - } - if impersonation.Groups != nil { - policyRules = append(policyRules, - rbacv1.PolicyRule{ - APIGroups: []string{""}, - Resources: []string{"groups"}, - ResourceNames: impersonation.Groups, - Verbs: []string{"impersonate"}, - }) - } - if impersonation.ServiceAccounts != nil { - policyRules = append(policyRules, - rbacv1.PolicyRule{ - APIGroups: []string{""}, - Resources: []string{"serviceaccounts"}, - ResourceNames: impersonation.ServiceAccounts, - Verbs: []string{"impersonate"}, - }) - } - } - - policyRules = append(policyRules, rulesForManagementClusterRequests(c.cfg.OpenShift)...) - - if c.cfg.OpenShift { - policyRules = append(policyRules, rbacv1.PolicyRule{ - APIGroups: []string{"security.openshift.io"}, - Resources: []string{"securitycontextconstraints"}, - Verbs: []string{"use"}, - ResourceNames: []string{securitycontextconstraints.NonRootV2}, - }) - } - } else { - policyRules = append(policyRules, - rbacv1.PolicyRule{ - APIGroups: []string{""}, - Resources: []string{"namespaces", "services", "pods"}, - Verbs: []string{"get", "list", "watch"}, - }, - rbacv1.PolicyRule{ - APIGroups: []string{"apps"}, - Resources: []string{"deployments", "replicasets", "statefulsets", "daemonsets"}, - Verbs: []string{"get", "list", "watch"}, - }, - rbacv1.PolicyRule{ - APIGroups: []string{"networking.k8s.io"}, - Resources: []string{"networkpolicies"}, - Verbs: []string{"get", "list", "watch"}, - }, - rbacv1.PolicyRule{ - APIGroups: []string{"projectcalico.org"}, - Resources: []string{ - "clusterinformations", - "tiers", - "stagednetworkpolicies", - "tier.stagednetworkpolicies", - "stagedglobalnetworkpolicies", - "tier.stagedglobalnetworkpolicies", - "stagedkubernetesnetworkpolicies", - "tier.stagedkubernetesnetworkpolicies", - "networkpolicies", - "tier.networkpolicies", - "globalnetworkpolicies", - "tier.globalnetworkpolicies", - "globalnetworksets", - "networksets", - }, - Verbs: []string{"get", "list", "watch"}, + policyRules := []rbacv1.PolicyRule{ + { + APIGroups: []string{""}, + Resources: []string{"namespaces", "services", "pods"}, + Verbs: []string{"get", "list", "watch"}, + }, + { + APIGroups: []string{"apps"}, + Resources: []string{"deployments", "replicasets", "statefulsets", "daemonsets"}, + Verbs: []string{"get", "list", "watch"}, + }, + { + APIGroups: []string{"networking.k8s.io"}, + Resources: []string{"networkpolicies"}, + Verbs: []string{"get", "list", "watch"}, + }, + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{ + "clusterinformations", + "tiers", + "stagednetworkpolicies", + "tier.stagednetworkpolicies", + "stagedglobalnetworkpolicies", + "tier.stagedglobalnetworkpolicies", + "stagedkubernetesnetworkpolicies", + "tier.stagedkubernetesnetworkpolicies", + "networkpolicies", + "tier.networkpolicies", + "globalnetworkpolicies", + "tier.globalnetworkpolicies", + "globalnetworksets", + "networksets", }, - ) + Verbs: []string{"get", "list", "watch"}, + }, } return &rbacv1.ClusterRole{ @@ -354,47 +337,6 @@ func (c *GuardianComponent) clusterRoleBinding() *rbacv1.ClusterRoleBinding { } } -// secretRole creates a Role that allows the management cluster to provision secrets to the tigera-operator Namespace. -// This is used to push secrets used by the managed cluster to access / authenticate with the management cluster. -func (c *GuardianComponent) secretsRole() *rbacv1.Role { - return &rbacv1.Role{ - TypeMeta: metav1.TypeMeta{Kind: "Role", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{ - Name: GuardianSecretsRole, - Namespace: common.OperatorNamespace(), - }, - Rules: []rbacv1.PolicyRule{ - { - APIGroups: []string{""}, - Resources: []string{"secrets"}, - Verbs: []string{"create", "delete", "deletecollection", "update"}, - }, - }, - } -} - -func (c *GuardianComponent) secretRoleBinding() *rbacv1.RoleBinding { - return &rbacv1.RoleBinding{ - TypeMeta: metav1.TypeMeta{Kind: "RoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{ - Name: GuardianSecretsRoleBindingName, - Namespace: common.OperatorNamespace(), - }, - RoleRef: rbacv1.RoleRef{ - APIGroup: "rbac.authorization.k8s.io", - Kind: "Role", - Name: GuardianSecretsRole, - }, - Subjects: []rbacv1.Subject{ - { - Kind: "ServiceAccount", - Name: GuardianServiceAccountName, - Namespace: GuardianNamespace, - }, - }, - } -} - func (c *GuardianComponent) deployment() *appsv1.Deployment { var replicas int32 = 1 @@ -468,14 +410,6 @@ func (c *GuardianComponent) container() []corev1.Container { } envVars = append(envVars, c.cfg.Installation.Proxy.EnvVars()...) - if c.cfg.Installation.Variant.IsEnterprise() { - envVars = append(envVars, - corev1.EnvVar{Name: "GUARDIAN_PACKET_CAPTURE_CA_BUNDLE_PATH", Value: c.cfg.TrustedCertBundle.MountPath()}, - corev1.EnvVar{Name: "GUARDIAN_PROMETHEUS_CA_BUNDLE_PATH", Value: c.cfg.TrustedCertBundle.MountPath()}, - corev1.EnvVar{Name: "GUARDIAN_QUERYSERVER_CA_BUNDLE_PATH", Value: c.cfg.TrustedCertBundle.MountPath()}, - ) - } - if c.cfg.GuardianClientKeyPair != nil { envVars = append(envVars, corev1.EnvVar{ @@ -585,198 +519,6 @@ func ossNetworkPolicy(cfg *GuardianConfiguration) *v3.NetworkPolicy { } } -func guardianCalicoSystemPolicy(cfg *GuardianConfiguration) (*v3.NetworkPolicy, error) { - if !cfg.Installation.Variant.IsEnterprise() { - return ossNetworkPolicy(cfg), nil - } - - egressRules := []v3.Rule{ - { - Action: v3.Allow, - Protocol: &networkpolicy.TCPProtocol, - Destination: PacketCaptureEntityRule, - }, - } - egressRules = networkpolicy.AppendDNSEgressRules(egressRules, cfg.OpenShift) - egressRules = append(egressRules, []v3.Rule{ - { - Action: v3.Allow, - Protocol: &networkpolicy.TCPProtocol, - Destination: networkpolicy.KubeAPIServerEntityRule, - }, - { - Action: v3.Allow, - Protocol: &networkpolicy.TCPProtocol, - Destination: networkpolicy.PrometheusEntityRule, - }, - { - Action: v3.Allow, - Protocol: &networkpolicy.TCPProtocol, - Destination: TigeraAPIServerEntityRule, - }, - }...) - - // The loop below creates an egress rule for each unique destination that the Guardian pods connect to. If there are - // multiple guardian pods and their proxy settings differ, then there are multiple destinations that must have egress allowed. - allowedDestinations := map[string]bool{} - processedPodProxies := ProcessPodProxies(cfg.PodProxies) - for _, podProxyConfig := range processedPodProxies { - var proxyURL *url.URL - var err error - if podProxyConfig != nil && podProxyConfig.HTTPSProxy != "" { - targetURL := &url.URL{ - // The scheme should be HTTPS, as we are establishing an mTLS session with the target. - Scheme: "https", - - // We expect `target` to be of the form host:port. - Host: cfg.URL, - } - - proxyURL, err = podProxyConfig.ProxyFunc()(targetURL) - if err != nil { - return nil, err - } - } - - var tunnelDestinationHostPort string - if proxyURL != nil { - proxyHostPort, err := operatorurl.ParseHostPortFromHTTPProxyURL(proxyURL) - if err != nil { - return nil, err - } - - tunnelDestinationHostPort = proxyHostPort - } else { - // cfg.URL has host:port form - tunnelDestinationHostPort = cfg.URL - } - - // Check if we've already created an egress rule for this destination. - if allowedDestinations[tunnelDestinationHostPort] { - continue - } - - host, port, err := net.SplitHostPort(tunnelDestinationHostPort) - if err != nil { - return nil, err - } - parsedPort, err := numorstring.PortFromString(port) - if err != nil { - return nil, err - } - parsedIp := net.ParseIP(host) - if parsedIp == nil { - // Domain-based egress rules require the EgressAccessControl license feature. - if !cfg.IncludeEgressNetworkPolicy { - continue - } - // Assume host is a valid hostname. - egressRules = append(egressRules, v3.Rule{ - Action: v3.Allow, - Protocol: &networkpolicy.TCPProtocol, - Destination: v3.EntityRule{ - Domains: []string{host}, - Ports: []numorstring.Port{parsedPort}, - }, - }) - allowedDestinations[tunnelDestinationHostPort] = true - - } else { - var netSuffix string - if parsedIp.To4() != nil { - netSuffix = "/32" - } else { - netSuffix = "/128" - } - - egressRules = append(egressRules, v3.Rule{ - Action: v3.Allow, - Protocol: &networkpolicy.TCPProtocol, - Destination: v3.EntityRule{ - Nets: []string{parsedIp.String() + netSuffix}, - Ports: []numorstring.Port{parsedPort}, - }, - }) - allowedDestinations[tunnelDestinationHostPort] = true - } - } - - egressRules = append(egressRules, v3.Rule{Action: v3.Pass}) - - guardianIngressDestinationEntityRule := v3.EntityRule{Ports: networkpolicy.Ports(GuardianTargetPort)} - networkpolicyHelper := networkpolicy.DefaultHelper() - var ingressRules []v3.Rule - if cfg.Installation.Variant.IsEnterprise() { - ingressRules = append(ingressRules, []v3.Rule{ - { - Action: v3.Allow, - Protocol: &networkpolicy.TCPProtocol, - Source: FluentdSourceEntityRule, - Destination: guardianIngressDestinationEntityRule, - }, - { - Action: v3.Allow, - Protocol: &networkpolicy.TCPProtocol, - Source: networkpolicyHelper.ComplianceBenchmarkerSourceEntityRule(), - Destination: guardianIngressDestinationEntityRule, - }, - { - Action: v3.Allow, - Protocol: &networkpolicy.TCPProtocol, - Source: networkpolicyHelper.ComplianceReporterSourceEntityRule(), - Destination: guardianIngressDestinationEntityRule, - }, - { - Action: v3.Allow, - Protocol: &networkpolicy.TCPProtocol, - Source: networkpolicyHelper.ComplianceSnapshotterSourceEntityRule(), - Destination: guardianIngressDestinationEntityRule, - }, - { - Action: v3.Allow, - Protocol: &networkpolicy.TCPProtocol, - Source: networkpolicyHelper.ComplianceControllerSourceEntityRule(), - Destination: guardianIngressDestinationEntityRule, - }, - { - Action: v3.Allow, - Protocol: &networkpolicy.TCPProtocol, - Source: IntrusionDetectionSourceEntityRule, - Destination: guardianIngressDestinationEntityRule, - }, - { - Action: v3.Allow, - Protocol: &networkpolicy.TCPProtocol, - Source: IntrusionDetectionInstallerSourceEntityRule, - Destination: guardianIngressDestinationEntityRule, - }, - { - Action: v3.Allow, - Protocol: &networkpolicy.TCPProtocol, - Destination: guardianIngressDestinationEntityRule, - }, - }...) - } - - policy := &v3.NetworkPolicy{ - TypeMeta: metav1.TypeMeta{Kind: "NetworkPolicy", APIVersion: "projectcalico.org/v3"}, - ObjectMeta: metav1.ObjectMeta{ - Name: GuardianPolicyName, - Namespace: GuardianNamespace, - }, - Spec: v3.NetworkPolicySpec{ - Order: &networkpolicy.HighPrecedenceOrder, - Tier: networkpolicy.CalicoTierName, - Selector: networkpolicy.KubernetesAppSelector(GuardianName), - Types: []v3.PolicyType{v3.PolicyTypeIngress, v3.PolicyTypeEgress}, - Ingress: ingressRules, - Egress: egressRules, - }, - } - - return policy, nil -} - func ProcessPodProxies(podProxies []*httpproxy.Config) []*httpproxy.Config { // If pod proxies are empty, then pod proxy resolution has not yet occurred. // Assume that a single Guardian pod is running without a proxy. @@ -791,304 +533,6 @@ func GuardianService(clusterDomain string) string { return fmt.Sprintf("https://%s.%s.svc.%s:%d", GuardianServiceName, GuardianNamespace, clusterDomain, 443) } -// rulesForManagementClusterRequests returns the set of RBAC rules needed by Guardian in order to -// satisfy requests from the management cluster over the tunnel. -func rulesForManagementClusterRequests(isOpenShift bool) []rbacv1.PolicyRule { - rules := []rbacv1.PolicyRule{ - // Common rules required to handle requests from multiple components in the management cluster. - { - // ID uses read-only permissions and kube-controllers uses both read and write verbs. - APIGroups: []string{""}, - Resources: []string{"configmaps"}, - Verbs: []string{"create", "delete", "get", "list", "update", "watch"}, - }, - { - // Allows Linseed to watch namespaces before copying its token. - // Also enables PolicyRecommendation to watch namespaces, - // and Manager/kube-controllers to list them. - APIGroups: []string{""}, - Resources: []string{"namespaces"}, - Verbs: []string{"get", "list", "watch"}, - }, - { - // kube-controllers watches Nodes to monitor for deletions. - // Manager performs a list operation on Nodes. - APIGroups: []string{""}, - Resources: []string{"nodes"}, - Verbs: []string{"get", "list", "watch"}, - }, - { - // kube-controllers watches Pods to verify existence for IPAM garbage collection. - // Manager performs get operations on Pods. - APIGroups: []string{""}, - Resources: []string{"pods"}, - Verbs: []string{"get", "list", "watch"}, - }, - { - // The Federated Services Controller needs access to the remote kubeconfig secret - // in order to create a remote syncer. - APIGroups: []string{""}, - Resources: []string{"secrets"}, - Verbs: []string{"get", "list", "watch"}, - }, - { - // Manager uses list; kube-controllers uses 'get', 'list', 'watch', 'update'. - APIGroups: []string{""}, - Resources: []string{"services"}, - Verbs: []string{"get", "list", "update", "watch"}, - }, - { - // Needed by kube-controllers to validate licenses; also used by ID. - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{"licensekeys"}, - Verbs: []string{"get", "watch"}, - }, - { - // Manager uses list; PolicyRecommendation & ID uses all verbs. - APIGroups: []string{"projectcalico.org"}, - Resources: []string{ - "globalnetworksets", - "networkpolicies", - "tier.networkpolicies", - "stagednetworkpolicies", - "tier.stagednetworkpolicies", - }, - Verbs: []string{"create", "delete", "get", "list", "patch", "update", "watch"}, - }, - { - // Manager uses list; PolicyRecommendation uses all verbs. - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"tiers"}, - Verbs: []string{"create", "delete", "get", "list", "patch", "update", "watch"}, - }, - // Rules needed by guardian to handle manager authorization reviews. - { - APIGroups: []string{"rbac.authorization.k8s.io"}, - Resources: []string{"clusterroles", "clusterrolebindings", "roles", "rolebindings"}, - Verbs: []string{"list", "get"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"uisettings", "uisettingsgroups"}, - Verbs: []string{"list", "get"}, - }, - - // Rules needed by guardian to handle other manager requests. - { - APIGroups: []string{""}, - Resources: []string{"events"}, - Verbs: []string{"list"}, - }, - { - APIGroups: []string{""}, - Resources: []string{"serviceaccounts"}, - Verbs: []string{"list"}, - }, - { - // Allow query server talk to Prometheus via the manager user. - APIGroups: []string{""}, - Resources: []string{"services/proxy"}, - ResourceNames: []string{ - "calico-node-prometheus:9090", - "https:calico-api:8080", - }, - Verbs: []string{"create", "get"}, - }, - { - APIGroups: []string{"apps"}, - Resources: []string{"daemonsets", "replicasets", "statefulsets"}, - Verbs: []string{"list"}, - }, - { - APIGroups: []string{"authentication.k8s.io"}, - Resources: []string{"tokenreviews"}, - Verbs: []string{"create"}, - }, - { - APIGroups: []string{"authorization.k8s.io"}, - Resources: []string{"subjectaccessreviews"}, - Verbs: []string{"create"}, - }, - { - APIGroups: []string{"networking.k8s.io"}, - Resources: []string{"networkpolicies"}, - Verbs: []string{"get", "list"}, - }, - { - APIGroups: []string{"policy.networking.k8s.io"}, - Resources: []string{ - "clusternetworkpolicies", - "adminnetworkpolicies", - "baselineadminnetworkpolicies", - }, - Verbs: []string{"list"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"alertexceptions"}, - Verbs: []string{"get", "list", "update"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"felixconfigurations"}, - ResourceNames: []string{"default"}, - Verbs: []string{"get"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{ - "globalnetworkpolicies", - "networksets", - "stagedglobalnetworkpolicies", - "stagedkubernetesnetworkpolicies", - "tier.globalnetworkpolicies", - "tier.stagedglobalnetworkpolicies", - }, - Verbs: []string{"list"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"hostendpoints"}, - Verbs: []string{"list"}, - }, - - // Rules needed by guardian to handle policy recommendation requests. - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{ - "policyrecommendationscopes", - "policyrecommendationscopes/status", - }, - Verbs: []string{"create", "delete", "get", "list", "patch", "update", "watch"}, - }, - - // Rules needed by guardian to handle calico-kube-controller requests. - { - // Nodes are watched to monitor for deletions. - APIGroups: []string{""}, - Resources: []string{"endpoints"}, - Verbs: []string{"create", "delete", "get", "list", "update", "watch"}, - }, - { - APIGroups: []string{""}, - Resources: []string{"services/status"}, - Verbs: []string{"get", "list", "update", "watch"}, - }, - { - // Needs to manage hostendpoints. - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{"hostendpoints"}, - Verbs: []string{"create", "delete", "get", "list", "update", "watch"}, - }, - { - // Needs access to update clusterinformations. - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{"clusterinformations"}, - Verbs: []string{"create", "get", "list", "update", "watch"}, - }, - { - // Needs to manipulate kubecontrollersconfiguration, which contains its config. - // It creates a default if none exists, and updates status as well. - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{"kubecontrollersconfigurations"}, - Verbs: []string{"create", "get", "list", "update", "watch"}, - }, - { - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{"tiers"}, - Verbs: []string{"create"}, - }, - { - APIGroups: []string{"crd.projectcalico.org", "projectcalico.org"}, - Resources: []string{"deeppacketinspections"}, - Verbs: []string{"get", "list", "watch"}, - }, - { - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{"deeppacketinspections/status"}, - Verbs: []string{"update"}, - }, - { - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{"packetcaptures"}, - Verbs: []string{"get", "list", "update"}, - }, - { - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{"remoteclusterconfigurations"}, - Verbs: []string{"get", "list", "watch"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"licensekeys"}, - Verbs: []string{"create", "get", "list", "update", "watch"}, - }, - { - // Grant permissions to access ClusterInformation resources in managed clusters. - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"clusterinformations"}, - Verbs: []string{"get", "list", "watch"}, - }, - { - APIGroups: []string{"usage.tigera.io"}, - Resources: []string{"licenseusagereports"}, - Verbs: []string{"create", "delete", "get", "list", "update", "watch"}, - }, - - // Rules needed by guardian to handle Intrusion detection requests. - { - APIGroups: []string{""}, - Resources: []string{"podtemplates"}, - Verbs: []string{"get"}, - }, - { - APIGroups: []string{"apps"}, - Resources: []string{"deployments"}, - Verbs: []string{"get"}, - }, - { - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{"alertexceptions"}, - Verbs: []string{"get", "list"}, - }, - { - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{"securityeventwebhooks"}, - Verbs: []string{"get", "list", "update", "watch"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{ - "globalalerts", - "globalalerts/status", - "globalthreatfeeds", - "globalthreatfeeds/status", - }, - Verbs: []string{"create", "delete", "get", "list", "patch", "update", "watch"}, - }, - // Rules needed to fetch the compliance reports - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"globalreporttypes", "globalreports"}, - Verbs: []string{"get", "list", "watch"}, - }, - } - - // Rules needed by policy recommendation in openshift. - if isOpenShift { - rules = append(rules, - rbacv1.PolicyRule{ - APIGroups: []string{"security.openshift.io"}, - Resources: []string{"securitycontextconstraints"}, - Verbs: []string{"use"}, - ResourceNames: []string{securitycontextconstraints.HostNetworkV2}, - }, - ) - } - - return rules -} - func deprecatedObjects() []client.Object { return []client.Object{ // All the Guardian objects were moved to "calico-system" circa Calico v3.30, and so the legacy tigera-guardian diff --git a/pkg/render/guardian_test.go b/pkg/render/guardian_test.go index 76f7fa56ea..c99c1e485f 100644 --- a/pkg/render/guardian_test.go +++ b/pkg/render/guardian_test.go @@ -20,8 +20,6 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" - netv1 "k8s.io/api/networking/v1" - rbacv1 "k8s.io/api/rbac/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -33,398 +31,77 @@ import ( operatorv1 "github.com/tigera/operator/api/v1" "github.com/tigera/operator/pkg/apis" "github.com/tigera/operator/pkg/common" - "github.com/tigera/operator/pkg/components" "github.com/tigera/operator/pkg/controller/certificatemanager" ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" "github.com/tigera/operator/pkg/render" - rmeta "github.com/tigera/operator/pkg/render/common/meta" - "github.com/tigera/operator/pkg/render/common/networkpolicy" rtest "github.com/tigera/operator/pkg/render/common/test" "github.com/tigera/operator/pkg/render/testutils" ) -var _ = Describe("Rendering tests", func() { - var cfg *render.GuardianConfiguration - var g render.Component - var resources []client.Object - var deleteResources []client.Object - - createGuardianConfig := func(i operatorv1.InstallationSpec, addr string, openshift bool) *render.GuardianConfiguration { - i.Variant = operatorv1.CalicoEnterprise - secret := &corev1.Secret{ - TypeMeta: metav1.TypeMeta{Kind: "Secret", APIVersion: "v1"}, - ObjectMeta: metav1.ObjectMeta{ - Name: render.GuardianSecretName, - Namespace: common.OperatorNamespace(), - }, - Data: map[string][]byte{ - "cert": []byte("foo"), - "key": []byte("bar"), - }, - } - scheme := runtime.NewScheme() - Expect(apis.AddToScheme(scheme, false)).NotTo(HaveOccurred()) - cli := ctrlrfake.DefaultFakeClientBuilder(scheme).Build() - - certificateManager, err := certificatemanager.Create(cli, nil, clusterDomain, common.OperatorNamespace(), certificatemanager.AllowCACreation()) - Expect(err).NotTo(HaveOccurred()) - - bundle := certificateManager.CreateTrustedBundle() - - return &render.GuardianConfiguration{ - URL: addr, - PullSecrets: []*corev1.Secret{{ - TypeMeta: metav1.TypeMeta{Kind: "Secret", APIVersion: "v1"}, - ObjectMeta: metav1.ObjectMeta{ - Name: "pull-secret", - Namespace: common.OperatorNamespace(), - }, - }}, - Installation: &i, - TunnelSecret: secret, - TrustedCertBundle: bundle, - OpenShift: openshift, - ManagementClusterConnection: &operatorv1.ManagementClusterConnection{}, - IncludeEgressNetworkPolicy: true, - } +// guardianObjects renders the base guardian component. The enterprise modifier is +// exercised in the pkg/enterprise/guardian tests; these tests cover the OSS render +// path, which never runs the modifier. +func guardianObjects(cfg *render.GuardianConfiguration) []client.Object { + g := render.Guardian(cfg) + ExpectWithOffset(1, g.ResolveImages(nil)).To(BeNil()) + objs, _ := g.Objects() + return objs +} + +func newGuardianConfig(addr string) *render.GuardianConfiguration { + secret := &corev1.Secret{ + TypeMeta: metav1.TypeMeta{Kind: "Secret", APIVersion: "v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: render.GuardianSecretName, + Namespace: common.OperatorNamespace(), + }, + Data: map[string][]byte{ + "cert": []byte("foo"), + "key": []byte("bar"), + }, } + scheme := runtime.NewScheme() + Expect(apis.AddToScheme(scheme, false)).NotTo(HaveOccurred()) + cli := ctrlrfake.DefaultFakeClientBuilder(scheme).Build() + + certificateManager, err := certificatemanager.Create(cli, nil, clusterDomain, common.OperatorNamespace(), certificatemanager.AllowCACreation()) + Expect(err).NotTo(HaveOccurred()) + + return &render.GuardianConfiguration{ + URL: addr, + Installation: &operatorv1.InstallationSpec{Registry: "my-reg/"}, + TunnelSecret: secret, + TrustedCertBundle: certificateManager.CreateTrustedBundle(), + ManagementClusterConnection: &operatorv1.ManagementClusterConnection{}, + IncludeEgressNetworkPolicy: true, + } +} - Context("Guardian component", func() { - renderGuardian := func(i operatorv1.InstallationSpec) { - cfg = createGuardianConfig(i, "127.0.0.1:1234", false) - g = render.Guardian(cfg) - Expect(g.ResolveImages(nil)).To(BeNil()) - resources, deleteResources = g.Objects() - } - - BeforeEach(func() { - renderGuardian(operatorv1.InstallationSpec{Registry: "my-reg/"}) - }) - - It("should render all resources for a managed cluster", func() { - expectedResources := []client.Object{ - &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: render.GuardianServiceAccountName, Namespace: render.GuardianNamespace}, TypeMeta: metav1.TypeMeta{Kind: "ServiceAccount", APIVersion: "v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: render.GuardianClusterRoleName}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: render.GuardianClusterRoleBindingName}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.Role{ObjectMeta: metav1.ObjectMeta{Name: render.GuardianSecretsRole, Namespace: "tigera-operator"}, TypeMeta: metav1.TypeMeta{Kind: "Role", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.RoleBinding{ObjectMeta: metav1.ObjectMeta{Name: render.GuardianSecretsRoleBindingName, Namespace: "tigera-operator"}, TypeMeta: metav1.TypeMeta{Kind: "RoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: render.GuardianDeploymentName, Namespace: render.GuardianNamespace}, TypeMeta: metav1.TypeMeta{Kind: "Deployment", APIVersion: "apps/v1"}}, - &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: render.GuardianServiceName, Namespace: render.GuardianNamespace}, TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: ""}}, - &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: render.GuardianSecretName, Namespace: render.GuardianNamespace}, TypeMeta: metav1.TypeMeta{Kind: "Secret", APIVersion: "v1"}}, - &v3.UISettingsGroup{ObjectMeta: metav1.ObjectMeta{Name: render.ManagerClusterSettings}, TypeMeta: metav1.TypeMeta{Kind: "UISettingsGroup", APIVersion: "projectcalico.org/v3"}}, - &v3.UISettingsGroup{ObjectMeta: metav1.ObjectMeta{Name: render.ManagerUserSettings}, TypeMeta: metav1.TypeMeta{Kind: "UISettingsGroup", APIVersion: "projectcalico.org/v3"}}, - &v3.UISettings{ObjectMeta: metav1.ObjectMeta{Name: render.ManagerClusterSettingsLayerTigera}, TypeMeta: metav1.TypeMeta{Kind: "UISettings", APIVersion: "projectcalico.org/v3"}}, - &v3.UISettings{ObjectMeta: metav1.ObjectMeta{Name: render.ManagerClusterSettingsViewDefault}, TypeMeta: metav1.TypeMeta{Kind: "UISettings", APIVersion: "projectcalico.org/v3"}}, - } - - expectedDeleteResources := []client.Object{ - &corev1.Namespace{TypeMeta: metav1.TypeMeta{Kind: "Namespace", APIVersion: "v1"}, ObjectMeta: metav1.ObjectMeta{Name: "tigera-guardian"}}, - &rbacv1.ClusterRole{TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, ObjectMeta: metav1.ObjectMeta{Name: "tigera-guardian"}}, - &rbacv1.ClusterRoleBinding{TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, ObjectMeta: metav1.ObjectMeta{Name: "tigera-guardian"}}, - &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "tigera-manager"}, TypeMeta: metav1.TypeMeta{Kind: "Namespace", APIVersion: "v1"}}, - &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: "tigera-manager", Namespace: "tigera-manager"}, TypeMeta: metav1.TypeMeta{Kind: "ServiceAccount", APIVersion: "v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "tigera-manager-role"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "tigera-manager-binding"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &netv1.NetworkPolicy{ObjectMeta: metav1.ObjectMeta{Name: "guardian", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "NetworkPolicy", APIVersion: "networking.k8s.io/v1"}}, - } - - rtest.ExpectResources(resources, expectedResources) - rtest.ExpectResources(deleteResources, expectedDeleteResources) - - deployment := rtest.GetResource(resources, render.GuardianDeploymentName, render.GuardianNamespace, "apps", "v1", "Deployment").(*appsv1.Deployment) - Expect(deployment.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(deployment.Spec.Template.Spec.Containers[0].Image).Should(Equal("my-reg/tigera/calico:" + components.ComponentTigeraCalico.Version)) - - Expect(*deployment.Spec.Template.Spec.Containers[0].SecurityContext.AllowPrivilegeEscalation).To(BeFalse()) - Expect(*deployment.Spec.Template.Spec.Containers[0].SecurityContext.Privileged).To(BeFalse()) - Expect(*deployment.Spec.Template.Spec.Containers[0].SecurityContext.RunAsGroup).To(BeEquivalentTo(10001)) - Expect(*deployment.Spec.Template.Spec.Containers[0].SecurityContext.RunAsNonRoot).To(BeTrue()) - Expect(*deployment.Spec.Template.Spec.Containers[0].SecurityContext.RunAsUser).To(BeEquivalentTo(10001)) - Expect(deployment.Spec.Template.Spec.Containers[0].SecurityContext.SeccompProfile).To(Equal( - &corev1.SeccompProfile{ - Type: corev1.SeccompProfileTypeRuntimeDefault, - })) - Expect(deployment.Spec.Template.Spec.Containers[0].SecurityContext.Capabilities).To(Equal( - &corev1.Capabilities{ - Drop: []corev1.Capability{"ALL"}, - }, - )) - }) - - It("should render controlPlaneTolerations", func() { - t := corev1.Toleration{ - Key: "foo", - Operator: corev1.TolerationOpEqual, - Value: "bar", - } - renderGuardian(operatorv1.InstallationSpec{ - ControlPlaneTolerations: []corev1.Toleration{t}, - }) - deployment := rtest.GetResource(resources, render.GuardianDeploymentName, render.GuardianNamespace, "apps", "v1", "Deployment").(*appsv1.Deployment) - Expect(deployment.Spec.Template.Spec.Tolerations).Should(ContainElements(append(rmeta.TolerateCriticalAddonsAndControlPlane, t))) - }) - - It("should render toleration on GKE", func() { - renderGuardian(operatorv1.InstallationSpec{ - KubernetesProvider: operatorv1.ProviderGKE, - }) - deployment := rtest.GetResource(resources, render.GuardianDeploymentName, render.GuardianNamespace, "apps", "v1", "Deployment").(*appsv1.Deployment) - Expect(deployment).NotTo(BeNil()) - Expect(deployment.Spec.Template.Spec.Tolerations).To(ContainElements(corev1.Toleration{ - Key: "kubernetes.io/arch", - Operator: corev1.TolerationOpEqual, - Value: "arm64", - Effect: corev1.TaintEffectNoSchedule, - })) - }) - - It("should render guardian with unlimited impersonation", func() { - cfg.ManagementClusterConnection = &operatorv1.ManagementClusterConnection{ - Spec: operatorv1.ManagementClusterConnectionSpec{ - Impersonation: &operatorv1.Impersonation{ - Users: []string{}, - Groups: []string{}, - ServiceAccounts: []string{}, - }, - }, - } - - g := render.Guardian(cfg) - resources, _ := g.Objects() - Expect(resources).ToNot(BeNil()) - - clusterRole, ok := rtest.GetResource(resources, render.GuardianClusterRoleName, "", "rbac.authorization.k8s.io", "v1", "ClusterRole").(*rbacv1.ClusterRole) - Expect(ok).To(BeTrue()) - - foundUserImp, foundGroupImp, foundSaImp := false, false, false - for _, rule := range clusterRole.Rules { - if rule.Verbs[0] == "impersonate" { - if rule.Resources[0] == "users" { - Expect(rule.ResourceNames).To(Equal([]string{})) - foundUserImp = true - } - if rule.Resources[0] == "groups" { - Expect(rule.ResourceNames).To(Equal([]string{})) - foundGroupImp = true - } - if rule.Resources[0] == "serviceaccounts" { - Expect(rule.ResourceNames).To(Equal([]string{})) - foundSaImp = true - } - } - } - - Expect(foundUserImp).To(BeTrue()) - Expect(foundGroupImp).To(BeTrue()) - Expect(foundSaImp).To(BeTrue()) - }) - - It("should render guardian with specific impersonation", func() { - cfg.ManagementClusterConnection = &operatorv1.ManagementClusterConnection{ - Spec: operatorv1.ManagementClusterConnectionSpec{ - Impersonation: &operatorv1.Impersonation{ - Users: []string{"foo"}, - Groups: []string{"bar"}, - ServiceAccounts: []string{"zaz"}, - }, - }, - } - - g := render.Guardian(cfg) - resources, _ := g.Objects() - Expect(resources).ToNot(BeNil()) - - clusterRole, ok := rtest.GetResource(resources, render.GuardianClusterRoleName, "", "rbac.authorization.k8s.io", "v1", "ClusterRole").(*rbacv1.ClusterRole) - Expect(ok).To(BeTrue()) - - foundUserImp, foundGroupImp, foundSaImp := false, false, false - for _, rule := range clusterRole.Rules { - if rule.Verbs[0] == "impersonate" { - if rule.Resources[0] == "users" { - Expect(rule.ResourceNames).To(Equal([]string{"foo"})) - foundUserImp = true - } - if rule.Resources[0] == "groups" { - Expect(rule.ResourceNames).To(Equal([]string{"bar"})) - foundGroupImp = true - } - if rule.Resources[0] == "serviceaccounts" { - Expect(rule.ResourceNames).To(Equal([]string{"zaz"})) - foundSaImp = true - } - } - } - - Expect(foundUserImp).To(BeTrue()) - Expect(foundGroupImp).To(BeTrue()) - Expect(foundSaImp).To(BeTrue()) - }) - - It("should render guardian with specific no sa permissions but with user and group", func() { - cfg.ManagementClusterConnection = &operatorv1.ManagementClusterConnection{ - Spec: operatorv1.ManagementClusterConnectionSpec{ - Impersonation: &operatorv1.Impersonation{ - Users: []string{}, - Groups: []string{}, - }, - }, - } - - g := render.Guardian(cfg) - resources, _ := g.Objects() - Expect(resources).ToNot(BeNil()) - - clusterRole, ok := rtest.GetResource(resources, render.GuardianClusterRoleName, "", "rbac.authorization.k8s.io", "v1", "ClusterRole").(*rbacv1.ClusterRole) - Expect(ok).To(BeTrue()) - - foundUserImp, foundGroupImp, foundSaImp := false, false, false - for _, rule := range clusterRole.Rules { - if rule.Verbs[0] == "impersonate" { - if rule.Resources[0] == "users" { - foundUserImp = true - } - if rule.Resources[0] == "groups" { - foundGroupImp = true - } - if rule.Resources[0] == "serviceaccounts" { - foundSaImp = true - } - } - } - - Expect(foundUserImp).To(BeTrue()) - Expect(foundGroupImp).To(BeTrue()) - Expect(foundSaImp).To(BeFalse()) - }) - }) - - It("should render SecurityContextConstrains properly when provider is OpenShift", func() { - cfg.Installation.KubernetesProvider = operatorv1.ProviderOpenShift - cfg.OpenShift = true - component := render.Guardian(cfg) - Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() - - role := rtest.GetResource(resources, render.GuardianClusterRoleName, "", "rbac.authorization.k8s.io", "v1", "ClusterRole").(*rbacv1.ClusterRole) - Expect(role.Rules).To(ContainElement(rbacv1.PolicyRule{ - APIGroups: []string{"security.openshift.io"}, - Resources: []string{"securitycontextconstraints"}, - Verbs: []string{"use"}, - ResourceNames: []string{"nonroot-v2"}, - })) - }) - +var _ = Describe("Guardian OSS rendering tests", func() { Context("GuardianPolicy component", func() { - guardianPolicy := testutils.GetExpectedPolicyFromFile("./testutils/expected_policies/guardian.json") - guardianPolicyForOCP := testutils.GetExpectedPolicyFromFile("./testutils/expected_policies/guardian_ocp.json") - - renderGuardianPolicy := func(addr string, openshift bool, variant operatorv1.ProductVariant, includeEgressNetworkPolicy bool) { - installation := operatorv1.InstallationSpec{ - Registry: "my-reg/", - } - cfg := createGuardianConfig(installation, addr, openshift) - cfg.Installation.Variant = variant - cfg.IncludeEgressNetworkPolicy = includeEgressNetworkPolicy + It("should render OSS network policy regardless of IncludeEgressNetworkPolicy flag", func() { + // OSS variant should always render a network policy, even when IncludeEgressNetworkPolicy is false + cfg := newGuardianConfig("127.0.0.1:1234") + cfg.Installation.Variant = operatorv1.Calico + cfg.IncludeEgressNetworkPolicy = false g, err := render.GuardianPolicy(cfg) Expect(err).NotTo(HaveOccurred()) - resources, _ = g.Objects() - } - - Context("policy rendering based on variant and IncludeEgressNetworkPolicy", func() { - It("should render OSS network policy regardless of IncludeEgressNetworkPolicy flag", func() { - // OSS variant should always render a network policy, even when IncludeEgressNetworkPolicy is false - renderGuardianPolicy("127.0.0.1:1234", false, operatorv1.Calico, false) - - policyName := types.NamespacedName{Name: "calico-system.guardian-access", Namespace: "calico-system"} - policy := testutils.GetCalicoSystemPolicyFromResources(policyName, resources) - Expect(policy).NotTo(BeNil(), "OSS variant should always render a network policy") - - // The OSS policy has both Ingress and Egress, ending in a Pass so the - // tunnel to the management cluster isn't dropped by the default-deny. - Expect(policy.Spec.Types).To(ConsistOf(v3.PolicyTypeIngress, v3.PolicyTypeEgress)) - Expect(policy.Spec.Egress).NotTo(BeEmpty()) - Expect(policy.Spec.Egress[len(policy.Spec.Egress)-1].Action).To(Equal(v3.Pass)) - - // OSS can't express domain-based egress rules. - for _, rule := range policy.Spec.Egress { - Expect(rule.Destination.Domains).To(BeEmpty()) - } - }) - - It("should render Enterprise network policy without domain-based egress when IncludeEgressNetworkPolicy is false", func() { - // Enterprise variant with IncludeEgressNetworkPolicy=false should render a policy but skip domain-based egress rules - renderGuardianPolicy("my-management.example.com:1234", false, operatorv1.CalicoEnterprise, false) - - policyName := types.NamespacedName{Name: "calico-system.guardian-access", Namespace: "calico-system"} - policy := testutils.GetCalicoSystemPolicyFromResources(policyName, resources) - Expect(policy).NotTo(BeNil(), "Enterprise variant should always render a network policy when tier exists") - - // Verify it's the Enterprise policy (should have both Ingress and Egress types) - Expect(policy.Spec.Types).To(ConsistOf(v3.PolicyTypeIngress, v3.PolicyTypeEgress)) - Expect(policy.Spec.Egress).NotTo(BeEmpty()) - - // Verify no domain-based egress rules are present - for _, rule := range policy.Spec.Egress { - Expect(rule.Destination.Domains).To(BeEmpty(), - "Domain-based egress rules should not be present when IncludeEgressNetworkPolicy is false") - } - }) - - It("should render Enterprise network policy with domain-based egress when IncludeEgressNetworkPolicy is true", func() { - // Enterprise variant with IncludeEgressNetworkPolicy=true should render the full policy including domain-based egress - renderGuardianPolicy("my-management.example.com:1234", false, operatorv1.CalicoEnterprise, true) - - policyName := types.NamespacedName{Name: "calico-system.guardian-access", Namespace: "calico-system"} - policy := testutils.GetCalicoSystemPolicyFromResources(policyName, resources) - Expect(policy).NotTo(BeNil(), "Enterprise variant with IncludeEgressNetworkPolicy=true should render a network policy") - - // Verify it's the Enterprise policy (should have both Ingress and Egress types) - Expect(policy.Spec.Types).To(ConsistOf(v3.PolicyTypeIngress, v3.PolicyTypeEgress)) - Expect(policy.Spec.Egress).NotTo(BeEmpty()) - - // Verify domain-based egress rule is present - hasDomainRule := false - for _, rule := range policy.Spec.Egress { - if len(rule.Destination.Domains) > 0 { - hasDomainRule = true - break - } - } - Expect(hasDomainRule).To(BeTrue(), "Domain-based egress rule should be present when IncludeEgressNetworkPolicy is true") - }) - }) + resources, _ := g.Objects() - Context("calico-system rendering", func() { policyName := types.NamespacedName{Name: "calico-system.guardian-access", Namespace: "calico-system"} - - getExpectedPolicy := func(name types.NamespacedName, scenario testutils.CalicoSystemScenario) *v3.NetworkPolicy { - if name.Name == "calico-system.guardian-access" && scenario.ManagedCluster { - return testutils.SelectPolicyByProvider(scenario, guardianPolicy, guardianPolicyForOCP) - } - - return nil + policy := testutils.GetCalicoSystemPolicyFromResources(policyName, resources) + Expect(policy).NotTo(BeNil(), "OSS variant should always render a network policy") + + // The OSS policy has both Ingress and Egress, ending in a Pass so the + // tunnel to the management cluster isn't dropped by the default-deny. + Expect(policy.Spec.Types).To(ConsistOf(v3.PolicyTypeIngress, v3.PolicyTypeEgress)) + Expect(policy.Spec.Egress).NotTo(BeEmpty()) + Expect(policy.Spec.Egress[len(policy.Spec.Egress)-1].Action).To(Equal(v3.Pass)) + + // OSS can't express domain-based egress rules. + for _, rule := range policy.Spec.Egress { + Expect(rule.Destination.Domains).To(BeEmpty()) } - - DescribeTable("should render calico-system policy", - func(scenario testutils.CalicoSystemScenario) { - renderGuardianPolicy("127.0.0.1:1234", scenario.OpenShift, operatorv1.CalicoEnterprise, true) - policy := testutils.GetCalicoSystemPolicyFromResources(policyName, resources) - expectedPolicy := getExpectedPolicy(policyName, scenario) - Expect(policy).To(Equal(expectedPolicy)) - }, - Entry("for managed, kube-dns", testutils.CalicoSystemScenario{ManagedCluster: true, OpenShift: false}), - Entry("for managed, openshift-dns", testutils.CalicoSystemScenario{ManagedCluster: true, OpenShift: true}), - ) - - // The test matrix above validates against an IP-based management cluster address. - // Validate policy adaptation for domain-based management cluster address here. - It("should adapt Guardian policy if ManagementClusterAddr is domain-based", func() { - renderGuardianPolicy("mydomain.io:8080", false, operatorv1.CalicoEnterprise, true) - policy := testutils.GetCalicoSystemPolicyFromResources(policyName, resources) - managementClusterEgressRule := policy.Spec.Egress[5] - Expect(managementClusterEgressRule.Destination.Domains).To(Equal([]string{"mydomain.io"})) - Expect(managementClusterEgressRule.Destination.Ports).To(Equal(networkpolicy.Ports(8080))) - }) }) }) }) @@ -448,8 +125,7 @@ var _ = Describe("guardian", func() { } }) It("should render when disabled", func() { - g := render.Guardian(cfg) - resources, _ := g.Objects() + resources := guardianObjects(cfg) Expect(resources).ToNot(BeNil()) deployment := rtest.GetResource(resources, render.GuardianDeploymentName, render.GuardianNamespace, "apps", "v1", "Deployment").(*appsv1.Deployment) @@ -459,8 +135,7 @@ var _ = Describe("guardian", func() { It("should render when set to disabled", func() { cfg.TunnelCAType = operatorv1.CATypeTigera - g := render.Guardian(cfg) - resources, _ := g.Objects() + resources := guardianObjects(cfg) Expect(resources).ToNot(BeNil()) deployment := rtest.GetResource(resources, render.GuardianDeploymentName, render.GuardianNamespace, "apps", "v1", "Deployment").(*appsv1.Deployment) @@ -471,8 +146,7 @@ var _ = Describe("guardian", func() { It("should render when enabled", func() { cfg.TunnelCAType = operatorv1.CATypePublic - g := render.Guardian(cfg) - resources, _ := g.Objects() + resources := guardianObjects(cfg) Expect(resources).ToNot(BeNil()) deployment := rtest.GetResource(resources, render.GuardianDeploymentName, render.GuardianNamespace, "apps", "v1", "Deployment").(*appsv1.Deployment) @@ -510,8 +184,7 @@ var _ = Describe("guardian", func() { }, } - g := render.Guardian(cfg) - resources, _ := g.Objects() + resources := guardianObjects(cfg) Expect(resources).ToNot(BeNil()) deployment, ok := rtest.GetResource(resources, render.GuardianDeploymentName, render.GuardianNamespace, "apps", "v1", "Deployment").(*appsv1.Deployment) diff --git a/pkg/render/kubecontrollers/kube-controllers.go b/pkg/render/kubecontrollers/kube-controllers.go index ad3c9f93cf..18311cf60b 100644 --- a/pkg/render/kubecontrollers/kube-controllers.go +++ b/pkg/render/kubecontrollers/kube-controllers.go @@ -16,7 +16,6 @@ package kubecontrollers import ( "fmt" - "path/filepath" "slices" "strconv" "strings" @@ -36,17 +35,13 @@ import ( "github.com/tigera/operator/pkg/components" "github.com/tigera/operator/pkg/controller/k8sapi" "github.com/tigera/operator/pkg/render" - "github.com/tigera/operator/pkg/render/applicationlayer" rcomp "github.com/tigera/operator/pkg/render/common/components" - relasticsearch "github.com/tigera/operator/pkg/render/common/elasticsearch" rmeta "github.com/tigera/operator/pkg/render/common/meta" "github.com/tigera/operator/pkg/render/common/networkpolicy" "github.com/tigera/operator/pkg/render/common/secret" "github.com/tigera/operator/pkg/render/common/securitycontext" "github.com/tigera/operator/pkg/render/common/securitycontextconstraints" - "github.com/tigera/operator/pkg/render/monitor" "github.com/tigera/operator/pkg/tls/certificatemanagement" - "github.com/tigera/operator/pkg/url" ) const ( @@ -57,33 +52,12 @@ const ( KubeControllerMetrics = "calico-kube-controllers-metrics" KubeControllerNetworkPolicyName = networkpolicy.CalicoComponentPolicyPrefix + "kube-controller-access" - // WASMPullSecretName is the dedicated image-pull Secret (a renamed copy of - // the install pull secret) that the WAF reconciler replicates into tenant - // namespaces for the Coraza wasm OCI pull. A dedicated name avoids clashing - // with the operator-managed tigera-pull-secret the GatewayAPI render also - // copies into those namespaces (EV-6386). - WASMPullSecretName = "tigera-waf-pull-secret" - - // WASMCACertName is the dedicated CA-bundle ConfigMap (in the controller - // namespace) the WAF reconciler replicates into tenant namespaces for the - // Coraza wasm OCI registry TLS check — a dedicated name avoids clashing with - // the operator-managed tigera-ca-bundle ConfigMap the GatewayAPI render also - // copies there (EV-6386). The source copy is a renamed copy of the trusted - // bundle, provisioned by the core controller and passed in as WASMCACert. - WASMCACertName = "tigera-waf-ca-bundle" - - EsKubeController = "es-calico-kube-controllers" - EsKubeControllerRole = "es-calico-kube-controllers" - EsKubeControllerRoleBinding = "es-calico-kube-controllers" - EsKubeControllerMetrics = "es-calico-kube-controllers-metrics" - EsKubeControllerNetworkPolicyName = networkpolicy.CalicoComponentPolicyPrefix + "es-kube-controller-access" + // ManagedClustersWatchRoleBindingName binds kube-controllers to the managed-cluster + // watch ClusterRole. Used by both calico-kube-controllers (in a management cluster) + // and the enterprise es-calico-kube-controllers, so the binding stays generic here. ManagedClustersWatchRoleBindingName = "es-calico-kube-controllers-managed-cluster-watch" - ElasticsearchKubeControllersUserSecret = "tigera-ee-kube-controllers-elasticsearch-access" - ElasticsearchKubeControllersUserName = "tigera-ee-kube-controllers" - ElasticsearchKubeControllersSecureUserSecret = "tigera-ee-kube-controllers-elasticsearch-access-gateway" - ElasticsearchKubeControllersVerificationUserSecret = "tigera-ee-kube-controllers-gateway-verification-credentials" - KubeControllerPrometheusTLSSecret = "calico-kube-controllers-metrics-tls" + KubeControllerPrometheusTLSSecret = "calico-kube-controllers-metrics-tls" // KubeControllersHealthPort is the port the kube-controllers HealthAggregator listens on when run from the // combined calico binary. The legacy per-component image uses file-based health checks instead. @@ -99,9 +73,6 @@ type KubeControllersConfiguration struct { ManagementClusterConnection *operatorv1.ManagementClusterConnection Authentication *operatorv1.Authentication - // Whether or not the LogStorage CRD is present in the cluster. - LogStorageExists bool - ClusterDomain string MetricsPort int @@ -112,12 +83,8 @@ type KubeControllersConfiguration struct { // namespace to be returned by the rendered. Expected that the calling code // take care to pass the same secret on each reconcile where possible. KubeControllersGatewaySecret *corev1.Secret - WASMPullSecret *corev1.Secret - WASMCACert *corev1.ConfigMap TrustedBundle certificatemanagement.TrustedBundleRO - MetricsServerTLS certificatemanagement.KeyPairInterface - // Namespace to be installed into. Namespace string @@ -128,28 +95,51 @@ type KubeControllersConfiguration struct { // If this is nil, then we should run in zero-tenant mode. Tenant *operatorv1.Tenant - // WAFGatewayExtensionEnabled gates the WAF v3 (Gateway API add-on) surface - // on calico-kube-controllers: the applicationlayer controller enablement, - // the WAF / Gateway-API / EnvoyExtensionPolicy / event / secret-replication - // RBAC, the WASM_IMAGE / WASM_PULL_SECRET / WASM_CA_CERT env vars, and the - // gateway envoy-proxy wasm image resolution. Sourced from - // `GatewayAPI.spec.extensions.waf.state == Enabled` (default off). - // See design `tigera/designs#25` (PMREQ-384). - WAFGatewayExtensionEnabled bool - - // WAFWebhookServerTLS is the serving certificate for the in-process WAF - // SecLang validating admission webhook hosted by calico-kube-controllers. - // When set (WAF enabled), it is mounted into the Pod and the webhook server - // reads it from WAF_WEBHOOK_CERT_DIR. Issued for the tigera-waf-webhook - // Service DNS name. Nil leaves the Deployment untouched (and the in-process - // server self-disables when the cert is absent). - WAFWebhookServerTLS certificatemanagement.KeyPairInterface - - // WAFWebhookCABundle is the PEM of the CA that issued WAFWebhookServerTLS - // (the operator CA), stamped into the ValidatingWebhookConfiguration's - // caBundle so the apiserver can verify the in-process webhook endpoint. - // Only consulted when WAFGatewayExtensionEnabled is true. - WAFWebhookCABundle []byte + // The fields below parameterize the generic kube-controllers component. The + // variant assemblers (NewCalicoKubeControllers, the enterprise es builder) + // fill them; the component renders them without any variant or component-name + // branching. + + // Name is the deployment / pod / container name (and the value the metrics + // Service selects on). + Name string + // ConfigName is the KUBE_CONTROLLERS_CONFIG_NAME the binary reconciles. + ConfigName string + // RoleName / RoleBindingName / MetricsName name the ClusterRole, its binding, + // and the Prometheus metrics Service. + RoleName string + RoleBindingName string + MetricsName string + // EnabledControllers is the ENABLED_CONTROLLERS list. The deployment is only + // rendered when it is non-empty. + EnabledControllers []string + // Rules are the ClusterRole policy rules. + Rules []rbacv1.PolicyRule + // NetworkPolicy, when set, is rendered into the install namespace (and the + // deprecated allow-tigera policy named DeprecatedNetworkPolicyName is deleted). + NetworkPolicy *v3.NetworkPolicy + DeprecatedNetworkPolicyName string + // ExtraEnv is appended to the deployment's container env. + ExtraEnv []corev1.EnvVar + // DisableConfigAPI sets DISABLE_KUBE_CONTROLLERS_CONFIG_API. + DisableConfigAPI bool + + // ModifierKey is the extension modifier key the component reports through + // render.Extensible. calico-kube-controllers sets it so the enterprise modifier + // can layer on the enterprise surface; es-calico-kube-controllers leaves it empty + // so it is never decorated. + ModifierKey string +} + +// calicoKubeControllersPolicyComponent wraps the calico-kube-controllers network +// policy passthrough so it is render.Extensible: the enterprise modifier adds the WAF +// admission webhook ingress rule. The base policy carries no WAF. +type calicoKubeControllersPolicyComponent struct { + render.Component +} + +func (calicoKubeControllersPolicyComponent) ModifierKey() string { + return render.ComponentNameKubeControllersPolicy } func NewCalicoKubeControllersPolicy(cfg *KubeControllersConfiguration, defaultDeny *v3.NetworkPolicy) render.Component { @@ -159,106 +149,40 @@ func NewCalicoKubeControllersPolicy(cfg *KubeControllersConfiguration, defaultDe toCreate = append(toCreate, defaultDeny) } - return render.NewPassthrough( + return calicoKubeControllersPolicyComponent{render.NewPassthrough( toCreate, []client.Object{ // allow-tigera Tier was renamed to calico-system networkpolicy.DeprecatedAllowTigeraNetworkPolicyObject("kube-controller-access", cfg.Namespace), networkpolicy.DeprecatedAllowTigeraNetworkPolicyObject("default-deny", common.CalicoNamespace), }, - ) + )} } -func NewCalicoKubeControllers(cfg *KubeControllersConfiguration) *kubeControllersComponent { - kubeControllerRolePolicyRules := kubeControllersRoleCommonRules(cfg) - enabledControllers := []string{"node", "loadbalancer"} - if cfg.Installation.Variant.IsEnterprise() { - kubeControllerRolePolicyRules = append(kubeControllerRolePolicyRules, kubeControllersRoleEnterpriseCommonRules(cfg)...) - kubeControllerRolePolicyRules = append(kubeControllerRolePolicyRules, - rbacv1.PolicyRule{ - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{"remoteclusterconfigurations"}, - Verbs: []string{"watch", "list", "get"}, - }, - rbacv1.PolicyRule{ - APIGroups: []string{""}, - Resources: []string{"endpoints"}, - Verbs: []string{"create", "update", "delete"}, - }, - rbacv1.PolicyRule{ - APIGroups: []string{""}, - Resources: []string{"namespaces"}, - Verbs: []string{"get"}, - }, - rbacv1.PolicyRule{ - APIGroups: []string{"usage.tigera.io"}, - Resources: []string{"licenseusagereports"}, - Verbs: []string{"create", "update", "delete", "watch", "list", "get"}, - }, - ) - enabledControllers = append(enabledControllers, "service", "federatedservices", "usage") - if cfg.WAFGatewayExtensionEnabled { - enabledControllers = append(enabledControllers, "applicationlayer") - } - } - - return &kubeControllersComponent{ - cfg: cfg, - kubeControllerServiceAccountName: KubeControllerServiceAccount, - kubeControllerRoleName: KubeControllerRole, - kubeControllerRoleBindingName: KubeControllerRoleBinding, - kubeControllerName: KubeController, - kubeControllerConfigName: "default", - kubeControllerMetricsName: KubeControllerMetrics, - kubeControllersRules: kubeControllerRolePolicyRules, - enabledControllers: enabledControllers, - } +// NewKubeControllers builds a kube-controllers component from a fully-populated +// configuration. Callers (NewCalicoKubeControllers, the enterprise es-kube-controllers +// builder) fill the generic Name/Rules/EnabledControllers/ExtraEnv/NetworkPolicy fields; +// the component renders them with no variant branching. +func NewKubeControllers(cfg *KubeControllersConfiguration) render.Component { + return &kubeControllersComponent{cfg: cfg} } -func NewElasticsearchKubeControllers(cfg *KubeControllersConfiguration) *kubeControllersComponent { - var kubeControllerCalicoSystemPolicy *v3.NetworkPolicy - kubeControllerRolePolicyRules := kubeControllersRoleCommonRules(cfg) - - if cfg.Installation.Variant.IsEnterprise() { - kubeControllerRolePolicyRules = append(kubeControllerRolePolicyRules, kubeControllersRoleEnterpriseCommonRules(cfg)...) - kubeControllerRolePolicyRules = append(kubeControllerRolePolicyRules, - rbacv1.PolicyRule{ - APIGroups: []string{"elasticsearch.k8s.elastic.co"}, - Resources: []string{"elasticsearches"}, - Verbs: []string{"watch", "get", "list"}, - }, - rbacv1.PolicyRule{ - APIGroups: []string{"rbac.authorization.k8s.io"}, - Resources: []string{"clusterroles", "clusterrolebindings"}, - Verbs: []string{"watch", "list", "get"}, - }, - ) - - kubeControllerCalicoSystemPolicy = esKubeControllersCalicoSystemPolicy(cfg) - } - - var enabledControllers []string - if !cfg.Tenant.MultiTenant() { - // Zero and single tenant cluster needs elasticsearch configuration - enabledControllers = append(enabledControllers, "authorization", "elasticsearchconfiguration") - if cfg.ManagementCluster != nil && cfg.Tenant == nil { - // Enterprise will require the managedcluster controller to push licenses - enabledControllers = append(enabledControllers, "managedcluster") - } - } - - return &kubeControllersComponent{ - cfg: cfg, - kubeControllerServiceAccountName: KubeControllerServiceAccount, - kubeControllerRoleName: EsKubeControllerRole, - kubeControllerRoleBindingName: EsKubeControllerRoleBinding, - kubeControllerName: EsKubeController, - kubeControllerConfigName: "elasticsearch", - kubeControllerMetricsName: EsKubeControllerMetrics, - kubeControllersRules: kubeControllerRolePolicyRules, - kubeControllerCalicoSystemPolicy: kubeControllerCalicoSystemPolicy, - enabledControllers: enabledControllers, - } +// NewCalicoKubeControllers builds the calico-kube-controllers component. The base is +// pure OSS (the common rules plus the node and loadbalancer controllers); the Calico +// Enterprise additions (extra RBAC, enterprise controllers, metrics TLS, the WAF v3 +// surface) are layered on by the enterprise modifier keyed by ModifierKey. +func NewCalicoKubeControllers(cfg *KubeControllersConfiguration) render.Component { + cfg.Name = KubeController + cfg.ConfigName = "default" + cfg.RoleName = KubeControllerRole + cfg.RoleBindingName = KubeControllerRoleBinding + cfg.MetricsName = KubeControllerMetrics + cfg.ModifierKey = render.ComponentNameKubeControllers + + cfg.Rules = KubeControllersRoleCommonRules(cfg) + cfg.EnabledControllers = []string{"node", "loadbalancer"} + + return NewKubeControllers(cfg) } type kubeControllersComponent struct { @@ -267,24 +191,6 @@ type kubeControllersComponent struct { // Internal state generated by the given configuration. calicoImage string - - kubeControllerServiceAccountName string - kubeControllerRoleName string - kubeControllerRoleBindingName string - kubeControllerName string - kubeControllerConfigName string - kubeControllerMetricsName string - - kubeControllersRules []rbacv1.PolicyRule - kubeControllerCalicoSystemPolicy *v3.NetworkPolicy - - enabledControllers []string - - // wasmImage is the fully-resolved OCI reference for the Coraza WAF wasm - // binary (Enterprise only). Surfaced to the kube-controllers binary via - // the WASM_IMAGE env var; consumed by the applicationlayer reconcilers - // in tigera/calico-private to program WAF policy attachments. - wasmImage string } func (c *kubeControllersComponent) ResolveImages(is *operatorv1.ImageSet) error { @@ -296,15 +202,6 @@ func (c *kubeControllersComponent) ResolveImages(is *operatorv1.ImageSet) error if err != nil { return err } - if c.cfg.Installation.Variant.IsEnterprise() && c.cfg.WAFGatewayExtensionEnabled { - // The Coraza WAF wasm is baked into the gateway envoy-proxy image as its - // final layer; Envoy Gateway extracts it from there. Point WASM_IMAGE at - // that same image (no standalone coraza-wasm image needed). - c.wasmImage, err = components.GetReference(components.ComponentGatewayAPIEnvoyProxy, reg, path, prefix, is) - if err != nil { - return err - } - } return nil } @@ -316,12 +213,14 @@ func (c *kubeControllersComponent) Objects() ([]client.Object, []client.Object) objectsToCreate := []client.Object{} objectsToDelete := []client.Object{} - if c.kubeControllerCalicoSystemPolicy != nil { - objectsToCreate = append(objectsToCreate, c.kubeControllerCalicoSystemPolicy) - // allow-tigera Tier was renamed to calico-system - objectsToDelete = append(objectsToDelete, - networkpolicy.DeprecatedAllowTigeraNetworkPolicyObject("es-kube-controller-access", c.cfg.Namespace), - ) + if c.cfg.NetworkPolicy != nil { + objectsToCreate = append(objectsToCreate, c.cfg.NetworkPolicy) + if c.cfg.DeprecatedNetworkPolicyName != "" { + // allow-tigera Tier was renamed to calico-system + objectsToDelete = append(objectsToDelete, + networkpolicy.DeprecatedAllowTigeraNetworkPolicyObject(c.cfg.DeprecatedNetworkPolicyName, c.cfg.Namespace), + ) + } } objectsToCreate = append(objectsToCreate, @@ -331,7 +230,7 @@ func (c *kubeControllersComponent) Objects() ([]client.Object, []client.Object) ) objectsToCreate = append(objectsToCreate, c.managedClusterRoleBindings()...) - if len(c.enabledControllers) > 0 { + if len(c.cfg.EnabledControllers) > 0 { // There's something to run, so create the deployment. objectsToCreate = append(objectsToCreate, c.controllersDeployment()) } else { @@ -346,26 +245,6 @@ func (c *kubeControllersComponent) Objects() ([]client.Object, []client.Object) objectsToCreate = append(objectsToCreate, secret.ToRuntimeObjects( secret.CopyToNamespace(c.cfg.Namespace, c.cfg.KubeControllersGatewaySecret)...)...) } - if c.cfg.WASMPullSecret != nil { - objectsToCreate = append(objectsToCreate, secret.ToRuntimeObjects( - secret.CopyToNamespace(c.cfg.Namespace, c.cfg.WASMPullSecret)...)...) - } - if c.cfg.WASMCACert != nil { - objectsToCreate = append(objectsToCreate, c.cfg.WASMCACert) - } - - // The in-process WAF admission webhook surface (Service fronting this Pod + - // ValidatingWebhookConfiguration). Rendered here, rather than as a - // passthrough in the core controller, so the objects are cleaned up when the - // WAF extension is disabled or the GatewayAPI CR is removed. - if c.kubeControllerName == KubeController { - webhookObjs := applicationlayer.WAFAdmissionWebhookComponents(c.cfg.WAFWebhookCABundle) - if c.cfg.WAFGatewayExtensionEnabled { - objectsToCreate = append(objectsToCreate, webhookObjs...) - } else { - objectsToDelete = append(objectsToDelete, webhookObjs...) - } - } if c.cfg.MetricsPort != 0 { objectsToCreate = append(objectsToCreate, c.prometheusService()) @@ -385,7 +264,13 @@ func (c *kubeControllersComponent) Ready() bool { return true } -func kubeControllersRoleCommonRules(cfg *KubeControllersConfiguration) []rbacv1.PolicyRule { +// ModifierKey implements render.Extensible. It is empty for es-calico-kube-controllers +// (never decorated) and set for calico-kube-controllers. +func (c *kubeControllersComponent) ModifierKey() string { + return c.cfg.ModifierKey +} + +func KubeControllersRoleCommonRules(cfg *KubeControllersConfiguration) []rbacv1.PolicyRule { rules := []rbacv1.PolicyRule{ { // Nodes are watched to monitor for deletions. @@ -512,158 +397,11 @@ func kubeControllersRoleCommonRules(cfg *KubeControllersConfiguration) []rbacv1. return rules } -func kubeControllersRoleEnterpriseCommonRules(cfg *KubeControllersConfiguration) []rbacv1.PolicyRule { - rules := []rbacv1.PolicyRule{ - { - APIGroups: []string{""}, - Resources: []string{"configmaps"}, - Verbs: []string{"watch", "list", "get", "update", "create", "delete"}, - }, - { - // The Federated Services Controller needs access to the remote kubeconfig secret - // in order to create a remote syncer. - APIGroups: []string{""}, - Resources: []string{"secrets"}, - Verbs: []string{"watch", "list", "get"}, - }, - { - // Needed to validate the license - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{"licensekeys"}, - Verbs: []string{"get", "watch", "list"}, - }, - { - // Needed to update the status of the LicenseKey with the result of license validation. - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"licensekeys/status"}, - Verbs: []string{"update"}, - }, - { - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{"deeppacketinspections"}, - Verbs: []string{"get", "watch", "list"}, - }, - { - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{"deeppacketinspections/status"}, - Verbs: []string{"update"}, - }, - { - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{"packetcaptures"}, - Verbs: []string{"get", "list", "update"}, - }, - { - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{"packetcaptures/status"}, - Verbs: []string{"update"}, - }, - } - - if cfg.WAFGatewayExtensionEnabled { - // WAF v3 (Gateway API add-on) RBAC. Gated by - // GatewayAPI.spec.extensions.waf.state == Enabled. - rules = append(rules, - // Application-layer (gateway-addons) reconcilers reconcile WAF resources - // against Gateway API targetRefs and emit events on the policy objects. - rbacv1.PolicyRule{ - APIGroups: []string{"applicationlayer.projectcalico.org"}, - Resources: []string{ - "wafpolicies", "globalwafpolicies", - "wafplugins", "globalwafplugins", - "wafvalidationpolicies", "globalwafvalidationpolicies", - }, - Verbs: []string{"get", "list", "watch", "create", "update", "patch", "delete"}, - }, - rbacv1.PolicyRule{ - APIGroups: []string{"applicationlayer.projectcalico.org"}, - Resources: []string{ - "wafpolicies/status", "globalwafpolicies/status", - "wafplugins/status", "globalwafplugins/status", - "wafvalidationpolicies/status", "globalwafvalidationpolicies/status", - }, - Verbs: []string{"get", "update", "patch"}, - }, - rbacv1.PolicyRule{ - APIGroups: []string{"applicationlayer.projectcalico.org"}, - Resources: []string{ - "wafpolicies/finalizers", "globalwafpolicies/finalizers", - "wafplugins/finalizers", "globalwafplugins/finalizers", - "wafvalidationpolicies/finalizers", "globalwafvalidationpolicies/finalizers", - }, - Verbs: []string{"update"}, - }, - rbacv1.PolicyRule{ - // Validate Gateway API targetRefs and surface attachment status. - APIGroups: []string{"gateway.networking.k8s.io"}, - Resources: []string{"gateways", "httproutes", "tcproutes", "tlsroutes", "grpcroutes"}, - Verbs: []string{"get", "list", "watch", "update", "patch"}, - }, - rbacv1.PolicyRule{ - APIGroups: []string{"gateway.networking.k8s.io"}, - Resources: []string{"gateways/status", "httproutes/status", "tcproutes/status", "tlsroutes/status", "grpcroutes/status"}, - Verbs: []string{"get", "update", "patch"}, - }, - // controller-runtime Reconcilers (e.g. the applicationlayer manager) record - // events on watched objects via Recorder.Eventf; both core and events.k8s.io - // API groups are emitted depending on the kubernetes version. - rbacv1.PolicyRule{ - APIGroups: []string{""}, - Resources: []string{"events"}, - Verbs: []string{"create", "patch"}, - }, - rbacv1.PolicyRule{ - APIGroups: []string{"events.k8s.io"}, - Resources: []string{"events"}, - Verbs: []string{"create", "patch"}, - }, - // Application-layer reconciler replicates the WAF wasm pull Secret from - // the controller namespace (calico-system) into each WAFPolicy's - // namespace so the rendered EnvoyExtensionPolicy can reference it. Also - // replicates CA-cert ConfigMaps when WASM_CA_CERT is set. - rbacv1.PolicyRule{ - APIGroups: []string{""}, - Resources: []string{"secrets", "configmaps"}, - Verbs: []string{"get", "list", "watch", "create", "update", "patch", "delete"}, - }, - // Application-layer reconciler emits one EnvoyExtensionPolicy per WAF - // targetRef to bind the Coraza wasm filter at the gateway / route. - rbacv1.PolicyRule{ - APIGroups: []string{"gateway.envoyproxy.io"}, - Resources: []string{"envoyextensionpolicies"}, - Verbs: []string{"get", "list", "watch", "create", "update", "patch", "delete"}, - }, - // Application-layer reconciler stamps each namespace with its - // allocated WAF rule-id range (applicationlayer.projectcalico.org/waf-id-range - // annotation) so application operators can author in-range rules. The - // base role already grants namespaces get/list/watch; the annotation - // write needs patch/update, gated to the WAF path. - rbacv1.PolicyRule{ - APIGroups: []string{""}, - Resources: []string{"namespaces"}, - Verbs: []string{"get", "patch", "update"}, - }, - ) - } - - if cfg.ManagementClusterConnection != nil { - rules = append(rules, - rbacv1.PolicyRule{ - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{"licensekeys"}, - Verbs: []string{"get", "create", "update", "list", "watch"}, - }, - ) - } - - return rules -} - func (c *kubeControllersComponent) controllersServiceAccount() *corev1.ServiceAccount { return &corev1.ServiceAccount{ TypeMeta: metav1.TypeMeta{Kind: "ServiceAccount", APIVersion: "v1"}, ObjectMeta: metav1.ObjectMeta{ - Name: c.kubeControllerServiceAccountName, + Name: KubeControllerServiceAccount, Namespace: c.cfg.Namespace, Labels: map[string]string{}, }, @@ -674,9 +412,9 @@ func (c *kubeControllersComponent) controllersClusterRole() *rbacv1.ClusterRole role := &rbacv1.ClusterRole{ TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, ObjectMeta: metav1.ObjectMeta{ - Name: c.kubeControllerRoleName, + Name: c.cfg.RoleName, }, - Rules: c.kubeControllersRules, + Rules: c.cfg.Rules, } return role @@ -699,7 +437,7 @@ func (c *kubeControllersComponent) controllersOCPFederationRoleBinding() *rbacv1 Subjects: []rbacv1.Subject{ { Kind: "ServiceAccount", - Name: KubeController, + Name: KubeControllerServiceAccount, Namespace: c.cfg.Namespace, }, }, @@ -708,95 +446,20 @@ func (c *kubeControllersComponent) controllersOCPFederationRoleBinding() *rbacv1 func (c *kubeControllersComponent) controllersDeployment() *appsv1.Deployment { env := []corev1.EnvVar{ - {Name: "KUBE_CONTROLLERS_CONFIG_NAME", Value: c.kubeControllerConfigName}, + {Name: "KUBE_CONTROLLERS_CONFIG_NAME", Value: c.cfg.ConfigName}, {Name: "DATASTORE_TYPE", Value: "kubernetes"}, - {Name: "ENABLED_CONTROLLERS", Value: strings.Join(c.enabledControllers, ",")}, - {Name: "DISABLE_KUBE_CONTROLLERS_CONFIG_API", Value: strconv.FormatBool(c.cfg.Tenant.MultiTenant() && c.kubeControllerConfigName == "elasticsearch")}, + {Name: "ENABLED_CONTROLLERS", Value: strings.Join(c.cfg.EnabledControllers, ",")}, + {Name: "DISABLE_KUBE_CONTROLLERS_CONFIG_API", Value: strconv.FormatBool(c.cfg.DisableConfigAPI)}, } env = append(env, c.cfg.K8sServiceEpPodNetwork.EnvVars()...) + env = append(env, c.cfg.ExtraEnv...) - if c.cfg.Installation.Variant.IsEnterprise() { - if c.cfg.Tenant != nil { - env = append(env, corev1.EnvVar{Name: "TENANT_ID", Value: c.cfg.Tenant.Spec.ID}) - } - - if c.kubeControllerName == EsKubeController { - // What started as a workaround is now the default behaviour. This feature uses our backend in order to - // log into Kibana for users from external identity providers, rather than configuring an authn realm - // in the Elastic stack. - env = append(env, corev1.EnvVar{Name: "ENABLE_ELASTICSEARCH_OIDC_WORKAROUND", Value: "true"}) - - if c.cfg.Authentication != nil { - env = append(env, - corev1.EnvVar{Name: "OIDC_AUTH_USERNAME_PREFIX", Value: c.cfg.Authentication.Spec.UsernamePrefix}, - corev1.EnvVar{Name: "OIDC_AUTH_GROUP_PREFIX", Value: c.cfg.Authentication.Spec.GroupsPrefix}, - ) - } - } - if c.cfg.TrustedBundle != nil { - env = append(env, corev1.EnvVar{Name: "MULTI_CLUSTER_FORWARDING_CA", Value: c.cfg.TrustedBundle.MountPath()}) - } - - if c.cfg.Installation.CalicoNetwork != nil && c.cfg.Installation.CalicoNetwork.MultiInterfaceMode != nil { - env = append(env, corev1.EnvVar{Name: "MULTI_INTERFACE_MODE", Value: c.cfg.Installation.CalicoNetwork.MultiInterfaceMode.Value()}) - } - - // Application-layer (gateway-addons / WAF v3) env vars, gated by - // GatewayAPI.spec.extensions.waf.state == Enabled. When the gate is - // off (default), none of the WASM_* env vars are rendered and the - // kube-controllers binary skips the WAF reconcilers entirely (see the - // applicationlayer entry in enabledControllers). - if c.cfg.WAFGatewayExtensionEnabled { - // Application-layer (gateway-addons) reconcilers consume the Coraza WAF - // wasm OCI reference from this env var to program WAF policy attachments. - // Empty when ResolveImages was not called for the Calico variant; the - // reconciler stamps Programmed=False/WASMUnavailable in that case. - if c.wasmImage != "" { - env = append(env, corev1.EnvVar{Name: "WASM_IMAGE", Value: c.wasmImage}) - } - - // WASM_PULL_SECRET names the imagePullSecret the reconciler replicates - // from the kube-controllers namespace into a WAFPolicy's namespace so - // the rendered EnvoyExtensionPolicy can pull the wasm OCI artifact from - // a private Tigera registry. Source the name from the first - // Installation.ImagePullSecrets entry so multi-tenant / BYO-registry - // installs reuse whatever pull secret operator already attaches here. - if c.cfg.WASMPullSecret != nil { - env = append(env, corev1.EnvVar{Name: "WASM_PULL_SECRET", Value: c.cfg.WASMPullSecret.Name}) - } - - // WASM_CA_CERT names the dedicated CA bundle ConfigMap (provisioned as - // WASMCACert) that the reconciler replicates alongside WASM_PULL_SECRET - // so the EnvoyExtensionPolicy wasm fetcher trusts the registry's TLS - // chain. Only set when the source ConfigMap is actually rendered. - if c.cfg.WASMCACert != nil { - env = append(env, corev1.EnvVar{Name: "WASM_CA_CERT", Value: c.cfg.WASMCACert.Name}) - } - } - } - - if c.cfg.MetricsServerTLS != nil { - env = append(env, - corev1.EnvVar{Name: "TLS_KEY_PATH", Value: c.cfg.MetricsServerTLS.VolumeMountKeyFilePath()}, - corev1.EnvVar{Name: "TLS_CRT_PATH", Value: c.cfg.MetricsServerTLS.VolumeMountCertificateFilePath()}, - corev1.EnvVar{Name: "CLIENT_COMMON_NAME", Value: monitor.PrometheusClientTLSSecretName}, - ) - } if c.cfg.TrustedBundle != nil { env = append(env, corev1.EnvVar{Name: "CA_CRT_PATH", Value: c.cfg.TrustedBundle.MountPath()}, ) } - if c.cfg.WAFWebhookServerTLS != nil { - // The in-process WAF admission webhook server (calico-private - // applicationlayer manager) reads its serving cert (tls.crt/tls.key) - // from this directory; the controller-runtime webhook server only - // registers when the cert is present. - env = append(env, - corev1.EnvVar{Name: "WAF_WEBHOOK_CERT_DIR", Value: filepath.Dir(c.cfg.WAFWebhookServerTLS.VolumeMountCertificateFilePath())}, - ) - } // UID 999 is used in kube-controller Dockerfile. sc := securitycontext.NewNonRootContext() @@ -829,7 +492,7 @@ func (c *kubeControllersComponent) controllersDeployment() *appsv1.Deployment { } container := corev1.Container{ - Name: c.kubeControllerName, + Name: c.cfg.Name, Image: c.calicoImage, Command: containerCommand, Env: env, @@ -840,34 +503,7 @@ func (c *kubeControllersComponent) controllersDeployment() *appsv1.Deployment { VolumeMounts: c.kubeControllersVolumeMounts(), } - if c.cfg.WAFWebhookServerTLS != nil { - // Expose the in-process WAF admission-webhook port that the - // tigera-waf-webhook Service forwards to. - container.Ports = append(container.Ports, corev1.ContainerPort{ - Name: "waf-webhook", - ContainerPort: applicationlayer.WAFWebhookContainerPort, - Protocol: corev1.ProtocolTCP, - }) - } - - if c.kubeControllerName == EsKubeController && !c.cfg.Tenant.MultiTenant() { - _, esHost, esPort, _ := url.ParseEndpoint(relasticsearch.GatewayEndpoint(c.SupportedOSType(), c.cfg.ClusterDomain, render.ElasticsearchNamespace)) - container.Env = append(container.Env, []corev1.EnvVar{ - relasticsearch.ElasticHostEnvVar(esHost), - relasticsearch.ElasticPortEnvVar(esPort), - relasticsearch.ElasticUsernameEnvVar(ElasticsearchKubeControllersUserSecret), - relasticsearch.ElasticPasswordEnvVar(ElasticsearchKubeControllersUserSecret), - relasticsearch.ElasticCAEnvVar(c.SupportedOSType()), - }...) - } - var initContainers []corev1.Container - if c.cfg.MetricsServerTLS != nil && c.cfg.MetricsServerTLS.UseCertificateManagement() { - initContainers = append(initContainers, c.cfg.MetricsServerTLS.InitContainer(c.cfg.Namespace, sc)) - } - if c.cfg.WAFWebhookServerTLS != nil && c.cfg.WAFWebhookServerTLS.UseCertificateManagement() { - initContainers = append(initContainers, c.cfg.WAFWebhookServerTLS.InitContainer(c.cfg.Namespace, sc)) - } tolerations := appendUniqueTolerations(c.cfg.Installation.ControlPlaneTolerations, rmeta.TolerateCriticalAddonsAndControlPlane...) if c.cfg.Installation.KubernetesProvider.IsGKE() { tolerations = appendUniqueTolerations(tolerations, rmeta.TolerateGKEARM64NoSchedule) @@ -876,7 +512,7 @@ func (c *kubeControllersComponent) controllersDeployment() *appsv1.Deployment { NodeSelector: c.cfg.Installation.ControlPlaneNodeSelector, Tolerations: tolerations, ImagePullSecrets: c.cfg.Installation.ImagePullSecrets, - ServiceAccountName: c.kubeControllerServiceAccountName, + ServiceAccountName: KubeControllerServiceAccount, InitContainers: initContainers, Containers: []corev1.Container{container}, Volumes: c.kubeControllersVolumes(), @@ -887,7 +523,7 @@ func (c *kubeControllersComponent) controllersDeployment() *appsv1.Deployment { d := appsv1.Deployment{ TypeMeta: metav1.TypeMeta{Kind: "Deployment", APIVersion: "apps/v1"}, ObjectMeta: metav1.ObjectMeta{ - Name: c.kubeControllerName, + Name: c.cfg.Name, Namespace: c.cfg.Namespace, }, Spec: appsv1.DeploymentSpec{ @@ -897,7 +533,7 @@ func (c *kubeControllersComponent) controllersDeployment() *appsv1.Deployment { }, Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ - Name: c.kubeControllerName, + Name: c.cfg.Name, Namespace: c.cfg.Namespace, Annotations: c.annotations(), }, @@ -929,20 +565,20 @@ func (c *kubeControllersComponent) controllersClusterRoleBinding() *rbacv1.Clust for _, ns := range c.cfg.BindingNamespaces { subjects = append(subjects, rbacv1.Subject{ Kind: "ServiceAccount", - Name: c.kubeControllerServiceAccountName, + Name: KubeControllerServiceAccount, Namespace: ns, }) } return &rbacv1.ClusterRoleBinding{ TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, ObjectMeta: metav1.ObjectMeta{ - Name: c.kubeControllerRoleBindingName, + Name: c.cfg.RoleBindingName, Labels: map[string]string{}, }, RoleRef: rbacv1.RoleRef{ APIGroup: "rbac.authorization.k8s.io", Kind: "ClusterRole", - Name: c.kubeControllerRoleName, + Name: c.cfg.RoleName, }, Subjects: subjects, } @@ -951,7 +587,7 @@ func (c *kubeControllersComponent) controllersClusterRoleBinding() *rbacv1.Clust func (c *kubeControllersComponent) managedClusterRoleBindings() []client.Object { if c.cfg.ManagementCluster != nil { return []client.Object{ - rcomp.ClusterRoleBinding(ManagedClustersWatchRoleBindingName, render.ManagedClustersWatchClusterRoleName, c.kubeControllerServiceAccountName, []string{c.cfg.Namespace}), + rcomp.ClusterRoleBinding(ManagedClustersWatchRoleBindingName, render.ManagedClustersWatchClusterRoleName, KubeControllerServiceAccount, []string{c.cfg.Namespace}), } } return []client.Object{} @@ -963,16 +599,16 @@ func (c *kubeControllersComponent) prometheusService() *corev1.Service { return &corev1.Service{ TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}, ObjectMeta: metav1.ObjectMeta{ - Name: c.kubeControllerMetricsName, + Name: c.cfg.MetricsName, Namespace: c.cfg.Namespace, Annotations: map[string]string{ "prometheus.io/scrape": "true", "prometheus.io/port": fmt.Sprintf("%d", c.cfg.MetricsPort), }, - Labels: map[string]string{"k8s-app": c.kubeControllerName}, + Labels: map[string]string{"k8s-app": c.cfg.Name}, }, Spec: corev1.ServiceSpec{ - Selector: map[string]string{"k8s-app": c.kubeControllerName}, + Selector: map[string]string{"k8s-app": c.cfg.Name}, // "Headless" service; prevent kube-proxy from rendering any rules for this service // (which is only intended for Prometheus to scrape). ClusterIP: "None", @@ -1001,9 +637,6 @@ func (c *kubeControllersComponent) annotations() map[string]string { am = make(map[string]string) } - if c.cfg.MetricsServerTLS != nil { - am[c.cfg.MetricsServerTLS.HashAnnotationKey()] = c.cfg.MetricsServerTLS.HashAnnotationValue() - } if c.cfg.KubeControllersGatewaySecret != nil { am[render.ElasticsearchUserHashAnnotation] = rmeta.AnnotationHash(c.cfg.KubeControllersGatewaySecret.Data) } @@ -1015,12 +648,6 @@ func (c *kubeControllersComponent) kubeControllersVolumeMounts() []corev1.Volume if c.cfg.TrustedBundle != nil { mounts = append(mounts, c.cfg.TrustedBundle.VolumeMounts(c.SupportedOSType())...) } - if c.cfg.MetricsServerTLS != nil { - mounts = append(mounts, c.cfg.MetricsServerTLS.VolumeMount(c.SupportedOSType())) - } - if c.cfg.WAFWebhookServerTLS != nil { - mounts = append(mounts, c.cfg.WAFWebhookServerTLS.VolumeMount(c.SupportedOSType())) - } return mounts } @@ -1029,12 +656,6 @@ func (c *kubeControllersComponent) kubeControllersVolumes() []corev1.Volume { if c.cfg.TrustedBundle != nil { volumes = append(volumes, c.cfg.TrustedBundle.Volume()) } - if c.cfg.MetricsServerTLS != nil { - volumes = append(volumes, c.cfg.MetricsServerTLS.Volume()) - } - if c.cfg.WAFWebhookServerTLS != nil { - volumes = append(volumes, c.cfg.WAFWebhookServerTLS.Volume()) - } return volumes } @@ -1077,20 +698,6 @@ func kubeControllersCalicoSystemPolicy(cfg *KubeControllersConfiguration) *v3.Ne }) } - // Allow the kube-apiserver to reach the in-process WAF admission webhook on - // :9443 (EV-6386). render-v3 wires the webhook Service/config/cert + the - // server, but without this ingress rule the calico-system default-deny drops - // the apiserver→:9443 call and every WAFPolicy/WAFPlugin admission times out. - if cfg.WAFGatewayExtensionEnabled { - ingressRules = append(ingressRules, v3.Rule{ - Action: v3.Allow, - Protocol: &networkpolicy.TCPProtocol, - Destination: v3.EntityRule{ - Ports: networkpolicy.Ports(uint16(applicationlayer.WAFWebhookContainerPort)), - }, - }) - } - if r, err := cfg.K8sServiceEp.DestinationEntityRule(); r != nil && err == nil { egressRules = append(egressRules, v3.Rule{ Action: v3.Allow, @@ -1115,53 +722,3 @@ func kubeControllersCalicoSystemPolicy(cfg *KubeControllersConfiguration) *v3.Ne }, } } - -func esKubeControllersCalicoSystemPolicy(cfg *KubeControllersConfiguration) *v3.NetworkPolicy { - if cfg.ManagementClusterConnection != nil { - return nil - } - - egressRules := []v3.Rule{} - egressRules = networkpolicy.AppendDNSEgressRules(egressRules, cfg.Installation.KubernetesProvider.IsOpenShift()) - egressRules = append(egressRules, []v3.Rule{ - { - Action: v3.Allow, - Protocol: &networkpolicy.TCPProtocol, - Destination: v3.EntityRule{ - Ports: networkpolicy.Ports(443, 6443, 12388), - }, - }, - }...) - - egressRules = append(egressRules, []v3.Rule{ - { - Action: v3.Allow, - Protocol: &networkpolicy.TCPProtocol, - Destination: networkpolicy.DefaultHelper().ESGatewayEntityRule(), - }, - }...) - - networkpolicyHelper := networkpolicy.Helper(cfg.Tenant.MultiTenant(), cfg.Namespace) - egressRules = append(egressRules, []v3.Rule{ - { - Action: v3.Allow, - Protocol: &networkpolicy.TCPProtocol, - Destination: networkpolicyHelper.ManagerEntityRule(), - }, - }...) - - return &v3.NetworkPolicy{ - TypeMeta: metav1.TypeMeta{Kind: "NetworkPolicy", APIVersion: "projectcalico.org/v3"}, - ObjectMeta: metav1.ObjectMeta{ - Name: EsKubeControllerNetworkPolicyName, - Namespace: cfg.Namespace, - }, - Spec: v3.NetworkPolicySpec{ - Order: &networkpolicy.HighPrecedenceOrder, - Tier: networkpolicy.CalicoTierName, - Selector: networkpolicy.KubernetesAppSelector(EsKubeController), - Types: []v3.PolicyType{v3.PolicyTypeEgress}, - Egress: egressRules, - }, - } -} diff --git a/pkg/render/kubecontrollers/kube-controllers_test.go b/pkg/render/kubecontrollers/kube-controllers_test.go index 42e65b1c23..19101def32 100644 --- a/pkg/render/kubecontrollers/kube-controllers_test.go +++ b/pkg/render/kubecontrollers/kube-controllers_test.go @@ -16,12 +16,10 @@ package kubecontrollers_test import ( "fmt" - "path/filepath" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - admissionregistrationv1 "k8s.io/api/admissionregistration/v1" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" @@ -41,15 +39,11 @@ import ( "github.com/tigera/operator/pkg/controller/k8sapi" ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" "github.com/tigera/operator/pkg/dns" - "github.com/tigera/operator/pkg/render" - "github.com/tigera/operator/pkg/render/applicationlayer" rmeta "github.com/tigera/operator/pkg/render/common/meta" "github.com/tigera/operator/pkg/render/common/networkpolicy" rtest "github.com/tigera/operator/pkg/render/common/test" "github.com/tigera/operator/pkg/render/kubecontrollers" "github.com/tigera/operator/pkg/render/testutils" - "github.com/tigera/operator/pkg/tls" - "github.com/tigera/operator/pkg/tls/certificatemanagement" ) var _ = Describe("kube-controllers rendering tests", func() { @@ -60,40 +54,10 @@ var _ = Describe("kube-controllers rendering tests", func() { cli client.Client ) - esEnvs := []corev1.EnvVar{ - {Name: "ELASTIC_HOST", Value: "tigera-secure-es-gateway-http.tigera-elasticsearch.svc"}, - {Name: "ELASTIC_PORT", Value: "9200", ValueFrom: nil}, - { - Name: "ELASTIC_USERNAME", Value: "", - ValueFrom: &corev1.EnvVarSource{ - SecretKeyRef: &corev1.SecretKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: "tigera-ee-kube-controllers-elasticsearch-access", - }, - Key: "username", - }, - }, - }, - { - Name: "ELASTIC_PASSWORD", Value: "", - ValueFrom: &corev1.EnvVarSource{ - SecretKeyRef: &corev1.SecretKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: "tigera-ee-kube-controllers-elasticsearch-access", - }, - Key: "password", - }, - }, - }, - {Name: "ELASTIC_CA", Value: certificatemanagement.TrustedCertBundleMountPath}, - } - expectedPolicyForUnmanaged := testutils.GetExpectedPolicyFromFile("../testutils/expected_policies/kubecontrollers.json") expectedPolicyForUnmanagedOCP := testutils.GetExpectedPolicyFromFile("../testutils/expected_policies/kubecontrollers_ocp.json") expectedPolicyForManaged := testutils.GetExpectedPolicyFromFile("../testutils/expected_policies/kubecontrollers_managed.json") expectedPolicyForManagedOCP := testutils.GetExpectedPolicyFromFile("../testutils/expected_policies/kubecontrollers_managed_ocp.json") - expectedESPolicy := testutils.GetExpectedPolicyFromFile("../testutils/expected_policies/es-kubecontrollers.json") - expectedESPolicyForOpenshift := testutils.GetExpectedPolicyFromFile("../testutils/expected_policies/es-kubecontrollers_ocp.json") BeforeEach(func() { // Initialize a default instance to use. Each test can override this to its @@ -240,175 +204,6 @@ var _ = Describe("kube-controllers rendering tests", func() { Expect(ds.Spec.Template.Spec.Tolerations).To(ConsistOf(rmeta.TolerateCriticalAddonsAndControlPlane)) }) - It("should render all calico kube-controllers resources for a default configuration (standalone) using CalicoEnterprise", func() { - expectedResources := []struct { - name string - ns string - group string - version string - kind string - }{ - {name: kubecontrollers.KubeControllerServiceAccount, ns: common.CalicoNamespace, group: "", version: "v1", kind: "ServiceAccount"}, - {name: kubecontrollers.KubeControllerRole, ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRole"}, - {name: kubecontrollers.KubeControllerRoleBinding, ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRoleBinding"}, - {name: kubecontrollers.KubeController, ns: common.CalicoNamespace, group: "apps", version: "v1", kind: "Deployment"}, - {name: kubecontrollers.WASMPullSecretName, ns: common.CalicoNamespace, group: "", version: "v1", kind: "Secret"}, - {name: kubecontrollers.WASMCACertName, ns: common.CalicoNamespace, group: "", version: "v1", kind: "ConfigMap"}, - {name: applicationlayer.WAFWebhookServiceName, ns: common.CalicoNamespace, group: "", version: "v1", kind: "Service"}, - {name: "tigera-waf.applicationlayer.projectcalico.org", ns: "", group: "admissionregistration.k8s.io", version: "v1", kind: "ValidatingWebhookConfiguration"}, - {name: kubecontrollers.KubeControllerMetrics, ns: common.CalicoNamespace, group: "", version: "v1", kind: "Service"}, - } - - instance.Variant = operatorv1.CalicoEnterprise - instance.ImagePullSecrets = []corev1.LocalObjectReference{{Name: "tigera-pull-secret"}} - cfg.MetricsPort = 9094 - // Opt in to the WAF Gateway API add-on so the WAF env vars + RBAC are rendered. - cfg.WAFGatewayExtensionEnabled = true - cfg.WAFWebhookCABundle = []byte("fake-ca-bundle") - // core_controller provisions a dedicated WAF wasm pull secret (a renamed - // copy of the install pull secret) so the reconciler can replicate it into - // WAFPolicy namespaces without clashing with the operator-managed - // tigera-pull-secret; surface it here so it renders and WASM_PULL_SECRET is set. - cfg.WASMPullSecret = &corev1.Secret{TypeMeta: metav1.TypeMeta{Kind: "Secret", APIVersion: "v1"}, ObjectMeta: metav1.ObjectMeta{Name: kubecontrollers.WASMPullSecretName, Namespace: common.CalicoNamespace}} - // Likewise core_controller provisions the dedicated WAF wasm CA-bundle - // ConfigMap (a renamed copy of the trusted bundle); surface it here so it - // renders and WASM_CA_CERT is set. - cfg.WASMCACert = &corev1.ConfigMap{TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"}, ObjectMeta: metav1.ObjectMeta{Name: kubecontrollers.WASMCACertName, Namespace: common.CalicoNamespace}} - - component := kubecontrollers.NewCalicoKubeControllers(&cfg) - Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() - Expect(len(resources)).To(Equal(len(expectedResources))) - - // Should render the correct resources. - i := 0 - for _, expectedRes := range expectedResources { - rtest.ExpectResourceTypeAndObjectMetadata(resources[i], expectedRes.name, expectedRes.ns, expectedRes.group, expectedRes.version, expectedRes.kind) - i++ - } - - // The Deployment should have the correct configuration. - dp := rtest.GetResource(resources, kubecontrollers.KubeController, common.CalicoNamespace, "apps", "v1", "Deployment").(*appsv1.Deployment) - - Expect(dp.Spec.Template.Spec.Containers[0].Image).To(Equal("test-reg/tigera/calico:" + components.ComponentTigeraCalico.Version)) - Expect(dp.Spec.Template.Spec.ImagePullSecrets).To(ContainElement(corev1.LocalObjectReference{Name: "tigera-pull-secret"})) - envs := dp.Spec.Template.Spec.Containers[0].Env - Expect(envs).To(ContainElement(corev1.EnvVar{ - Name: "ENABLED_CONTROLLERS", Value: "node,loadbalancer,service,federatedservices,usage,applicationlayer", - })) - // Application-layer reconcilers consume these env vars to program WAF - // EnvoyExtensionPolicy attachments. - Expect(envs).To(ContainElement(corev1.EnvVar{ - Name: "WASM_IMAGE", Value: "test-reg/tigera/envoy-proxy:" + components.ComponentGatewayAPIEnvoyProxy.Version, - })) - Expect(envs).To(ContainElement(corev1.EnvVar{ - Name: "WASM_PULL_SECRET", Value: kubecontrollers.WASMPullSecretName, - })) - // WASM_CA_CERT names the dedicated WAF trusted-bundle ConfigMap that the - // reconciler replicates into WAFPolicy namespaces (kept separate from the - // operator-managed tigera-ca-bundle the GatewayAPI render also copies there). - Expect(envs).To(ContainElement(corev1.EnvVar{ - Name: "WASM_CA_CERT", Value: kubecontrollers.WASMCACertName, - })) - - Expect(len(dp.Spec.Template.Spec.Containers[0].VolumeMounts)).To(Equal(1)) - Expect(len(dp.Spec.Template.Spec.Volumes)).To(Equal(1)) - - clusterRole := rtest.GetResource(resources, kubecontrollers.KubeControllerRole, "", "rbac.authorization.k8s.io", "v1", "ClusterRole").(*rbacv1.ClusterRole) - Expect(clusterRole.Rules).To(HaveLen(38), "cluster role should have 38 rules") - - // Application-layer reconciler RBAC: WAF CRDs (resources, /status, /finalizers). - Expect(clusterRole.Rules).To(ContainElement(rbacv1.PolicyRule{ - APIGroups: []string{"applicationlayer.projectcalico.org"}, - Resources: []string{ - "wafpolicies", "globalwafpolicies", - "wafplugins", "globalwafplugins", - "wafvalidationpolicies", "globalwafvalidationpolicies", - }, - Verbs: []string{"get", "list", "watch", "create", "update", "patch", "delete"}, - })) - Expect(clusterRole.Rules).To(ContainElement(rbacv1.PolicyRule{ - APIGroups: []string{"applicationlayer.projectcalico.org"}, - Resources: []string{ - "wafpolicies/status", "globalwafpolicies/status", - "wafplugins/status", "globalwafplugins/status", - "wafvalidationpolicies/status", "globalwafvalidationpolicies/status", - }, - Verbs: []string{"get", "update", "patch"}, - })) - Expect(clusterRole.Rules).To(ContainElement(rbacv1.PolicyRule{ - APIGroups: []string{"applicationlayer.projectcalico.org"}, - Resources: []string{ - "wafpolicies/finalizers", "globalwafpolicies/finalizers", - "wafplugins/finalizers", "globalwafplugins/finalizers", - "wafvalidationpolicies/finalizers", "globalwafvalidationpolicies/finalizers", - }, - Verbs: []string{"update"}, - })) - // Gateway API targetRef validation + status patching. - Expect(clusterRole.Rules).To(ContainElement(rbacv1.PolicyRule{ - APIGroups: []string{"gateway.networking.k8s.io"}, - Resources: []string{"gateways", "httproutes", "tcproutes", "tlsroutes", "grpcroutes"}, - Verbs: []string{"get", "list", "watch", "update", "patch"}, - })) - Expect(clusterRole.Rules).To(ContainElement(rbacv1.PolicyRule{ - APIGroups: []string{"gateway.networking.k8s.io"}, - Resources: []string{"gateways/status", "httproutes/status", "tcproutes/status", "tlsroutes/status", "grpcroutes/status"}, - Verbs: []string{"get", "update", "patch"}, - })) - // Recorder.Eventf emits to both core/events and events.k8s.io/events. - Expect(clusterRole.Rules).To(ContainElement(rbacv1.PolicyRule{ - APIGroups: []string{""}, - Resources: []string{"events"}, - Verbs: []string{"create", "patch"}, - })) - Expect(clusterRole.Rules).To(ContainElement(rbacv1.PolicyRule{ - APIGroups: []string{"events.k8s.io"}, - Resources: []string{"events"}, - Verbs: []string{"create", "patch"}, - })) - // Cluster-wide secrets+configmaps CRUD: reconciler replicates pull - // secrets and CA bundles from the controller namespace into target - // WAFPolicy namespaces. - Expect(clusterRole.Rules).To(ContainElement(rbacv1.PolicyRule{ - APIGroups: []string{""}, - Resources: []string{"secrets", "configmaps"}, - Verbs: []string{"get", "list", "watch", "create", "update", "patch", "delete"}, - })) - // EnvoyExtensionPolicy CRUD: reconciler renders one EEP per WAF targetRef. - Expect(clusterRole.Rules).To(ContainElement(rbacv1.PolicyRule{ - APIGroups: []string{"gateway.envoyproxy.io"}, - Resources: []string{"envoyextensionpolicies"}, - Verbs: []string{"get", "list", "watch", "create", "update", "patch", "delete"}, - })) - - ms := rtest.GetResource(resources, kubecontrollers.KubeControllerMetrics, common.CalicoNamespace, "", "v1", "Service").(*corev1.Service) - Expect(ms.Spec.ClusterIP).To(Equal("None"), "metrics service should be headless") - - // The webhook surface is rendered with the operator CA stamped into the - // ValidatingWebhookConfiguration caBundle. - vwc := rtest.GetResource(resources, "tigera-waf.applicationlayer.projectcalico.org", "", "admissionregistration.k8s.io", "v1", "ValidatingWebhookConfiguration").(*admissionregistrationv1.ValidatingWebhookConfiguration) - Expect(vwc.Webhooks).To(HaveLen(1)) - Expect(vwc.Webhooks[0].ClientConfig.CABundle).To(Equal([]byte("fake-ca-bundle"))) - }) - - It("should delete the WAF admission webhook surface when the WAF Gateway API add-on is disabled", func() { - instance.Variant = operatorv1.CalicoEnterprise - cfg.WAFGatewayExtensionEnabled = false - - component := kubecontrollers.NewCalicoKubeControllers(&cfg) - Expect(component.ResolveImages(nil)).To(BeNil()) - toCreate, toDelete := component.Objects() - - // Neither webhook object is created... - Expect(rtest.GetResource(toCreate, applicationlayer.WAFWebhookServiceName, common.CalicoNamespace, "", "v1", "Service")).To(BeNil()) - Expect(rtest.GetResource(toCreate, "tigera-waf.applicationlayer.projectcalico.org", "", "admissionregistration.k8s.io", "v1", "ValidatingWebhookConfiguration")).To(BeNil()) - // ...and both are queued for deletion, so disabling the feature (or - // removing the GatewayAPI CR) cleans up an earlier enabled render. - Expect(rtest.GetResource(toDelete, applicationlayer.WAFWebhookServiceName, common.CalicoNamespace, "", "v1", "Service")).NotTo(BeNil()) - Expect(rtest.GetResource(toDelete, "tigera-waf.applicationlayer.projectcalico.org", "", "admissionregistration.k8s.io", "v1", "ValidatingWebhookConfiguration")).NotTo(BeNil()) - }) - It("should render all calico kube-controllers resources using CalicoEnterprise on Openshift", func() { expectedResources := []struct { name string @@ -437,130 +232,7 @@ var _ = Describe("kube-controllers rendering tests", func() { } }) - It("should render all es-calico-kube-controllers resources for a default configuration (standalone) using CalicoEnterprise when logstorage and secrets exist", func() { - expectedResources := []struct { - name string - ns string - group string - version string - kind string - }{ - {name: kubecontrollers.EsKubeControllerNetworkPolicyName, ns: common.CalicoNamespace, group: "projectcalico.org", version: "v3", kind: "NetworkPolicy"}, - {name: "calico-kube-controllers", ns: common.CalicoNamespace, group: "", version: "v1", kind: "ServiceAccount"}, - {name: kubecontrollers.EsKubeControllerRole, ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRole"}, - {name: kubecontrollers.EsKubeControllerRoleBinding, ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRoleBinding"}, - {name: kubecontrollers.EsKubeController, ns: common.CalicoNamespace, group: "apps", version: "v1", kind: "Deployment"}, - {name: kubecontrollers.ElasticsearchKubeControllersUserSecret, ns: common.CalicoNamespace, group: "", version: "v1", kind: "Secret"}, - {name: kubecontrollers.EsKubeControllerMetrics, ns: common.CalicoNamespace, group: "", version: "v1", kind: "Service"}, - } - - instance.Variant = operatorv1.CalicoEnterprise - cfg.LogStorageExists = true - cfg.KubeControllersGatewaySecret = &testutils.KubeControllersUserSecret - cfg.MetricsPort = 9094 - // Opt in to the WAF Gateway API add-on so the WAF env vars + RBAC are rendered. - cfg.WAFGatewayExtensionEnabled = true - - component := kubecontrollers.NewElasticsearchKubeControllers(&cfg) - Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() - Expect(len(resources)).To(Equal(len(expectedResources))) - - // Should render the correct resources. - i := 0 - for _, expectedRes := range expectedResources { - rtest.ExpectResourceTypeAndObjectMetadata(resources[i], expectedRes.name, expectedRes.ns, expectedRes.group, expectedRes.version, expectedRes.kind) - i++ - } - - // The Deployment should have the correct configuration. - dp := rtest.GetResource(resources, kubecontrollers.EsKubeController, common.CalicoNamespace, "apps", "v1", "Deployment").(*appsv1.Deployment) - - Expect(dp.Spec.Template.Spec.Containers[0].Image).To(Equal("test-reg/tigera/calico:" + components.ComponentTigeraCalico.Version)) - envs := dp.Spec.Template.Spec.Containers[0].Env - Expect(envs).To(ContainElement(corev1.EnvVar{ - Name: "ENABLED_CONTROLLERS", Value: "authorization,elasticsearchconfiguration", - })) - Expect(envs).To(ContainElements(esEnvs)) - - Expect(dp.Spec.Template.Spec.Containers[0].VolumeMounts).To(HaveLen(1)) - Expect(dp.Spec.Template.Spec.Containers[0].VolumeMounts[0].Name).To(Equal("tigera-ca-bundle")) - Expect(dp.Spec.Template.Spec.Containers[0].VolumeMounts[0].MountPath).To(Equal("/etc/pki/tls/certs")) - - Expect(dp.Spec.Template.Spec.Volumes).To(HaveLen(1)) - Expect(dp.Spec.Template.Spec.Volumes[0].Name).To(Equal("tigera-ca-bundle")) - Expect(dp.Spec.Template.Spec.Volumes[0].ConfigMap.Name).To(Equal("tigera-ca-bundle")) - - clusterRole := rtest.GetResource(resources, kubecontrollers.EsKubeControllerRole, "", "rbac.authorization.k8s.io", "v1", "ClusterRole").(*rbacv1.ClusterRole) - Expect(clusterRole.Rules).To(HaveLen(36), "cluster role should have 36 rules") - Expect(clusterRole.Rules).To(ContainElement( - rbacv1.PolicyRule{ - APIGroups: []string{""}, - Resources: []string{"configmaps"}, - Verbs: []string{"watch", "list", "get", "update", "create", "delete"}, - })) - Expect(clusterRole.Rules).To(ContainElement( - rbacv1.PolicyRule{ - APIGroups: []string{""}, - Resources: []string{"secrets"}, - Verbs: []string{"watch", "list", "get"}, - })) - }) - - It("should render all calico-kube-controllers resources for a default configuration using CalicoEnterprise and ClusterType is Management", func() { - expectedResources := []struct { - name string - ns string - group string - version string - kind string - }{ - {name: kubecontrollers.KubeControllerServiceAccount, ns: common.CalicoNamespace, group: "", version: "v1", kind: "ServiceAccount"}, - {name: kubecontrollers.KubeControllerRole, ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRole"}, - {name: kubecontrollers.KubeControllerRoleBinding, ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRoleBinding"}, - {name: kubecontrollers.ManagedClustersWatchRoleBindingName, ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRoleBinding"}, - {name: kubecontrollers.KubeController, ns: common.CalicoNamespace, group: "apps", version: "v1", kind: "Deployment"}, - {name: applicationlayer.WAFWebhookServiceName, ns: common.CalicoNamespace, group: "", version: "v1", kind: "Service"}, - {name: "tigera-waf.applicationlayer.projectcalico.org", ns: "", group: "admissionregistration.k8s.io", version: "v1", kind: "ValidatingWebhookConfiguration"}, - {name: kubecontrollers.KubeControllerMetrics, ns: common.CalicoNamespace, group: "", version: "v1", kind: "Service"}, - } - - // Override configuration to match expected Enterprise config. - instance.Variant = operatorv1.CalicoEnterprise - cfg.ManagementCluster = &operatorv1.ManagementCluster{} - cfg.MetricsPort = 9094 - // Opt in to the WAF Gateway API add-on so the WAF env vars + RBAC are rendered. - cfg.WAFGatewayExtensionEnabled = true - - component := kubecontrollers.NewCalicoKubeControllers(&cfg) - Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() - Expect(len(resources)).To(Equal(len(expectedResources))) - - // Should render the correct resources. - i := 0 - for _, expectedRes := range expectedResources { - rtest.ExpectResourceTypeAndObjectMetadata(resources[i], expectedRes.name, expectedRes.ns, expectedRes.group, expectedRes.version, expectedRes.kind) - i++ - } - - // The Deployment should have the correct configuration. - dp := rtest.GetResource(resources, kubecontrollers.KubeController, common.CalicoNamespace, "apps", "v1", "Deployment").(*appsv1.Deployment) - - envs := dp.Spec.Template.Spec.Containers[0].Env - Expect(envs).To(ContainElement(corev1.EnvVar{ - Name: "ENABLED_CONTROLLERS", - Value: "node,loadbalancer,service,federatedservices,usage,applicationlayer", - })) - - Expect(len(dp.Spec.Template.Spec.Containers[0].VolumeMounts)).To(Equal(1)) - - Expect(len(dp.Spec.Template.Spec.Volumes)).To(Equal(1)) - Expect(dp.Spec.Template.Spec.Containers[0].Image).To(Equal("test-reg/tigera/calico:" + components.ComponentTigeraCalico.Version)) - }) It("should render all calico-kube-controllers resources for a default configuration using CalicoEnterprise", func() { - var defaultMode int32 = 420 - var kubeControllerTLS certificatemanagement.KeyPairInterface expectedResources := []struct { name string ns string @@ -575,15 +247,14 @@ var _ = Describe("kube-controllers rendering tests", func() { {name: kubecontrollers.KubeControllerMetrics, ns: common.CalicoNamespace, group: "", version: "v1", kind: "Service"}, } + // The metrics serving TLS (TLS_KEY_PATH/TLS_CRT_PATH/CLIENT_COMMON_NAME env, + // the keypair volume + mount) is layered on by the enterprise modifier, so + // the base render here carries only the trusted bundle. expectedEnv := []corev1.EnvVar{ - {Name: "TLS_KEY_PATH", Value: "/calico-kube-controllers-metrics-tls/tls.key"}, - {Name: "TLS_CRT_PATH", Value: "/calico-kube-controllers-metrics-tls/tls.crt"}, - {Name: "CLIENT_COMMON_NAME", Value: "calico-node-prometheus-client-tls"}, {Name: "CA_CRT_PATH", Value: "/etc/pki/tls/certs/tigera-ca-bundle.crt"}, } expectedVolumeMounts := []corev1.VolumeMount{ {Name: "tigera-ca-bundle", MountPath: "/etc/pki/tls/certs", ReadOnly: true}, - {Name: "calico-kube-controllers-metrics-tls", MountPath: "/calico-kube-controllers-metrics-tls", ReadOnly: true}, } expectedVolume := []corev1.Volume{ { @@ -594,34 +265,11 @@ var _ = Describe("kube-controllers rendering tests", func() { }, }, }, - { - Name: "calico-kube-controllers-metrics-tls", - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: "calico-kube-controllers-metrics-tls", - DefaultMode: &defaultMode, - }, - }, - }, } - scheme := runtime.NewScheme() - Expect(apis.AddToScheme(scheme, false)).NotTo(HaveOccurred()) - cli := ctrlrfake.DefaultFakeClientBuilder(scheme).Build() - - certificateManager, err := certificatemanager.Create(cli, nil, dns.DefaultClusterDomain, common.OperatorNamespace(), certificatemanager.AllowCACreation()) - Expect(err).NotTo(HaveOccurred()) - - kubeControllerTLS, err = certificateManager.GetOrCreateKeyPair(cli, - kubecontrollers.KubeControllerPrometheusTLSSecret, - common.OperatorNamespace(), - dns.GetServiceDNSNames(kubecontrollers.KubeControllerMetrics, common.CalicoNamespace, dns.DefaultClusterDomain)) - Expect(err).NotTo(HaveOccurred()) - // Override configuration to match expected Enterprise config. instance.Variant = operatorv1.CalicoEnterprise cfg.MetricsPort = 9094 - cfg.MetricsServerTLS = kubeControllerTLS component := kubecontrollers.NewCalicoKubeControllers(&cfg) Expect(component.ResolveImages(nil)).To(BeNil()) @@ -641,142 +289,15 @@ var _ = Describe("kube-controllers rendering tests", func() { envs := dp.Spec.Template.Spec.Containers[0].Env Expect(envs).To(ContainElements(expectedEnv)) - Expect(len(dp.Spec.Template.Spec.Containers[0].VolumeMounts)).To(Equal(2)) + Expect(len(dp.Spec.Template.Spec.Containers[0].VolumeMounts)).To(Equal(1)) Expect(dp.Spec.Template.Spec.Containers[0].VolumeMounts).To(ContainElements(expectedVolumeMounts)) - Expect(len(dp.Spec.Template.Spec.Volumes)).To(Equal(2)) + Expect(len(dp.Spec.Template.Spec.Volumes)).To(Equal(1)) Expect(dp.Spec.Template.Spec.Volumes).To(ContainElements(expectedVolume)) Expect(dp.Spec.Template.Spec.Containers[0].Image).To(Equal("test-reg/tigera/calico:" + components.ComponentTigeraCalico.Version)) }) - It("should mount the WAF admission webhook serving cert and expose its port when WAF is enabled", func() { - certificateManager, err := certificatemanager.Create(cli, nil, dns.DefaultClusterDomain, common.OperatorNamespace(), certificatemanager.AllowCACreation()) - Expect(err).NotTo(HaveOccurred()) - wafTLS, err := certificateManager.GetOrCreateKeyPair(cli, - applicationlayer.WAFWebhookServerTLSSecretName, - common.OperatorNamespace(), - dns.GetServiceDNSNames(applicationlayer.WAFWebhookServiceName, common.CalicoNamespace, dns.DefaultClusterDomain)) - Expect(err).NotTo(HaveOccurred()) - - instance.Variant = operatorv1.CalicoEnterprise - cfg.WAFGatewayExtensionEnabled = true - cfg.WAFWebhookServerTLS = wafTLS - - component := kubecontrollers.NewCalicoKubeControllers(&cfg) - Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() - - dp := rtest.GetResource(resources, kubecontrollers.KubeController, common.CalicoNamespace, "apps", "v1", "Deployment").(*appsv1.Deployment) - c := dp.Spec.Template.Spec.Containers[0] - - // Serving cert is mounted and advertised to the in-process webhook server. - Expect(dp.Spec.Template.Spec.Volumes).To(ContainElement(wafTLS.Volume())) - Expect(c.VolumeMounts).To(ContainElement(wafTLS.VolumeMount(rmeta.OSTypeLinux))) - Expect(c.Env).To(ContainElement(corev1.EnvVar{ - Name: "WAF_WEBHOOK_CERT_DIR", - Value: filepath.Dir(wafTLS.VolumeMountCertificateFilePath()), - })) - - // In-process webhook port exposed for the tigera-waf-webhook Service. - Expect(c.Ports).To(ContainElement(corev1.ContainerPort{ - Name: "waf-webhook", - ContainerPort: int32(9443), - Protocol: corev1.ProtocolTCP, - })) - - // namespaces patch/update RBAC for the waf-id-range annotation. - clusterRole := rtest.GetResource(resources, "calico-kube-controllers", "", "rbac.authorization.k8s.io", "v1", "ClusterRole").(*rbacv1.ClusterRole) - Expect(clusterRole.Rules).To(ContainElement(rbacv1.PolicyRule{ - APIGroups: []string{""}, - Resources: []string{"namespaces"}, - Verbs: []string{"get", "patch", "update"}, - })) - }) - - It("should render all es-calico-kube-controllers resources for a default configuration using CalicoEnterprise and ClusterType is Management", func() { - expectedResources := []struct { - name string - ns string - group string - version string - kind string - }{ - {name: kubecontrollers.EsKubeControllerNetworkPolicyName, ns: common.CalicoNamespace, group: "projectcalico.org", version: "v3", kind: "NetworkPolicy"}, - {name: "calico-kube-controllers", ns: common.CalicoNamespace, group: "", version: "v1", kind: "ServiceAccount"}, - {name: kubecontrollers.EsKubeControllerRole, ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRole"}, - {name: kubecontrollers.EsKubeControllerRoleBinding, ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRoleBinding"}, - {name: kubecontrollers.ManagedClustersWatchRoleBindingName, ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRoleBinding"}, - {name: kubecontrollers.EsKubeController, ns: common.CalicoNamespace, group: "apps", version: "v1", kind: "Deployment"}, - {name: kubecontrollers.ElasticsearchKubeControllersUserSecret, ns: common.CalicoNamespace, group: "", version: "v1", kind: "Secret"}, - {name: kubecontrollers.EsKubeControllerMetrics, ns: common.CalicoNamespace, group: "", version: "v1", kind: "Service"}, - } - - // Override configuration to match expected Enterprise config. - instance.Variant = operatorv1.CalicoEnterprise - cfg.LogStorageExists = true - cfg.ManagementCluster = &operatorv1.ManagementCluster{} - cfg.KubeControllersGatewaySecret = &testutils.KubeControllersUserSecret - cfg.MetricsPort = 9094 - // Opt in to the WAF Gateway API add-on so the WAF env vars + RBAC are rendered. - cfg.WAFGatewayExtensionEnabled = true - - component := kubecontrollers.NewElasticsearchKubeControllers(&cfg) - Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() - Expect(len(resources)).To(Equal(len(expectedResources))) - - // Should render the correct resources. - i := 0 - for _, expectedRes := range expectedResources { - rtest.ExpectResourceTypeAndObjectMetadata(resources[i], expectedRes.name, expectedRes.ns, expectedRes.group, expectedRes.version, expectedRes.kind) - i++ - } - - // The Deployment should have the correct configuration. - dp := rtest.GetResource(resources, kubecontrollers.EsKubeController, common.CalicoNamespace, "apps", "v1", "Deployment").(*appsv1.Deployment) - - envs := dp.Spec.Template.Spec.Containers[0].Env - Expect(envs).To(ContainElement(corev1.EnvVar{ - Name: "ENABLED_CONTROLLERS", - Value: "authorization,elasticsearchconfiguration,managedcluster", - })) - - Expect(dp.Spec.Template.Spec.Containers[0].VolumeMounts).To(HaveLen(1)) - Expect(dp.Spec.Template.Spec.Containers[0].VolumeMounts[0].Name).To(Equal("tigera-ca-bundle")) - Expect(dp.Spec.Template.Spec.Containers[0].VolumeMounts[0].MountPath).To(Equal("/etc/pki/tls/certs")) - - Expect(dp.Spec.Template.Spec.Volumes).To(HaveLen(1)) - Expect(dp.Spec.Template.Spec.Volumes[0].Name).To(Equal("tigera-ca-bundle")) - Expect(dp.Spec.Template.Spec.Volumes[0].ConfigMap.Name).To(Equal("tigera-ca-bundle")) - - Expect(dp.Spec.Template.Spec.Containers[0].Image).To(Equal("test-reg/tigera/calico:" + components.ComponentTigeraCalico.Version)) - - clusterRole := rtest.GetResource(resources, kubecontrollers.EsKubeControllerRole, "", "rbac.authorization.k8s.io", "v1", "ClusterRole").(*rbacv1.ClusterRole) - Expect(clusterRole.Rules).To(HaveLen(36), "cluster role should have 36 rules") - Expect(clusterRole.Rules).To(ContainElement( - rbacv1.PolicyRule{ - APIGroups: []string{""}, - Resources: []string{"configmaps"}, - Verbs: []string{"watch", "list", "get", "update", "create", "delete"}, - })) - Expect(clusterRole.Rules).To(ContainElement( - rbacv1.PolicyRule{ - APIGroups: []string{""}, - Resources: []string{"secrets"}, - Verbs: []string{"watch", "list", "get"}, - })) - roleBindingWatch := rtest.GetResource(resources, kubecontrollers.ManagedClustersWatchRoleBindingName, "", "rbac.authorization.k8s.io", "v1", "ClusterRoleBinding").(*rbacv1.ClusterRoleBinding) - Expect(roleBindingWatch.RoleRef.Name).To(Equal(render.ManagedClustersWatchClusterRoleName)) - Expect(roleBindingWatch.Subjects).To(ConsistOf([]rbacv1.Subject{ - { - Kind: "ServiceAccount", - Name: kubecontrollers.KubeControllerServiceAccount, - Namespace: common.CalicoNamespace, - }, - })) - }) - It("should include a ControlPlaneNodeSelector when specified", func() { expectedResources := []struct { name string @@ -872,44 +393,6 @@ var _ = Describe("kube-controllers rendering tests", func() { Expect(passed).To(Equal(true)) }) - It("should add the OIDC prefix env variables", func() { - instance.Variant = operatorv1.CalicoEnterprise - cfg.LogStorageExists = true - cfg.ManagementCluster = &operatorv1.ManagementCluster{} - cfg.KubeControllersGatewaySecret = &testutils.KubeControllersUserSecret - cfg.MetricsPort = 9094 - cfg.Authentication = &operatorv1.Authentication{Spec: operatorv1.AuthenticationSpec{ - UsernamePrefix: "uOIDC:", - GroupsPrefix: "gOIDC:", - Openshift: &operatorv1.AuthenticationOpenshift{IssuerURL: "https://api.example.com"}, - }} - - component := kubecontrollers.NewElasticsearchKubeControllers(&cfg) - Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() - - depResource := rtest.GetResource(resources, kubecontrollers.EsKubeController, common.CalicoNamespace, "apps", "v1", "Deployment") - Expect(depResource).ToNot(BeNil()) - deployment := depResource.(*appsv1.Deployment) - - var usernamePrefix, groupPrefix string - for _, container := range deployment.Spec.Template.Spec.Containers { - if container.Name == kubecontrollers.EsKubeController { - for _, env := range container.Env { - switch env.Name { - case "OIDC_AUTH_USERNAME_PREFIX": - usernamePrefix = env.Value - case "OIDC_AUTH_GROUP_PREFIX": - groupPrefix = env.Value - } - } - } - } - - Expect(usernamePrefix).To(Equal("uOIDC:")) - Expect(groupPrefix).To(Equal("gOIDC:")) - }) - Context("With calico-kube-controllers overrides", func() { rr1 := corev1.ResourceRequirements{ Limits: corev1.ResourceList{ @@ -1138,35 +621,6 @@ var _ = Describe("kube-controllers rendering tests", func() { }) }) - When("enableESOIDCWorkaround is true", func() { - It("should set the ENABLE_ELASTICSEARCH_OIDC_WORKAROUND env variable to true", func() { - instance.Variant = operatorv1.CalicoEnterprise - cfg.LogStorageExists = true - cfg.ManagementCluster = &operatorv1.ManagementCluster{} - cfg.KubeControllersGatewaySecret = &testutils.KubeControllersUserSecret - cfg.MetricsPort = 9094 - component := kubecontrollers.NewElasticsearchKubeControllers(&cfg) - resources, _ := component.Objects() - - depResource := rtest.GetResource(resources, kubecontrollers.EsKubeController, common.CalicoNamespace, "apps", "v1", "Deployment") - Expect(depResource).ToNot(BeNil()) - deployment := depResource.(*appsv1.Deployment) - - var esLicenseType string - for _, container := range deployment.Spec.Template.Spec.Containers { - if container.Name == kubecontrollers.EsKubeController { - for _, env := range container.Env { - if env.Name == "ENABLE_ELASTICSEARCH_OIDC_WORKAROUND" { - esLicenseType = env.Value - } - } - } - } - - Expect(esLicenseType).To(Equal("true")) - }) - }) - It("should add the KUBERNETES_SERVICE_... variables", func() { cfg.K8sServiceEpPodNetwork = k8sapi.ServiceEndpoint{ Host: "k8shost", @@ -1274,70 +728,6 @@ var _ = Describe("kube-controllers rendering tests", func() { }) }) - Context("es-kube-controllers calico-system rendering", func() { - policyName := types.NamespacedName{Name: "calico-system.es-kube-controller-access", Namespace: common.CalicoNamespace} - - getExpectedPolicy := func(scenario testutils.CalicoSystemScenario) *v3.NetworkPolicy { - if scenario.ManagedCluster { - return nil - } - - return testutils.SelectPolicyByProvider(scenario, expectedESPolicy, expectedESPolicyForOpenshift) - } - - DescribeTable("should render calico-system policy", - func(scenario testutils.CalicoSystemScenario) { - if scenario.OpenShift { - cfg.Installation.KubernetesProvider = operatorv1.ProviderOpenShift - } else { - cfg.Installation.KubernetesProvider = operatorv1.ProviderNone - } - if scenario.ManagedCluster { - cfg.ManagementClusterConnection = &operatorv1.ManagementClusterConnection{} - } else { - cfg.ManagementClusterConnection = nil - } - instance.Variant = operatorv1.CalicoEnterprise - cfg.LogStorageExists = true - cfg.KubeControllersGatewaySecret = &testutils.KubeControllersUserSecret - - component := kubecontrollers.NewElasticsearchKubeControllers(&cfg) - resources, _ := component.Objects() - - policy := testutils.GetCalicoSystemPolicyFromResources(policyName, resources) - expectedPolicy := getExpectedPolicy(scenario) - Expect(policy).To(Equal(expectedPolicy)) - }, - Entry("for management/standalone, kube-dns", testutils.CalicoSystemScenario{ManagedCluster: false, OpenShift: false}), - Entry("for management/standalone, openshift-dns", testutils.CalicoSystemScenario{ManagedCluster: false, OpenShift: true}), - Entry("for managed, kube-dns", testutils.CalicoSystemScenario{ManagedCluster: true, OpenShift: false}), - Entry("for managed, openshift-dns", testutils.CalicoSystemScenario{ManagedCluster: true, OpenShift: true}), - ) - }) - - It("should render init containers when certificate management is enabled", func() { - instance.Variant = operatorv1.CalicoEnterprise - cfg.MetricsPort = 9094 - ca, _ := tls.MakeCA(rmeta.DefaultOperatorCASignerName()) - cert, _, _ := ca.Config.GetPEMBytes() // create a valid pem block - cfg.Installation.CertificateManagement = &operatorv1.CertificateManagement{CACert: cert} - - certificateManager, err := certificatemanager.Create(cli, cfg.Installation, dns.DefaultClusterDomain, common.OperatorNamespace(), certificatemanager.AllowCACreation()) - Expect(err).NotTo(HaveOccurred()) - - tls, err := certificateManager.GetOrCreateKeyPair(cli, kubecontrollers.KubeControllerPrometheusTLSSecret, common.OperatorNamespace(), []string{""}) - Expect(err).NotTo(HaveOccurred()) - - cfg.MetricsServerTLS = tls - - resources, _ := kubecontrollers.NewCalicoKubeControllers(&cfg).Objects() - - dp := rtest.GetResource(resources, kubecontrollers.KubeController, common.CalicoNamespace, "apps", "v1", "Deployment").(*appsv1.Deployment) - Expect(dp.Spec.Template.Spec.InitContainers).To(HaveLen(1)) - csrInitContainer := dp.Spec.Template.Spec.InitContainers[0] - Expect(csrInitContainer.Name).To(Equal(fmt.Sprintf("%v-key-cert-provisioner", kubecontrollers.KubeControllerPrometheusTLSSecret))) - }) - It("should add egress policy with Enterprise variant and K8SServiceEndpoint defined", func() { cfg.K8sServiceEp.Host = "k8shost" cfg.K8sServiceEp.Port = "1234" diff --git a/pkg/render/kubecontrollers/waf_pull_secret.go b/pkg/render/kubecontrollers/waf_pull_secret.go deleted file mode 100644 index 02ada09f8a..0000000000 --- a/pkg/render/kubecontrollers/waf_pull_secret.go +++ /dev/null @@ -1,101 +0,0 @@ -// Copyright (c) 2026 Tigera, Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package kubecontrollers - -import ( - "encoding/json" - "fmt" - - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - "github.com/tigera/operator/pkg/common" -) - -// MergeWAFPullSecret synthesizes the dedicated WAF wasm pull secret -// (tigera-waf-pull-secret) by merging the registry auths of every Installation -// pull secret. The EnvoyExtensionPolicy image source takes a single -// pullSecretRef, so a merged secret is the only way to honor multiple -// Installation pull secrets for the Coraza wasm OCI pull (e.g. the Tigera pull -// secret plus credentials for a private registry mirror). -// -// If the same registry appears in more than one secret, the first secret in -// Installation order wins. Secrets that cannot be parsed are skipped and their -// names returned, so the caller can log them without failing the reconcile. -// Returns a nil Secret when no registry auths could be collected. -func MergeWAFPullSecret(pullSecrets []*corev1.Secret) (*corev1.Secret, []string) { - merged := map[string]json.RawMessage{} - var skipped []string - for _, s := range pullSecrets { - auths, err := registryAuths(s) - if err != nil { - skipped = append(skipped, s.Name) - continue - } - for registry, auth := range auths { - if _, ok := merged[registry]; !ok { - merged[registry] = auth - } - } - } - if len(merged) == 0 { - return nil, skipped - } - - // Marshalling a map sorts its keys, so the rendered bytes are deterministic - // and do not churn the object on every reconcile. - data, err := json.Marshal(map[string]map[string]json.RawMessage{"auths": merged}) - if err != nil { - // Each auth entry round-trips from a successful Unmarshal above, so - // this cannot fail in practice; treat it as nothing to render. - return nil, skipped - } - - return &corev1.Secret{ - TypeMeta: metav1.TypeMeta{Kind: "Secret", APIVersion: "v1"}, - ObjectMeta: metav1.ObjectMeta{Name: WASMPullSecretName, Namespace: common.CalicoNamespace}, - Type: corev1.SecretTypeDockerConfigJson, - Data: map[string][]byte{corev1.DockerConfigJsonKey: data}, - }, skipped -} - -// registryAuths extracts the per-registry auth entries from a pull secret of -// either the dockerconfigjson type (auths nested under an "auths" key) or the -// legacy dockercfg type (a bare registry -> auth map). -func registryAuths(s *corev1.Secret) (map[string]json.RawMessage, error) { - if raw, ok := s.Data[corev1.DockerConfigJsonKey]; ok { - var cfg struct { - Auths map[string]json.RawMessage `json:"auths"` - } - if err := json.Unmarshal(raw, &cfg); err != nil { - return nil, err - } - if len(cfg.Auths) == 0 { - return nil, fmt.Errorf("secret %s has no auths entries", s.Name) - } - return cfg.Auths, nil - } - if raw, ok := s.Data[corev1.DockerConfigKey]; ok { - var auths map[string]json.RawMessage - if err := json.Unmarshal(raw, &auths); err != nil { - return nil, err - } - if len(auths) == 0 { - return nil, fmt.Errorf("secret %s has no auths entries", s.Name) - } - return auths, nil - } - return nil, fmt.Errorf("secret %s has neither a %s nor a %s key", s.Name, corev1.DockerConfigJsonKey, corev1.DockerConfigKey) -} diff --git a/pkg/render/logstorage/esgateway/esgateway_test.go b/pkg/render/logstorage/esgateway/esgateway_test.go index 7f29d298d7..ce81684e66 100644 --- a/pkg/render/logstorage/esgateway/esgateway_test.go +++ b/pkg/render/logstorage/esgateway/esgateway_test.go @@ -37,12 +37,12 @@ import ( "github.com/tigera/operator/pkg/controller/certificatemanager" ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" "github.com/tigera/operator/pkg/dns" + entkubecontrollers "github.com/tigera/operator/pkg/enterprise/kubecontrollers" "github.com/tigera/operator/pkg/render" relasticsearch "github.com/tigera/operator/pkg/render/common/elasticsearch" rmeta "github.com/tigera/operator/pkg/render/common/meta" "github.com/tigera/operator/pkg/render/common/podaffinity" rtest "github.com/tigera/operator/pkg/render/common/test" - "github.com/tigera/operator/pkg/render/kubecontrollers" "github.com/tigera/operator/pkg/render/testutils" "github.com/tigera/operator/pkg/tls" "github.com/tigera/operator/pkg/tls/certificatemanagement" @@ -81,9 +81,9 @@ var _ = Describe("ES Gateway rendering tests", func() { ESGatewayKeyPair: kp, TrustedBundle: bundle, KubeControllersUserSecrets: []*corev1.Secret{ - {ObjectMeta: metav1.ObjectMeta{Name: kubecontrollers.ElasticsearchKubeControllersUserSecret, Namespace: common.OperatorNamespace()}}, - {ObjectMeta: metav1.ObjectMeta{Name: kubecontrollers.ElasticsearchKubeControllersVerificationUserSecret, Namespace: render.ElasticsearchNamespace}}, - {ObjectMeta: metav1.ObjectMeta{Name: kubecontrollers.ElasticsearchKubeControllersSecureUserSecret, Namespace: render.ElasticsearchNamespace}}, + {ObjectMeta: metav1.ObjectMeta{Name: entkubecontrollers.ElasticsearchKubeControllersUserSecret, Namespace: common.OperatorNamespace()}}, + {ObjectMeta: metav1.ObjectMeta{Name: entkubecontrollers.ElasticsearchKubeControllersVerificationUserSecret, Namespace: render.ElasticsearchNamespace}}, + {ObjectMeta: metav1.ObjectMeta{Name: entkubecontrollers.ElasticsearchKubeControllersSecureUserSecret, Namespace: render.ElasticsearchNamespace}}, }, ClusterDomain: clusterDomain, EsAdminUserName: "elastic", @@ -95,9 +95,9 @@ var _ = Describe("ES Gateway rendering tests", func() { It("should render an ES Gateway deployment and all supporting resources", func() { expectedResources := []client.Object{ &v3.NetworkPolicy{ObjectMeta: metav1.ObjectMeta{Name: PolicyName, Namespace: render.ElasticsearchNamespace}}, - &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: kubecontrollers.ElasticsearchKubeControllersUserSecret, Namespace: common.OperatorNamespace()}}, - &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: kubecontrollers.ElasticsearchKubeControllersVerificationUserSecret, Namespace: render.ElasticsearchNamespace}}, - &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: kubecontrollers.ElasticsearchKubeControllersSecureUserSecret, Namespace: render.ElasticsearchNamespace}}, + &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: entkubecontrollers.ElasticsearchKubeControllersUserSecret, Namespace: common.OperatorNamespace()}}, + &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: entkubecontrollers.ElasticsearchKubeControllersVerificationUserSecret, Namespace: render.ElasticsearchNamespace}}, + &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: entkubecontrollers.ElasticsearchKubeControllersSecureUserSecret, Namespace: render.ElasticsearchNamespace}}, &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: ServiceName, Namespace: render.ElasticsearchNamespace}}, &rbacv1.Role{ObjectMeta: metav1.ObjectMeta{Name: RoleName, Namespace: render.ElasticsearchNamespace}}, &rbacv1.RoleBinding{ObjectMeta: metav1.ObjectMeta{Name: RoleName, Namespace: render.ElasticsearchNamespace}}, @@ -134,9 +134,9 @@ var _ = Describe("ES Gateway rendering tests", func() { installation.CertificateManagement = &operatorv1.CertificateManagement{CACert: secret.Data[corev1.TLSCertKey]} expectedResources := []client.Object{ &v3.NetworkPolicy{ObjectMeta: metav1.ObjectMeta{Name: PolicyName, Namespace: render.ElasticsearchNamespace}}, - &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: kubecontrollers.ElasticsearchKubeControllersUserSecret, Namespace: common.OperatorNamespace()}}, - &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: kubecontrollers.ElasticsearchKubeControllersVerificationUserSecret, Namespace: render.ElasticsearchNamespace}}, - &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: kubecontrollers.ElasticsearchKubeControllersSecureUserSecret, Namespace: render.ElasticsearchNamespace}}, + &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: entkubecontrollers.ElasticsearchKubeControllersUserSecret, Namespace: common.OperatorNamespace()}}, + &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: entkubecontrollers.ElasticsearchKubeControllersVerificationUserSecret, Namespace: render.ElasticsearchNamespace}}, + &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: entkubecontrollers.ElasticsearchKubeControllersSecureUserSecret, Namespace: render.ElasticsearchNamespace}}, &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: ServiceName, Namespace: render.ElasticsearchNamespace}}, &rbacv1.Role{ObjectMeta: metav1.ObjectMeta{Name: RoleName, Namespace: render.ElasticsearchNamespace}}, &rbacv1.RoleBinding{ObjectMeta: metav1.ObjectMeta{Name: RoleName, Namespace: render.ElasticsearchNamespace}}, diff --git a/pkg/render/manager.go b/pkg/render/manager.go index 765918beb3..f002cb1aff 100644 --- a/pkg/render/manager.go +++ b/pkg/render/manager.go @@ -262,10 +262,10 @@ func (c *managerComponent) Objects() ([]client.Object, []client.Object) { // For multi-tenant environments, the management cluster itself isn't shown in the UI so we only need to create these // when there is no tenant. objsToCreate = append(objsToCreate, - managerClusterWideSettingsGroup(), - managerUserSpecificSettingsGroup(), - managerClusterWideTigeraLayer(), - managerClusterWideDefaultView(), + ManagerClusterWideSettingsGroup(), + ManagerUserSpecificSettingsGroup(), + ManagerClusterWideTigeraLayer(), + ManagerClusterWideDefaultView(), ) // Continue to create the legacy namespace so that we can create our external name service that points to the new // manager service. This will help ease transition for customers and avoid outages caused by the name and namespace @@ -1358,10 +1358,10 @@ func (c *managerComponent) multiTenantManagedClustersAccess() []client.Object { return objects } -// managerClusterWideSettingsGroup returns a UISettingsGroup with the description "cluster-wide settings" +// ManagerClusterWideSettingsGroup returns a UISettingsGroup with the description "cluster-wide settings" // // Calico Enterprise only -func managerClusterWideSettingsGroup() *v3.UISettingsGroup { +func ManagerClusterWideSettingsGroup() *v3.UISettingsGroup { return &v3.UISettingsGroup{ TypeMeta: metav1.TypeMeta{Kind: "UISettingsGroup", APIVersion: "projectcalico.org/v3"}, ObjectMeta: metav1.ObjectMeta{ @@ -1373,10 +1373,10 @@ func managerClusterWideSettingsGroup() *v3.UISettingsGroup { } } -// managerUserSpecificSettingsGroup returns a UISettingsGroup with the description "user settings" +// ManagerUserSpecificSettingsGroup returns a UISettingsGroup with the description "user settings" // // Calico Enterprise only -func managerUserSpecificSettingsGroup() *v3.UISettingsGroup { +func ManagerUserSpecificSettingsGroup() *v3.UISettingsGroup { return &v3.UISettingsGroup{ TypeMeta: metav1.TypeMeta{Kind: "UISettingsGroup", APIVersion: "projectcalico.org/v3"}, ObjectMeta: metav1.ObjectMeta{ @@ -1389,11 +1389,11 @@ func managerUserSpecificSettingsGroup() *v3.UISettingsGroup { } } -// managerClusterWideTigeraLayer returns a UISettings layer belonging to the cluster-wide settings group that contains +// ManagerClusterWideTigeraLayer returns a UISettings layer belonging to the cluster-wide settings group that contains // all of the tigera namespaces. // // Calico Enterprise only -func managerClusterWideTigeraLayer() *v3.UISettings { +func ManagerClusterWideTigeraLayer() *v3.UISettings { namespaces := []string{ "tigera-compliance", "tigera-dex", @@ -1439,11 +1439,11 @@ func managerClusterWideTigeraLayer() *v3.UISettings { } } -// managerClusterWideDefaultView returns a UISettings view belonging to the cluster-wide settings group that shows +// ManagerClusterWideDefaultView returns a UISettings view belonging to the cluster-wide settings group that shows // everything and uses the tigera-infrastructure layer. // // Calico Enterprise only -func managerClusterWideDefaultView() *v3.UISettings { +func ManagerClusterWideDefaultView() *v3.UISettings { return &v3.UISettings{ TypeMeta: metav1.TypeMeta{Kind: "UISettings", APIVersion: "projectcalico.org/v3"}, ObjectMeta: metav1.ObjectMeta{ diff --git a/pkg/render/node.go b/pkg/render/node.go index 06282683f4..6d2a607248 100644 --- a/pkg/render/node.go +++ b/pkg/render/node.go @@ -35,6 +35,7 @@ import ( "github.com/tigera/operator/pkg/components" "github.com/tigera/operator/pkg/controller/k8sapi" "github.com/tigera/operator/pkg/controller/migration" + "github.com/tigera/operator/pkg/imageoverride" rcomp "github.com/tigera/operator/pkg/render/common/components" "github.com/tigera/operator/pkg/render/common/configmap" rmeta "github.com/tigera/operator/pkg/render/common/meta" @@ -72,9 +73,9 @@ const ( ) var ( - // The port used by calico/node to report Calico Enterprise BGP metrics. + // NodeBGPReporterPort is the port used by calico/node to report Calico Enterprise BGP metrics. // This is currently not intended to be user configurable. - nodeBGPReporterPort int32 = 9900 + NodeBGPReporterPort int32 = 9900 NodeTLSSecretName = "node-certs" NodeTLSSecretNameNonClusterHost = NodeTLSSecretName + TyphaNonClusterHostSuffix @@ -115,11 +116,9 @@ type NodeConfiguration struct { GoldmaneIP string // Optional fields. - LogCollector *operatorv1.LogCollector - MigrateNamespaces bool - NodeAppArmorProfile string - BirdTemplates map[string]string - NodeReporterMetricsPort int + MigrateNamespaces bool + NodeAppArmorProfile string + BirdTemplates map[string]string // CanRemoveCNIFinalizer specifies whether CNI plugin is still needed during uninstall since the CNI plugin and // associated RBAC resources are required for pod teardown to succeed. Setting this to true removes @@ -127,8 +126,6 @@ type NodeConfiguration struct { // For details on why this is needed see 'Node and Installation finalizer' in the core_controller. CanRemoveCNIFinalizer bool - PrometheusServerTLS certificatemanagement.KeyPairInterface - // BGPLayouts is returned by the rendering code after modifying its namespace // so that it can be deployed into the cluster. // TODO: The controller should pass the contents, the renderer should build its own @@ -146,11 +143,12 @@ type NodeConfiguration struct { // should this value change. BindMode string - FelixPrometheusMetricsEnabled bool - - FelixPrometheusMetricsPort int - V3CRDs bool + + // ImageOverrides lets a variant swap the node and cni-plugins images. The + // controller wires in the operator's image overrides; nil resolves to the + // core images. + ImageOverrides *imageoverride.Overrides } // Node creates the node daemonset and other resources for the daemonset to operate normally. @@ -185,18 +183,11 @@ func (c *nodeComponent) ResolveImages(is *operatorv1.ImageSet) error { } c.calicoImage = appendIfErr(components.GetReference(components.CombinedCalicoImage(c.cfg.Installation), reg, path, prefix, is)) + nodeImage := c.cfg.ImageOverrides.Resolve(ComponentNameNode, components.ComponentCalicoNode, c.cfg.Installation) + c.nodeImage = appendIfErr(components.GetReference(nodeImage, reg, path, prefix, is)) if c.installUpstreamPlugins() { - if c.cfg.Installation.Variant.IsEnterprise() { - c.cniPluginsImage = appendIfErr(components.GetReference(components.ComponentTigeraCNIPlugins, reg, path, prefix, is)) - } else { - c.cniPluginsImage = appendIfErr(components.GetReference(components.ComponentCalicoCNIPlugins, reg, path, prefix, is)) - } - } - switch { - case c.cfg.Installation.Variant.IsEnterprise(): - c.nodeImage = appendIfErr(components.GetReference(components.ComponentTigeraNode, reg, path, prefix, is)) - default: - c.nodeImage = appendIfErr(components.GetReference(components.ComponentCalicoNode, reg, path, prefix, is)) + cniPluginsImage := c.cfg.ImageOverrides.Resolve(ComponentNameCNIPlugins, components.ComponentCalicoCNIPlugins, c.cfg.Installation) + c.cniPluginsImage = appendIfErr(components.GetReference(cniPluginsImage, reg, path, prefix, is)) } if len(errMsgs) != 0 { @@ -209,6 +200,10 @@ func (c *nodeComponent) SupportedOSType() rmeta.OSType { return rmeta.OSTypeLinux } +func (c *nodeComponent) ModifierKey() string { + return ComponentNameNode +} + func (c *nodeComponent) Objects() ([]client.Object, []client.Object) { objs := []client.Object{ c.nodeServiceAccount(), @@ -234,11 +229,6 @@ func (c *nodeComponent) Objects() ([]client.Object, []client.Object) { var objsToDelete []client.Object - if c.cfg.Installation.Variant.IsEnterprise() { - // Include Service for exposing node metrics. - objs = append(objs, c.nodeMetricsService()) - } - cniConfig := c.nodeCNIConfigMap() if cniConfig != nil { objs = append(objs, cniConfig) @@ -567,34 +557,6 @@ func (c *nodeComponent) nodeRole() *rbacv1.ClusterRole { }, }, } - if c.cfg.Installation.Variant.IsEnterprise() { - extraRules := []rbacv1.PolicyRule{ - { - // Calico Enterprise needs to be able to read additional resources. - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{ - "bfdconfigurations", - "egressgatewaypolicies", - "externalnetworks", - "licensekeys", - "networks", - "packetcaptures", - "remoteclusterconfigurations", - }, - Verbs: []string{"get", "list", "watch"}, - }, - { - // Tigera Secure updates status for packet captures. - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{ - "packetcaptures", - "packetcaptures/status", - }, - Verbs: []string{"update"}, - }, - } - role.Rules = append(role.Rules, extraRules...) - } if c.cfg.Installation.KubernetesProvider.IsOpenShift() { role.Rules = append(role.Rules, rbacv1.PolicyRule{ APIGroups: []string{"security.openshift.io"}, @@ -656,14 +618,6 @@ func (c *nodeComponent) cniPluginRole() *rbacv1.ClusterRole { }, }, } - if c.cfg.Installation.Variant.IsEnterprise() { - // The Network resource is only available in Enterprise / Cloud at this time. - role.Rules = append(role.Rules, rbacv1.PolicyRule{ - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{"networks"}, - Verbs: []string{"get"}, - }) - } return role } @@ -951,18 +905,11 @@ func (c *nodeComponent) nodeDaemonset(cniCfgMap *corev1.ConfigMap) *appsv1.Daemo if len(c.cfg.BirdTemplates) != 0 { annotations[birdTemplateHashAnnotation] = rmeta.AnnotationHash(c.cfg.BirdTemplates) } - if c.cfg.PrometheusServerTLS != nil { - annotations[c.cfg.PrometheusServerTLS.HashAnnotationKey()] = c.cfg.PrometheusServerTLS.HashAnnotationValue() - } if c.cfg.TLS.NodeSecret.UseCertificateManagement() { initContainers = append(initContainers, c.cfg.TLS.NodeSecret.InitContainer(common.CalicoNamespace, nodeContainer.SecurityContext)) } - if c.cfg.PrometheusServerTLS != nil && c.cfg.PrometheusServerTLS.UseCertificateManagement() { - initContainers = append(initContainers, c.cfg.PrometheusServerTLS.InitContainer(common.CalicoNamespace, nodeContainer.SecurityContext)) - } - if cniCfgMap != nil { annotations[nodeCniConfigAnnotation] = rmeta.AnnotationHash(cniCfgMap.Data) } @@ -1078,10 +1025,6 @@ func (c *nodeComponent) nodeDaemonset(cniCfgMap *corev1.ConfigMap) *appsv1.Daemo ds.Spec.Template.Spec.InitContainers = append(ds.Spec.Template.Spec.InitContainers, c.cniContainer()) } - if c.collectProcessPathEnabled() { - ds.Spec.Template.Spec.HostPID = true - } - setNodeCriticalPod(&(ds.Spec.Template)) if c.cfg.MigrateNamespaces { migration.LimitDaemonSetToMigratedNodes(&ds) @@ -1112,13 +1055,16 @@ func (c *nodeComponent) nodeVolumes() []corev1.Volume { c.cfg.TLS.TrustedBundle.Volume(), c.cfg.TLS.NodeSecret.Volume(), c.varRunCalicoVolume(), - corev1.Volume{Name: "var-lib-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/lib/calico", Type: &dirOrCreate}}}, + {Name: "var-lib-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/lib/calico", Type: &dirOrCreate}}}, + // The Calico log directory. The CNI plugin logs to the cni/ subdirectory of + // this, and Felix writes its flow/DNS logs here on the enterprise variant. + {Name: "var-log-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico", Type: &dirOrCreate}}}, // Volume for the containing directory so that the init container can mount the child bpf directory if needed. - corev1.Volume{Name: "sys-fs", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/sys/fs", Type: &dirOrCreate}}}, + {Name: "sys-fs", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/sys/fs", Type: &dirOrCreate}}}, // Volume for the bpffs itself, used by the main node container. - corev1.Volume{Name: "bpffs", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/sys/fs/bpf", Type: &dirMustExist}}}, + {Name: "bpffs", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/sys/fs/bpf", Type: &dirMustExist}}}, // Volume used by mount-cgroupv2 init container to access root cgroup name space of node. - corev1.Volume{Name: "nodeproc", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/proc"}}}, + {Name: "nodeproc", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/proc"}}}, } if c.vppDataplaneEnabled() { @@ -1132,7 +1078,6 @@ func (c *nodeComponent) nodeVolumes() []corev1.Volume { if c.cfg.Installation.CNI.Type == operatorv1.PluginCalico { volumes = append(volumes, corev1.Volume{Name: "cni-bin-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: *c.cfg.Installation.CNI.BinDir, Type: &dirOrCreate}}}) volumes = append(volumes, corev1.Volume{Name: "cni-net-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: *c.cfg.Installation.CNI.ConfDir}}}) - volumes = append(volumes, corev1.Volume{Name: "cni-log-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico/cni"}}}) } if c.installUpstreamPlugins() { // Staging volume populated by the cni-plugins init container and read @@ -1140,16 +1085,6 @@ func (c *nodeComponent) nodeVolumes() []corev1.Volume { volumes = append(volumes, corev1.Volume{Name: "cni-plugins-stage", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}) } - // Override with Tigera-specific config. - if c.cfg.Installation.Variant.IsEnterprise() { - // Add volume for calico logs. - calicoLogVol := corev1.Volume{ - Name: "var-log-calico", - VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico", Type: &dirOrCreate}}, - } - volumes = append(volumes, calicoLogVol) - } - // Create and append flexvolume if c.cfg.Installation.FlexVolumePath != "None" { volumes = append(volumes, corev1.Volume{ @@ -1186,10 +1121,6 @@ func (c *nodeComponent) nodeVolumes() []corev1.Volume { }, }) } - if c.cfg.PrometheusServerTLS != nil { - volumes = append(volumes, c.cfg.PrometheusServerTLS.Volume()) - } - return volumes } @@ -1209,12 +1140,6 @@ func (c *nodeComponent) vppDataplaneEnabled() bool { *c.cfg.Installation.CalicoNetwork.LinuxDataplane == operatorv1.LinuxDataplaneVPP } -func (c *nodeComponent) collectProcessPathEnabled() bool { - return c.cfg.LogCollector != nil && - c.cfg.LogCollector.Spec.CollectProcessPath != nil && - *c.cfg.LogCollector.Spec.CollectProcessPath == operatorv1.CollectProcessPathEnable -} - // cniContainer creates the node's init container that installs CNI. func (c *nodeComponent) cniContainer() corev1.Container { // Determine environment to pass to the CNI init container. @@ -1383,12 +1308,6 @@ func (c *nodeComponent) cniEnvvars() []corev1.EnvVar { envVars = append(envVars, c.cfg.K8sServiceEp.EnvVars()...) - if c.cfg.Installation.Variant.IsEnterprise() { - if c.cfg.Installation.CalicoNetwork != nil && c.cfg.Installation.CalicoNetwork.MultiInterfaceMode != nil { - envVars = append(envVars, corev1.EnvVar{Name: "MULTI_INTERFACE_MODE", Value: c.cfg.Installation.CalicoNetwork.MultiInterfaceMode.Value()}) - } - } - return envVars } @@ -1432,15 +1351,9 @@ func (c *nodeComponent) nodeVolumeMounts() []corev1.VolumeMount { if c.vppDataplaneEnabled() { nodeVolumeMounts = append(nodeVolumeMounts, corev1.VolumeMount{MountPath: "/usr/local/bin/felix-plugins", Name: "felix-plugins", ReadOnly: true}) } - if c.cfg.Installation.Variant.IsEnterprise() { - extraNodeMounts := []corev1.VolumeMount{ - {MountPath: "/var/log/calico", Name: "var-log-calico"}, - } - nodeVolumeMounts = append(nodeVolumeMounts, extraNodeMounts...) - } else if c.cfg.Installation.CNI.Type == operatorv1.PluginCalico { - cniLogMount := corev1.VolumeMount{MountPath: "/var/log/calico/cni", Name: "cni-log-dir", ReadOnly: false} - nodeVolumeMounts = append(nodeVolumeMounts, cniLogMount) - } + // Mount the Calico log directory. The CNI plugin writes to the cni/ subdirectory + // and, on the enterprise variant, Felix writes its flow/DNS logs here too. + nodeVolumeMounts = append(nodeVolumeMounts, corev1.VolumeMount{MountPath: "/var/log/calico", Name: "var-log-calico"}) if c.cfg.Installation.CNI.Type == operatorv1.PluginCalico { nodeVolumeMounts = append(nodeVolumeMounts, corev1.VolumeMount{MountPath: "/host/etc/cni/net.d", Name: "cni-net-dir"}) @@ -1476,9 +1389,6 @@ func (c *nodeComponent) nodeVolumeMounts() []corev1.VolumeMount { SubPath: BGPLayoutConfigMapKey, }) } - if c.cfg.PrometheusServerTLS != nil { - nodeVolumeMounts = append(nodeVolumeMounts, c.cfg.PrometheusServerTLS.VolumeMount(c.SupportedOSType())) - } return nodeVolumeMounts } @@ -1589,10 +1499,6 @@ func (c *nodeComponent) nodeEnvVars() []corev1.EnvVar { } } - if c.collectProcessPathEnabled() { - nodeEnv = append(nodeEnv, corev1.EnvVar{Name: "FELIX_FLOWLOGSCOLLECTPROCESSPATH", Value: "true"}) - } - // Determine MTU to use. If specified explicitly, use that. Otherwise, set defaults based on an overall // MTU of 1460. mtu := getMTU(c.cfg.Installation) @@ -1686,35 +1592,6 @@ func (c *nodeComponent) nodeEnvVars() []corev1.EnvVar { nodeEnv = append(nodeEnv, corev1.EnvVar{Name: "FELIX_IPV6SUPPORT", Value: "false"}) } - if c.cfg.Installation.Variant.IsEnterprise() { - // Add in Calico Enterprise specific configuration. - extraNodeEnv := []corev1.EnvVar{ - {Name: "FELIX_PROMETHEUSREPORTERENABLED", Value: "true"}, - {Name: "FELIX_PROMETHEUSREPORTERPORT", Value: fmt.Sprintf("%d", c.cfg.NodeReporterMetricsPort)}, - {Name: "FELIX_FLOWLOGSFILEENABLED", Value: "true"}, - {Name: "FELIX_FLOWLOGSFILEINCLUDELABELS", Value: "true"}, - {Name: "FELIX_FLOWLOGSFILEINCLUDEPOLICIES", Value: "true"}, - {Name: "FELIX_FLOWLOGSFILEINCLUDESERVICE", Value: "true"}, - {Name: "FELIX_FLOWLOGSENABLENETWORKSETS", Value: "true"}, - {Name: "FELIX_FLOWLOGSCOLLECTPROCESSINFO", Value: "true"}, - {Name: "FELIX_DNSLOGSFILEENABLED", Value: "true"}, - {Name: "FELIX_DNSLOGSFILEPERNODELIMIT", Value: "1000"}, - } - - if c.cfg.Installation.CalicoNetwork != nil && c.cfg.Installation.CalicoNetwork.MultiInterfaceMode != nil { - extraNodeEnv = append(extraNodeEnv, corev1.EnvVar{Name: "MULTI_INTERFACE_MODE", Value: c.cfg.Installation.CalicoNetwork.MultiInterfaceMode.Value()}) - } - - if c.cfg.PrometheusServerTLS != nil { - extraNodeEnv = append(extraNodeEnv, - corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERCERTFILE", Value: c.cfg.PrometheusServerTLS.VolumeMountCertificateFilePath()}, - corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERKEYFILE", Value: c.cfg.PrometheusServerTLS.VolumeMountKeyFilePath()}, - corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERCAFILE", Value: c.cfg.TLS.TrustedBundle.MountPath()}, - ) - } - nodeEnv = append(nodeEnv, extraNodeEnv...) - } - if c.cfg.Installation.NodeMetricsPort != nil { // If a node metrics port was given, then enable felix prometheus metrics and set the port. // Note that this takes precedence over any FelixConfiguration resources in the cluster. @@ -1793,10 +1670,7 @@ func (c *nodeComponent) nodeLivenessReadinessProbes() (*corev1.Probe, *corev1.Pr var readinessCmd []string readinessCmd = []string{components.CalicoBinaryPath, "component", "node", "health", "--bird-ready", "--felix-ready"} - if c.cfg.Installation.Variant.IsEnterprise() { - readinessCmd = append(readinessCmd, "--bgp-metrics-ready") - } - // If not using BGP or using VPP, don't check bird status (or bgp metrics server for enterprise). + // If not using BGP or using VPP, don't check bird status. if !bgpEnabled(c.cfg.Installation) || c.vppDataplaneEnabled() { readinessCmd = []string{components.CalicoBinaryPath, "component", "node", "health", "--felix-ready"} } @@ -1822,56 +1696,6 @@ func (c *nodeComponent) nodeLivenessReadinessProbes() (*corev1.Probe, *corev1.Pr return lp, rp } -// nodeMetricsService creates a Service which exposes two endpoints on calico/node for -// reporting Prometheus metrics (for policy enforcement activity and BGP stats). -// This service is used internally by Calico Enterprise and is separate from general -// Prometheus metrics which are user-configurable. -func (c *nodeComponent) nodeMetricsService() *corev1.Service { - ports := []corev1.ServicePort{ - { - Name: "calico-metrics-port", - Port: int32(c.cfg.NodeReporterMetricsPort), - TargetPort: intstr.FromInt(c.cfg.NodeReporterMetricsPort), - Protocol: corev1.ProtocolTCP, - }, - { - Name: "calico-bgp-metrics-port", - Port: nodeBGPReporterPort, - TargetPort: intstr.FromInt(int(nodeBGPReporterPort)), - Protocol: corev1.ProtocolTCP, - }, - } - - if c.cfg.FelixPrometheusMetricsEnabled { - felixMetricsPort := int32(c.cfg.FelixPrometheusMetricsPort) - - ports = append(ports, corev1.ServicePort{ - Name: "felix-metrics-port", - Port: felixMetricsPort, - TargetPort: intstr.FromInt(int(felixMetricsPort)), - Protocol: corev1.ProtocolTCP, - }) - } - - return &corev1.Service{ - TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}, - ObjectMeta: metav1.ObjectMeta{ - Name: CalicoNodeMetricsService, - Namespace: common.CalicoNamespace, - Labels: map[string]string{"k8s-app": CalicoNodeObjectName}, - }, - Spec: corev1.ServiceSpec{ - Selector: map[string]string{"k8s-app": CalicoNodeObjectName}, - // Important: "None" tells Kubernetes that we want a headless service with - // no kube-proxy load balancer. If we omit this then kube-proxy will render - // a huge set of iptables rules for this service since there's an instance - // on every node. - ClusterIP: "None", - Ports: ports, - }, - } -} - // getAutodetectionMethod returns the IP auto detection method in a form understandable by the calico/node // startup processing. It returns an empty string if IP auto detection should not be enabled. func getAutodetectionMethod(ad *operatorv1.NodeAddressAutodetection) string { diff --git a/pkg/render/node_test.go b/pkg/render/node_test.go index 5c5c0f50eb..40f4712f97 100644 --- a/pkg/render/node_test.go +++ b/pkg/render/node_test.go @@ -38,6 +38,7 @@ import ( "github.com/tigera/operator/pkg/controller/certificatemanager" "github.com/tigera/operator/pkg/controller/k8sapi" ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" + "github.com/tigera/operator/pkg/imageoverride" "github.com/tigera/operator/pkg/render" rmeta "github.com/tigera/operator/pkg/render/common/meta" rtest "github.com/tigera/operator/pkg/render/common/test" @@ -134,14 +135,13 @@ var _ = Describe("Node rendering tests", func() { // Create a default configuration. cfg = render.NodeConfiguration{ - K8sServiceEp: k8sServiceEp, - Installation: defaultInstance, - TLS: typhaNodeTLS, - ClusterDomain: defaultClusterDomain, - FelixHealthPort: 9099, - IPPools: defaultInstance.CalicoNetwork.IPPools, - FelixPrometheusMetricsEnabled: false, - FelixPrometheusMetricsPort: 9098, + K8sServiceEp: k8sServiceEp, + Installation: defaultInstance, + TLS: typhaNodeTLS, + ClusterDomain: defaultClusterDomain, + FelixHealthPort: 9099, + IPPools: defaultInstance.CalicoNetwork.IPPools, + ImageOverrides: imageoverride.New(), } }) @@ -321,7 +321,7 @@ var _ = Describe("Node rendering tests", func() { {Name: "xtables-lock", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/run/xtables.lock", Type: &fileOrCreate}}}, {Name: "cni-bin-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/opt/cni/bin", Type: &dirOrCreate}}}, {Name: "cni-net-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/etc/cni/net.d"}}}, - {Name: "cni-log-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico/cni"}}}, + {Name: "var-log-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico", Type: &dirOrCreate}}}, {Name: "cni-plugins-stage", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}, {Name: "policysync", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/run/nodeagent", Type: &dirOrCreate}}}, {Name: "sys-fs", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/sys/fs", Type: &dirOrCreate}}}, @@ -360,7 +360,7 @@ var _ = Describe("Node rendering tests", func() { {MountPath: "/var/run/nodeagent", Name: "policysync"}, {MountPath: "/etc/pki/tls/certs", Name: "tigera-ca-bundle", ReadOnly: true}, {MountPath: "/node-certs", Name: render.NodeTLSSecretName, ReadOnly: true}, - {MountPath: "/var/log/calico/cni", Name: "cni-log-dir", ReadOnly: false}, + {MountPath: "/var/log/calico", Name: "var-log-calico"}, {MountPath: "/sys/fs/bpf", Name: "bpffs"}, } Expect(ds.Spec.Template.Spec.Containers[0].VolumeMounts).To(ConsistOf(expectedNodeVolumeMounts)) @@ -368,7 +368,7 @@ var _ = Describe("Node rendering tests", func() { // Verify tolerations. Expect(ds.Spec.Template.Spec.Tolerations).To(ConsistOf(rmeta.TolerateAll)) - verifyProbesAndLifecycle(ds, false, false) + verifyProbesAndLifecycle(ds, false) }) It("should render node correctly for BPF dataplane", func() { @@ -513,7 +513,7 @@ var _ = Describe("Node rendering tests", func() { {Name: "xtables-lock", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/run/xtables.lock", Type: &fileOrCreate}}}, {Name: "cni-bin-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/opt/cni/bin", Type: &dirOrCreate}}}, {Name: "cni-net-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/etc/cni/net.d"}}}, - {Name: "cni-log-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico/cni"}}}, + {Name: "var-log-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico", Type: &dirOrCreate}}}, {Name: "cni-plugins-stage", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}, {Name: "sys-fs", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/sys/fs", Type: &dirOrCreate}}}, {Name: "bpffs", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/sys/fs/bpf", Type: &dirMustExist}}}, @@ -552,7 +552,7 @@ var _ = Describe("Node rendering tests", func() { {MountPath: "/var/run/nodeagent", Name: "policysync"}, {MountPath: "/etc/pki/tls/certs", Name: "tigera-ca-bundle", ReadOnly: true}, {MountPath: "/node-certs", Name: render.NodeTLSSecretName, ReadOnly: true}, - {MountPath: "/var/log/calico/cni", Name: "cni-log-dir", ReadOnly: false}, + {MountPath: "/var/log/calico", Name: "var-log-calico"}, {MountPath: "/sys/fs/bpf", Name: "bpffs"}, } Expect(ds.Spec.Template.Spec.Containers[0].VolumeMounts).To(ConsistOf(expectedNodeVolumeMounts)) @@ -560,7 +560,7 @@ var _ = Describe("Node rendering tests", func() { // Verify tolerations. Expect(ds.Spec.Template.Spec.Tolerations).To(ConsistOf(rmeta.TolerateAll)) - verifyProbesAndLifecycle(ds, false, false) + verifyProbesAndLifecycle(ds, false) }) It("should properly render an explicitly configured MTU", func() { @@ -639,143 +639,6 @@ var _ = Describe("Node rendering tests", func() { } }) - It("should render all resources for a default configuration using CalicoEnterprise", func() { - expectedResources := []struct { - name string - ns string - group string - version string - kind string - }{ - {name: "calico-node", ns: common.CalicoNamespace, group: "", version: "v1", kind: "ServiceAccount"}, - {name: "calico-node", ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRole"}, - {name: "calico-node", ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRoleBinding"}, - {name: "calico-cni-plugin", ns: common.CalicoNamespace, group: "", version: "v1", kind: "ServiceAccount"}, - {name: "calico-cni-plugin", ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRole"}, - {name: "calico-cni-plugin", ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRoleBinding"}, - {name: "calico-node-metrics", ns: "calico-system", group: "", version: "v1", kind: "Service"}, - {name: "cni-config", ns: common.CalicoNamespace, group: "", version: "v1", kind: "ConfigMap"}, - {name: common.NodeDaemonSetName, ns: common.CalicoNamespace, group: "apps", version: "v1", kind: "DaemonSet"}, - } - defaultInstance.Variant = operatorv1.CalicoEnterprise - cfg.NodeReporterMetricsPort = 9081 - - component := render.Node(&cfg) - Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() - Expect(len(resources)).To(Equal(len(expectedResources))) - - // Should render the correct resources. - i := 0 - for _, expectedRes := range expectedResources { - rtest.ExpectResourceTypeAndObjectMetadata(resources[i], expectedRes.name, expectedRes.ns, expectedRes.group, expectedRes.version, expectedRes.kind) - i++ - } - - // The DaemonSet should have the correct configuration. - ds := rtest.GetResource(resources, "calico-node", "calico-system", "apps", "v1", "DaemonSet").(*appsv1.DaemonSet) - - // The pod template should have node critical priority - Expect(ds.Spec.Template.Spec.PriorityClassName).To(Equal(render.NodePriorityClassName)) - Expect(ds.Spec.Template.Spec.Containers[0].Image).To(Equal(components.TigeraRegistry + "tigera/node:" + components.ComponentTigeraNode.Version)) - verifyInitContainers(ds, defaultInstance) - - expectedNodeEnv := []corev1.EnvVar{ - // Default envvars. - {Name: "DATASTORE_TYPE", Value: "kubernetes"}, - {Name: "WAIT_FOR_DATASTORE", Value: "true"}, - {Name: "CALICO_MANAGE_CNI", Value: "true"}, - {Name: "CALICO_NETWORKING_BACKEND", Value: "bird"}, - {Name: "CLUSTER_TYPE", Value: "k8s,operator,bgp"}, - {Name: "CALICO_DISABLE_FILE_LOGGING", Value: "false"}, - {Name: "FELIX_DEFAULTENDPOINTTOHOSTACTION", Value: "ACCEPT"}, - {Name: "FELIX_HEALTHENABLED", Value: "true"}, - {Name: "FELIX_HEALTHPORT", Value: "9099"}, - { - Name: "NODENAME", - ValueFrom: &corev1.EnvVarSource{ - FieldRef: &corev1.ObjectFieldSelector{FieldPath: "spec.nodeName"}, - }, - }, - { - Name: "NAMESPACE", - ValueFrom: &corev1.EnvVarSource{ - FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.namespace"}, - }, - }, - {Name: "FELIX_TYPHAK8SNAMESPACE", Value: "calico-system"}, - {Name: "FELIX_TYPHAK8SSERVICENAME", Value: "calico-typha"}, - {Name: "FELIX_TYPHACAFILE", Value: certificatemanagement.TrustedCertBundleMountPath}, - {Name: "FELIX_TYPHACERTFILE", Value: "/node-certs/tls.crt"}, - {Name: "FELIX_TYPHACN", Value: "typha-server"}, - {Name: "FELIX_TYPHAKEYFILE", Value: "/node-certs/tls.key"}, - // Tigera-specific envvars - {Name: "FELIX_PROMETHEUSREPORTERENABLED", Value: "true"}, - {Name: "FELIX_PROMETHEUSREPORTERPORT", Value: "9081"}, - {Name: "FELIX_FLOWLOGSFILEENABLED", Value: "true"}, - {Name: "FELIX_FLOWLOGSFILEINCLUDELABELS", Value: "true"}, - {Name: "FELIX_FLOWLOGSFILEINCLUDEPOLICIES", Value: "true"}, - {Name: "FELIX_FLOWLOGSFILEINCLUDESERVICE", Value: "true"}, - {Name: "FELIX_FLOWLOGSENABLENETWORKSETS", Value: "true"}, - {Name: "FELIX_FLOWLOGSCOLLECTPROCESSINFO", Value: "true"}, - {Name: "FELIX_DNSLOGSFILEENABLED", Value: "true"}, - {Name: "FELIX_DNSLOGSFILEPERNODELIMIT", Value: "1000"}, - {Name: "MULTI_INTERFACE_MODE", Value: operatorv1.MultiInterfaceModeNone.Value()}, - {Name: "NO_DEFAULT_POOLS", Value: "true"}, - } - expectedNodeEnv = configureExpectedNodeEnvIPVersions(expectedNodeEnv, defaultInstance, enableIPv4, enableIPv6) - Expect(ds.Spec.Template.Spec.Containers[0].Env).To(ConsistOf(expectedNodeEnv)) - Expect(len(ds.Spec.Template.Spec.Containers[0].Env)).To(Equal(len(expectedNodeEnv))) - - // Expect 2 Ports when FelixPrometheusMetricsEnabled is false - ms := rtest.GetResource(resources, "calico-node-metrics", "calico-system", "", "v1", "Service").(*corev1.Service) - Expect(len(ms.Spec.Ports)).To(Equal(2)) - - dirMustExist := corev1.HostPathDirectory - bpfVol := corev1.Volume{Name: "bpffs", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/sys/fs/bpf", Type: &dirMustExist}}} - Expect(ds.Spec.Template.Spec.Volumes).To(ContainElement(bpfVol)) - - bpfVolMount := corev1.VolumeMount{MountPath: "/sys/fs/bpf", Name: "bpffs"} - Expect(ds.Spec.Template.Spec.Containers[0].VolumeMounts).To(ContainElement(bpfVolMount)) - - verifyProbesAndLifecycle(ds, false, true) - }) - - It("should render felix service metric with FelixPrometheusMetricPort when FelixPrometheusMetricsEnabled is true", func() { - defaultInstance.Variant = operatorv1.CalicoEnterprise - cfg.NodeReporterMetricsPort = 9081 - cfg.FelixPrometheusMetricsEnabled = true - - component := render.Node(&cfg) - Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() - - expectedServicePorts := []corev1.ServicePort{ - { - Name: "calico-metrics-port", - Port: int32(cfg.NodeReporterMetricsPort), - TargetPort: intstr.FromInt(cfg.NodeReporterMetricsPort), - Protocol: corev1.ProtocolTCP, - }, - { - Name: "calico-bgp-metrics-port", - Port: 9900, - TargetPort: intstr.FromInt(int(9900)), - Protocol: corev1.ProtocolTCP, - }, - { - Name: "felix-metrics-port", - Port: 9098, - TargetPort: intstr.FromInt(int(9098)), - Protocol: corev1.ProtocolTCP, - }, - } - - // Expect 3 Ports when FelixPrometheusMetricsEnabled is true - ms := rtest.GetResource(resources, "calico-node-metrics", "calico-system", "", "v1", "Service").(*corev1.Service) - Expect(ms.Spec.Ports).To(Equal(expectedServicePorts)) - }) - It("should render all resources when using Calico CNI on EKS", func() { expectedResources := []struct { name string @@ -913,7 +776,7 @@ var _ = Describe("Node rendering tests", func() { {Name: "xtables-lock", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/run/xtables.lock", Type: &fileOrCreate}}}, {Name: "cni-bin-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/opt/cni/bin", Type: &dirOrCreate}}}, {Name: "cni-net-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/etc/cni/net.d"}}}, - {Name: "cni-log-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico/cni"}}}, + {Name: "var-log-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico", Type: &dirOrCreate}}}, {Name: "cni-plugins-stage", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}, {Name: "policysync", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/run/nodeagent", Type: &dirOrCreate}}}, {Name: "sys-fs", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/sys/fs", Type: &dirOrCreate}}}, @@ -952,7 +815,7 @@ var _ = Describe("Node rendering tests", func() { {MountPath: "/var/run/nodeagent", Name: "policysync"}, {MountPath: "/etc/pki/tls/certs", Name: "tigera-ca-bundle", ReadOnly: true}, {MountPath: "/node-certs", Name: render.NodeTLSSecretName, ReadOnly: true}, - {MountPath: "/var/log/calico/cni", Name: "cni-log-dir", ReadOnly: false}, + {MountPath: "/var/log/calico", Name: "var-log-calico"}, {MountPath: "/sys/fs/bpf", Name: "bpffs"}, } Expect(ds.Spec.Template.Spec.Containers[0].VolumeMounts).To(ConsistOf(expectedNodeVolumeMounts)) @@ -962,7 +825,7 @@ var _ = Describe("Node rendering tests", func() { // Verify readiness and liveness probes. - verifyProbesAndLifecycle(ds, false, false) + verifyProbesAndLifecycle(ds, false) }) It("should properly render a configuration using the AmazonVPC CNI plugin", func() { @@ -1051,6 +914,7 @@ var _ = Describe("Node rendering tests", func() { {Name: "lib-modules", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/lib/modules"}}}, {Name: "var-run-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/run/calico", Type: &dirOrCreate}}}, {Name: "var-lib-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/lib/calico", Type: &dirOrCreate}}}, + {Name: "var-log-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico", Type: &dirOrCreate}}}, {Name: "xtables-lock", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/run/xtables.lock", Type: &fileOrCreate}}}, {Name: "policysync", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/run/nodeagent", Type: &dirOrCreate}}}, {Name: "sys-fs", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/sys/fs", Type: &dirOrCreate}}}, @@ -1085,6 +949,7 @@ var _ = Describe("Node rendering tests", func() { {MountPath: "/run/xtables.lock", Name: "xtables-lock"}, {MountPath: "/var/run/calico", Name: "var-run-calico"}, {MountPath: "/var/lib/calico", Name: "var-lib-calico"}, + {MountPath: "/var/log/calico", Name: "var-log-calico"}, {MountPath: "/var/run/nodeagent", Name: "policysync"}, {MountPath: "/etc/pki/tls/certs", Name: "tigera-ca-bundle", ReadOnly: true}, {MountPath: "/node-certs", Name: render.NodeTLSSecretName, ReadOnly: true}, @@ -1096,7 +961,7 @@ var _ = Describe("Node rendering tests", func() { Expect(ds.Spec.Template.Spec.Tolerations).To(ConsistOf(rmeta.TolerateAll)) // Verify readiness and liveness probes. - verifyProbesAndLifecycle(ds, false, false) + verifyProbesAndLifecycle(ds, false) }) It("should return customized CNI directories when specified", func() { @@ -1116,7 +981,7 @@ var _ = Describe("Node rendering tests", func() { expectedVols := []corev1.Volume{ {Name: "cni-bin-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/custom/cni/bin", Type: &dirOrCreate}}}, {Name: "cni-net-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/custom/cni/net.d"}}}, - {Name: "cni-log-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico/cni"}}}, + {Name: "var-log-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico", Type: &dirOrCreate}}}, {Name: "cni-plugins-stage", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}, } Expect(ds.Spec.Template.Spec.Volumes).To(ContainElements(expectedVols)) @@ -1165,7 +1030,7 @@ var _ = Describe("Node rendering tests", func() { } // Verify readiness and liveness probes. - verifyProbesAndLifecycle(ds, false, false) + verifyProbesAndLifecycle(ds, false) }, Entry("GKE", operatorv1.PluginGKE, operatorv1.IPAMPluginHostLocal, []corev1.EnvVar{ {Name: "FELIX_INTERFACEPREFIX", Value: "gke"}, @@ -1317,7 +1182,7 @@ var _ = Describe("Node rendering tests", func() { {Name: "xtables-lock", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/run/xtables.lock", Type: &fileOrCreate}}}, {Name: "cni-bin-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/opt/cni/bin", Type: &dirOrCreate}}}, {Name: "cni-net-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/etc/cni/net.d"}}}, - {Name: "cni-log-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico/cni"}}}, + {Name: "var-log-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico", Type: &dirOrCreate}}}, {Name: "cni-plugins-stage", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}, {Name: "policysync", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/run/nodeagent", Type: &dirOrCreate}}}, {Name: "sys-fs", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/sys/fs", Type: &dirOrCreate}}}, @@ -1356,7 +1221,7 @@ var _ = Describe("Node rendering tests", func() { {MountPath: "/var/run/nodeagent", Name: "policysync"}, {MountPath: "/etc/pki/tls/certs", Name: "tigera-ca-bundle", ReadOnly: true}, {MountPath: "/node-certs", Name: render.NodeTLSSecretName, ReadOnly: true}, - {MountPath: "/var/log/calico/cni", Name: "cni-log-dir", ReadOnly: false}, + {MountPath: "/var/log/calico", Name: "var-log-calico"}, {MountPath: "/sys/fs/bpf", Name: "bpffs"}, } Expect(ds.Spec.Template.Spec.Containers[0].VolumeMounts).To(ConsistOf(expectedNodeVolumeMounts)) @@ -1365,7 +1230,7 @@ var _ = Describe("Node rendering tests", func() { Expect(ds.Spec.Template.Spec.Tolerations).To(ConsistOf(rmeta.TolerateAll)) // Verify readiness and liveness probes. - verifyProbesAndLifecycle(ds, false, false) + verifyProbesAndLifecycle(ds, false) }) It("should properly render a configuration using the AmazonVPC CNI plugin", func() { @@ -1452,6 +1317,7 @@ var _ = Describe("Node rendering tests", func() { {Name: "lib-modules", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/lib/modules"}}}, {Name: "var-run-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/run/calico", Type: &dirOrCreate}}}, {Name: "var-lib-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/lib/calico", Type: &dirOrCreate}}}, + {Name: "var-log-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico", Type: &dirOrCreate}}}, {Name: "xtables-lock", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/run/xtables.lock", Type: &fileOrCreate}}}, {Name: "policysync", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/run/nodeagent", Type: &dirOrCreate}}}, {Name: "sys-fs", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/sys/fs", Type: &dirOrCreate}}}, @@ -1486,6 +1352,7 @@ var _ = Describe("Node rendering tests", func() { {MountPath: "/run/xtables.lock", Name: "xtables-lock"}, {MountPath: "/var/run/calico", Name: "var-run-calico"}, {MountPath: "/var/lib/calico", Name: "var-lib-calico"}, + {MountPath: "/var/log/calico", Name: "var-log-calico"}, {MountPath: "/var/run/nodeagent", Name: "policysync"}, {MountPath: "/etc/pki/tls/certs", Name: "tigera-ca-bundle", ReadOnly: true}, {MountPath: "/node-certs", Name: render.NodeTLSSecretName, ReadOnly: true}, @@ -1497,7 +1364,7 @@ var _ = Describe("Node rendering tests", func() { Expect(ds.Spec.Template.Spec.Tolerations).To(ConsistOf(rmeta.TolerateAll)) // Verify readiness and liveness probes. - verifyProbesAndLifecycle(ds, false, false) + verifyProbesAndLifecycle(ds, false) }) It("should render all resources when running on openshift", func() { @@ -1564,7 +1431,7 @@ var _ = Describe("Node rendering tests", func() { {Name: "xtables-lock", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/run/xtables.lock", Type: &fileOrCreate}}}, {Name: "cni-bin-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/lib/cni/bin", Type: &dirOrCreate}}}, {Name: "cni-net-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/run/multus/cni/net.d"}}}, - {Name: "cni-log-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico/cni"}}}, + {Name: "var-log-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico", Type: &dirOrCreate}}}, {Name: "cni-plugins-stage", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}, {Name: "policysync", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/run/nodeagent", Type: &dirOrCreate}}}, {Name: "flexvol-driver-host", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/etc/kubernetes/kubelet-plugins/volume/exec/nodeagent~uds", Type: &dirOrCreate}}}, @@ -1628,215 +1495,7 @@ var _ = Describe("Node rendering tests", func() { Expect(ds.Spec.Template.Spec.Containers[0].Env).To(ConsistOf(expectedNodeEnv)) Expect(len(ds.Spec.Template.Spec.Containers[0].Env)).To(Equal(len(expectedNodeEnv))) - verifyProbesAndLifecycle(ds, true, false) - }) - - It("should render all resources when variant is CalicoEnterprise and running on openshift", func() { - expectedResources := []struct { - name string - ns string - group string - version string - kind string - }{ - {name: "calico-node", ns: common.CalicoNamespace, group: "", version: "v1", kind: "ServiceAccount"}, - {name: "calico-node", ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRole"}, - {name: "calico-node", ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRoleBinding"}, - {name: "calico-cni-plugin", ns: common.CalicoNamespace, group: "", version: "v1", kind: "ServiceAccount"}, - {name: "calico-cni-plugin", ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRole"}, - {name: "calico-cni-plugin", ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRoleBinding"}, - {name: "calico-node-metrics", ns: "calico-system", group: "", version: "v1", kind: "Service"}, - {name: "cni-config", ns: common.CalicoNamespace, group: "", version: "v1", kind: "ConfigMap"}, - {name: common.NodeDaemonSetName, ns: common.CalicoNamespace, group: "apps", version: "v1", kind: "DaemonSet"}, - } - - defaultInstance.Variant = operatorv1.CalicoEnterprise - defaultInstance.KubernetesProvider = operatorv1.ProviderOpenShift - defaultCNIConfDir, defaultCNIBinDir := render.DefaultCNIDirectories(defaultInstance.KubernetesProvider) - defaultInstance.CNI.ConfDir, defaultInstance.CNI.BinDir = &defaultCNIConfDir, &defaultCNIBinDir - cfg.NodeReporterMetricsPort = 9081 - cfg.FelixHealthPort = 9199 - - component := render.Node(&cfg) - Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() - Expect(len(resources)).To(Equal(len(expectedResources))) - - // Should render the correct resources. - i := 0 - for _, expectedRes := range expectedResources { - rtest.ExpectResourceTypeAndObjectMetadata(resources[i], expectedRes.name, expectedRes.ns, expectedRes.group, expectedRes.version, expectedRes.kind) - i++ - } - - // calico-node clusterRole should have openshift securitycontextconstraints PolicyRule - nodeRole := rtest.GetResource(resources, "calico-node", "", "rbac.authorization.k8s.io", "v1", "ClusterRole").(*rbacv1.ClusterRole) - Expect(nodeRole.Rules).To(ContainElement(rbacv1.PolicyRule{ - APIGroups: []string{"security.openshift.io"}, - Resources: []string{"securitycontextconstraints"}, - Verbs: []string{"use"}, - ResourceNames: []string{"privileged"}, - })) - - // The DaemonSet should have the correct configuration. - ds := rtest.GetResource(resources, "calico-node", "calico-system", "apps", "v1", "DaemonSet").(*appsv1.DaemonSet) - Expect(ds.Spec.Template.Spec.Containers[0].Image).To(Equal(components.TigeraRegistry + "tigera/node:" + components.ComponentTigeraNode.Version)) - - // The pod template should have node critical priority - Expect(ds.Spec.Template.Spec.PriorityClassName).To(Equal(render.NodePriorityClassName)) - - verifyInitContainers(ds, defaultInstance) - expectedNodeEnv := []corev1.EnvVar{ - // Default envvars. - {Name: "DATASTORE_TYPE", Value: "kubernetes"}, - {Name: "WAIT_FOR_DATASTORE", Value: "true"}, - {Name: "CALICO_MANAGE_CNI", Value: "true"}, - {Name: "CALICO_NETWORKING_BACKEND", Value: "bird"}, - {Name: "CLUSTER_TYPE", Value: "k8s,operator,openshift,bgp"}, - {Name: "CALICO_DISABLE_FILE_LOGGING", Value: "false"}, - {Name: "FELIX_DEFAULTENDPOINTTOHOSTACTION", Value: "ACCEPT"}, - {Name: "FELIX_HEALTHENABLED", Value: "true"}, - {Name: "FELIX_HEALTHPORT", Value: "9199"}, - { - Name: "NODENAME", - ValueFrom: &corev1.EnvVarSource{ - FieldRef: &corev1.ObjectFieldSelector{FieldPath: "spec.nodeName"}, - }, - }, - { - Name: "NAMESPACE", - ValueFrom: &corev1.EnvVarSource{ - FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.namespace"}, - }, - }, - {Name: "FELIX_TYPHAK8SNAMESPACE", Value: "calico-system"}, - {Name: "FELIX_TYPHAK8SSERVICENAME", Value: "calico-typha"}, - {Name: "FELIX_TYPHACAFILE", Value: certificatemanagement.TrustedCertBundleMountPath}, - {Name: "FELIX_TYPHACERTFILE", Value: "/node-certs/tls.crt"}, - {Name: "FELIX_TYPHACN", Value: "typha-server"}, - {Name: "FELIX_TYPHAKEYFILE", Value: "/node-certs/tls.key"}, - // Tigera-specific envvars - {Name: "FELIX_PROMETHEUSREPORTERENABLED", Value: "true"}, - {Name: "FELIX_PROMETHEUSREPORTERPORT", Value: "9081"}, - {Name: "FELIX_FLOWLOGSFILEENABLED", Value: "true"}, - {Name: "FELIX_FLOWLOGSFILEINCLUDELABELS", Value: "true"}, - {Name: "FELIX_FLOWLOGSFILEINCLUDEPOLICIES", Value: "true"}, - {Name: "FELIX_FLOWLOGSFILEINCLUDESERVICE", Value: "true"}, - {Name: "FELIX_FLOWLOGSENABLENETWORKSETS", Value: "true"}, - {Name: "FELIX_FLOWLOGSCOLLECTPROCESSINFO", Value: "true"}, - {Name: "FELIX_DNSLOGSFILEENABLED", Value: "true"}, - {Name: "FELIX_DNSLOGSFILEPERNODELIMIT", Value: "1000"}, - {Name: "MULTI_INTERFACE_MODE", Value: operatorv1.MultiInterfaceModeNone.Value()}, - {Name: "NO_DEFAULT_POOLS", Value: "true"}, - } - expectedNodeEnv = configureExpectedNodeEnvIPVersions(expectedNodeEnv, defaultInstance, enableIPv4, enableIPv6) - Expect(ds.Spec.Template.Spec.Containers[0].Env).To(ConsistOf(expectedNodeEnv)) - Expect(len(ds.Spec.Template.Spec.Containers[0].Env)).To(Equal(len(expectedNodeEnv))) - - verifyProbesAndLifecycle(ds, true, true) - }) - - It("should render all resources when variant is CalicoEnterprise and running on RKE2", func() { - expectedResources := []struct { - name string - ns string - group string - version string - kind string - }{ - {name: "calico-node", ns: common.CalicoNamespace, group: "", version: "v1", kind: "ServiceAccount"}, - {name: "calico-node", ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRole"}, - {name: "calico-node", ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRoleBinding"}, - {name: "calico-cni-plugin", ns: common.CalicoNamespace, group: "", version: "v1", kind: "ServiceAccount"}, - {name: "calico-cni-plugin", ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRole"}, - {name: "calico-cni-plugin", ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRoleBinding"}, - {name: "calico-node-metrics", ns: "calico-system", group: "", version: "v1", kind: "Service"}, - {name: "cni-config", ns: common.CalicoNamespace, group: "", version: "v1", kind: "ConfigMap"}, - {name: common.NodeDaemonSetName, ns: common.CalicoNamespace, group: "apps", version: "v1", kind: "DaemonSet"}, - } - - defaultInstance.Variant = operatorv1.CalicoEnterprise - defaultInstance.KubernetesProvider = operatorv1.ProviderRKE2 - defaultCNIConfDir, defaultCNIBinDir := render.DefaultCNIDirectories(defaultInstance.KubernetesProvider) - defaultInstance.CNI.ConfDir, defaultInstance.CNI.BinDir = &defaultCNIConfDir, &defaultCNIBinDir - cfg.NodeReporterMetricsPort = 9081 - cfg.FelixHealthPort = 9199 - - component := render.Node(&cfg) - Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() - Expect(len(resources)).To(Equal(len(expectedResources)), fmt.Sprintf("Actual resources: %#v", resources)) - - // Should render the correct resources. - i := 0 - for _, expectedRes := range expectedResources { - rtest.ExpectResourceTypeAndObjectMetadata(resources[i], expectedRes.name, expectedRes.ns, expectedRes.group, expectedRes.version, expectedRes.kind) - i++ - } - - // The DaemonSet should have the correct configuration. - ds := rtest.GetResource(resources, "calico-node", "calico-system", "apps", "v1", "DaemonSet").(*appsv1.DaemonSet) - Expect(ds.Spec.Template.Spec.Containers[0].Image).To(Equal(components.TigeraRegistry + "tigera/node:" + components.ComponentTigeraNode.Version)) - - // The pod template should have node critical priority - Expect(ds.Spec.Template.Spec.PriorityClassName).To(Equal(render.NodePriorityClassName)) - - verifyInitContainers(ds, defaultInstance) - - expectedNodeEnv := []corev1.EnvVar{ - // Default envvars. - {Name: "DATASTORE_TYPE", Value: "kubernetes"}, - {Name: "WAIT_FOR_DATASTORE", Value: "true"}, - {Name: "CALICO_MANAGE_CNI", Value: "true"}, - {Name: "CALICO_NETWORKING_BACKEND", Value: "bird"}, - {Name: "CLUSTER_TYPE", Value: "k8s,operator,bgp"}, - {Name: "CALICO_DISABLE_FILE_LOGGING", Value: "false"}, - {Name: "FELIX_DEFAULTENDPOINTTOHOSTACTION", Value: "ACCEPT"}, - {Name: "FELIX_HEALTHENABLED", Value: "true"}, - {Name: "FELIX_HEALTHPORT", Value: "9199"}, - { - Name: "NODENAME", - ValueFrom: &corev1.EnvVarSource{ - FieldRef: &corev1.ObjectFieldSelector{FieldPath: "spec.nodeName"}, - }, - }, - { - Name: "NAMESPACE", - ValueFrom: &corev1.EnvVarSource{ - FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.namespace"}, - }, - }, - {Name: "FELIX_TYPHAK8SNAMESPACE", Value: "calico-system"}, - {Name: "FELIX_TYPHAK8SSERVICENAME", Value: "calico-typha"}, - {Name: "FELIX_TYPHACAFILE", Value: certificatemanagement.TrustedCertBundleMountPath}, - {Name: "FELIX_TYPHACERTFILE", Value: "/node-certs/tls.crt"}, - {Name: "FELIX_TYPHACN", Value: "typha-server"}, - {Name: "FELIX_TYPHAKEYFILE", Value: "/node-certs/tls.key"}, - {Name: "NO_DEFAULT_POOLS", Value: "true"}, - // Tigera-specific envvars - {Name: "FELIX_PROMETHEUSREPORTERENABLED", Value: "true"}, - {Name: "FELIX_PROMETHEUSREPORTERPORT", Value: "9081"}, - {Name: "FELIX_FLOWLOGSFILEENABLED", Value: "true"}, - {Name: "FELIX_FLOWLOGSFILEINCLUDELABELS", Value: "true"}, - {Name: "FELIX_FLOWLOGSFILEINCLUDEPOLICIES", Value: "true"}, - {Name: "FELIX_FLOWLOGSFILEINCLUDESERVICE", Value: "true"}, - {Name: "FELIX_FLOWLOGSENABLENETWORKSETS", Value: "true"}, - {Name: "FELIX_FLOWLOGSCOLLECTPROCESSINFO", Value: "true"}, - {Name: "FELIX_DNSLOGSFILEENABLED", Value: "true"}, - {Name: "FELIX_DNSLOGSFILEPERNODELIMIT", Value: "1000"}, - - // The RKE2 envvar overrides. - {Name: "MULTI_INTERFACE_MODE", Value: operatorv1.MultiInterfaceModeNone.Value()}, - } - expectedNodeEnv = configureExpectedNodeEnvIPVersions(expectedNodeEnv, defaultInstance, enableIPv4, enableIPv6) - Expect(ds.Spec.Template.Spec.Containers[0].Env).To(ConsistOf(expectedNodeEnv)) - Expect(len(ds.Spec.Template.Spec.Containers[0].Env)).To(Equal(len(expectedNodeEnv))) - - verifyProbesAndLifecycle(ds, true, true) - - // The metrics service should have the correct configuration. - ms := rtest.GetResource(resources, "calico-node-metrics", "calico-system", "", "v1", "Service").(*corev1.Service) - Expect(ms.Spec.ClusterIP).To(Equal("None"), "metrics service should be headless to prevent kube-proxy from rendering too many iptables rules") + verifyProbesAndLifecycle(ds, true) }) It("should render volumes and node volumemounts when bird templates are provided", func() { @@ -2095,12 +1754,11 @@ var _ = Describe("Node rendering tests", func() { It("should not enable prometheus metrics if NodeMetricsPort is nil", func() { defaultInstance.Variant = operatorv1.CalicoEnterprise defaultInstance.NodeMetricsPort = nil - cfg.NodeReporterMetricsPort = 9081 component := render.Node(&cfg) Expect(component.ResolveImages(nil)).To(BeNil()) resources, _ := component.Objects() - Expect(len(resources)).To(Equal(defaultNumExpectedResources + 1)) + Expect(len(resources)).To(Equal(defaultNumExpectedResources)) dsResource := rtest.GetResource(resources, "calico-node", "calico-system", "apps", "v1", "DaemonSet") Expect(dsResource).ToNot(BeNil()) @@ -2109,7 +1767,8 @@ var _ = Describe("Node rendering tests", func() { ds := dsResource.(*appsv1.DaemonSet) Expect(ds.Spec.Template.Spec.Containers[0].Env).ToNot(ContainElement(notExpectedEnvVar)) - // It should have the reporter port, though. + // The reporter port env is added by the enterprise node modifier, not the + // base render, so it should be absent here. expected := corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERPORT"} Expect(ds.Spec.Template.Spec.Containers[0].Env).ToNot(ContainElement(expected)) }) @@ -2121,7 +1780,7 @@ var _ = Describe("Node rendering tests", func() { component := render.Node(&cfg) Expect(component.ResolveImages(nil)).To(BeNil()) resources, _ := component.Objects() - Expect(len(resources)).To(Equal(defaultNumExpectedResources + 1)) + Expect(len(resources)).To(Equal(defaultNumExpectedResources)) dsResource := rtest.GetResource(resources, "calico-node", "calico-system", "apps", "v1", "DaemonSet") Expect(dsResource).ToNot(BeNil()) @@ -2949,7 +2608,7 @@ var _ = Describe("Node rendering tests", func() { Expect(ds.Spec.Template.Spec.Containers[0].Env).To(ConsistOf(expectedNodeEnv)) // Verify readiness and liveness probes. - verifyProbesAndLifecycle(ds, false, false) + verifyProbesAndLifecycle(ds, false) }) DescribeTable("test node probes", @@ -2974,7 +2633,7 @@ var _ = Describe("Node rendering tests", func() { Expect(dsResource).ToNot(BeNil()) ds := dsResource.(*appsv1.DaemonSet) - verifyProbesAndLifecycle(ds, isOpenshift, isEnterprise) + verifyProbesAndLifecycle(ds, isOpenshift) }, Entry("k8s Calico OS no BGP", false, false, operatorv1.BGPDisabled), @@ -3293,7 +2952,7 @@ var _ = Describe("Node rendering tests", func() { }) // verifyProbesAndLifecycle asserts the expected node liveness and readiness probe plus pod lifecycle settings. -func verifyProbesAndLifecycle(ds *appsv1.DaemonSet, isOpenshift, isEnterprise bool) { +func verifyProbesAndLifecycle(ds *appsv1.DaemonSet, isOpenshift bool) { // Verify readiness and liveness probes. expectedReadiness := &corev1.Probe{ PeriodSeconds: 10, @@ -3327,14 +2986,14 @@ func verifyProbesAndLifecycle(ds *appsv1.DaemonSet, isOpenshift, isEnterprise bo } ExpectWithOffset(1, found).To(BeTrue()) + // The base render produces the same readiness command for all variants; the + // enterprise --bgp-metrics-ready check is added by the node modifier and is + // covered in the enterprise package tests. var expectedReadinessCmd []string - switch { - case !bgp: - expectedReadinessCmd = []string{"/usr/bin/calico", "component", "node", "health", "--felix-ready"} - case bgp && isEnterprise: - expectedReadinessCmd = []string{"/usr/bin/calico", "component", "node", "health", "--bird-ready", "--felix-ready", "--bgp-metrics-ready"} - case bgp: + if bgp { expectedReadinessCmd = []string{"/usr/bin/calico", "component", "node", "health", "--bird-ready", "--felix-ready"} + } else { + expectedReadinessCmd = []string{"/usr/bin/calico", "component", "node", "health", "--felix-ready"} } expectedReadiness.ProbeHandler = corev1.ProbeHandler{Exec: &corev1.ExecAction{Command: expectedReadinessCmd}} diff --git a/pkg/render/render_test.go b/pkg/render/render_test.go index 2b97cfaeb9..beef705146 100644 --- a/pkg/render/render_test.go +++ b/pkg/render/render_test.go @@ -66,9 +66,7 @@ func allCalicoComponents( nodeAppArmorProfile string, clusterDomain string, kubeControllersMetricsPort int, - nodeReporterMetricsPort int, bgpLayout *corev1.ConfigMap, - logCollector *operatorv1.LogCollector, ) ([]render.Component, error) { namespaces := render.Namespaces(&render.NamespaceConfiguration{Installation: cr, PullSecrets: pullSecrets}) @@ -79,17 +77,15 @@ func allCalicoComponents( secretsAndConfigMaps := render.NewCreationPassthrough(objs...) nodeCfg := &render.NodeConfiguration{ - K8sServiceEp: k8sServiceEp, - Installation: cr, - TLS: typhaNodeTLS, - NodeAppArmorProfile: nodeAppArmorProfile, - ClusterDomain: clusterDomain, - NodeReporterMetricsPort: nodeReporterMetricsPort, - BGPLayouts: bgpLayout, - LogCollector: logCollector, - BirdTemplates: bt, - MigrateNamespaces: up, - FelixHealthPort: 9099, + K8sServiceEp: k8sServiceEp, + Installation: cr, + TLS: typhaNodeTLS, + NodeAppArmorProfile: nodeAppArmorProfile, + ClusterDomain: clusterDomain, + BGPLayouts: bgpLayout, + BirdTemplates: bt, + MigrateNamespaces: up, + FelixHealthPort: 9099, } typhaCfg := &render.TyphaConfiguration{ K8sServiceEp: k8sServiceEp, @@ -111,13 +107,12 @@ func allCalicoComponents( } winCfg := &render.WindowsConfiguration{ - K8sServiceEp: k8sServiceEp, - K8sDNSServers: []string{}, - Installation: cr, - ClusterDomain: clusterDomain, - TLS: typhaNodeTLS, - NodeReporterMetricsPort: nodeReporterMetricsPort, - VXLANVNI: 4096, + K8sServiceEp: k8sServiceEp, + K8sDNSServers: []string{}, + Installation: cr, + ClusterDomain: clusterDomain, + TLS: typhaNodeTLS, + VXLANVNI: 4096, } nodeCertComponent := rcertificatemanagement.CertificateManagement(&rcertificatemanagement.Config{ @@ -219,22 +214,22 @@ var _ = Describe("Rendering tests", func() { // - 6 kube-controllers resources (ServiceAccount, ClusterRole, Binding, Deployment, Service, Secret,RoleBinding) // - 1 namespace // - 2 Windows node resources (ConfigMap, DaemonSet) - c, err := allCalicoComponents(k8sServiceEp, instance, nil, nil, nil, typhaNodeTLS, nil, nil, false, "", dns.DefaultClusterDomain, 9094, 0, nil, nil) + c, err := allCalicoComponents(k8sServiceEp, instance, nil, nil, nil, typhaNodeTLS, nil, nil, false, "", dns.DefaultClusterDomain, 9094, nil) Expect(err).To(BeNil(), "Expected Calico to create successfully %s", err) Expect(componentCount(c)).To(Equal(5 + 3 + 4 + 1 + 6 + 6 + 1 + 2)) }) It("should render all resources when variant is Tigera Secure", func() { - // For this scenario, we expect the basic resources plus the following for Tigera Secure: - // - X Same as default config - // - 1 Service to expose calico/node metrics. - // - 1 Service to expose Windows calico/node metrics. + // For this scenario, we expect the basic resources plus the following for Tigera Secure. + // The calico/node and Windows calico/node metrics Services are added by the + // enterprise modifiers at the componentHandler, not by Objects(), so they do + // not appear in this render-only aggregation. var nodeMetricsPort int32 = 9081 instance.Variant = operatorv1.CalicoEnterprise instance.NodeMetricsPort = &nodeMetricsPort - c, err := allCalicoComponents(k8sServiceEp, instance, nil, nil, nil, typhaNodeTLS, nil, nil, false, "", dns.DefaultClusterDomain, 9094, 0, nil, nil) + c, err := allCalicoComponents(k8sServiceEp, instance, nil, nil, nil, typhaNodeTLS, nil, nil, false, "", dns.DefaultClusterDomain, 9094, nil) Expect(err).To(BeNil(), "Expected Calico to create successfully %s", err) - Expect(componentCount(c)).To(Equal((5 + 3 + 4 + 1 + 6 + 6 + 1 + 2) + 1 + 1)) + Expect(componentCount(c)).To(Equal(5 + 3 + 4 + 1 + 6 + 6 + 1 + 2)) }) It("should render all resources when variant is Tigera Secure and Management Cluster", func() { @@ -245,7 +240,7 @@ var _ = Describe("Rendering tests", func() { instance.Variant = operatorv1.CalicoEnterprise instance.NodeMetricsPort = &nodeMetricsPort - c, err := allCalicoComponents(k8sServiceEp, instance, &operatorv1.ManagementCluster{}, nil, nil, typhaNodeTLS, internalManagerKeyPair, nil, false, "", dns.DefaultClusterDomain, 9094, 0, nil, nil) + c, err := allCalicoComponents(k8sServiceEp, instance, &operatorv1.ManagementCluster{}, nil, nil, typhaNodeTLS, internalManagerKeyPair, nil, false, "", dns.DefaultClusterDomain, 9094, nil) Expect(err).To(BeNil(), "Expected Calico to create successfully %s", err) expectedResources := []client.Object{ @@ -267,7 +262,6 @@ var _ = Describe("Rendering tests", func() { &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: "calico-cni-plugin", Namespace: common.CalicoNamespace}, TypeMeta: metav1.TypeMeta{Kind: "ServiceAccount", APIVersion: "v1"}}, &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-cni-plugin"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-cni-plugin"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: "calico-node-metrics", Namespace: common.CalicoNamespace}, TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}}, &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "cni-config", Namespace: common.CalicoNamespace}, TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"}}, &appsv1.DaemonSet{ObjectMeta: metav1.ObjectMeta{Name: common.NodeDaemonSetName, Namespace: common.CalicoNamespace}, TypeMeta: metav1.TypeMeta{Kind: "DaemonSet", APIVersion: "apps/v1"}}, @@ -279,8 +273,8 @@ var _ = Describe("Rendering tests", func() { &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: common.KubeControllersDeploymentName, Namespace: common.CalicoNamespace}, TypeMeta: metav1.TypeMeta{Kind: "Deployment", APIVersion: "apps/v1"}}, &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: "calico-kube-controllers-metrics", Namespace: common.CalicoNamespace}, TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}}, - // Windows node objects. - &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: render.WindowsNodeMetricsService, Namespace: common.CalicoNamespace}, TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}}, + // Windows node objects. The Windows node-metrics Service is added by the + // enterprise modifier at the componentHandler, so it is not in this output. &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "cni-config-windows", Namespace: common.CalicoNamespace}, TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"}}, &appsv1.DaemonSet{ObjectMeta: metav1.ObjectMeta{Name: common.WindowsDaemonSetName, Namespace: common.CalicoNamespace}, TypeMeta: metav1.TypeMeta{Kind: "DaemonSet", APIVersion: "apps/v1"}}, @@ -307,7 +301,7 @@ var _ = Describe("Rendering tests", func() { It("should render calico with a apparmor profile if annotation is present in installation", func() { apparmorProf := "foobar" - comps, err := allCalicoComponents(k8sServiceEp, instance, nil, nil, nil, typhaNodeTLS, nil, nil, false, apparmorProf, dns.DefaultClusterDomain, 0, 0, nil, nil) + comps, err := allCalicoComponents(k8sServiceEp, instance, nil, nil, nil, typhaNodeTLS, nil, nil, false, apparmorProf, dns.DefaultClusterDomain, 0, nil) Expect(err).To(BeNil(), "Expected Calico to create successfully %s", err) var cn *appsv1.DaemonSet for _, comp := range comps { @@ -331,7 +325,7 @@ var _ = Describe("Rendering tests", func() { } bgpLayout.Name = "bgp-layout" bgpLayout.Namespace = common.OperatorNamespace() - comps, err := allCalicoComponents(k8sServiceEp, instance, nil, nil, nil, typhaNodeTLS, nil, nil, false, "", dns.DefaultClusterDomain, 0, 0, bgpLayout, nil) + comps, err := allCalicoComponents(k8sServiceEp, instance, nil, nil, nil, typhaNodeTLS, nil, nil, false, "", dns.DefaultClusterDomain, 0, bgpLayout) Expect(err).To(BeNil(), "Expected Calico to create successfully %s", err) var cm *corev1.ConfigMap var ds *appsv1.DaemonSet @@ -352,42 +346,8 @@ var _ = Describe("Rendering tests", func() { Expect(ds.Spec.Template.Annotations["hash.operator.tigera.io/bgp-layout"]).NotTo(BeEmpty()) }) - It("should handle collectProcessPath in logCollector", func() { - testNode := func(processPath operatorv1.CollectProcessPathOption, expectedHostPID bool) { - var logCollector operatorv1.LogCollector - logCollector.Spec.CollectProcessPath = &processPath - comps, err := allCalicoComponents(k8sServiceEp, instance, nil, nil, nil, typhaNodeTLS, nil, nil, false, "", dns.DefaultClusterDomain, 0, 0, nil, &logCollector) - Expect(err).To(BeNil(), "Expected Calico to create successfully %s", err) - var ds *appsv1.DaemonSet - for _, comp := range comps { - resources, _ := comp.Objects() - r := rtest.GetResource(resources, "calico-node", "calico-system", "apps", "v1", "DaemonSet") - if r != nil { - ds = r.(*appsv1.DaemonSet) - } - } - checkEnvVar := func(ds *appsv1.DaemonSet) bool { - envPresent := false - for _, env := range ds.Spec.Template.Spec.Containers[0].Env { - if env.Name == "FELIX_FLOWLOGSCOLLECTPROCESSPATH" { - envPresent = true - if env.Value == "true" { - return true - } - } - } - return !envPresent - } - Expect(ds).ToNot(BeNil()) - Expect(ds.Spec.Template.Spec.HostPID).To(Equal(expectedHostPID)) - Expect(checkEnvVar(ds)).To(Equal(true)) - } - testNode(operatorv1.CollectProcessPathEnable, true) - testNode(operatorv1.CollectProcessPathDisable, false) - }) - It("should set node priority class to system-node-critical", func() { - comps, err := allCalicoComponents(k8sServiceEp, instance, nil, nil, nil, typhaNodeTLS, nil, nil, false, "", dns.DefaultClusterDomain, 0, 0, nil, nil) + comps, err := allCalicoComponents(k8sServiceEp, instance, nil, nil, nil, typhaNodeTLS, nil, nil, false, "", dns.DefaultClusterDomain, 0, nil) Expect(err).To(BeNil(), "Expected Calico to create successfully %s", err) var cn *appsv1.DaemonSet for _, comp := range comps { @@ -403,7 +363,7 @@ var _ = Describe("Rendering tests", func() { }) It("should set typha priority class to system-cluster-critical", func() { - comps, err := allCalicoComponents(k8sServiceEp, instance, nil, nil, nil, typhaNodeTLS, nil, nil, false, "", dns.DefaultClusterDomain, 0, 0, nil, nil) + comps, err := allCalicoComponents(k8sServiceEp, instance, nil, nil, nil, typhaNodeTLS, nil, nil, false, "", dns.DefaultClusterDomain, 0, nil) Expect(err).To(BeNil(), "Expected Calico to create successfully %s", err) var cn *appsv1.Deployment for _, comp := range comps { @@ -419,7 +379,7 @@ var _ = Describe("Rendering tests", func() { }) It("should set kube controllers priority class to system-cluster-critical", func() { - comps, err := allCalicoComponents(k8sServiceEp, instance, nil, nil, nil, typhaNodeTLS, nil, nil, false, "", dns.DefaultClusterDomain, 0, 0, nil, nil) + comps, err := allCalicoComponents(k8sServiceEp, instance, nil, nil, nil, typhaNodeTLS, nil, nil, false, "", dns.DefaultClusterDomain, 0, nil) Expect(err).To(BeNil(), "Expected Calico to create successfully %s", err) var cn *appsv1.Deployment for _, comp := range comps { diff --git a/pkg/render/rendercontext.go b/pkg/render/rendercontext.go new file mode 100644 index 0000000000..7101a0e10d --- /dev/null +++ b/pkg/render/rendercontext.go @@ -0,0 +1,65 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package render + +import ( + v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/tls/certificatemanagement" +) + +// RenderContext carries reconcile-derived inputs from controllers into render +// modifiers. Core operator code never reads these fields - only registered +// modifiers do. It carries raw cluster state gathered generically (Installation, +// FelixConfiguration, ClusterDomain) that modifiers derive their own values from, +// the shared TrustedBundle, and an opaque Extension slot for controller-produced +// data specific to one extension. +// +// It lives in render (not the extensions package) because it is the render-phase +// input a modifier consumes - the render-side corollary to the controller-phase +// ControllerContext. The extensions package wires modifiers to it; it is not +// itself part of the extension mechanism. +// +// Per-component config a modifier needs but can't derive from these fields is +// not carried here; it flows to the modifier as a typed argument (see +// extensions.RegisterModifier), supplied by the component via +// ExtensionContextProvider. +type RenderContext struct { + Installation *operatorv1.InstallationSpec + FelixConfiguration *v3.FelixConfiguration + ClusterDomain string + + // TrustedBundle is the shared CA bundle for the calico-system namespace. + TrustedBundle certificatemanagement.TrustedBundle + + // Extension is opaque, extension-owned data that the controller extension + // produced. Usually a modifier reads it back out (an artifact that can only be + // created controller-side because it has cluster side effects, e.g. a keypair), + // in which case it holds an extension-package type core code never names. When a + // controller needs the extension's reconcile-derived inputs back (e.g. the + // clusterconnection controller's variant-specific Guardian inputs), the payload + // is instead a render-package type the controller reads generically without + // depending on the extension. Nil when no extension is active. + Extension any +} + +// ExtractExtensionData returns the extension-owned data a controller extension +// stashed in the render context, asserted to T, or the zero value of T when the +// slot is empty or holds a different type. Extensions use it instead of repeating +// the type assertion in a per-component accessor. +func ExtractExtensionData[T any](rc RenderContext) T { + data, _ := rc.Extension.(T) + return data +} diff --git a/pkg/render/typha.go b/pkg/render/typha.go index 264851785f..9a17c7115f 100644 --- a/pkg/render/typha.go +++ b/pkg/render/typha.go @@ -113,6 +113,8 @@ func (c *typhaComponent) SupportedOSType() rmeta.OSType { return rmeta.OSTypeLinux } +func (c *typhaComponent) ModifierKey() string { return ComponentNameTypha } + func (c *typhaComponent) Objects() ([]client.Object, []client.Object) { pdb := c.typhaPodDisruptionBudget() if overrides := c.cfg.Installation.TyphaPodDisruptionBudget; overrides != nil { @@ -356,26 +358,6 @@ func (c *typhaComponent) typhaRole() *rbacv1.ClusterRole { }, }, } - if c.cfg.Installation.Variant.IsEnterprise() { - extraRules := []rbacv1.PolicyRule{ - { - // Tigera Secure needs to be able to read licenses, and config. - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{ - "bfdconfigurations", - "deeppacketinspections", - "egressgatewaypolicies", - "externalnetworks", - "licensekeys", - "networks", - "packetcaptures", - "remoteclusterconfigurations", - }, - Verbs: []string{"get", "list", "watch"}, - }, - } - role.Rules = append(role.Rules, extraRules...) - } if c.cfg.Installation.KubernetesProvider.IsOpenShift() { role.Rules = append(role.Rules, rbacv1.PolicyRule{ APIGroups: []string{"security.openshift.io"}, @@ -633,15 +615,6 @@ func (c *typhaComponent) typhaEnvVars(typhaSecret certificatemanagement.KeyPairI typhaEnv = append(typhaEnv, corev1.EnvVar{Name: "FELIX_INTERFACEPREFIX", Value: "azv"}) } - if c.cfg.Installation.Variant.IsEnterprise() { - if c.cfg.Installation.CalicoNetwork != nil && c.cfg.Installation.CalicoNetwork.MultiInterfaceMode != nil { - typhaEnv = append(typhaEnv, corev1.EnvVar{ - Name: "MULTI_INTERFACE_MODE", - Value: c.cfg.Installation.CalicoNetwork.MultiInterfaceMode.Value(), - }) - } - } - // If host-local IPAM is in use, we need to configure typha to use the Kubernetes pod CIDR. cni := c.cfg.Installation.CNI if cni != nil && cni.IPAM != nil && cni.IPAM.Type == operatorv1.IPAMPluginHostLocal { diff --git a/pkg/render/windows.go b/pkg/render/windows.go index 28df89c78d..dd9692c77f 100644 --- a/pkg/render/windows.go +++ b/pkg/render/windows.go @@ -24,17 +24,16 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/intstr" "sigs.k8s.io/controller-runtime/pkg/client" operatorv1 "github.com/tigera/operator/api/v1" "github.com/tigera/operator/pkg/common" "github.com/tigera/operator/pkg/components" "github.com/tigera/operator/pkg/controller/k8sapi" + "github.com/tigera/operator/pkg/imageoverride" rcomp "github.com/tigera/operator/pkg/render/common/components" rmeta "github.com/tigera/operator/pkg/render/common/meta" "github.com/tigera/operator/pkg/render/common/securitycontext" - "github.com/tigera/operator/pkg/tls/certificatemanagement" ) const ( @@ -49,14 +48,17 @@ func Windows( } type WindowsConfiguration struct { - K8sServiceEp k8sapi.ServiceEndpoint - K8sDNSServers []string - Installation *operatorv1.InstallationSpec - ClusterDomain string - TLS *TyphaNodeTLS - PrometheusServerTLS certificatemanagement.KeyPairInterface - NodeReporterMetricsPort int - VXLANVNI int + K8sServiceEp k8sapi.ServiceEndpoint + K8sDNSServers []string + Installation *operatorv1.InstallationSpec + ClusterDomain string + TLS *TyphaNodeTLS + VXLANVNI int + + // ImageOverrides lets a variant swap the windows node and CNI images. The + // controller wires in the operator's image overrides; nil resolves to the + // core images. + ImageOverrides *imageoverride.Overrides } type windowsComponent struct { @@ -77,13 +79,10 @@ func (c *windowsComponent) ResolveImages(is *operatorv1.ImageSet) error { return imageName } - if c.cfg.Installation.Variant.IsEnterprise() { - c.cniImage = appendIfErr(components.GetReference(components.ComponentTigeraCNIWindows, reg, path, prefix, is)) - c.nodeImage = appendIfErr(components.GetReference(components.ComponentTigeraNodeWindows, reg, path, prefix, is)) - } else { - c.cniImage = appendIfErr(components.GetReference(components.ComponentCalicoCNIWindows, reg, path, prefix, is)) - c.nodeImage = appendIfErr(components.GetReference(components.ComponentCalicoNodeWindows, reg, path, prefix, is)) - } + cniImage := c.cfg.ImageOverrides.Resolve(ComponentNameWindowsCNIImg, components.ComponentCalicoCNIWindows, c.cfg.Installation) + nodeImage := c.cfg.ImageOverrides.Resolve(ComponentNameWindowsNodeImg, components.ComponentCalicoNodeWindows, c.cfg.Installation) + c.cniImage = appendIfErr(components.GetReference(cniImage, reg, path, prefix, is)) + c.nodeImage = appendIfErr(components.GetReference(nodeImage, reg, path, prefix, is)) if len(errMsgs) != 0 { return fmt.Errorf("%s", strings.Join(errMsgs, ",")) @@ -95,6 +94,8 @@ func (c *windowsComponent) SupportedOSType() rmeta.OSType { return rmeta.OSTypeWindows } +func (c *windowsComponent) ModifierKey() string { return ComponentNameWindows } + func (c *windowsComponent) Objects() ([]client.Object, []client.Object) { // Clean up old windows upgrader daemonset if present objsToDelete := []client.Object{ @@ -116,11 +117,6 @@ func (c *windowsComponent) Objects() ([]client.Object, []client.Object) { objs := []client.Object{} - if c.cfg.Installation.Variant.IsEnterprise() { - // Include Service for exposing node metrics. - objs = append(objs, c.nodeMetricsService()) - } - cniConfig := c.windowsCNIConfigMap() if cniConfig != nil { objs = append(objs, cniConfig) @@ -135,43 +131,6 @@ func (c *windowsComponent) Ready() bool { return true } -// nodeMetricsService creates a Service which exposes two endpoints on calico/node for -// reporting Prometheus metrics (for policy enforcement activity and BGP stats). -// This service is used internally by Calico Enterprise and is separate from general -// Prometheus metrics which are user-configurable. -func (c *windowsComponent) nodeMetricsService() *corev1.Service { - return &corev1.Service{ - TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}, - ObjectMeta: metav1.ObjectMeta{ - Name: WindowsNodeMetricsService, - Namespace: common.CalicoNamespace, - Labels: map[string]string{"k8s-app": WindowsNodeObjectName}, - }, - Spec: corev1.ServiceSpec{ - Selector: map[string]string{"k8s-app": WindowsNodeObjectName}, - // Important: "None" tells Kubernetes that we want a headless service with - // no kube-proxy load balancer. If we omit this then kube-proxy will render - // a huge set of iptables rules for this service since there's an instance - // on every node. - ClusterIP: "None", - Ports: []corev1.ServicePort{ - { - Name: "calico-metrics-port", - Port: int32(c.cfg.NodeReporterMetricsPort), - TargetPort: intstr.FromInt(c.cfg.NodeReporterMetricsPort), - Protocol: corev1.ProtocolTCP, - }, - { - Name: "calico-bgp-metrics-port", - Port: nodeBGPReporterPort, - TargetPort: intstr.FromInt(int(nodeBGPReporterPort)), - Protocol: corev1.ProtocolTCP, - }, - }, - }, - } -} - // windowsCNIConfigMap returns a config map containing the CNI network config to be installed on each node. // Returns nil if no configmap is needed. func (c *windowsComponent) windowsCNIConfigMap() *corev1.ConfigMap { @@ -380,8 +339,8 @@ func (c *windowsComponent) windowsVolumes() []corev1.Volume { {Name: "policysync", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/run/nodeagent", Type: &dirOrCreate}}}, c.cfg.TLS.TrustedBundle.Volume(), c.cfg.TLS.NodeSecret.Volume(), - corev1.Volume{Name: "var-run-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/run/calico", Type: &dirOrCreate}}}, - corev1.Volume{Name: "var-lib-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/lib/calico", Type: &dirOrCreate}}}, + {Name: "var-run-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/run/calico", Type: &dirOrCreate}}}, + {Name: "var-lib-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/lib/calico", Type: &dirOrCreate}}}, } // If needed for this configuration, then include the CNI volumes. @@ -392,20 +351,6 @@ func (c *windowsComponent) windowsVolumes() []corev1.Volume { volumes = append(volumes, corev1.Volume{Name: "cni-log-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: c.cfg.Installation.WindowsNodes.CNILogDir, Type: &dirOrCreate}}}) } - // Override with Tigera-specific config. - if c.cfg.Installation.Variant.IsEnterprise() { - // Add volume for calico logs. - calicoLogVol := corev1.Volume{ - Name: "var-log-calico", - VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico", Type: &dirOrCreate}}, - } - volumes = append(volumes, calicoLogVol) - } - - if c.cfg.PrometheusServerTLS != nil { - volumes = append(volumes, c.cfg.PrometheusServerTLS.Volume()) - } - return volumes } @@ -483,7 +428,6 @@ func (c *windowsComponent) nodeContainer() corev1.Container { // felixContainer creates the windows felix container. func (c *windowsComponent) felixContainer() corev1.Container { - lp, rp := c.windowsLivenessReadinessProbes() return corev1.Container{ @@ -663,31 +607,6 @@ func (c *windowsComponent) windowsEnvVars() []corev1.EnvVar { windowsEnv = append(windowsEnv, corev1.EnvVar{Name: "FELIX_IPV6SUPPORT", Value: "false"}) } - if c.cfg.Installation.Variant.IsEnterprise() { - // Add in Calico Enterprise specific configuration. - extraNodeEnv := []corev1.EnvVar{ - {Name: "FELIX_PROMETHEUSREPORTERENABLED", Value: "true"}, - {Name: "FELIX_PROMETHEUSREPORTERPORT", Value: fmt.Sprintf("%d", c.cfg.NodeReporterMetricsPort)}, - {Name: "FELIX_FLOWLOGSFILEENABLED", Value: "true"}, - {Name: "FELIX_FLOWLOGSFILEINCLUDELABELS", Value: "true"}, - {Name: "FELIX_FLOWLOGSFILEINCLUDEPOLICIES", Value: "true"}, - {Name: "FELIX_FLOWLOGSFILEINCLUDESERVICE", Value: "true"}, - {Name: "FELIX_FLOWLOGSENABLENETWORKSETS", Value: "true"}, - {Name: "FELIX_FLOWLOGSCOLLECTPROCESSINFO", Value: "true"}, - {Name: "FELIX_DNSLOGSFILEENABLED", Value: "true"}, - {Name: "FELIX_DNSLOGSFILEPERNODELIMIT", Value: "1000"}, - } - - if c.cfg.PrometheusServerTLS != nil { - extraNodeEnv = append(extraNodeEnv, - corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERCERTFILE", Value: c.cfg.PrometheusServerTLS.VolumeMountCertificateFilePath()}, - corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERKEYFILE", Value: c.cfg.PrometheusServerTLS.VolumeMountKeyFilePath()}, - corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERCAFILE", Value: c.cfg.TLS.TrustedBundle.MountPath()}, - ) - } - windowsEnv = append(windowsEnv, extraNodeEnv...) - } - if c.cfg.Installation.NodeMetricsPort != nil { // If a node metrics port was given, then enable felix prometheus metrics and set the port. // Note that this takes precedence over any FelixConfiguration resources in the cluster. @@ -698,20 +617,6 @@ func (c *windowsComponent) windowsEnvVars() []corev1.EnvVar { windowsEnv = append(windowsEnv, extraNodeEnv...) } - // Configure provider specific environment variables here. - switch c.cfg.Installation.KubernetesProvider { - case operatorv1.ProviderOpenShift: - if c.cfg.Installation.Variant.IsEnterprise() { - // We need to configure a non-default trusted DNS server, since there's no kube-dns. - windowsEnv = append(windowsEnv, corev1.EnvVar{Name: "FELIX_DNSTRUSTEDSERVERS", Value: "k8s-service:openshift-dns/dns-default"}) - } - case operatorv1.ProviderRKE2: - // For RKE2, configure a non-default trusted DNS server, as the DNS service is not named "kube-dns". - if c.cfg.Installation.Variant.IsEnterprise() { - windowsEnv = append(windowsEnv, corev1.EnvVar{Name: "FELIX_DNSTRUSTEDSERVERS", Value: "k8s-service:kube-system/rke2-coredns-rke2-coredns"}) - } - } - if c.cfg.Installation.CNI.Type != operatorv1.PluginCalico { windowsEnv = append(windowsEnv, corev1.EnvVar{Name: "FELIX_ROUTESOURCE", Value: "WorkloadIPs"}) } @@ -730,12 +635,7 @@ func (c *windowsComponent) windowsVolumeMounts() []corev1.VolumeMount { corev1.VolumeMount{MountPath: "/var/run/calico", Name: "var-run-calico"}, corev1.VolumeMount{MountPath: "/var/lib/calico", Name: "var-lib-calico"}) - if c.cfg.Installation.Variant.IsEnterprise() { - extraNodeMounts := []corev1.VolumeMount{ - {MountPath: "/var/log/calico", Name: "var-log-calico"}, - } - windowsVolumeMounts = append(windowsVolumeMounts, extraNodeMounts...) - } else if c.cfg.Installation.CNI.Type == operatorv1.PluginCalico { + if c.cfg.Installation.CNI.Type == operatorv1.PluginCalico { windowsVolumeMounts = append(windowsVolumeMounts, corev1.VolumeMount{MountPath: "/var/log/calico/cni", Name: "cni-log-dir", ReadOnly: false}) } @@ -743,9 +643,6 @@ func (c *windowsComponent) windowsVolumeMounts() []corev1.VolumeMount { windowsVolumeMounts = append(windowsVolumeMounts, corev1.VolumeMount{MountPath: "/host/etc/cni/net.d", Name: "cni-net-dir"}) } - if c.cfg.PrometheusServerTLS != nil { - windowsVolumeMounts = append(windowsVolumeMounts, c.cfg.PrometheusServerTLS.VolumeMount(c.SupportedOSType())) - } return windowsVolumeMounts } @@ -801,9 +698,6 @@ func (c *windowsComponent) windowsDaemonset(cniCfgMap *corev1.ConfigMap) *appsv1 initContainers := []corev1.Container{c.uninstallContainer()} annotations := c.cfg.TLS.TrustedBundle.HashAnnotations() - if c.cfg.PrometheusServerTLS != nil { - annotations[c.cfg.PrometheusServerTLS.HashAnnotationKey()] = c.cfg.PrometheusServerTLS.HashAnnotationValue() - } if cniCfgMap != nil { annotations[nodeCniConfigAnnotation] = rmeta.AnnotationHash(cniCfgMap.Data) diff --git a/pkg/render/windows_test.go b/pkg/render/windows_test.go index f0afe2007c..868334d55d 100644 --- a/pkg/render/windows_test.go +++ b/pkg/render/windows_test.go @@ -35,6 +35,7 @@ import ( "github.com/tigera/operator/pkg/controller/certificatemanager" "github.com/tigera/operator/pkg/controller/k8sapi" ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" + "github.com/tigera/operator/pkg/imageoverride" "github.com/tigera/operator/pkg/render" rmeta "github.com/tigera/operator/pkg/render/common/meta" rtest "github.com/tigera/operator/pkg/render/common/test" @@ -107,12 +108,13 @@ var _ = Describe("Windows rendering tests", func() { // Create a default configuration. cfg = render.WindowsConfiguration{ - K8sServiceEp: k8sServiceEp, - K8sDNSServers: []string{"10.96.0.10"}, - Installation: defaultInstance, - ClusterDomain: defaultClusterDomain, - TLS: typhaNodeTLS, - VXLANVNI: 4096, + K8sServiceEp: k8sServiceEp, + K8sDNSServers: []string{"10.96.0.10"}, + Installation: defaultInstance, + ClusterDomain: defaultClusterDomain, + TLS: typhaNodeTLS, + VXLANVNI: 4096, + ImageOverrides: imageoverride.New(), } }) @@ -662,393 +664,6 @@ var _ = Describe("Windows rendering tests", func() { } }) - It("should render all resources for a default configuration using CalicoEnterprise", func() { - type testConf struct { - EnableBGP bool - EnableVXLAN bool - } - for _, testConfig := range []testConf{ - {true, false}, - {false, true}, - {true, true}, - } { - enableBGP := testConfig.EnableBGP - enableVXLAN := testConfig.EnableVXLAN - - if enableBGP { - defaultInstance.CalicoNetwork.BGP = &bgpEnabled - } else { - defaultInstance.CalicoNetwork.BGP = &bgpDisabled - } - - if enableVXLAN { - defaultInstance.CalicoNetwork.IPPools[0].Encapsulation = operatorv1.EncapsulationVXLAN - } else { - defaultInstance.CalicoNetwork.IPPools[0].Encapsulation = operatorv1.EncapsulationNone - } - By(fmt.Sprintf("BGP enabled: %v, VXLAN enabled: %v", enableBGP, enableVXLAN), func() { - expectedResources := []struct { - name string - ns string - group string - version string - kind string - }{ - {name: "calico-node-metrics-windows", ns: "calico-system", group: "", version: "v1", kind: "Service"}, - {name: "cni-config-windows", ns: common.CalicoNamespace, group: "", version: "v1", kind: "ConfigMap"}, - {name: common.WindowsDaemonSetName, ns: common.CalicoNamespace, group: "apps", version: "v1", kind: "DaemonSet"}, - } - defaultInstance.Variant = operatorv1.CalicoEnterprise - cfg.NodeReporterMetricsPort = 9081 - - component := render.Windows(&cfg) - Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() - Expect(len(resources)).To(Equal(len(expectedResources))) - - // Should render the correct resources. - i := 0 - for _, expectedRes := range expectedResources { - rtest.ExpectResourceTypeAndObjectMetadata(resources[i], expectedRes.name, expectedRes.ns, expectedRes.group, expectedRes.version, expectedRes.kind) - i++ - } - - // The DaemonSet should have the correct configuration. - ds := rtest.GetResource(resources, "calico-node-windows", "calico-system", "apps", "v1", "DaemonSet").(*appsv1.DaemonSet) - - // The pod template should have node critical priority - Expect(ds.Spec.Template.Spec.PriorityClassName).To(Equal(render.NodePriorityClassName)) - - // The calico-node-windows daemonset has 3 containers (felix, node and confd). - // confd is only instantiated if using BGP. - numContainers := 3 - if !enableBGP { - numContainers = 2 - } - Expect(ds.Spec.Template.Spec.Containers).To(HaveLen(numContainers)) - for _, container := range ds.Spec.Template.Spec.Containers { - - // Windows node image override results in correct image. - Expect(container.Image).To(Equal(components.TigeraRegistry + "tigera/node-windows:" + components.ComponentTigeraNodeWindows.Version)) - Expect(container.SecurityContext.Capabilities).To(BeNil()) - Expect(container.SecurityContext.Privileged).To(BeNil()) - Expect(container.SecurityContext.SELinuxOptions).To(BeNil()) - Expect(container.SecurityContext.WindowsOptions).To(Not(BeNil())) - Expect(container.SecurityContext.WindowsOptions.GMSACredentialSpecName).To(BeNil()) - Expect(container.SecurityContext.WindowsOptions.GMSACredentialSpec).To(BeNil()) - Expect(*container.SecurityContext.WindowsOptions.RunAsUserName).To(Equal("NT AUTHORITY\\system")) - Expect(*container.SecurityContext.WindowsOptions.HostProcess).To(BeTrue()) - Expect(container.SecurityContext.RunAsUser).To(BeNil()) - Expect(container.SecurityContext.RunAsGroup).To(BeNil()) - Expect(container.SecurityContext.RunAsNonRoot).To(BeNil()) - Expect(container.SecurityContext.ReadOnlyRootFilesystem).To(BeNil()) - Expect(container.SecurityContext.AllowPrivilegeEscalation).To(BeNil()) - Expect(container.SecurityContext.ProcMount).To(BeNil()) - Expect(container.SecurityContext.SeccompProfile).To(BeNil()) - } - - felixContainer := rtest.GetContainer(ds.Spec.Template.Spec.Containers, "felix") - - // Windows node image override results in correct image. - Expect(felixContainer.Image).To(Equal(components.TigeraRegistry + "tigera/node-windows:" + components.ComponentTigeraNodeWindows.Version)) - Expect(felixContainer.SecurityContext.Capabilities).To(BeNil()) - Expect(felixContainer.SecurityContext.Privileged).To(BeNil()) - Expect(felixContainer.SecurityContext.SELinuxOptions).To(BeNil()) - Expect(felixContainer.SecurityContext.WindowsOptions).To(Not(BeNil())) - Expect(felixContainer.SecurityContext.WindowsOptions.GMSACredentialSpecName).To(BeNil()) - Expect(felixContainer.SecurityContext.WindowsOptions.GMSACredentialSpec).To(BeNil()) - Expect(*felixContainer.SecurityContext.WindowsOptions.RunAsUserName).To(Equal("NT AUTHORITY\\system")) - Expect(*felixContainer.SecurityContext.WindowsOptions.HostProcess).To(BeTrue()) - Expect(felixContainer.SecurityContext.RunAsUser).To(BeNil()) - Expect(felixContainer.SecurityContext.RunAsGroup).To(BeNil()) - Expect(felixContainer.SecurityContext.RunAsNonRoot).To(BeNil()) - Expect(felixContainer.SecurityContext.ReadOnlyRootFilesystem).To(BeNil()) - Expect(felixContainer.SecurityContext.AllowPrivilegeEscalation).To(BeNil()) - Expect(felixContainer.SecurityContext.ProcMount).To(BeNil()) - Expect(felixContainer.SecurityContext.SeccompProfile).To(BeNil()) - - nodeContainer := rtest.GetContainer(ds.Spec.Template.Spec.Containers, "node") - - // Windows node image override results in correct image. - Expect(nodeContainer.Image).To(Equal(components.TigeraRegistry + "tigera/node-windows:" + components.ComponentTigeraNodeWindows.Version)) - Expect(nodeContainer.SecurityContext.Capabilities).To(BeNil()) - Expect(nodeContainer.SecurityContext.Privileged).To(BeNil()) - Expect(nodeContainer.SecurityContext.SELinuxOptions).To(BeNil()) - Expect(nodeContainer.SecurityContext.WindowsOptions).To(Not(BeNil())) - Expect(nodeContainer.SecurityContext.WindowsOptions.GMSACredentialSpecName).To(BeNil()) - Expect(nodeContainer.SecurityContext.WindowsOptions.GMSACredentialSpec).To(BeNil()) - Expect(*nodeContainer.SecurityContext.WindowsOptions.RunAsUserName).To(Equal("NT AUTHORITY\\system")) - Expect(*nodeContainer.SecurityContext.WindowsOptions.HostProcess).To(BeTrue()) - Expect(nodeContainer.SecurityContext.RunAsUser).To(BeNil()) - Expect(nodeContainer.SecurityContext.RunAsGroup).To(BeNil()) - Expect(nodeContainer.SecurityContext.RunAsNonRoot).To(BeNil()) - Expect(nodeContainer.SecurityContext.ReadOnlyRootFilesystem).To(BeNil()) - Expect(nodeContainer.SecurityContext.AllowPrivilegeEscalation).To(BeNil()) - Expect(nodeContainer.SecurityContext.ProcMount).To(BeNil()) - Expect(nodeContainer.SecurityContext.SeccompProfile).To(BeNil()) - - if enableBGP { - confdContainer := rtest.GetContainer(ds.Spec.Template.Spec.Containers, "confd") - - // Windows node image override results in correct image. - Expect(confdContainer.Image).To(Equal(components.TigeraRegistry + "tigera/node-windows:" + components.ComponentTigeraNodeWindows.Version)) - Expect(confdContainer.SecurityContext.Capabilities).To(BeNil()) - Expect(confdContainer.SecurityContext.Privileged).To(BeNil()) - Expect(confdContainer.SecurityContext.SELinuxOptions).To(BeNil()) - Expect(confdContainer.SecurityContext.WindowsOptions).To(Not(BeNil())) - Expect(confdContainer.SecurityContext.WindowsOptions.GMSACredentialSpecName).To(BeNil()) - Expect(confdContainer.SecurityContext.WindowsOptions.GMSACredentialSpec).To(BeNil()) - Expect(*confdContainer.SecurityContext.WindowsOptions.RunAsUserName).To(Equal("NT AUTHORITY\\system")) - Expect(*confdContainer.SecurityContext.WindowsOptions.HostProcess).To(BeTrue()) - Expect(confdContainer.SecurityContext.RunAsUser).To(BeNil()) - Expect(confdContainer.SecurityContext.RunAsGroup).To(BeNil()) - Expect(confdContainer.SecurityContext.RunAsNonRoot).To(BeNil()) - Expect(confdContainer.SecurityContext.ReadOnlyRootFilesystem).To(BeNil()) - Expect(confdContainer.SecurityContext.AllowPrivilegeEscalation).To(BeNil()) - Expect(confdContainer.SecurityContext.ProcMount).To(BeNil()) - Expect(confdContainer.SecurityContext.SeccompProfile).To(BeNil()) - } - - // Validate correct number of init containers. - Expect(ds.Spec.Template.Spec.InitContainers).To(HaveLen(2)) - - // CNI container uses image override. - cniContainer := rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "install-cni") - rtest.ExpectEnv(cniContainer.Env, "CNI_NET_DIR", "/etc/cni/net.d") - Expect(cniContainer.Image).To(Equal(components.TigeraRegistry + "tigera/cni-windows:" + components.ComponentTigeraCNIWindows.Version)) - - Expect(cniContainer.SecurityContext.Capabilities).To(BeNil()) - Expect(cniContainer.SecurityContext.Privileged).To(BeNil()) - Expect(cniContainer.SecurityContext.SELinuxOptions).To(BeNil()) - Expect(cniContainer.SecurityContext.WindowsOptions).To(Not(BeNil())) - Expect(cniContainer.SecurityContext.WindowsOptions.GMSACredentialSpecName).To(BeNil()) - Expect(cniContainer.SecurityContext.WindowsOptions.GMSACredentialSpec).To(BeNil()) - Expect(*cniContainer.SecurityContext.WindowsOptions.RunAsUserName).To(Equal("NT AUTHORITY\\system")) - Expect(*cniContainer.SecurityContext.WindowsOptions.HostProcess).To(BeTrue()) - Expect(cniContainer.SecurityContext.RunAsUser).To(BeNil()) - Expect(cniContainer.SecurityContext.RunAsGroup).To(BeNil()) - Expect(cniContainer.SecurityContext.RunAsNonRoot).To(BeNil()) - Expect(cniContainer.SecurityContext.ReadOnlyRootFilesystem).To(BeNil()) - Expect(cniContainer.SecurityContext.AllowPrivilegeEscalation).To(BeNil()) - Expect(cniContainer.SecurityContext.ProcMount).To(BeNil()) - Expect(cniContainer.SecurityContext.SeccompProfile).To(BeNil()) - - // uninstall container uses image override. - uninstallContainer := rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "uninstall-calico") - Expect(uninstallContainer.Image).To(Equal(components.TigeraRegistry + "tigera/node-windows:" + components.ComponentTigeraNodeWindows.Version)) - - Expect(uninstallContainer.SecurityContext.Capabilities).To(BeNil()) - Expect(uninstallContainer.SecurityContext.Privileged).To(BeNil()) - Expect(uninstallContainer.SecurityContext.SELinuxOptions).To(BeNil()) - Expect(uninstallContainer.SecurityContext.WindowsOptions).To(Not(BeNil())) - Expect(uninstallContainer.SecurityContext.WindowsOptions.GMSACredentialSpecName).To(BeNil()) - Expect(uninstallContainer.SecurityContext.WindowsOptions.GMSACredentialSpec).To(BeNil()) - Expect(*uninstallContainer.SecurityContext.WindowsOptions.RunAsUserName).To(Equal("NT AUTHORITY\\system")) - Expect(*uninstallContainer.SecurityContext.WindowsOptions.HostProcess).To(BeTrue()) - Expect(uninstallContainer.SecurityContext.RunAsUser).To(BeNil()) - Expect(uninstallContainer.SecurityContext.RunAsGroup).To(BeNil()) - Expect(uninstallContainer.SecurityContext.RunAsNonRoot).To(BeNil()) - Expect(uninstallContainer.SecurityContext.ReadOnlyRootFilesystem).To(BeNil()) - Expect(uninstallContainer.SecurityContext.AllowPrivilegeEscalation).To(BeNil()) - Expect(uninstallContainer.SecurityContext.ProcMount).To(BeNil()) - Expect(uninstallContainer.SecurityContext.SeccompProfile).To(BeNil()) - - // Verify env - expectedNodeEnv := []corev1.EnvVar{ - {Name: "CNI_PLUGIN_TYPE", Value: "Calico"}, - {Name: "DATASTORE_TYPE", Value: "kubernetes"}, - {Name: "WAIT_FOR_DATASTORE", Value: "true"}, - {Name: "CALICO_MANAGE_CNI", Value: "true"}, - {Name: "CALICO_DISABLE_FILE_LOGGING", Value: "false"}, - {Name: "FELIX_DEFAULTENDPOINTTOHOSTACTION", Value: "ACCEPT"}, - {Name: "FELIX_HEALTHENABLED", Value: "true"}, - { - Name: "NODENAME", - ValueFrom: &corev1.EnvVarSource{ - FieldRef: &corev1.ObjectFieldSelector{FieldPath: "spec.nodeName"}, - }, - }, - { - Name: "NAMESPACE", - ValueFrom: &corev1.EnvVarSource{ - FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.namespace"}, - }, - }, - {Name: "IP", Value: "autodetect"}, - {Name: "IP_AUTODETECTION_METHOD", Value: "first-found"}, - {Name: "IP6", Value: "none"}, - {Name: "FELIX_IPV6SUPPORT", Value: "false"}, - {Name: "FELIX_TYPHAK8SNAMESPACE", Value: "calico-system"}, - {Name: "FELIX_TYPHAK8SSERVICENAME", Value: "calico-typha"}, - {Name: "FELIX_TYPHACAFILE", Value: certificatemanagement.TrustedCertBundleMountPath}, - {Name: "FELIX_TYPHACERTFILE", Value: "/node-certs/tls.crt"}, - {Name: "FELIX_TYPHACN", Value: "typha-server"}, - {Name: "FELIX_TYPHAKEYFILE", Value: "/node-certs/tls.key"}, - - {Name: "VXLAN_VNI", Value: "4096"}, - {Name: "VXLAN_ADAPTER", Value: ""}, - {Name: "KUBE_NETWORK", Value: "Calico.*"}, - {Name: "KUBERNETES_SERVICE_HOST", Value: "1.2.3.4"}, - {Name: "KUBERNETES_SERVICE_PORT", Value: "6443"}, - - // Tigera-specific envvars - {Name: "FELIX_PROMETHEUSREPORTERENABLED", Value: "true"}, - {Name: "FELIX_PROMETHEUSREPORTERPORT", Value: "9081"}, - {Name: "FELIX_FLOWLOGSFILEENABLED", Value: "true"}, - {Name: "FELIX_FLOWLOGSFILEINCLUDELABELS", Value: "true"}, - {Name: "FELIX_FLOWLOGSFILEINCLUDEPOLICIES", Value: "true"}, - {Name: "FELIX_FLOWLOGSFILEINCLUDESERVICE", Value: "true"}, - {Name: "FELIX_FLOWLOGSENABLENETWORKSETS", Value: "true"}, - {Name: "FELIX_FLOWLOGSCOLLECTPROCESSINFO", Value: "true"}, - {Name: "FELIX_DNSLOGSFILEENABLED", Value: "true"}, - {Name: "FELIX_DNSLOGSFILEPERNODELIMIT", Value: "1000"}, - } - - // Set CALICO_NETWORKING_BACKEND - if enableBGP { - expectedNodeEnv = append(expectedNodeEnv, corev1.EnvVar{Name: "CALICO_NETWORKING_BACKEND", Value: "windows-bgp"}) - } else if enableVXLAN { - expectedNodeEnv = append(expectedNodeEnv, corev1.EnvVar{Name: "CALICO_NETWORKING_BACKEND", Value: "vxlan"}) - } else { - expectedNodeEnv = append(expectedNodeEnv, corev1.EnvVar{Name: "CALICO_NETWORKING_BACKEND", Value: "none"}) - } - - // Set CLUSTER_TYPE - if enableBGP { - expectedNodeEnv = append(expectedNodeEnv, corev1.EnvVar{Name: "CLUSTER_TYPE", Value: "k8s,operator,bgp,windows"}) - } else { - expectedNodeEnv = append(expectedNodeEnv, corev1.EnvVar{Name: "CLUSTER_TYPE", Value: "k8s,operator,windows"}) - } - - Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "node").Env).To(ConsistOf(expectedNodeEnv)) - Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "felix").Env).To(ConsistOf(expectedNodeEnv)) - if enableBGP { - Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "confd").Env).To(ConsistOf(expectedNodeEnv)) - } - - // Expect the SECURITY_GROUP env variables to not be set - Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "node").Env).NotTo(ContainElement(gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{"Name": Equal("TIGERA_DEFAULT_SECURITY_GROUPS")}))) - Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "node").Env).NotTo(ContainElement(gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{"Name": Equal("TIGERA_POD_SECURITY_GROUP")}))) - Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "felix").Env).NotTo(ContainElement(gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{"Name": Equal("TIGERA_DEFAULT_SECURITY_GROUPS")}))) - Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "felix").Env).NotTo(ContainElement(gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{"Name": Equal("TIGERA_POD_SECURITY_GROUP")}))) - if enableBGP { - Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "confd").Env).NotTo(ContainElement(gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{"Name": Equal("TIGERA_DEFAULT_SECURITY_GROUPS")}))) - Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "confd").Env).NotTo(ContainElement(gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{"Name": Equal("TIGERA_POD_SECURITY_GROUP")}))) - } - - expectedCNIEnv := []corev1.EnvVar{ - {Name: "SLEEP", Value: "false"}, - {Name: "CNI_PLUGIN_TYPE", Value: "Calico"}, - {Name: "CNI_BIN_DIR", Value: "/host/opt/cni/bin"}, - {Name: "CNI_CONF_NAME", Value: "10-calico.conflist"}, - {Name: "CNI_NET_DIR", Value: "/etc/cni/net.d"}, - {Name: "VXLAN_VNI", Value: "4096"}, - { - Name: "KUBERNETES_NODE_NAME", - Value: "", - ValueFrom: &corev1.EnvVarSource{ - FieldRef: &corev1.ObjectFieldSelector{FieldPath: "spec.nodeName"}, - }, - }, - { - Name: "CNI_NETWORK_CONFIG", - ValueFrom: &corev1.EnvVarSource{ - ConfigMapKeyRef: &corev1.ConfigMapKeySelector{ - Key: "config", - LocalObjectReference: corev1.LocalObjectReference{ - Name: "cni-config-windows", - }, - }, - }, - }, - - {Name: "KUBERNETES_SERVICE_HOST", Value: "1.2.3.4"}, - {Name: "KUBERNETES_SERVICE_PORT", Value: "6443"}, - {Name: "KUBERNETES_SERVICE_CIDRS", Value: "10.96.0.0/12"}, - {Name: "KUBERNETES_DNS_SERVERS", Value: "10.96.0.10"}, - } - Expect(rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "install-cni").Env).To(ConsistOf(expectedCNIEnv)) - - expectedUninstallEnv := []corev1.EnvVar{ - {Name: "SLEEP", Value: "false"}, - {Name: "CNI_PLUGIN_TYPE", Value: "Calico"}, - {Name: "CNI_BIN_DIR", Value: "/host/opt/cni/bin"}, - {Name: "CNI_CONF_NAME", Value: "10-calico.conflist"}, - {Name: "CNI_NET_DIR", Value: "/host/etc/cni/net.d"}, - } - Expect(rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "uninstall-calico").Env).To(ConsistOf(expectedUninstallEnv)) - - // Verify volumes. - fileOrCreate := corev1.HostPathFileOrCreate - dirOrCreate := corev1.HostPathDirectoryOrCreate - expectedVols := []corev1.Volume{ - {Name: "lib-modules", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/lib/modules"}}}, - {Name: "var-run-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/run/calico", Type: &dirOrCreate}}}, - {Name: "var-lib-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/lib/calico", Type: &dirOrCreate}}}, - {Name: "xtables-lock", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/run/xtables.lock", Type: &fileOrCreate}}}, - {Name: "cni-bin-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/opt/cni/bin", Type: &dirOrCreate}}}, - {Name: "cni-net-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/etc/cni/net.d"}}}, - {Name: "cni-log-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico/cni", Type: &dirOrCreate}}}, - {Name: "policysync", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/run/nodeagent", Type: &dirOrCreate}}}, - { - Name: "tigera-ca-bundle", - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: "tigera-ca-bundle", - }, - }, - }, - }, - { - Name: render.NodeTLSSecretName, - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: render.NodeTLSSecretName, - DefaultMode: &defaultMode, - }, - }, - }, - {Name: "var-log-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico", Type: &dirOrCreate}}}, - } - Expect(ds.Spec.Template.Spec.Volumes).To(ConsistOf(expectedVols)) - - // Verify volume mounts. - expectedNodeVolumeMounts := []corev1.VolumeMount{ - {MountPath: "/host/etc/cni/net.d", Name: "cni-net-dir"}, - {MountPath: "/var/run/calico", Name: "var-run-calico"}, - {MountPath: "/var/lib/calico", Name: "var-lib-calico"}, - {MountPath: "c:/etc/pki/tls/certs", Name: "tigera-ca-bundle", ReadOnly: true}, - {MountPath: "c:/node-certs", Name: render.NodeTLSSecretName, ReadOnly: true}, - {MountPath: "/var/log/calico", Name: "var-log-calico", ReadOnly: false}, - } - Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "node").VolumeMounts).To(ConsistOf(expectedNodeVolumeMounts)) - Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "felix").VolumeMounts).To(ConsistOf(expectedNodeVolumeMounts)) - if enableBGP { - Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "confd").VolumeMounts).To(ConsistOf(expectedNodeVolumeMounts)) - } - - expectedCNIVolumeMounts := []corev1.VolumeMount{ - {MountPath: "/host/opt/cni/bin", Name: "cni-bin-dir"}, - {MountPath: "/host/etc/cni/net.d", Name: "cni-net-dir"}, - } - Expect(rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "install-cni").VolumeMounts).To(ConsistOf(expectedCNIVolumeMounts)) - - expectedUninstallVolumeMounts := []corev1.VolumeMount{ - {MountPath: "/host/opt/cni/bin", Name: "cni-bin-dir"}, - {MountPath: "/host/etc/cni/net.d", Name: "cni-net-dir"}, - } - Expect(rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "uninstall-calico").VolumeMounts).To(ConsistOf(expectedUninstallVolumeMounts)) - - // Verify tolerations. - Expect(ds.Spec.Template.Spec.Tolerations).To(ConsistOf(rmeta.TolerateAll)) - - // Verify readiness and liveness probes. - verifyWindowsProbesAndLifecycle(ds, false) - }) - } - }) - It("should render all resources when using Calico CNI on EKS", func() { expectedResources := []struct { name string @@ -1162,240 +777,34 @@ var _ = Describe("Windows rendering tests", func() { // The calico-node-windows daemonset has 2 containers (felix, node) when using VXLAN Expect(ds.Spec.Template.Spec.Containers).To(HaveLen(2)) - for _, container := range ds.Spec.Template.Spec.Containers { - // Windows image override results in correct image. - Expect(container.Image).To(Equal(fmt.Sprintf("quay.io/%s%s:%s", components.CalicoImagePath, components.ComponentCalicoNodeWindows.Image, components.ComponentCalicoNodeWindows.Version))) - } - - // Validate correct number of init containers. - Expect(len(ds.Spec.Template.Spec.InitContainers)).To(Equal(2)) - - cniContainer := rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "install-cni") - rtest.ExpectEnv(cniContainer.Env, "CNI_NET_DIR", "/etc/cni/net.d") - - // CNI container uses image override. - Expect(cniContainer.Image).To(Equal(fmt.Sprintf("quay.io/%s%s:%s", components.CalicoImagePath, components.ComponentCalicoCNIWindows.Image, components.ComponentCalicoCNIWindows.Version))) - - // uninstall container uses image override. - uninstallContainer := rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "uninstall-calico") - Expect(uninstallContainer.Image).To(Equal(fmt.Sprintf("quay.io/%s%s:%s", components.CalicoImagePath, components.ComponentCalicoNodeWindows.Image, components.ComponentCalicoNodeWindows.Version))) - - // Verify env - expectedNodeEnv := []corev1.EnvVar{ - {Name: "CNI_PLUGIN_TYPE", Value: "Calico"}, - {Name: "DATASTORE_TYPE", Value: "kubernetes"}, - {Name: "WAIT_FOR_DATASTORE", Value: "true"}, - {Name: "CALICO_MANAGE_CNI", Value: "true"}, - {Name: "CALICO_NETWORKING_BACKEND", Value: "vxlan"}, - {Name: "CALICO_DISABLE_FILE_LOGGING", Value: "false"}, - {Name: "CLUSTER_TYPE", Value: "k8s,operator,ecs,windows"}, - {Name: "FELIX_DEFAULTENDPOINTTOHOSTACTION", Value: "ACCEPT"}, - {Name: "FELIX_HEALTHENABLED", Value: "true"}, - { - Name: "NODENAME", - ValueFrom: &corev1.EnvVarSource{ - FieldRef: &corev1.ObjectFieldSelector{FieldPath: "spec.nodeName"}, - }, - }, - { - Name: "NAMESPACE", - ValueFrom: &corev1.EnvVarSource{ - FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.namespace"}, - }, - }, - {Name: "IP", Value: "autodetect"}, - {Name: "IP_AUTODETECTION_METHOD", Value: "first-found"}, - {Name: "IP6", Value: "none"}, - {Name: "FELIX_IPV6SUPPORT", Value: "false"}, - {Name: "FELIX_TYPHAK8SNAMESPACE", Value: "calico-system"}, - {Name: "FELIX_TYPHAK8SSERVICENAME", Value: "calico-typha"}, - {Name: "FELIX_TYPHACAFILE", Value: certificatemanagement.TrustedCertBundleMountPath}, - {Name: "FELIX_TYPHACERTFILE", Value: "/node-certs/tls.crt"}, - {Name: "FELIX_TYPHACN", Value: "typha-server"}, - {Name: "FELIX_TYPHAKEYFILE", Value: "/node-certs/tls.key"}, - - {Name: "VXLAN_VNI", Value: "4096"}, - {Name: "VXLAN_ADAPTER", Value: ""}, - {Name: "KUBE_NETWORK", Value: "Calico.*"}, - {Name: "KUBERNETES_SERVICE_HOST", Value: "1.2.3.4"}, - {Name: "KUBERNETES_SERVICE_PORT", Value: "6443"}, - } - - Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "node").Env).To(ConsistOf(expectedNodeEnv)) - Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "felix").Env).To(ConsistOf(expectedNodeEnv)) - - // Expect the SECURITY_GROUP env variables to not be set - Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "node").Env).NotTo(ContainElement(gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{"Name": Equal("TIGERA_DEFAULT_SECURITY_GROUPS")}))) - Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "node").Env).NotTo(ContainElement(gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{"Name": Equal("TIGERA_POD_SECURITY_GROUP")}))) - Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "felix").Env).NotTo(ContainElement(gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{"Name": Equal("TIGERA_DEFAULT_SECURITY_GROUPS")}))) - Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "felix").Env).NotTo(ContainElement(gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{"Name": Equal("TIGERA_POD_SECURITY_GROUP")}))) - - expectedCNIEnv := []corev1.EnvVar{ - {Name: "SLEEP", Value: "false"}, - {Name: "CNI_PLUGIN_TYPE", Value: "Calico"}, - {Name: "CNI_BIN_DIR", Value: "/host/opt/cni/bin"}, - {Name: "CNI_CONF_NAME", Value: "10-calico.conflist"}, - {Name: "CNI_NET_DIR", Value: "/etc/cni/net.d"}, - {Name: "VXLAN_VNI", Value: "4096"}, - - { - Name: "KUBERNETES_NODE_NAME", - Value: "", - ValueFrom: &corev1.EnvVarSource{ - FieldRef: &corev1.ObjectFieldSelector{FieldPath: "spec.nodeName"}, - }, - }, - { - Name: "CNI_NETWORK_CONFIG", - ValueFrom: &corev1.EnvVarSource{ - ConfigMapKeyRef: &corev1.ConfigMapKeySelector{ - Key: "config", - LocalObjectReference: corev1.LocalObjectReference{ - Name: "cni-config-windows", - }, - }, - }, - }, - - {Name: "KUBERNETES_SERVICE_HOST", Value: "1.2.3.4"}, - {Name: "KUBERNETES_SERVICE_PORT", Value: "6443"}, - {Name: "KUBERNETES_SERVICE_CIDRS", Value: "10.96.0.0/12"}, - {Name: "KUBERNETES_DNS_SERVERS", Value: "10.96.0.10"}, - } - Expect(rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "install-cni").Env).To(ConsistOf(expectedCNIEnv)) - - expectedUninstallEnv := []corev1.EnvVar{ - {Name: "SLEEP", Value: "false"}, - {Name: "CNI_PLUGIN_TYPE", Value: "Calico"}, - {Name: "CNI_BIN_DIR", Value: "/host/opt/cni/bin"}, - {Name: "CNI_CONF_NAME", Value: "10-calico.conflist"}, - {Name: "CNI_NET_DIR", Value: "/host/etc/cni/net.d"}, - } - Expect(rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "uninstall-calico").Env).To(ConsistOf(expectedUninstallEnv)) - - // Verify volumes. - fileOrCreate := corev1.HostPathFileOrCreate - dirOrCreate := corev1.HostPathDirectoryOrCreate - expectedVols := []corev1.Volume{ - {Name: "lib-modules", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/lib/modules"}}}, - {Name: "var-run-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/run/calico", Type: &dirOrCreate}}}, - {Name: "var-lib-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/lib/calico", Type: &dirOrCreate}}}, - {Name: "xtables-lock", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/run/xtables.lock", Type: &fileOrCreate}}}, - {Name: "cni-bin-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/opt/cni/bin", Type: &dirOrCreate}}}, - {Name: "cni-net-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/etc/cni/net.d"}}}, - {Name: "cni-log-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico/cni", Type: &dirOrCreate}}}, - {Name: "policysync", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/run/nodeagent", Type: &dirOrCreate}}}, - { - Name: "tigera-ca-bundle", - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: "tigera-ca-bundle", - }, - }, - }, - }, - { - Name: render.NodeTLSSecretName, - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: render.NodeTLSSecretName, - DefaultMode: &defaultMode, - }, - }, - }, - } - Expect(ds.Spec.Template.Spec.Volumes).To(ConsistOf(expectedVols)) - - // Verify volume mounts. - expectedNodeVolumeMounts := []corev1.VolumeMount{ - {MountPath: "/host/etc/cni/net.d", Name: "cni-net-dir"}, - {MountPath: "/var/run/calico", Name: "var-run-calico"}, - {MountPath: "/var/lib/calico", Name: "var-lib-calico"}, - {MountPath: "c:/etc/pki/tls/certs", Name: "tigera-ca-bundle", ReadOnly: true}, - {MountPath: "c:/node-certs", Name: render.NodeTLSSecretName, ReadOnly: true}, - {MountPath: "/var/log/calico/cni", Name: "cni-log-dir", ReadOnly: false}, - } - Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "node").VolumeMounts).To(ConsistOf(expectedNodeVolumeMounts)) - Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "felix").VolumeMounts).To(ConsistOf(expectedNodeVolumeMounts)) - - expectedCNIVolumeMounts := []corev1.VolumeMount{ - {MountPath: "/host/opt/cni/bin", Name: "cni-bin-dir"}, - {MountPath: "/host/etc/cni/net.d", Name: "cni-net-dir"}, - } - Expect(rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "install-cni").VolumeMounts).To(ConsistOf(expectedCNIVolumeMounts)) - - expectedUninstallVolumeMounts := []corev1.VolumeMount{ - {MountPath: "/host/opt/cni/bin", Name: "cni-bin-dir"}, - {MountPath: "/host/etc/cni/net.d", Name: "cni-net-dir"}, - } - Expect(rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "uninstall-calico").VolumeMounts).To(ConsistOf(expectedUninstallVolumeMounts)) - - // Verify tolerations. - Expect(ds.Spec.Template.Spec.Tolerations).To(ConsistOf(rmeta.TolerateAll)) - - // Verify readiness and liveness probes. - verifyWindowsProbesAndLifecycle(ds, true) - }) - - It("should properly render a configuration using the AmazonVPC CNI plugin", func() { - // Override the installation with one configured for AmazonVPC CNI. - amazonVPCInstalllation := &operatorv1.InstallationSpec{ - KubernetesProvider: operatorv1.ProviderEKS, - CNI: &operatorv1.CNISpec{Type: operatorv1.PluginAmazonVPC}, - ServiceCIDRs: []string{"10.96.0.0/12"}, - WindowsNodes: &operatorv1.WindowsNodeSpec{ - CNIBinDir: "/opt/cni/bin", - CNIConfigDir: "/etc/cni/net.d", - CNILogDir: "/var/log/calico/cni", - }, - } - cfg.Installation = amazonVPCInstalllation - - component := render.Windows(&cfg) - Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() - Expect(len(resources)).To(Equal(defaultNumExpectedResources - 1)) - - // Should render the correct resources. - Expect(rtest.GetResource(resources, "calico-node-windows", "calico-system", "apps", "v1", "DaemonSet")).ToNot(BeNil()) - dsResource := rtest.GetResource(resources, "calico-node-windows", "calico-system", "apps", "v1", "DaemonSet") - Expect(dsResource).ToNot(BeNil()) - - // Should not render CNI configuration. - cniCmResource := rtest.GetResource(resources, "cni-config-windows", "calico-system", "", "v1", "ConfigMap") - Expect(cniCmResource).To(BeNil()) - - // The DaemonSet should have the correct configuration. - ds := dsResource.(*appsv1.DaemonSet) - - // The pod template should have node critical priority - Expect(ds.Spec.Template.Spec.PriorityClassName).To(Equal(render.NodePriorityClassName)) + for _, container := range ds.Spec.Template.Spec.Containers { + // Windows image override results in correct image. + Expect(container.Image).To(Equal(fmt.Sprintf("quay.io/%s%s:%s", components.CalicoImagePath, components.ComponentCalicoNodeWindows.Image, components.ComponentCalicoNodeWindows.Version))) + } + + // Validate correct number of init containers. + Expect(len(ds.Spec.Template.Spec.InitContainers)).To(Equal(2)) - // CNI install container should not be present. cniContainer := rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "install-cni") - Expect(cniContainer).To(BeNil()) + rtest.ExpectEnv(cniContainer.Env, "CNI_NET_DIR", "/etc/cni/net.d") - // uninstall container should still be present. - uninstallContainer := rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "uninstall-calico") - Expect(uninstallContainer).NotTo(BeNil()) + // CNI container uses image override. + Expect(cniContainer.Image).To(Equal(fmt.Sprintf("quay.io/%s%s:%s", components.CalicoImagePath, components.ComponentCalicoCNIWindows.Image, components.ComponentCalicoCNIWindows.Version))) - // Validate correct number of init containers. - Expect(len(ds.Spec.Template.Spec.InitContainers)).To(Equal(1)) + // uninstall container uses image override. + uninstallContainer := rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "uninstall-calico") + Expect(uninstallContainer.Image).To(Equal(fmt.Sprintf("quay.io/%s%s:%s", components.CalicoImagePath, components.ComponentCalicoNodeWindows.Image, components.ComponentCalicoNodeWindows.Version))) // Verify env expectedNodeEnv := []corev1.EnvVar{ - {Name: "CNI_PLUGIN_TYPE", Value: "AmazonVPC"}, + {Name: "CNI_PLUGIN_TYPE", Value: "Calico"}, {Name: "DATASTORE_TYPE", Value: "kubernetes"}, {Name: "WAIT_FOR_DATASTORE", Value: "true"}, - {Name: "CALICO_NETWORKING_BACKEND", Value: "none"}, + {Name: "CALICO_MANAGE_CNI", Value: "true"}, + {Name: "CALICO_NETWORKING_BACKEND", Value: "vxlan"}, {Name: "CALICO_DISABLE_FILE_LOGGING", Value: "false"}, {Name: "CLUSTER_TYPE", Value: "k8s,operator,ecs,windows"}, - {Name: "IP", Value: "none"}, - {Name: "IP6", Value: "none"}, - {Name: "CALICO_MANAGE_CNI", Value: "false"}, {Name: "FELIX_DEFAULTENDPOINTTOHOSTACTION", Value: "ACCEPT"}, - {Name: "FELIX_IPV6SUPPORT", Value: "false"}, {Name: "FELIX_HEALTHENABLED", Value: "true"}, { Name: "NODENAME", @@ -1409,18 +818,20 @@ var _ = Describe("Windows rendering tests", func() { FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.namespace"}, }, }, + {Name: "IP", Value: "autodetect"}, + {Name: "IP_AUTODETECTION_METHOD", Value: "first-found"}, + {Name: "IP6", Value: "none"}, + {Name: "FELIX_IPV6SUPPORT", Value: "false"}, {Name: "FELIX_TYPHAK8SNAMESPACE", Value: "calico-system"}, {Name: "FELIX_TYPHAK8SSERVICENAME", Value: "calico-typha"}, {Name: "FELIX_TYPHACAFILE", Value: certificatemanagement.TrustedCertBundleMountPath}, {Name: "FELIX_TYPHACERTFILE", Value: "/node-certs/tls.crt"}, {Name: "FELIX_TYPHACN", Value: "typha-server"}, {Name: "FELIX_TYPHAKEYFILE", Value: "/node-certs/tls.key"}, - {Name: "FELIX_ROUTESOURCE", Value: "WorkloadIPs"}, - {Name: "FELIX_BPFEXTTOSERVICECONNMARK", Value: "0x80"}, {Name: "VXLAN_VNI", Value: "4096"}, {Name: "VXLAN_ADAPTER", Value: ""}, - {Name: "KUBE_NETWORK", Value: "vpc.*"}, + {Name: "KUBE_NETWORK", Value: "Calico.*"}, {Name: "KUBERNETES_SERVICE_HOST", Value: "1.2.3.4"}, {Name: "KUBERNETES_SERVICE_PORT", Value: "6443"}, } @@ -1434,6 +845,49 @@ var _ = Describe("Windows rendering tests", func() { Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "felix").Env).NotTo(ContainElement(gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{"Name": Equal("TIGERA_DEFAULT_SECURITY_GROUPS")}))) Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "felix").Env).NotTo(ContainElement(gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{"Name": Equal("TIGERA_POD_SECURITY_GROUP")}))) + expectedCNIEnv := []corev1.EnvVar{ + {Name: "SLEEP", Value: "false"}, + {Name: "CNI_PLUGIN_TYPE", Value: "Calico"}, + {Name: "CNI_BIN_DIR", Value: "/host/opt/cni/bin"}, + {Name: "CNI_CONF_NAME", Value: "10-calico.conflist"}, + {Name: "CNI_NET_DIR", Value: "/etc/cni/net.d"}, + {Name: "VXLAN_VNI", Value: "4096"}, + + { + Name: "KUBERNETES_NODE_NAME", + Value: "", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{FieldPath: "spec.nodeName"}, + }, + }, + { + Name: "CNI_NETWORK_CONFIG", + ValueFrom: &corev1.EnvVarSource{ + ConfigMapKeyRef: &corev1.ConfigMapKeySelector{ + Key: "config", + LocalObjectReference: corev1.LocalObjectReference{ + Name: "cni-config-windows", + }, + }, + }, + }, + + {Name: "KUBERNETES_SERVICE_HOST", Value: "1.2.3.4"}, + {Name: "KUBERNETES_SERVICE_PORT", Value: "6443"}, + {Name: "KUBERNETES_SERVICE_CIDRS", Value: "10.96.0.0/12"}, + {Name: "KUBERNETES_DNS_SERVERS", Value: "10.96.0.10"}, + } + Expect(rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "install-cni").Env).To(ConsistOf(expectedCNIEnv)) + + expectedUninstallEnv := []corev1.EnvVar{ + {Name: "SLEEP", Value: "false"}, + {Name: "CNI_PLUGIN_TYPE", Value: "Calico"}, + {Name: "CNI_BIN_DIR", Value: "/host/opt/cni/bin"}, + {Name: "CNI_CONF_NAME", Value: "10-calico.conflist"}, + {Name: "CNI_NET_DIR", Value: "/host/etc/cni/net.d"}, + } + Expect(rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "uninstall-calico").Env).To(ConsistOf(expectedUninstallEnv)) + // Verify volumes. fileOrCreate := corev1.HostPathFileOrCreate dirOrCreate := corev1.HostPathDirectoryOrCreate @@ -1442,6 +896,9 @@ var _ = Describe("Windows rendering tests", func() { {Name: "var-run-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/run/calico", Type: &dirOrCreate}}}, {Name: "var-lib-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/lib/calico", Type: &dirOrCreate}}}, {Name: "xtables-lock", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/run/xtables.lock", Type: &fileOrCreate}}}, + {Name: "cni-bin-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/opt/cni/bin", Type: &dirOrCreate}}}, + {Name: "cni-net-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/etc/cni/net.d"}}}, + {Name: "cni-log-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico/cni", Type: &dirOrCreate}}}, {Name: "policysync", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/run/nodeagent", Type: &dirOrCreate}}}, { Name: "tigera-ca-bundle", @@ -1467,14 +924,28 @@ var _ = Describe("Windows rendering tests", func() { // Verify volume mounts. expectedNodeVolumeMounts := []corev1.VolumeMount{ + {MountPath: "/host/etc/cni/net.d", Name: "cni-net-dir"}, {MountPath: "/var/run/calico", Name: "var-run-calico"}, {MountPath: "/var/lib/calico", Name: "var-lib-calico"}, {MountPath: "c:/etc/pki/tls/certs", Name: "tigera-ca-bundle", ReadOnly: true}, {MountPath: "c:/node-certs", Name: render.NodeTLSSecretName, ReadOnly: true}, + {MountPath: "/var/log/calico/cni", Name: "cni-log-dir", ReadOnly: false}, } Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "node").VolumeMounts).To(ConsistOf(expectedNodeVolumeMounts)) Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "felix").VolumeMounts).To(ConsistOf(expectedNodeVolumeMounts)) + expectedCNIVolumeMounts := []corev1.VolumeMount{ + {MountPath: "/host/opt/cni/bin", Name: "cni-bin-dir"}, + {MountPath: "/host/etc/cni/net.d", Name: "cni-net-dir"}, + } + Expect(rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "install-cni").VolumeMounts).To(ConsistOf(expectedCNIVolumeMounts)) + + expectedUninstallVolumeMounts := []corev1.VolumeMount{ + {MountPath: "/host/opt/cni/bin", Name: "cni-bin-dir"}, + {MountPath: "/host/etc/cni/net.d", Name: "cni-net-dir"}, + } + Expect(rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "uninstall-calico").VolumeMounts).To(ConsistOf(expectedUninstallVolumeMounts)) + // Verify tolerations. Expect(ds.Spec.Template.Spec.Tolerations).To(ConsistOf(rmeta.TolerateAll)) @@ -1482,163 +953,64 @@ var _ = Describe("Windows rendering tests", func() { verifyWindowsProbesAndLifecycle(ds, true) }) - DescribeTable("should properly render configuration using non-Calico CNI plugin", - func(cni operatorv1.CNIPluginType, ipam operatorv1.IPAMPluginType) { - installlation := &operatorv1.InstallationSpec{ - CNI: &operatorv1.CNISpec{ - Type: cni, - IPAM: &operatorv1.IPAMSpec{Type: ipam}, - }, - WindowsNodes: &operatorv1.WindowsNodeSpec{ - CNIBinDir: "/opt/cni/bin", - CNIConfigDir: "/etc/cni/net.d", - CNILogDir: "/var/log/calico/cni", - }, - } - cfg.Installation = installlation - - component := render.Windows(&cfg) - Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() - - // Should render the correct resources. - Expect(rtest.GetResource(resources, "calico-node-windows", "calico-system", "apps", "v1", "DaemonSet")).ToNot(BeNil()) - dsResource := rtest.GetResource(resources, "calico-node-windows", "calico-system", "apps", "v1", "DaemonSet") - Expect(dsResource).ToNot(BeNil()) - - // Should not render CNI configuration. - cniCmResource := rtest.GetResource(resources, "cni-config-windows", "calico-system", "", "v1", "ConfigMap") - Expect(cniCmResource).To(BeNil()) - - // The DaemonSet should have the correct configuration. - ds := dsResource.(*appsv1.DaemonSet) - - // The pod template should have node critical priority - Expect(ds.Spec.Template.Spec.PriorityClassName).To(Equal(render.NodePriorityClassName)) - - // CNI install container should not be present. - cniContainer := rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "install-cni") - Expect(cniContainer).To(BeNil()) - // Validate correct number of init containers. - Expect(len(ds.Spec.Template.Spec.InitContainers)).To(Equal(1)) - - // Verify env - expectedEnvs := []corev1.EnvVar{ - {Name: "CNI_PLUGIN_TYPE", Value: string(cni)}, - {Name: "CALICO_NETWORKING_BACKEND", Value: "none"}, - {Name: "FELIX_DEFAULTENDPOINTTOHOSTACTION", Value: "ACCEPT"}, - } - for _, expected := range expectedEnvs { - Expect(ds.Spec.Template.Spec.Containers[0].Env).To(ContainElement(expected)) - } - - // Verify readiness and liveness probes. - verifyWindowsProbesAndLifecycle(ds, true) - }, - Entry("GKE", operatorv1.PluginGKE, operatorv1.IPAMPluginHostLocal), - Entry("AmazonVPC", operatorv1.PluginAmazonVPC, operatorv1.IPAMPluginAmazonVPC), - Entry("AzureVNET", operatorv1.PluginAzureVNET, operatorv1.IPAMPluginAzureVNET), - ) - - It("should render all resources when running on openshift", func() { - expectedResources := []struct { - name string - ns string - group string - version string - kind string - }{ - {name: "cni-config-windows", ns: common.CalicoNamespace, group: "", version: "v1", kind: "ConfigMap"}, - {name: common.WindowsDaemonSetName, ns: common.CalicoNamespace, group: "apps", version: "v1", kind: "DaemonSet"}, + It("should properly render a configuration using the AmazonVPC CNI plugin", func() { + // Override the installation with one configured for AmazonVPC CNI. + amazonVPCInstalllation := &operatorv1.InstallationSpec{ + KubernetesProvider: operatorv1.ProviderEKS, + CNI: &operatorv1.CNISpec{Type: operatorv1.PluginAmazonVPC}, + ServiceCIDRs: []string{"10.96.0.0/12"}, + WindowsNodes: &operatorv1.WindowsNodeSpec{ + CNIBinDir: "/opt/cni/bin", + CNIConfigDir: "/etc/cni/net.d", + CNILogDir: "/var/log/calico/cni", + }, } + cfg.Installation = amazonVPCInstalllation - defaultInstance.FlexVolumePath = "/etc/kubernetes/kubelet-plugins/volume/exec/" - defaultInstance.KubernetesProvider = operatorv1.ProviderOpenShift component := render.Windows(&cfg) Expect(component.ResolveImages(nil)).To(BeNil()) resources, _ := component.Objects() - Expect(len(resources)).To(Equal(len(expectedResources))) + Expect(len(resources)).To(Equal(defaultNumExpectedResources - 1)) // Should render the correct resources. - i := 0 - for _, expectedRes := range expectedResources { - rtest.ExpectResourceTypeAndObjectMetadata(resources[i], expectedRes.name, expectedRes.ns, expectedRes.group, expectedRes.version, expectedRes.kind) - i++ - } - - // The DaemonSet should have the correct configuration. - ds := rtest.GetResource(resources, "calico-node-windows", "calico-system", "apps", "v1", "DaemonSet").(*appsv1.DaemonSet) - - // The pod template should have node critical priority - Expect(ds.Spec.Template.Spec.PriorityClassName).To(Equal(render.NodePriorityClassName)) - - felixContainer := rtest.GetContainer(ds.Spec.Template.Spec.Containers, "felix") - Expect(felixContainer.Image).To(Equal(fmt.Sprintf("quay.io/%s%s:%s", components.CalicoImagePath, components.ComponentCalicoNodeWindows.Image, components.ComponentCalicoNodeWindows.Version))) - nodeContainer := rtest.GetContainer(ds.Spec.Template.Spec.Containers, "node") - Expect(nodeContainer.Image).To(Equal(fmt.Sprintf("quay.io/%s%s:%s", components.CalicoImagePath, components.ComponentCalicoNodeWindows.Image, components.ComponentCalicoNodeWindows.Version))) - cniContainer := rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "install-cni") - Expect(cniContainer.Image).To(Equal(fmt.Sprintf("quay.io/%s%s:%s", components.CalicoImagePath, components.ComponentCalicoCNIWindows.Image, components.ComponentCalicoCNIWindows.Version))) - uninstallContainer := rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "uninstall-calico") - Expect(uninstallContainer.Image).To(Equal(fmt.Sprintf("quay.io/%s%s:%s", components.CalicoImagePath, components.ComponentCalicoNodeWindows.Image, components.ComponentCalicoNodeWindows.Version))) - - // FIXME: confirm openshift CNI path defaults - expectedCNIVolumeMounts := []corev1.VolumeMount{ - {MountPath: "/host/opt/cni/bin", Name: "cni-bin-dir"}, - {MountPath: "/host/etc/cni/net.d", Name: "cni-net-dir"}, - } - Expect(rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "install-cni").VolumeMounts).To(ConsistOf(expectedCNIVolumeMounts)) - - // FIXME: confirm openshift CNI path defaults - expectedUninstallVolumeMounts := []corev1.VolumeMount{ - {MountPath: "/host/opt/cni/bin", Name: "cni-bin-dir"}, - {MountPath: "/host/etc/cni/net.d", Name: "cni-net-dir"}, - } - Expect(rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "uninstall-calico").VolumeMounts).To(ConsistOf(expectedUninstallVolumeMounts)) + Expect(rtest.GetResource(resources, "calico-node-windows", "calico-system", "apps", "v1", "DaemonSet")).ToNot(BeNil()) + dsResource := rtest.GetResource(resources, "calico-node-windows", "calico-system", "apps", "v1", "DaemonSet") + Expect(dsResource).ToNot(BeNil()) - // Verify volumes - // FIXME: confirm openshift CNI path defaults - fileOrCreate := corev1.HostPathFileOrCreate - dirOrCreate := corev1.HostPathDirectoryOrCreate - expectedVols := []corev1.Volume{ - {Name: "lib-modules", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/lib/modules"}}}, - {Name: "var-run-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/run/calico", Type: &dirOrCreate}}}, - {Name: "var-lib-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/lib/calico", Type: &dirOrCreate}}}, - {Name: "xtables-lock", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/run/xtables.lock", Type: &fileOrCreate}}}, - {Name: "cni-bin-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/opt/cni/bin", Type: &dirOrCreate}}}, - {Name: "cni-net-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/etc/cni/net.d"}}}, - {Name: "cni-log-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico/cni", Type: &dirOrCreate}}}, - {Name: "policysync", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/run/nodeagent", Type: &dirOrCreate}}}, - { - Name: "tigera-ca-bundle", - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: "tigera-ca-bundle", - }, - }, - }, - }, - { - Name: render.NodeTLSSecretName, - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: render.NodeTLSSecretName, - DefaultMode: &defaultMode, - }, - }, - }, - } - Expect(ds.Spec.Template.Spec.Volumes).To(ConsistOf(expectedVols)) + // Should not render CNI configuration. + cniCmResource := rtest.GetResource(resources, "cni-config-windows", "calico-system", "", "v1", "ConfigMap") + Expect(cniCmResource).To(BeNil()) + + // The DaemonSet should have the correct configuration. + ds := dsResource.(*appsv1.DaemonSet) + + // The pod template should have node critical priority + Expect(ds.Spec.Template.Spec.PriorityClassName).To(Equal(render.NodePriorityClassName)) + + // CNI install container should not be present. + cniContainer := rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "install-cni") + Expect(cniContainer).To(BeNil()) + + // uninstall container should still be present. + uninstallContainer := rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "uninstall-calico") + Expect(uninstallContainer).NotTo(BeNil()) + + // Validate correct number of init containers. + Expect(len(ds.Spec.Template.Spec.InitContainers)).To(Equal(1)) + // Verify env expectedNodeEnv := []corev1.EnvVar{ - {Name: "CNI_PLUGIN_TYPE", Value: "Calico"}, + {Name: "CNI_PLUGIN_TYPE", Value: "AmazonVPC"}, {Name: "DATASTORE_TYPE", Value: "kubernetes"}, {Name: "WAIT_FOR_DATASTORE", Value: "true"}, - {Name: "CALICO_MANAGE_CNI", Value: "true"}, - {Name: "CALICO_NETWORKING_BACKEND", Value: "windows-bgp"}, - {Name: "CLUSTER_TYPE", Value: "k8s,operator,openshift,bgp,windows"}, + {Name: "CALICO_NETWORKING_BACKEND", Value: "none"}, {Name: "CALICO_DISABLE_FILE_LOGGING", Value: "false"}, + {Name: "CLUSTER_TYPE", Value: "k8s,operator,ecs,windows"}, + {Name: "IP", Value: "none"}, + {Name: "IP6", Value: "none"}, + {Name: "CALICO_MANAGE_CNI", Value: "false"}, {Name: "FELIX_DEFAULTENDPOINTTOHOSTACTION", Value: "ACCEPT"}, + {Name: "FELIX_IPV6SUPPORT", Value: "false"}, {Name: "FELIX_HEALTHENABLED", Value: "true"}, { Name: "NODENAME", @@ -1652,92 +1024,32 @@ var _ = Describe("Windows rendering tests", func() { FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.namespace"}, }, }, - {Name: "IP", Value: "autodetect"}, - {Name: "IP_AUTODETECTION_METHOD", Value: "first-found"}, - {Name: "IP6", Value: "none"}, - {Name: "FELIX_IPV6SUPPORT", Value: "false"}, {Name: "FELIX_TYPHAK8SNAMESPACE", Value: "calico-system"}, {Name: "FELIX_TYPHAK8SSERVICENAME", Value: "calico-typha"}, {Name: "FELIX_TYPHACAFILE", Value: certificatemanagement.TrustedCertBundleMountPath}, {Name: "FELIX_TYPHACERTFILE", Value: "/node-certs/tls.crt"}, {Name: "FELIX_TYPHACN", Value: "typha-server"}, {Name: "FELIX_TYPHAKEYFILE", Value: "/node-certs/tls.key"}, + {Name: "FELIX_ROUTESOURCE", Value: "WorkloadIPs"}, + {Name: "FELIX_BPFEXTTOSERVICECONNMARK", Value: "0x80"}, - // Calico Windows specific envvars {Name: "VXLAN_VNI", Value: "4096"}, {Name: "VXLAN_ADAPTER", Value: ""}, - {Name: "KUBE_NETWORK", Value: "Calico.*"}, + {Name: "KUBE_NETWORK", Value: "vpc.*"}, {Name: "KUBERNETES_SERVICE_HOST", Value: "1.2.3.4"}, {Name: "KUBERNETES_SERVICE_PORT", Value: "6443"}, } Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "node").Env).To(ConsistOf(expectedNodeEnv)) Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "felix").Env).To(ConsistOf(expectedNodeEnv)) - Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "confd").Env).To(ConsistOf(expectedNodeEnv)) - - verifyWindowsProbesAndLifecycle(ds, true) - }) - - It("should render all resources when variant is CalicoEnterprise and running on openshift", func() { - expectedResources := []struct { - name string - ns string - group string - version string - kind string - }{ - {name: "calico-node-metrics-windows", ns: "calico-system", group: "", version: "v1", kind: "Service"}, - {name: "cni-config-windows", ns: common.CalicoNamespace, group: "", version: "v1", kind: "ConfigMap"}, - {name: common.WindowsDaemonSetName, ns: common.CalicoNamespace, group: "apps", version: "v1", kind: "DaemonSet"}, - } - - defaultInstance.Variant = operatorv1.CalicoEnterprise - defaultInstance.KubernetesProvider = operatorv1.ProviderOpenShift - cfg.NodeReporterMetricsPort = 9081 - - component := render.Windows(&cfg) - Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() - Expect(len(resources)).To(Equal(len(expectedResources))) - - // Should render the correct resources. - i := 0 - for _, expectedRes := range expectedResources { - rtest.ExpectResourceTypeAndObjectMetadata(resources[i], expectedRes.name, expectedRes.ns, expectedRes.group, expectedRes.version, expectedRes.kind) - i++ - } - - // The DaemonSet should have the correct configuration. - ds := rtest.GetResource(resources, "calico-node-windows", "calico-system", "apps", "v1", "DaemonSet").(*appsv1.DaemonSet) - - // The pod template should have node critical priority - Expect(ds.Spec.Template.Spec.PriorityClassName).To(Equal(render.NodePriorityClassName)) - - felixContainer := rtest.GetContainer(ds.Spec.Template.Spec.Containers, "felix") - Expect(felixContainer.Image).To(Equal(fmt.Sprintf("%s%s%s:%s", components.TigeraRegistry, components.TigeraImagePath, components.ComponentTigeraNodeWindows.Image, components.ComponentTigeraNodeWindows.Version))) - nodeContainer := rtest.GetContainer(ds.Spec.Template.Spec.Containers, "node") - Expect(nodeContainer.Image).To(Equal(fmt.Sprintf("%s%s%s:%s", components.TigeraRegistry, components.TigeraImagePath, components.ComponentTigeraNodeWindows.Image, components.ComponentTigeraNodeWindows.Version))) - cniContainer := rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "install-cni") - Expect(cniContainer.Image).To(Equal(fmt.Sprintf("%s%s%s:%s", components.TigeraRegistry, components.TigeraImagePath, components.ComponentTigeraCNIWindows.Image, components.ComponentTigeraCNIWindows.Version))) - uninstallContainer := rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "uninstall-calico") - Expect(uninstallContainer.Image).To(Equal(fmt.Sprintf("%s%s%s:%s", components.TigeraRegistry, components.TigeraImagePath, components.ComponentTigeraNodeWindows.Image, components.ComponentTigeraNodeWindows.Version))) - - // FIXME: confirm openshift CNI path defaults - expectedCNIVolumeMounts := []corev1.VolumeMount{ - {MountPath: "/host/opt/cni/bin", Name: "cni-bin-dir"}, - {MountPath: "/host/etc/cni/net.d", Name: "cni-net-dir"}, - } - Expect(rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "install-cni").VolumeMounts).To(ConsistOf(expectedCNIVolumeMounts)) - // FIXME: confirm openshift CNI path defaults - expectedUninstallVolumeMounts := []corev1.VolumeMount{ - {MountPath: "/host/opt/cni/bin", Name: "cni-bin-dir"}, - {MountPath: "/host/etc/cni/net.d", Name: "cni-net-dir"}, - } - Expect(rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "uninstall-calico").VolumeMounts).To(ConsistOf(expectedUninstallVolumeMounts)) + // Expect the SECURITY_GROUP env variables to not be set + Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "node").Env).NotTo(ContainElement(gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{"Name": Equal("TIGERA_DEFAULT_SECURITY_GROUPS")}))) + Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "node").Env).NotTo(ContainElement(gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{"Name": Equal("TIGERA_POD_SECURITY_GROUP")}))) + Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "felix").Env).NotTo(ContainElement(gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{"Name": Equal("TIGERA_DEFAULT_SECURITY_GROUPS")}))) + Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "felix").Env).NotTo(ContainElement(gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{"Name": Equal("TIGERA_POD_SECURITY_GROUP")}))) - // Verify volumes - // FIXME: confirm openshift CNI path defaults + // Verify volumes. fileOrCreate := corev1.HostPathFileOrCreate dirOrCreate := corev1.HostPathDirectoryOrCreate expectedVols := []corev1.Volume{ @@ -1745,9 +1057,6 @@ var _ = Describe("Windows rendering tests", func() { {Name: "var-run-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/run/calico", Type: &dirOrCreate}}}, {Name: "var-lib-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/lib/calico", Type: &dirOrCreate}}}, {Name: "xtables-lock", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/run/xtables.lock", Type: &fileOrCreate}}}, - {Name: "cni-bin-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/opt/cni/bin", Type: &dirOrCreate}}}, - {Name: "cni-net-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/etc/cni/net.d"}}}, - {Name: "cni-log-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico/cni", Type: &dirOrCreate}}}, {Name: "policysync", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/run/nodeagent", Type: &dirOrCreate}}}, { Name: "tigera-ca-bundle", @@ -1768,73 +1077,85 @@ var _ = Describe("Windows rendering tests", func() { }, }, }, - {Name: "var-log-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico", Type: &dirOrCreate}}}, } Expect(ds.Spec.Template.Spec.Volumes).To(ConsistOf(expectedVols)) - expectedNodeEnv := []corev1.EnvVar{ - // Default envvars. - {Name: "CNI_PLUGIN_TYPE", Value: "Calico"}, - {Name: "DATASTORE_TYPE", Value: "kubernetes"}, - {Name: "WAIT_FOR_DATASTORE", Value: "true"}, - {Name: "CALICO_MANAGE_CNI", Value: "true"}, - {Name: "CALICO_NETWORKING_BACKEND", Value: "windows-bgp"}, - {Name: "CLUSTER_TYPE", Value: "k8s,operator,openshift,bgp,windows"}, - {Name: "CALICO_DISABLE_FILE_LOGGING", Value: "false"}, - {Name: "FELIX_DEFAULTENDPOINTTOHOSTACTION", Value: "ACCEPT"}, - {Name: "FELIX_HEALTHENABLED", Value: "true"}, - { - Name: "NODENAME", - ValueFrom: &corev1.EnvVarSource{ - FieldRef: &corev1.ObjectFieldSelector{FieldPath: "spec.nodeName"}, + // Verify volume mounts. + expectedNodeVolumeMounts := []corev1.VolumeMount{ + {MountPath: "/var/run/calico", Name: "var-run-calico"}, + {MountPath: "/var/lib/calico", Name: "var-lib-calico"}, + {MountPath: "c:/etc/pki/tls/certs", Name: "tigera-ca-bundle", ReadOnly: true}, + {MountPath: "c:/node-certs", Name: render.NodeTLSSecretName, ReadOnly: true}, + } + Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "node").VolumeMounts).To(ConsistOf(expectedNodeVolumeMounts)) + Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "felix").VolumeMounts).To(ConsistOf(expectedNodeVolumeMounts)) + + // Verify tolerations. + Expect(ds.Spec.Template.Spec.Tolerations).To(ConsistOf(rmeta.TolerateAll)) + + // Verify readiness and liveness probes. + verifyWindowsProbesAndLifecycle(ds, true) + }) + + DescribeTable("should properly render configuration using non-Calico CNI plugin", + func(cni operatorv1.CNIPluginType, ipam operatorv1.IPAMPluginType) { + installlation := &operatorv1.InstallationSpec{ + CNI: &operatorv1.CNISpec{ + Type: cni, + IPAM: &operatorv1.IPAMSpec{Type: ipam}, }, - }, - { - Name: "NAMESPACE", - ValueFrom: &corev1.EnvVarSource{ - FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.namespace"}, + WindowsNodes: &operatorv1.WindowsNodeSpec{ + CNIBinDir: "/opt/cni/bin", + CNIConfigDir: "/etc/cni/net.d", + CNILogDir: "/var/log/calico/cni", }, - }, - {Name: "IP", Value: "autodetect"}, - {Name: "IP_AUTODETECTION_METHOD", Value: "first-found"}, - {Name: "IP6", Value: "none"}, - {Name: "FELIX_IPV6SUPPORT", Value: "false"}, - {Name: "FELIX_TYPHAK8SNAMESPACE", Value: "calico-system"}, - {Name: "FELIX_TYPHAK8SSERVICENAME", Value: "calico-typha"}, - {Name: "FELIX_TYPHACAFILE", Value: certificatemanagement.TrustedCertBundleMountPath}, - {Name: "FELIX_TYPHACERTFILE", Value: "/node-certs/tls.crt"}, - {Name: "FELIX_TYPHACN", Value: "typha-server"}, - {Name: "FELIX_TYPHAKEYFILE", Value: "/node-certs/tls.key"}, + } + cfg.Installation = installlation - {Name: "FELIX_DNSTRUSTEDSERVERS", Value: "k8s-service:openshift-dns/dns-default"}, + component := render.Windows(&cfg) + Expect(component.ResolveImages(nil)).To(BeNil()) + resources, _ := component.Objects() - // Tigera-specific envvars - {Name: "FELIX_PROMETHEUSREPORTERENABLED", Value: "true"}, - {Name: "FELIX_PROMETHEUSREPORTERPORT", Value: "9081"}, - {Name: "FELIX_FLOWLOGSFILEENABLED", Value: "true"}, - {Name: "FELIX_FLOWLOGSFILEINCLUDELABELS", Value: "true"}, - {Name: "FELIX_FLOWLOGSFILEINCLUDEPOLICIES", Value: "true"}, - {Name: "FELIX_FLOWLOGSFILEINCLUDESERVICE", Value: "true"}, - {Name: "FELIX_FLOWLOGSENABLENETWORKSETS", Value: "true"}, - {Name: "FELIX_FLOWLOGSCOLLECTPROCESSINFO", Value: "true"}, - {Name: "FELIX_DNSLOGSFILEENABLED", Value: "true"}, - {Name: "FELIX_DNSLOGSFILEPERNODELIMIT", Value: "1000"}, + // Should render the correct resources. + Expect(rtest.GetResource(resources, "calico-node-windows", "calico-system", "apps", "v1", "DaemonSet")).ToNot(BeNil()) + dsResource := rtest.GetResource(resources, "calico-node-windows", "calico-system", "apps", "v1", "DaemonSet") + Expect(dsResource).ToNot(BeNil()) - // Calico Windows specific envvars - {Name: "VXLAN_VNI", Value: "4096"}, - {Name: "VXLAN_ADAPTER", Value: ""}, - {Name: "KUBE_NETWORK", Value: "Calico.*"}, - {Name: "KUBERNETES_SERVICE_HOST", Value: "1.2.3.4"}, - {Name: "KUBERNETES_SERVICE_PORT", Value: "6443"}, - } - Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "node").Env).To(ConsistOf(expectedNodeEnv)) - Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "felix").Env).To(ConsistOf(expectedNodeEnv)) - Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "confd").Env).To(ConsistOf(expectedNodeEnv)) + // Should not render CNI configuration. + cniCmResource := rtest.GetResource(resources, "cni-config-windows", "calico-system", "", "v1", "ConfigMap") + Expect(cniCmResource).To(BeNil()) - verifyWindowsProbesAndLifecycle(ds, false) - }) + // The DaemonSet should have the correct configuration. + ds := dsResource.(*appsv1.DaemonSet) + + // The pod template should have node critical priority + Expect(ds.Spec.Template.Spec.PriorityClassName).To(Equal(render.NodePriorityClassName)) + + // CNI install container should not be present. + cniContainer := rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "install-cni") + Expect(cniContainer).To(BeNil()) + // Validate correct number of init containers. + Expect(len(ds.Spec.Template.Spec.InitContainers)).To(Equal(1)) + + // Verify env + expectedEnvs := []corev1.EnvVar{ + {Name: "CNI_PLUGIN_TYPE", Value: string(cni)}, + {Name: "CALICO_NETWORKING_BACKEND", Value: "none"}, + {Name: "FELIX_DEFAULTENDPOINTTOHOSTACTION", Value: "ACCEPT"}, + } + for _, expected := range expectedEnvs { + Expect(ds.Spec.Template.Spec.Containers[0].Env).To(ContainElement(expected)) + } + + // Verify readiness and liveness probes. + verifyWindowsProbesAndLifecycle(ds, true) + }, + Entry("GKE", operatorv1.PluginGKE, operatorv1.IPAMPluginHostLocal), + Entry("AmazonVPC", operatorv1.PluginAmazonVPC, operatorv1.IPAMPluginAmazonVPC), + Entry("AzureVNET", operatorv1.PluginAzureVNET, operatorv1.IPAMPluginAzureVNET), + ) - It("should render all resources when variant is CalicoEnterprise and running on RKE2", func() { + It("should render all resources when running on openshift", func() { expectedResources := []struct { name string ns string @@ -1842,19 +1163,16 @@ var _ = Describe("Windows rendering tests", func() { version string kind string }{ - {name: "calico-node-metrics-windows", ns: "calico-system", group: "", version: "v1", kind: "Service"}, {name: "cni-config-windows", ns: common.CalicoNamespace, group: "", version: "v1", kind: "ConfigMap"}, {name: common.WindowsDaemonSetName, ns: common.CalicoNamespace, group: "apps", version: "v1", kind: "DaemonSet"}, } - defaultInstance.Variant = operatorv1.CalicoEnterprise - defaultInstance.KubernetesProvider = operatorv1.ProviderRKE2 - cfg.NodeReporterMetricsPort = 9081 - + defaultInstance.FlexVolumePath = "/etc/kubernetes/kubelet-plugins/volume/exec/" + defaultInstance.KubernetesProvider = operatorv1.ProviderOpenShift component := render.Windows(&cfg) Expect(component.ResolveImages(nil)).To(BeNil()) resources, _ := component.Objects() - Expect(len(resources)).To(Equal(len(expectedResources)), fmt.Sprintf("Actual resources: %#v", resources)) + Expect(len(resources)).To(Equal(len(expectedResources))) // Should render the correct resources. i := 0 @@ -1870,22 +1188,22 @@ var _ = Describe("Windows rendering tests", func() { Expect(ds.Spec.Template.Spec.PriorityClassName).To(Equal(render.NodePriorityClassName)) felixContainer := rtest.GetContainer(ds.Spec.Template.Spec.Containers, "felix") - Expect(felixContainer.Image).To(Equal(fmt.Sprintf("%s%s%s:%s", components.TigeraRegistry, components.TigeraImagePath, components.ComponentTigeraNodeWindows.Image, components.ComponentTigeraNodeWindows.Version))) + Expect(felixContainer.Image).To(Equal(fmt.Sprintf("quay.io/%s%s:%s", components.CalicoImagePath, components.ComponentCalicoNodeWindows.Image, components.ComponentCalicoNodeWindows.Version))) nodeContainer := rtest.GetContainer(ds.Spec.Template.Spec.Containers, "node") - Expect(nodeContainer.Image).To(Equal(fmt.Sprintf("%s%s%s:%s", components.TigeraRegistry, components.TigeraImagePath, components.ComponentTigeraNodeWindows.Image, components.ComponentTigeraNodeWindows.Version))) + Expect(nodeContainer.Image).To(Equal(fmt.Sprintf("quay.io/%s%s:%s", components.CalicoImagePath, components.ComponentCalicoNodeWindows.Image, components.ComponentCalicoNodeWindows.Version))) cniContainer := rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "install-cni") - Expect(cniContainer.Image).To(Equal(fmt.Sprintf("%s%s%s:%s", components.TigeraRegistry, components.TigeraImagePath, components.ComponentTigeraCNIWindows.Image, components.ComponentTigeraCNIWindows.Version))) + Expect(cniContainer.Image).To(Equal(fmt.Sprintf("quay.io/%s%s:%s", components.CalicoImagePath, components.ComponentCalicoCNIWindows.Image, components.ComponentCalicoCNIWindows.Version))) uninstallContainer := rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "uninstall-calico") - Expect(uninstallContainer.Image).To(Equal(fmt.Sprintf("%s%s%s:%s", components.TigeraRegistry, components.TigeraImagePath, components.ComponentTigeraNodeWindows.Image, components.ComponentTigeraNodeWindows.Version))) + Expect(uninstallContainer.Image).To(Equal(fmt.Sprintf("quay.io/%s%s:%s", components.CalicoImagePath, components.ComponentCalicoNodeWindows.Image, components.ComponentCalicoNodeWindows.Version))) - // FIXME: confirm RKE2 CNI path defaults + // FIXME: confirm openshift CNI path defaults expectedCNIVolumeMounts := []corev1.VolumeMount{ {MountPath: "/host/opt/cni/bin", Name: "cni-bin-dir"}, {MountPath: "/host/etc/cni/net.d", Name: "cni-net-dir"}, } Expect(rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "install-cni").VolumeMounts).To(ConsistOf(expectedCNIVolumeMounts)) - // FIXME: confirm RKE2 CNI path defaults + // FIXME: confirm openshift CNI path defaults expectedUninstallVolumeMounts := []corev1.VolumeMount{ {MountPath: "/host/opt/cni/bin", Name: "cni-bin-dir"}, {MountPath: "/host/etc/cni/net.d", Name: "cni-net-dir"}, @@ -1893,7 +1211,7 @@ var _ = Describe("Windows rendering tests", func() { Expect(rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "uninstall-calico").VolumeMounts).To(ConsistOf(expectedUninstallVolumeMounts)) // Verify volumes - // FIXME: confirm RKE2 CNI path defaults + // FIXME: confirm openshift CNI path defaults fileOrCreate := corev1.HostPathFileOrCreate dirOrCreate := corev1.HostPathDirectoryOrCreate expectedVols := []corev1.Volume{ @@ -1924,18 +1242,16 @@ var _ = Describe("Windows rendering tests", func() { }, }, }, - {Name: "var-log-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico", Type: &dirOrCreate}}}, } Expect(ds.Spec.Template.Spec.Volumes).To(ConsistOf(expectedVols)) expectedNodeEnv := []corev1.EnvVar{ - // Default envvars. {Name: "CNI_PLUGIN_TYPE", Value: "Calico"}, {Name: "DATASTORE_TYPE", Value: "kubernetes"}, {Name: "WAIT_FOR_DATASTORE", Value: "true"}, {Name: "CALICO_MANAGE_CNI", Value: "true"}, {Name: "CALICO_NETWORKING_BACKEND", Value: "windows-bgp"}, - {Name: "CLUSTER_TYPE", Value: "k8s,operator,bgp,windows"}, + {Name: "CLUSTER_TYPE", Value: "k8s,operator,openshift,bgp,windows"}, {Name: "CALICO_DISABLE_FILE_LOGGING", Value: "false"}, {Name: "FELIX_DEFAULTENDPOINTTOHOSTACTION", Value: "ACCEPT"}, {Name: "FELIX_HEALTHENABLED", Value: "true"}, @@ -1962,20 +1278,6 @@ var _ = Describe("Windows rendering tests", func() { {Name: "FELIX_TYPHACN", Value: "typha-server"}, {Name: "FELIX_TYPHAKEYFILE", Value: "/node-certs/tls.key"}, - {Name: "FELIX_DNSTRUSTEDSERVERS", Value: "k8s-service:kube-system/rke2-coredns-rke2-coredns"}, - - // Tigera-specific envvars - {Name: "FELIX_PROMETHEUSREPORTERENABLED", Value: "true"}, - {Name: "FELIX_PROMETHEUSREPORTERPORT", Value: "9081"}, - {Name: "FELIX_FLOWLOGSFILEENABLED", Value: "true"}, - {Name: "FELIX_FLOWLOGSFILEINCLUDELABELS", Value: "true"}, - {Name: "FELIX_FLOWLOGSFILEINCLUDEPOLICIES", Value: "true"}, - {Name: "FELIX_FLOWLOGSFILEINCLUDESERVICE", Value: "true"}, - {Name: "FELIX_FLOWLOGSENABLENETWORKSETS", Value: "true"}, - {Name: "FELIX_FLOWLOGSCOLLECTPROCESSINFO", Value: "true"}, - {Name: "FELIX_DNSLOGSFILEENABLED", Value: "true"}, - {Name: "FELIX_DNSLOGSFILEPERNODELIMIT", Value: "1000"}, - // Calico Windows specific envvars {Name: "VXLAN_VNI", Value: "4096"}, {Name: "VXLAN_ADAPTER", Value: ""}, @@ -1983,14 +1285,12 @@ var _ = Describe("Windows rendering tests", func() { {Name: "KUBERNETES_SERVICE_HOST", Value: "1.2.3.4"}, {Name: "KUBERNETES_SERVICE_PORT", Value: "6443"}, } - Expect(ds.Spec.Template.Spec.Containers[0].Env).To(ConsistOf(expectedNodeEnv)) - Expect(len(ds.Spec.Template.Spec.Containers[0].Env)).To(Equal(len(expectedNodeEnv))) - verifyWindowsProbesAndLifecycle(ds, false) + Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "node").Env).To(ConsistOf(expectedNodeEnv)) + Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "felix").Env).To(ConsistOf(expectedNodeEnv)) + Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "confd").Env).To(ConsistOf(expectedNodeEnv)) - // The metrics service should have the correct configuration. - ms := rtest.GetResource(resources, "calico-node-metrics-windows", "calico-system", "", "v1", "Service").(*corev1.Service) - Expect(ms.Spec.ClusterIP).To(Equal("None"), "metrics service should be headless to prevent kube-proxy from rendering too many iptables rules") + verifyWindowsProbesAndLifecycle(ds, true) }) Describe("AKS", func() { @@ -2133,55 +1433,6 @@ var _ = Describe("Windows rendering tests", func() { }) }) - It("should not enable prometheus metrics if NodeMetricsPort is nil", func() { - defaultInstance.Variant = operatorv1.CalicoEnterprise - defaultInstance.NodeMetricsPort = nil - cfg.NodeReporterMetricsPort = 9081 - - component := render.Windows(&cfg) - Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() - Expect(len(resources)).To(Equal(defaultNumExpectedResources + 1)) - - dsResource := rtest.GetResource(resources, "calico-node-windows", "calico-system", "apps", "v1", "DaemonSet") - Expect(dsResource).ToNot(BeNil()) - - notExpectedEnvVar := corev1.EnvVar{Name: "FELIX_PROMETHEUSMETRICSPORT"} - ds := dsResource.(*appsv1.DaemonSet) - Expect(ds.Spec.Template.Spec.Containers[0].Env).ToNot(ContainElement(notExpectedEnvVar)) - - // It should have the reporter port, though. - expected := corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERPORT"} - Expect(ds.Spec.Template.Spec.Containers[0].Env).ToNot(ContainElement(expected)) - }) - - It("should set FELIX_PROMETHEUSMETRICSPORT with a custom value if NodeMetricsPort is set", func() { - var nodeMetricsPort int32 = 1234 - defaultInstance.Variant = operatorv1.CalicoEnterprise - defaultInstance.NodeMetricsPort = &nodeMetricsPort - component := render.Windows(&cfg) - Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() - Expect(len(resources)).To(Equal(defaultNumExpectedResources + 1)) - - dsResource := rtest.GetResource(resources, "calico-node-windows", "calico-system", "apps", "v1", "DaemonSet") - Expect(dsResource).ToNot(BeNil()) - - // Assert on expected env vars. - expectedEnvVars := []corev1.EnvVar{ - {Name: "FELIX_PROMETHEUSMETRICSPORT", Value: "1234"}, - {Name: "FELIX_PROMETHEUSMETRICSENABLED", Value: "true"}, - } - ds := dsResource.(*appsv1.DaemonSet) - for _, v := range expectedEnvVars { - Expect(ds.Spec.Template.Spec.Containers[0].Env).To(ContainElement(v)) - } - - // Assert we set annotations properly. - Expect(ds.Spec.Template.Annotations["prometheus.io/scrape"]).To(Equal("true")) - Expect(ds.Spec.Template.Annotations["prometheus.io/port"]).To(Equal("1234")) - }) - It("should render MaxUnavailable if a custom value was set", func() { two := intstr.FromInt(2) defaultInstance.NodeUpdateStrategy.RollingUpdate.MaxUnavailable = &two