Extract variant branches into a generic extension mechanism

cmd/main.go

@@ -42,6 +42,7 @@ import (
 	"github.com/tigera/operator/pkg/controller/options"
 	"github.com/tigera/operator/pkg/controller/utils"
 	"github.com/tigera/operator/pkg/dns"
+	"github.com/tigera/operator/pkg/enterprise"
 	"github.com/tigera/operator/pkg/imports/admission"
 	"github.com/tigera/operator/pkg/imports/crds"
 	"github.com/tigera/operator/pkg/render"
@@ -509,6 +510,14 @@ If a value other than 'all' is specified, the first CRD with a prefix of the spe
 		os.Exit(1)
 	}
 
+	// Build the variant extensions and let them compute their controller-phase
+	// options (the core operator's Set computes none).
+	extensionSet := enterprise.New()
+	if err := extensionSet.ComputeOptions(ctx, clientset); err != nil {
+		setupLog.Error(err, "Failed to compute extension options")
+		os.Exit(1)
+	}
+
 	options := options.ControllerOptions{
 		DetectedProvider:    provider,
 		EnterpriseCRDExists: enterpriseCRDExists,
@@ -521,6 +530,7 @@ If a value other than 'all' is specified, the first CRD with a prefix of the spe
 		ElasticExternal:     discovery.UseExternalElastic(bootConfig),
 		UseV3CRDs:           v3CRDs,
 		APIDiscovery:        apiDiscovery,
+		Extensions:          extensionSet,
 	}
 
 	// Before we start any controllers, make sure our options are valid.

pkg/controller/apiserver/apiserver_controller.go

@@ -41,6 +41,7 @@ import (
 	apiserver "github.com/tigera/operator/pkg/common/validation/apiserver"
 	webhooksvalidation "github.com/tigera/operator/pkg/common/validation/webhooks"
 	"github.com/tigera/operator/pkg/controller/certificatemanager"
+	"github.com/tigera/operator/pkg/controller/contexts"
 	"github.com/tigera/operator/pkg/controller/k8sapi"
 	"github.com/tigera/operator/pkg/controller/migration/datastoremigration"
 	"github.com/tigera/operator/pkg/controller/options"
@@ -51,7 +52,6 @@ import (
 	"github.com/tigera/operator/pkg/dns"
 	"github.com/tigera/operator/pkg/render"
 	rcertificatemanagement "github.com/tigera/operator/pkg/render/certificatemanagement"
-	"github.com/tigera/operator/pkg/render/common/authentication"
 	"github.com/tigera/operator/pkg/render/common/networkpolicy"
 	"github.com/tigera/operator/pkg/render/monitor"
 	"github.com/tigera/operator/pkg/render/webhooks"
@@ -104,42 +104,11 @@ func Add(mgr manager.Manager, opts options.ControllerOptions) error {
 	}
 
 	if opts.EnterpriseCRDExists {
-		// Watch for changes to ApplicationLayer
-		err = c.WatchObject(&operatorv1.ApplicationLayer{ObjectMeta: metav1.ObjectMeta{Name: utils.DefaultEnterpriseInstanceKey.Name}}, &handler.EnqueueRequestForObject{})
-		if err != nil {
-			return fmt.Errorf("apiserver-controller failed to watch ApplicationLayer resource: %v", err)
-		}
-
-		// Watch for changes to primary resource ManagementCluster
-		err = c.WatchObject(&operatorv1.ManagementCluster{}, &handler.EnqueueRequestForObject{})
-		if err != nil {
-			return fmt.Errorf("apiserver-controller failed to watch primary resource: %v", err)
-		}
-
-		// Watch for changes to primary resource ManagementClusterConnection
-		err = c.WatchObject(&operatorv1.ManagementClusterConnection{}, &handler.EnqueueRequestForObject{})
-		if err != nil {
-			return fmt.Errorf("apiserver-controller failed to watch primary resource: %v", err)
-		}
-
-		for _, namespace := range []string{common.OperatorNamespace(), render.APIServerNamespace} {
-			for _, secretName := range []string{render.VoltronTunnelSecretName, render.ManagerTLSSecretName} {
-				if err = utils.AddSecretsWatch(c, secretName, namespace); err != nil {
-					return fmt.Errorf("apiserver-controller failed to watch the Secret resource: %v", err)
-				}
-			}
+		// The variant extension registers the enterprise watches it needs (the management
+		// cluster CRs, ApplicationLayer, Authentication, and the tunnel secrets).
+		if err = opts.Extensions.SetupWatches(contexts.APIServerController, c); err != nil {
+			return fmt.Errorf("apiserver-controller failed to set up extension watches: %w", err)
 		}
-
-		if err = utils.AddSecretsWatch(c, render.VoltronLinseedPublicCert, common.OperatorNamespace()); err != nil {
-			return fmt.Errorf("apiserver-controller failed to watch the Secret resource: %v", err)
-		}
-
-		// Watch for changes to authentication
-		err = c.WatchObject(&operatorv1.Authentication{}, &handler.EnqueueRequestForObject{})
-		if err != nil {
-			return fmt.Errorf("apiserver-controller failed to watch resource: %w", err)
-		}
-
 	}
 
 	// Watch for the namespace(s) managed by this controller.
@@ -318,16 +287,6 @@ func (r *ReconcileAPIServer) Reconcile(ctx context.Context, request reconcile.Re
 		return reconcile.Result{}, err
 	}
 
-	// Since apiserver and queryserver may have different UID:GID at run-time, we need to produce this secret in separate volumes and with different permissions.
-	var queryServerTLSSecretCertificateManagementOnly certificatemanagement.KeyPairInterface
-	if installationSpec.CertificateManagement != nil {
-		queryServerTLSSecretCertificateManagementOnly, err = certificateManager.GetOrCreateKeyPair(r.client, "query-server-tls", common.OperatorNamespace(), dns.GetServiceDNSNames(render.APIServerServiceName, render.APIServerNamespace, r.opts.ClusterDomain))
-		if err != nil {
-			r.status.SetDegraded(operatorv1.ResourceCreateError, "Unable to get or create tls key pair", err, reqLogger)
-			return reconcile.Result{}, err
-		}
-	}
-
 	certificateManager.AddToStatusManager(r.status, render.APIServerNamespace)
 
 	pullSecrets, err := utils.GetInstallationPullSecrets(installationSpec, r.client)
@@ -336,114 +295,44 @@ func (r *ReconcileAPIServer) Reconcile(ctx context.Context, request reconcile.Re
 		return reconcile.Result{}, err
 	}
 
-	// Query enterprise-only data.
-	var trustedBundle certificatemanagement.TrustedBundle
-	var applicationLayer *operatorv1.ApplicationLayer
-	var managementCluster *operatorv1.ManagementCluster
-	var managementClusterConnection *operatorv1.ManagementClusterConnection
-	var keyValidatorConfig authentication.KeyValidatorConfig
-	includeV3NetworkPolicy := false
-
-	if installationSpec.Variant.IsEnterprise() {
-		trustedBundle, err = certificateManager.CreateNamedTrustedBundleFromSecrets(render.APIServerResourceName, r.client,
-			common.OperatorNamespace(), false)
-		if err != nil {
-			r.status.SetDegraded(operatorv1.ResourceCreateError, "Unable to create the trusted bundle", err, reqLogger)
-			return reconcile.Result{}, err
-		}
-
-		applicationLayer, err = utils.GetApplicationLayer(ctx, r.client)
-		if err != nil {
-			r.status.SetDegraded(operatorv1.ResourceReadError, "Error reading ApplicationLayer", err, reqLogger)
-			return reconcile.Result{}, err
-		}
+	// Run the variant extension: it validates the configuration and produces the
+	// enterprise render data (the trusted bundle, the query server certificate, the
+	// management cluster CRs, the OIDC config, and the resolved L7 sidecar images). For
+	// the core operator this is a no-op and the render context carries no extension data.
+	cc := contexts.ControllerContext{
+		RenderContext: render.RenderContext{
+			Installation:  installationSpec,
+			ClusterDomain: r.opts.ClusterDomain,
+		},
+		Controller:         contexts.APIServerController,
+		Ctx:                ctx,
+		Client:             r.client,
+		CertificateManager: certificateManager,
+	}
+	if err := r.opts.Extensions.Validate(cc); err != nil {
+		r.status.SetDegraded(operatorv1.ResourceValidationError, "Invalid API server configuration", err, reqLogger)
+		return reconcile.Result{}, err
+	}
+	cc, managedKeyPairs, err := r.opts.Extensions.ExtendContext(cc)
+	if err != nil {
+		r.status.SetDegraded(operatorv1.ResourceCreateError, "Error preparing the API server extension", err, reqLogger)
+		return reconcile.Result{}, err
+	}
+	trustedBundle := cc.TrustedBundle
 
+	// The webhooks component (v3-CRD mode) needs the ManagementCluster to register the
+	// managed-cluster webhook. Reading it requires the enterprise CRDs.
+	var managementCluster *operatorv1.ManagementCluster
+	if r.opts.EnterpriseCRDExists {
 		managementCluster, err = utils.GetManagementCluster(ctx, r.client)
 		if err != nil {
 			r.status.SetDegraded(operatorv1.ResourceReadError, "Error reading ManagementCluster", err, reqLogger)
 			return reconcile.Result{}, err
 		}
-
-		managementClusterConnection, err = utils.GetManagementClusterConnection(ctx, r.client)
-		if err != nil {
-			r.status.SetDegraded(operatorv1.ResourceReadError, "Error reading ManagementClusterConnection", err, reqLogger)
-			return reconcile.Result{}, err
-		}
-
-		if managementClusterConnection != nil && managementCluster != nil {
-			err = fmt.Errorf("having both a ManagementCluster and a ManagementClusterConnection is not supported")
-			r.status.SetDegraded(operatorv1.ResourceValidationError, "", err, reqLogger)
-			return reconcile.Result{}, err
-		}
-
-		// Management cluster only: check if the tunnel CA secret has been created. The apiserver mounts this secret so
-		// it can sign certificates for managed clusters. If the managementCluster has not been defaulted then we should
-		// not degrade. This is because the manager_controller exits the reconcile loop if the apiserver is not available.
-		if managementCluster != nil && managementCluster.Spec.TLS != nil && !r.opts.MultiTenant {
-			tunnelSecretName := managementCluster.Spec.TLS.SecretName
-			// The manager_controller should have written this secret. We know this since spec.TLS has been defaulted.
-			// If the secret does not exist, we degrade this controller.
-			_, err := utils.GetSecret(ctx, r.client, tunnelSecretName, common.OperatorNamespace())
-			if err != nil {
-				r.status.SetDegraded(operatorv1.ResourceReadError, "Unable to fetch the tunnel secret", err, reqLogger)
-				return reconcile.Result{}, err
-			}
-		}
-
-		prometheusCertificate, err := certificateManager.GetCertificate(r.client, monitor.PrometheusClientTLSSecretName, common.OperatorNamespace())
-		if err != nil {
-			r.status.SetDegraded(operatorv1.ResourceReadError, "Failed to get certificate", err, reqLogger)
-			return reconcile.Result{}, err
-		} else if prometheusCertificate != nil {
-			trustedBundle.AddCertificates(prometheusCertificate)
-		}
-
-		if managementClusterConnection != nil {
-			voltronLinseedCert, err := certificateManager.GetCertificate(r.client, render.VoltronLinseedPublicCert, common.OperatorNamespace())
-			if err != nil {
-				r.status.SetDegraded(operatorv1.ResourceReadError, fmt.Sprintf("Failed to retrieve %s", render.VoltronLinseedPublicCert), err, reqLogger)
-				return reconcile.Result{}, err
-			}
-			if voltronLinseedCert != nil {
-				trustedBundle.AddCertificates(voltronLinseedCert)
-			}
-		}
-
-		var authenticationCR *operatorv1.Authentication
-		// Fetch the Authentication spec. If present, we use it to configure user authentication.
-		authenticationCR, err = utils.GetAuthentication(ctx, r.client)
-		if err != nil && !errors.IsNotFound(err) {
-			r.status.SetDegraded(operatorv1.ResourceReadError, "Error while fetching Authentication", err, reqLogger)
-			return reconcile.Result{}, err
-		}
-
-		if authenticationCR != nil && authenticationCR.Status.State == operatorv1.TigeraStatusReady {
-			if utils.DexEnabled(authenticationCR) {
-				// Do not include DEX TLS Secret Name if authentication CR does not have type Dex
-				secret := render.DexTLSSecretName
-				certificate, err := certificateManager.GetCertificate(r.client, secret, common.OperatorNamespace())
-				if err != nil {
-					r.status.SetDegraded(operatorv1.CertificateError, fmt.Sprintf("Failed to retrieve %s", secret),
-						err, reqLogger)
-					return reconcile.Result{}, err
-				} else if certificate == nil {
-					reqLogger.Info(fmt.Sprintf("Waiting for secret '%s' to become available", secret))
-					r.status.SetDegraded(operatorv1.ResourceNotReady,
-						fmt.Sprintf("Waiting for secret '%s' to become available", secret),
-						nil, reqLogger)
-					return reconcile.Result{}, nil
-				}
-				trustedBundle.AddCertificates(certificate)
-			}
-
-			keyValidatorConfig, err = utils.GetKeyValidatorConfig(ctx, r.client, authenticationCR, r.opts.ClusterDomain)
-			if err != nil {
-				r.status.SetDegraded(operatorv1.ResourceReadError, "Failed to get KeyValidator Config", err, reqLogger)
-				return reconcile.Result{}, err
-			}
-		}
 	}
 
+	includeV3NetworkPolicy := false
+
 	// Ensure the calico-system tier exists, before rendering any network policies within it.
 	//
 	// The creation of the Tier depends on this controller to reconcile it's non-NetworkPolicy resources so that
@@ -474,7 +363,14 @@ func (r *ReconcileAPIServer) Reconcile(ctx context.Context, request reconcile.Re
 	}
 
 	// Create a component handler to manage the rendered component.
-	handler := utils.NewComponentHandler(log, r.client, r.scheme, instance)
+	handler := utils.NewComponentHandler(
+		log,
+		r.client,
+		r.scheme,
+		instance,
+		utils.WithRenderContext(cc.RenderContext),
+		utils.WithExtensions(r.opts.Extensions),
+	)
 
 	// Render the desired objects from the CRD and create or update them.
 	reqLogger.V(3).Info("rendering components")
@@ -485,19 +381,14 @@ func (r *ReconcileAPIServer) Reconcile(ctx context.Context, request reconcile.Re
 		Installation:                 installationSpec,
 		APIServer:                    &instance.Spec,
 		ForceHostNetwork:             false,
-		ApplicationLayer:             applicationLayer,
-		ManagementCluster:            managementCluster,
-		ManagementClusterConnection:  managementClusterConnection,
 		TLSKeyPair:                   tlsSecret,
 		PullSecrets:                  pullSecrets,
 		OpenShift:                    r.opts.DetectedProvider.IsOpenShift(),
 		TrustedBundle:                trustedBundle,
 		MultiTenant:                  r.opts.MultiTenant,
-		KeyValidatorConfig:           keyValidatorConfig,
 		KubernetesVersion:            r.opts.KubernetesVersion,
 		ClusterDomain:                r.opts.ClusterDomain,
 		RequiresAggregationServer:    !r.opts.UseV3CRDs,
-		QueryServerTLSKeyPairCertificateManagementOnly: queryServerTLSSecretCertificateManagementOnly,
 	}
 
 	var components []render.Component
@@ -506,6 +397,9 @@ func (r *ReconcileAPIServer) Reconcile(ctx context.Context, request reconcile.Re
 	certKeyPairOptions := []rcertificatemanagement.KeyPairOption{
 		rcertificatemanagement.NewKeyPairOption(tlsSecret, true, true),
 	}
+	for _, kp := range managedKeyPairs {
+		certKeyPairOptions = append(certKeyPairOptions, rcertificatemanagement.NewKeyPairOption(kp, true, true))
+	}
 	if r.opts.UseV3CRDs {
 		// If using v3 CRDs, we render the webhooks component that handles various RBAC and validation
 		// responsibilities. The ordering of resources here is important to avoid a deadlock:
@@ -585,10 +479,14 @@ func (r *ReconcileAPIServer) Reconcile(ctx context.Context, request reconcile.Re
 	}
 
 	// Check BYO certificate expiry warnings.
-	certificatemanagement.CheckKeyPairWarnings(map[string]certificatemanagement.KeyPairInterface{
+	keyPairWarnings := map[string]certificatemanagement.KeyPairInterface{
 		render.CalicoAPIServerTLSSecretName: tlsSecret,
 		webhooks.WebhooksTLSSecretName:      webhooksTLS,
-	}, r.status)
+	}
+	for _, kp := range managedKeyPairs {
+		keyPairWarnings[kp.GetName()] = kp
+	}
+	certificatemanagement.CheckKeyPairWarnings(keyPairWarnings, r.status)
 
 	// Clear the degraded bit if we've reached this far.
 	r.status.ClearDegraded()