diff --git a/README.md b/README.md index 3875399..ab87faf 100644 --- a/README.md +++ b/README.md @@ -192,6 +192,17 @@ The status conditions include `SecretsReady`, `DatabaseReady`, endpoint is `-api-gw:8000`; the database endpoint is the CNPG `-pg-rw:5432` Service. +Each core service condition and its `availableReplicas` value reflects the +observed Deployment, not merely the existence of its Kubernetes objects. A +core service is ready only after the Deployment controller has observed its +current generation and all desired replicas are updated, ready, and available +with no unavailable replicas. During creation or a partial rollout, pending +components remain `Ready=False`, the project phase is `Provisioning`, and the +aggregate `Ready` condition is false. The project reports `Running` with +aggregate `Ready=True` only after the database, every core service, and any +enabled PowerSync workloads are ready; `observedGeneration` advances to the +current project generation at that point. + ## Development and verification ```bash diff --git a/internal/controller/powersync_lifecycle_test.go b/internal/controller/powersync_lifecycle_test.go index 6760448..b33037e 100644 --- a/internal/controller/powersync_lifecycle_test.go +++ b/internal/controller/powersync_lifecycle_test.go @@ -131,7 +131,7 @@ func ownedMeta(project *supabasev1alpha1.SupabaseProject, name string) metav1.Ob } } -func TestPowersyncDeploymentIsReadyForCurrentGeneration(t *testing.T) { +func TestDeploymentIsReadyForCurrentGeneration(t *testing.T) { t.Parallel() replicas := int32(2) @@ -145,27 +145,27 @@ func TestPowersyncDeploymentIsReadyForCurrentGeneration(t *testing.T) { AvailableReplicas: 2, }, } - if !powersyncDeploymentIsReady(deployment) { + if !deploymentIsReady(deployment) { t.Fatal("current rollout with all replicas ready must be ready") } deployment.Status.ObservedGeneration = 2 - if powersyncDeploymentIsReady(deployment) { + if deploymentIsReady(deployment) { t.Fatal("stale rollout status must not be ready") } deployment.Status.ObservedGeneration = 3 deployment.Status.UpdatedReplicas = 1 - if powersyncDeploymentIsReady(deployment) { + if deploymentIsReady(deployment) { t.Fatal("old ready pods must not make an incomplete rollout ready") } deployment.Status.UpdatedReplicas = 2 deployment.Status.ReadyReplicas = 1 - if powersyncDeploymentIsReady(deployment) { + if deploymentIsReady(deployment) { t.Fatal("partial rollout must not be ready") } deployment.Status.ReadyReplicas = 2 deployment.Status.AvailableReplicas = 1 deployment.Status.UnavailableReplicas = 1 - if powersyncDeploymentIsReady(deployment) { + if deploymentIsReady(deployment) { t.Fatal("unavailable rollout must not be ready") } } diff --git a/internal/controller/supabaseproject_controller.go b/internal/controller/supabaseproject_controller.go index 1256219..1254c85 100644 --- a/internal/controller/supabaseproject_controller.go +++ b/internal/controller/supabaseproject_controller.go @@ -73,7 +73,7 @@ type serviceReconcileConfig struct { conditionType string buildDeployment func() *appsv1.Deployment buildService func() *corev1.Service - setStatus func(ready bool) + setStatus func(status supabasev1alpha1.ServiceStatus) logFields []any // optional credentialHash string } @@ -85,7 +85,7 @@ func newServiceReconcileConfig( conditionType string, buildDeployment func() *appsv1.Deployment, buildService func() *corev1.Service, - setStatus func(ready bool), + setStatus func(status supabasev1alpha1.ServiceStatus), ) serviceReconcileConfig { return serviceReconcileConfig{ name: name, @@ -191,6 +191,9 @@ func (r *SupabaseProjectReconciler) Reconcile(ctx context.Context, req ctrl.Requ if err := r.reconcileServices(ctx, project, credentials); err != nil { return ctrl.Result{}, err } + if !coreServicesReady(project) { + return ctrl.Result{RequeueAfter: RequeueDelay}, nil + } // Phase 6: PowerSync (after core services) if project.Spec.Powersync != nil { @@ -897,34 +900,53 @@ func (r *SupabaseProjectReconciler) reconcileServices(ctx context.Context, proje // Deploy Auth (GoTrue) if err := r.reconcileAuth(ctx, project, secretNames, credentialHash); err != nil { - return err + return r.failCoreServices(ctx, project, err) } // Deploy REST (PostgREST) if err := r.reconcileRest(ctx, project, secretNames, credentialHash); err != nil { - return err + return r.failCoreServices(ctx, project, err) } // Deploy Studio if err := r.reconcileStudio(ctx, project, secretNames, credentialHash); err != nil { - return err + return r.failCoreServices(ctx, project, err) } // Deploy Meta (postgres-meta) // postgres-meta consumes no project credential material, so credential // rotation must not roll this deployment. if err := r.reconcileMeta(ctx, project, secretNames); err != nil { - return err + return r.failCoreServices(ctx, project, err) } // Deploy Envoy gateway if err := r.reconcileGateway(ctx, project, credentialHash); err != nil { - return err + return r.failCoreServices(ctx, project, err) + } + + if !coreServicesReady(project) { + project.Status.Phase = supabasev1alpha1.PhaseProvisioning + r.setCondition(project, supabasev1alpha1.ConditionTypeReady, metav1.ConditionFalse, "CoreServicesPending", "Waiting for all core service deployments to become ready") + if err := r.updateProjectStatus(ctx, project); err != nil { + return err + } } return nil } +// failCoreServices records the aggregate failure state before returning the +// original component reconciliation error to the controller runtime. +func (r *SupabaseProjectReconciler) failCoreServices(ctx context.Context, project *supabasev1alpha1.SupabaseProject, reconcileErr error) error { + project.Status.Phase = supabasev1alpha1.PhaseProvisioning + r.setCondition(project, supabasev1alpha1.ConditionTypeReady, metav1.ConditionFalse, "CoreServicesFailed", fmt.Sprintf("Core service reconciliation failed: %v", reconcileErr)) + if err := r.updateProjectStatus(ctx, project); err != nil { + logf.FromContext(ctx).Error(err, "persisting core service failure status") + } + return reconcileErr +} + // reconcileServiceComponent is a generic helper for reconciling a service component (deployment + service) func (r *SupabaseProjectReconciler) reconcileServiceComponent(ctx context.Context, project *supabasev1alpha1.SupabaseProject, config serviceReconcileConfig) error { log := logf.FromContext(ctx) @@ -937,6 +959,7 @@ func (r *SupabaseProjectReconciler) reconcileServiceComponent(ctx context.Contex } log.V(1).Info(fmt.Sprintf("Built %s deployment", config.name), config.logFields...) if err := r.createOrUpdateDeployment(ctx, project, deployment); err != nil { + config.setStatus(supabasev1alpha1.ServiceStatus{}) r.setCondition(project, config.conditionType, metav1.ConditionFalse, "DeploymentFailed", err.Error()) if statusErr := r.updateProjectStatus(ctx, project); statusErr != nil { return statusErr @@ -947,6 +970,7 @@ func (r *SupabaseProjectReconciler) reconcileServiceComponent(ctx context.Contex // Create service service := config.buildService() if err := r.createOrUpdateService(ctx, project, service); err != nil { + config.setStatus(supabasev1alpha1.ServiceStatus{}) r.setCondition(project, config.conditionType, metav1.ConditionFalse, "ServiceFailed", err.Error()) if statusErr := r.updateProjectStatus(ctx, project); statusErr != nil { return statusErr @@ -954,7 +978,21 @@ func (r *SupabaseProjectReconciler) reconcileServiceComponent(ctx context.Contex return err } - config.setStatus(true) + ready, availableReplicas, err := r.deploymentStatus(ctx, deployment) + if err != nil { + config.setStatus(supabasev1alpha1.ServiceStatus{}) + r.setCondition(project, config.conditionType, metav1.ConditionFalse, "DeploymentStatusFailed", err.Error()) + if statusErr := r.updateProjectStatus(ctx, project); statusErr != nil { + return statusErr + } + return err + } + + config.setStatus(supabasev1alpha1.ServiceStatus{Ready: ready, AvailableReplicas: availableReplicas}) + if !ready { + r.setCondition(project, config.conditionType, metav1.ConditionFalse, "DeploymentPending", fmt.Sprintf("Waiting for %s deployment to become ready", config.name)) + return nil + } r.setCondition(project, config.conditionType, metav1.ConditionTrue, "Ready", fmt.Sprintf("%s service is running", config.name)) return nil } @@ -983,7 +1021,7 @@ func (r *SupabaseProjectReconciler) reconcileAuth(ctx context.Context, project * supabasev1alpha1.ConditionTypeAuthReady, func() *appsv1.Deployment { return deployment }, func() *corev1.Service { return services.BuildAuthService(project) }, - func(ready bool) { project.Status.Services.Auth = supabasev1alpha1.ServiceStatus{Ready: ready} }, + func(status supabasev1alpha1.ServiceStatus) { project.Status.Services.Auth = status }, ) config.logFields = []any{ "image", fmt.Sprintf("%s:%s", "supabase/gotrue", project.Spec.Auth.ImageTag), @@ -1104,7 +1142,7 @@ func (r *SupabaseProjectReconciler) reconcileRest(ctx context.Context, project * supabasev1alpha1.ConditionTypeRestReady, func() *appsv1.Deployment { return deployments.BuildRestDeployment(project, secretNames) }, func() *corev1.Service { return services.BuildRestService(project) }, - func(ready bool) { project.Status.Services.Rest = supabasev1alpha1.ServiceStatus{Ready: ready} }, + func(status supabasev1alpha1.ServiceStatus) { project.Status.Services.Rest = status }, ) config.logFields = []any{ "image", fmt.Sprintf("%s:%s", "postgrest/postgrest", project.Spec.Rest.ImageTag), @@ -1121,7 +1159,7 @@ func (r *SupabaseProjectReconciler) reconcileStudio(ctx context.Context, project supabasev1alpha1.ConditionTypeStudioReady, func() *appsv1.Deployment { return deployments.BuildStudioDeployment(project, secretNames) }, func() *corev1.Service { return services.BuildStudioService(project) }, - func(ready bool) { project.Status.Services.Studio = supabasev1alpha1.ServiceStatus{Ready: ready} }, + func(status supabasev1alpha1.ServiceStatus) { project.Status.Services.Studio = status }, ) config.logFields = []any{ "image", fmt.Sprintf("%s:%s", "supabase/studio", project.Spec.Studio.ImageTag), @@ -1138,7 +1176,7 @@ func (r *SupabaseProjectReconciler) reconcileMeta(ctx context.Context, project * supabasev1alpha1.ConditionTypeMetaReady, func() *appsv1.Deployment { return deployments.BuildMetaDeployment(project, secretNames) }, func() *corev1.Service { return services.BuildMetaService(project) }, - func(ready bool) { project.Status.Services.Meta = supabasev1alpha1.ServiceStatus{Ready: ready} }, + func(status supabasev1alpha1.ServiceStatus) { project.Status.Services.Meta = status }, ) config.logFields = []any{ "image", fmt.Sprintf("%s:%s", "supabase/postgres-meta", project.Spec.Meta.ImageTag), @@ -1155,6 +1193,7 @@ func (r *SupabaseProjectReconciler) reconcileGateway(ctx context.Context, projec envoyConfig := configmaps.BuildEnvoyConfigMap(project) log.V(1).Info("Built Envoy ConfigMap", "name", envoyConfig.Name) if err := r.createOrUpdateConfigMap(ctx, project, envoyConfig); err != nil { + project.Status.Services.Gateway = supabasev1alpha1.ServiceStatus{} r.setCondition(project, supabasev1alpha1.ConditionTypeGatewayReady, metav1.ConditionFalse, "ConfigMapFailed", err.Error()) if statusErr := r.updateProjectStatus(ctx, project); statusErr != nil { return statusErr @@ -1167,7 +1206,7 @@ func (r *SupabaseProjectReconciler) reconcileGateway(ctx context.Context, projec supabasev1alpha1.ConditionTypeGatewayReady, func() *appsv1.Deployment { return deployments.BuildGatewayDeployment(project) }, func() *corev1.Service { return services.BuildGatewayService(project) }, - func(ready bool) { project.Status.Services.Gateway = supabasev1alpha1.ServiceStatus{Ready: ready} }, + func(status supabasev1alpha1.ServiceStatus) { project.Status.Services.Gateway = status }, ) config.credentialHash = credentialHash config.logFields = []any{ @@ -2044,11 +2083,11 @@ func (r *SupabaseProjectReconciler) reconcilePowersync(ctx context.Context, proj return ctrl.Result{}, err } - apiReady, apiAvailable, err := r.powersyncDeploymentStatus(ctx, apiDeployment) + apiReady, apiAvailable, err := r.deploymentStatus(ctx, apiDeployment) if err != nil { return ctrl.Result{}, err } - replicationReady, replicationAvailable, err := r.powersyncDeploymentStatus(ctx, replDeployment) + replicationReady, replicationAvailable, err := r.deploymentStatus(ctx, replDeployment) if err != nil { return ctrl.Result{}, err } @@ -2141,15 +2180,21 @@ func (r *SupabaseProjectReconciler) deletePowerSyncOwnedResource(ctx context.Con return client.IgnoreNotFound(r.Delete(ctx, object)) } -func (r *SupabaseProjectReconciler) powersyncDeploymentStatus(ctx context.Context, desired *appsv1.Deployment) (bool, int32, error) { +func (r *SupabaseProjectReconciler) deploymentStatus(ctx context.Context, desired *appsv1.Deployment) (bool, int32, error) { + if desired == nil { + return false, 0, nil + } existing := &appsv1.Deployment{} if err := r.Get(ctx, types.NamespacedName{Name: desired.Name, Namespace: desired.Namespace}, existing); err != nil { + if apierrors.IsNotFound(err) { + return false, 0, nil + } return false, 0, err } - return powersyncDeploymentIsReady(existing), existing.Status.AvailableReplicas, nil + return deploymentIsReady(existing), existing.Status.AvailableReplicas, nil } -func powersyncDeploymentIsReady(deployment *appsv1.Deployment) bool { +func deploymentIsReady(deployment *appsv1.Deployment) bool { expected := int32(1) if deployment.Spec.Replicas != nil { expected = *deployment.Spec.Replicas @@ -2161,6 +2206,15 @@ func powersyncDeploymentIsReady(deployment *appsv1.Deployment) bool { deployment.Status.UnavailableReplicas == 0 } +func coreServicesReady(project *supabasev1alpha1.SupabaseProject) bool { + serviceStatus := project.Status.Services + return serviceStatus.Auth.Ready && + serviceStatus.Rest.Ready && + serviceStatus.Studio.Ready && + serviceStatus.Meta.Ready && + serviceStatus.Gateway.Ready +} + func (r *SupabaseProjectReconciler) updateProjectStatus(ctx context.Context, project *supabasev1alpha1.SupabaseProject) error { current := &supabasev1alpha1.SupabaseProject{} if err := r.Get(ctx, client.ObjectKeyFromObject(project), current); err != nil { diff --git a/internal/controller/workload_readiness_test.go b/internal/controller/workload_readiness_test.go new file mode 100644 index 0000000..7490f34 --- /dev/null +++ b/internal/controller/workload_readiness_test.go @@ -0,0 +1,429 @@ +package controller + +import ( + "context" + "errors" + "testing" + + cnpgv1 "github.com/cloudnative-pg/cloudnative-pg/api/v1" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/utils/ptr" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + supabasev1alpha1 "github.com/GuionAI/cloudnative-supabase/api/v1alpha1" + "github.com/GuionAI/cloudnative-supabase/internal/resources/cnpg" + "github.com/GuionAI/cloudnative-supabase/internal/resources/deployments" + secretresources "github.com/GuionAI/cloudnative-supabase/internal/resources/secrets" + "github.com/GuionAI/cloudnative-supabase/internal/resources/services" +) + +func TestReconcileServiceComponentReportsPendingDeployment(t *testing.T) { + t.Parallel() + + project := &supabasev1alpha1.SupabaseProject{ + TypeMeta: metav1.TypeMeta{APIVersion: supabasev1alpha1.GroupVersion.String(), Kind: "SupabaseProject"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "pending", Namespace: "default", UID: "pending-project-uid", + }, + } + replicas := int32(2) + setStatus := supabasev1alpha1.ServiceStatus{} + config := newServiceReconcileConfig( + "Auth", + supabasev1alpha1.ConditionTypeAuthReady, + func() *appsv1.Deployment { + return &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{Name: "pending-auth", Namespace: project.Namespace}, + Spec: appsv1.DeploymentSpec{Replicas: &replicas}, + } + }, + func() *corev1.Service { + return &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: "pending-auth", Namespace: project.Namespace}} + }, + func(status supabasev1alpha1.ServiceStatus) { setStatus = status }, + ) + scheme := newIdempotencyTestScheme(t) + reconciler := &SupabaseProjectReconciler{ + Client: fake.NewClientBuilder().WithScheme(scheme).WithObjects(project).Build(), + Scheme: scheme, + } + + if err := reconciler.reconcileServiceComponent(context.Background(), project, config); err != nil { + t.Fatal(err) + } + if setStatus.Ready || setStatus.AvailableReplicas != 0 { + t.Fatalf("pending status = %#v, want not ready with zero available replicas", setStatus) + } + condition := meta.FindStatusCondition(project.Status.Conditions, supabasev1alpha1.ConditionTypeAuthReady) + if condition == nil || condition.Status != metav1.ConditionFalse || condition.Reason != "DeploymentPending" { + t.Fatalf("AuthReady condition = %#v, want DeploymentPending false", condition) + } +} + +func TestReconcileServicesReportsMixedDeploymentReadiness(t *testing.T) { + t.Parallel() + + project, reconciler := newCoreReadinessFixture(t, true) + + if err := reconciler.reconcileServices(context.Background(), project, &secretresources.ProjectCredentials{}); err != nil { + t.Fatal(err) + } + + status := &supabasev1alpha1.SupabaseProject{} + if err := reconciler.Get(context.Background(), client.ObjectKeyFromObject(project), status); err != nil { + t.Fatal(err) + } + if status.Status.Phase != supabasev1alpha1.PhaseProvisioning { + t.Fatalf("phase = %q, want Provisioning", status.Status.Phase) + } + if status.Status.ObservedGeneration != 4 { + t.Fatalf("observedGeneration = %d, want previous generation 4", status.Status.ObservedGeneration) + } + if status.Status.Services.Auth.AvailableReplicas != 2 || !status.Status.Services.Auth.Ready { + t.Fatalf("auth status = %#v, want ready with two available replicas", status.Status.Services.Auth) + } + if status.Status.Services.Rest.AvailableReplicas != 1 || status.Status.Services.Rest.Ready { + t.Fatalf("rest status = %#v, want pending with one available replica", status.Status.Services.Rest) + } + for name, service := range map[string]supabasev1alpha1.ServiceStatus{ + "studio": status.Status.Services.Studio, + "meta": status.Status.Services.Meta, + "gateway": status.Status.Services.Gateway, + } { + if !service.Ready || service.AvailableReplicas != 1 { + t.Fatalf("%s status = %#v, want ready with one available replica", name, service) + } + } + ready := meta.FindStatusCondition(status.Status.Conditions, supabasev1alpha1.ConditionTypeReady) + if ready == nil || ready.Status != metav1.ConditionFalse { + t.Fatalf("Ready condition = %#v, want false while one core deployment converges", ready) + } + for _, conditionType := range []string{ + supabasev1alpha1.ConditionTypeRestReady, + } { + condition := meta.FindStatusCondition(status.Status.Conditions, conditionType) + if condition == nil || condition.Status != metav1.ConditionFalse || condition.Reason != "DeploymentPending" { + t.Fatalf("%s condition = %#v, want DeploymentPending false", conditionType, condition) + } + } +} + +func TestReconcileServicesReportsAllCoreDeploymentsReady(t *testing.T) { + t.Parallel() + + project, reconciler := newCoreReadinessFixture(t, false) + if err := reconciler.reconcileServices(context.Background(), project, &secretresources.ProjectCredentials{}); err != nil { + t.Fatal(err) + } + if !coreServicesReady(project) { + t.Fatalf("core services = %#v, want all ready", project.Status.Services) + } + for _, conditionType := range []string{ + supabasev1alpha1.ConditionTypeAuthReady, + supabasev1alpha1.ConditionTypeRestReady, + supabasev1alpha1.ConditionTypeStudioReady, + supabasev1alpha1.ConditionTypeMetaReady, + supabasev1alpha1.ConditionTypeGatewayReady, + } { + condition := meta.FindStatusCondition(project.Status.Conditions, conditionType) + if condition == nil || condition.Status != metav1.ConditionTrue { + t.Fatalf("%s condition = %#v, want true", conditionType, condition) + } + } +} + +func TestReconcileReturnsRequeueUntilCoreDeploymentsAreReady(t *testing.T) { + t.Parallel() + + project := &supabasev1alpha1.SupabaseProject{ + TypeMeta: metav1.TypeMeta{APIVersion: supabasev1alpha1.GroupVersion.String(), Kind: "SupabaseProject"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "aggregate", Namespace: "default", UID: "aggregate-project-uid", Generation: 7, + }, + Spec: supabasev1alpha1.SupabaseProjectSpec{ + ProjectCredentialsSecret: "aggregate-credentials", + Database: supabasev1alpha1.DatabaseSpec{ + Instances: 1, + Storage: cnpgv1.StorageConfiguration{Size: "1Gi"}, + }, + Auth: supabasev1alpha1.AuthSpec{SiteURL: "https://app.example.com", ExternalURL: "https://auth.example.com"}, + }, + } + credentials := validProjectCredentialsSecret(t, project, project.Spec.ProjectCredentialsSecret) + generated, names, err := secretresources.GenerateSecrets(project) + if err != nil { + t.Fatal(err) + } + project.Status.SecretNames = names + cluster := cnpg.BuildCluster(project, &project.Status.SecretNames) + cluster.Default() + cluster.Status.ReadyInstances = cluster.Spec.Instances + + apiCredentials, err := secretresources.ValidateProjectCredentials(credentials) + if err != nil { + t.Fatal(err) + } + scheme := newIdempotencyTestScheme(t) + coreDeployments := []*appsv1.Deployment{ + deployments.BuildAuthDeployment(project, &project.Status.SecretNames), + deployments.BuildRestDeployment(project, &project.Status.SecretNames), + deployments.BuildStudioDeployment(project, &project.Status.SecretNames), + deployments.BuildMetaDeployment(project, &project.Status.SecretNames), + deployments.BuildGatewayDeployment(project), + } + for i, deployment := range coreDeployments { + if i != 3 { + applyProjectCredentialsHash(deployment, apiCredentials.PodTemplateHash) + } + markCoreDeploymentStatus(t, deployment, 7, i != 1, 0) + if err := setTestControllerReference(project, deployment, scheme); err != nil { + t.Fatal(err) + } + } + + objects := []client.Object{project, credentials, cluster} + for _, secret := range generated { + objects = append(objects, secret) + } + for _, deployment := range coreDeployments { + objects = append(objects, deployment) + } + reconciler := &SupabaseProjectReconciler{ + Client: fake.NewClientBuilder().WithScheme(scheme).WithStatusSubresource(project).WithObjects(objects...).Build(), + Scheme: scheme, + } + result, err := reconciler.Reconcile(context.Background(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(project)}) + if err != nil { + t.Fatal(err) + } + if result.RequeueAfter != RequeueDelay { + t.Fatalf("requeueAfter = %s, want %s", result.RequeueAfter, RequeueDelay) + } + status := &supabasev1alpha1.SupabaseProject{} + if err := reconciler.Get(context.Background(), client.ObjectKeyFromObject(project), status); err != nil { + t.Fatal(err) + } + if status.Status.Phase != supabasev1alpha1.PhaseProvisioning || status.Status.ObservedGeneration != 0 { + t.Fatalf("status = %#v, want Provisioning with old observedGeneration", status.Status) + } + if ready := meta.FindStatusCondition(status.Status.Conditions, supabasev1alpha1.ConditionTypeReady); ready == nil || ready.Status != metav1.ConditionFalse { + t.Fatalf("Ready condition = %#v, want false", ready) + } + + rest := &appsv1.Deployment{} + if err := reconciler.Get(context.Background(), types.NamespacedName{Name: deployments.RestDeploymentName(project), Namespace: project.Namespace}, rest); err != nil { + t.Fatal(err) + } + markCoreDeploymentStatus(t, rest, rest.Generation, true, 0) + if err := reconciler.Status().Update(context.Background(), rest); err != nil { + t.Fatal(err) + } + result, err = reconciler.Reconcile(context.Background(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(project)}) + if err != nil { + t.Fatal(err) + } + if result != (ctrl.Result{}) { + t.Fatalf("ready reconcile result = %#v, want no requeue", result) + } + if err := reconciler.Get(context.Background(), client.ObjectKeyFromObject(project), status); err != nil { + t.Fatal(err) + } + if status.Status.Phase != supabasev1alpha1.PhaseRunning || status.Status.ObservedGeneration != project.Generation { + t.Fatalf("final status = %#v, want Running at generation %d", status.Status, project.Generation) + } + ready := meta.FindStatusCondition(status.Status.Conditions, supabasev1alpha1.ConditionTypeReady) + if ready == nil || ready.Status != metav1.ConditionTrue { + t.Fatalf("final Ready condition = %#v, want true", ready) + } +} + +func TestReconcileClearsAggregateReadinessOnCoreServiceFailure(t *testing.T) { + t.Parallel() + + project := &supabasev1alpha1.SupabaseProject{ + TypeMeta: metav1.TypeMeta{APIVersion: supabasev1alpha1.GroupVersion.String(), Kind: "SupabaseProject"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "failure", Namespace: "default", UID: "failure-project-uid", Generation: 7, + }, + Status: supabasev1alpha1.SupabaseProjectStatus{ + Phase: supabasev1alpha1.PhaseRunning, + ObservedGeneration: 7, + Services: supabasev1alpha1.ServicesStatus{ + Auth: supabasev1alpha1.ServiceStatus{Ready: true, AvailableReplicas: 1}, + Rest: supabasev1alpha1.ServiceStatus{Ready: true, AvailableReplicas: 1}, + Studio: supabasev1alpha1.ServiceStatus{Ready: true, AvailableReplicas: 1}, + Meta: supabasev1alpha1.ServiceStatus{Ready: true, AvailableReplicas: 1}, + Gateway: supabasev1alpha1.ServiceStatus{Ready: true, AvailableReplicas: 1}, + }, + }, + Spec: supabasev1alpha1.SupabaseProjectSpec{ + ProjectCredentialsSecret: "failure-credentials", + Database: supabasev1alpha1.DatabaseSpec{ + Instances: 1, + Storage: cnpgv1.StorageConfiguration{Size: "1Gi"}, + }, + Auth: supabasev1alpha1.AuthSpec{SiteURL: "https://app.example.com", ExternalURL: "https://auth.example.com"}, + }, + } + for _, conditionType := range []string{ + supabasev1alpha1.ConditionTypeReady, + supabasev1alpha1.ConditionTypeAuthReady, + supabasev1alpha1.ConditionTypeRestReady, + supabasev1alpha1.ConditionTypeStudioReady, + supabasev1alpha1.ConditionTypeMetaReady, + supabasev1alpha1.ConditionTypeGatewayReady, + } { + project.Status.Conditions = append(project.Status.Conditions, metav1.Condition{ + Type: conditionType, Status: metav1.ConditionTrue, ObservedGeneration: project.Generation, + Reason: "Ready", Message: "component is running", + }) + } + + credentials := validProjectCredentialsSecret(t, project, project.Spec.ProjectCredentialsSecret) + generated, names, err := secretresources.GenerateSecrets(project) + if err != nil { + t.Fatal(err) + } + project.Status.SecretNames = names + cluster := cnpg.BuildCluster(project, &project.Status.SecretNames) + cluster.Default() + cluster.Status.ReadyInstances = cluster.Spec.Instances + apiCredentials, err := secretresources.ValidateProjectCredentials(credentials) + if err != nil { + t.Fatal(err) + } + scheme := newIdempotencyTestScheme(t) + coreDeployments := []*appsv1.Deployment{ + deployments.BuildAuthDeployment(project, &project.Status.SecretNames), + deployments.BuildRestDeployment(project, &project.Status.SecretNames), + deployments.BuildStudioDeployment(project, &project.Status.SecretNames), + deployments.BuildMetaDeployment(project, &project.Status.SecretNames), + deployments.BuildGatewayDeployment(project), + } + for i, deployment := range coreDeployments { + if i != 3 { + applyProjectCredentialsHash(deployment, apiCredentials.PodTemplateHash) + } + markCoreDeploymentStatus(t, deployment, project.Generation, true, 0) + if err := setTestControllerReference(project, deployment, scheme); err != nil { + t.Fatal(err) + } + } + objects := []client.Object{project, credentials, cluster} + for _, secret := range generated { + objects = append(objects, secret) + } + for _, deployment := range coreDeployments { + objects = append(objects, deployment) + } + injected := errors.New("injected auth service create failure") + baseClient := fake.NewClientBuilder().WithScheme(scheme).WithStatusSubresource(project).WithObjects(objects...).Build() + failingClient := &failCoreServiceCreateClient{ + Client: baseClient, + ServiceName: services.BuildAuthService(project).Name, + Err: injected, + } + reconciler := &SupabaseProjectReconciler{Client: failingClient, Scheme: scheme} + + _, err = reconciler.Reconcile(context.Background(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(project)}) + if !errors.Is(err, injected) { + t.Fatalf("reconcile error = %v, want injected service error", err) + } + status := &supabasev1alpha1.SupabaseProject{} + if err := reconciler.Get(context.Background(), client.ObjectKeyFromObject(project), status); err != nil { + t.Fatal(err) + } + if status.Status.Phase != supabasev1alpha1.PhaseProvisioning { + t.Fatalf("phase = %q, want Provisioning", status.Status.Phase) + } + if status.Status.Services.Auth.Ready || status.Status.Services.Auth.AvailableReplicas != 0 { + t.Fatalf("auth status = %#v, want cleared after service failure", status.Status.Services.Auth) + } + ready := meta.FindStatusCondition(status.Status.Conditions, supabasev1alpha1.ConditionTypeReady) + if ready == nil || ready.Status != metav1.ConditionFalse || ready.Reason != "CoreServicesFailed" { + t.Fatalf("Ready condition = %#v, want CoreServicesFailed false", ready) + } +} + +type failCoreServiceCreateClient struct { + client.Client + ServiceName string + Err error +} + +func (c *failCoreServiceCreateClient) Create(ctx context.Context, object client.Object, options ...client.CreateOption) error { + service, ok := object.(*corev1.Service) + if ok && service.Name == c.ServiceName { + return c.Err + } + return c.Client.Create(ctx, object, options...) +} + +func newCoreReadinessFixture(t *testing.T, pendingRest bool) (*supabasev1alpha1.SupabaseProject, *SupabaseProjectReconciler) { + t.Helper() + project := &supabasev1alpha1.SupabaseProject{ + TypeMeta: metav1.TypeMeta{APIVersion: supabasev1alpha1.GroupVersion.String(), Kind: "SupabaseProject"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "readiness", Namespace: "default", UID: "readiness-project-uid", Generation: 7, + }, + Status: supabasev1alpha1.SupabaseProjectStatus{ + Phase: supabasev1alpha1.PhaseProvisioning, + ObservedGeneration: 4, + }, + Spec: supabasev1alpha1.SupabaseProjectSpec{ + Auth: supabasev1alpha1.AuthSpec{Replicas: 2}, + Rest: supabasev1alpha1.RestSpec{Replicas: 2}, + Studio: supabasev1alpha1.StudioSpec{Replicas: 1}, + Meta: supabasev1alpha1.MetaSpec{Replicas: 1}, + Gateway: supabasev1alpha1.GatewaySpec{Replicas: 1}, + }, + } + secretNames := &project.Status.SecretNames + desired := []*appsv1.Deployment{ + deployments.BuildAuthDeployment(project, secretNames), + deployments.BuildRestDeployment(project, secretNames), + deployments.BuildStudioDeployment(project, secretNames), + deployments.BuildMetaDeployment(project, secretNames), + deployments.BuildGatewayDeployment(project), + } + scheme := newPowerSyncTestScheme(t) + objects := []client.Object{project} + for i, deployment := range desired { + markCoreDeploymentStatus(t, deployment, 7, !pendingRest || i != 1, 1) + if err := setTestControllerReference(project, deployment, scheme); err != nil { + t.Fatal(err) + } + objects = append(objects, deployment) + } + reconciler := &SupabaseProjectReconciler{ + Client: fake.NewClientBuilder().WithScheme(scheme).WithStatusSubresource(project).WithObjects(objects...).Build(), + Scheme: scheme, + } + return project, reconciler +} + +func markCoreDeploymentStatus(t *testing.T, desired *appsv1.Deployment, generation int64, ready bool, available int32) { + t.Helper() + if desired.Spec.Replicas == nil { + desired.Spec.Replicas = ptr.To(int32(1)) + } + desired.Generation = generation + desired.Status.ObservedGeneration = generation + desired.Status.AvailableReplicas = available + if ready { + desired.Status.UpdatedReplicas = *desired.Spec.Replicas + desired.Status.ReadyReplicas = *desired.Spec.Replicas + desired.Status.AvailableReplicas = *desired.Spec.Replicas + desired.Status.UnavailableReplicas = 0 + return + } + desired.Status.UpdatedReplicas = *desired.Spec.Replicas + desired.Status.ReadyReplicas = *desired.Spec.Replicas + desired.Status.UnavailableReplicas = *desired.Spec.Replicas - available +}