diff --git a/internal/controller/genstatus.go b/internal/controller/genstatus.go index c6caa95b..d1da7a61 100644 --- a/internal/controller/genstatus.go +++ b/internal/controller/genstatus.go @@ -12,9 +12,24 @@ import ( "github.com/temporalio/temporal-worker-controller/internal/k8s" "github.com/temporalio/temporal-worker-controller/internal/temporal" temporalclient "go.temporal.io/sdk/client" + corev1 "k8s.io/api/core/v1" ctrl "sigs.k8s.io/controller-runtime" ) +// isGateWorkflowTerminalFailure reports whether a test/gate workflow status +// represents an ended-but-not-successful terminal state worth alerting on. +func isGateWorkflowTerminalFailure(status temporaliov1alpha1.WorkflowExecutionStatus) bool { + switch status { + case temporaliov1alpha1.WorkflowExecutionStatusFailed, + temporaliov1alpha1.WorkflowExecutionStatusCanceled, + temporaliov1alpha1.WorkflowExecutionStatusTerminated, + temporaliov1alpha1.WorkflowExecutionStatusTimedOut: + return true + default: + return false + } +} + func (r *WorkerDeploymentReconciler) generateStatus( ctx context.Context, l logr.Logger, @@ -42,6 +57,25 @@ func (r *WorkerDeploymentReconciler) generateStatus( // Continue without test workflow status } + // Emit a Warning event the first time a gate/test workflow is observed to have + // ended in a non-successful terminal state. Compare against the previous + // reconcile's recorded status (still on workerDeploy.Status at this point, since + // it hasn't been overwritten yet) so this doesn't re-fire on every loop. + prevStatusByWorkflowID := make(map[string]temporaliov1alpha1.WorkflowExecutionStatus, len(workerDeploy.Status.TargetVersion.TestWorkflows)) + for _, wf := range workerDeploy.Status.TargetVersion.TestWorkflows { + prevStatusByWorkflowID[wf.WorkflowID] = wf.Status + } + for _, wf := range testWorkflows { + if !isGateWorkflowTerminalFailure(wf.Status) { + continue + } + if prevStatusByWorkflowID[wf.WorkflowID] == wf.Status { + continue + } + r.Recorder.Eventf(workerDeploy, corev1.EventTypeWarning, ReasonGateWorkflowFailed, + "Gate/test workflow %s for version %s ended with status %s", wf.WorkflowID, targetBuildID, wf.Status) + } + // Add test workflow status to version info if it doesn't exist if versionInfo, exists := temporalState.Versions[targetBuildID]; exists { versionInfo.TestWorkflows = append(versionInfo.TestWorkflows, testWorkflows...) diff --git a/internal/controller/util.go b/internal/controller/util.go index e91fcbb5..0f6525ce 100644 --- a/internal/controller/util.go +++ b/internal/controller/util.go @@ -28,6 +28,7 @@ const ( ReasonVersionPromotionFailed = "VersionPromotionFailed" ReasonMetadataUpdateFailed = "MetadataUpdateFailed" ReasonManagerIdentityClaimFailed = "ManagerIdentityClaimFailed" + ReasonGateWorkflowFailed = "GateWorkflowFailed" ) const (