feat(controller): surface poller and gate-workflow health as conditions and events #448
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,47 @@ | ||
| // Unless explicitly stated otherwise all files in this repository are licensed under the MIT License. | ||
| // | ||
| // This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2024 Datadog, Inc. | ||
|
|
||
| package controller | ||
|
|
||
| import ( | ||
| "sort" | ||
|
|
||
| temporaliov1alpha1 "github.com/temporalio/temporal-worker-controller/api/v1alpha1" | ||
| metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" | ||
| ) | ||
|
|
||
| // computePollerHealthCondition derives a poller-health status/reason from a version's | ||
| // poller health data, for use as part of the ConditionReady calculation (see | ||
| // syncConditions). It is a pure function so the decision logic can be unit tested | ||
| // without an envtest environment. | ||
| // | ||
| // - pollerHealth == nil: poller status was never checked for this version (e.g. not | ||
| // yet registered with Temporal) -> Unknown. | ||
| // - any task queue with a false value -> False, naming the affected queues. A known | ||
| // problem takes precedence over an unrelated unknown elsewhere. | ||
| // - no false values, but unknown == true (some task queues errored) -> Unknown. | ||
| // - all task queues true, unknown == false -> True. | ||
| func computePollerHealthCondition( | ||
| pollerHealth map[string]bool, | ||
| unknown bool, | ||
| ) (status metav1.ConditionStatus, reason string, affectedQueues []string) { | ||
| if pollerHealth == nil { | ||
| return metav1.ConditionUnknown, temporaliov1alpha1.ReasonPollerStatusUnknown, nil | ||
| } | ||
|
|
||
| for tq, healthy := range pollerHealth { | ||
| if !healthy { | ||
| affectedQueues = append(affectedQueues, tq) | ||
| } | ||
| } | ||
|
|
||
| if len(affectedQueues) > 0 { | ||
| sort.Strings(affectedQueues) | ||
| return metav1.ConditionFalse, temporaliov1alpha1.ReasonNoActivePollers, affectedQueues | ||
| } | ||
| if unknown { | ||
| return metav1.ConditionUnknown, temporaliov1alpha1.ReasonPollerStatusUnknown, nil | ||
| } | ||
| return metav1.ConditionTrue, temporaliov1alpha1.ReasonPollersHealthy, nil | ||
| } |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,78 @@ | ||
| // Unless explicitly stated otherwise all files in this repository are licensed under the MIT License. | ||
| // | ||
| // This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2024 Datadog, Inc. | ||
|
|
||
| package controller | ||
|
|
||
| import ( | ||
| "testing" | ||
|
|
||
| "github.com/stretchr/testify/assert" | ||
| temporaliov1alpha1 "github.com/temporalio/temporal-worker-controller/api/v1alpha1" | ||
| metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" | ||
| ) | ||
|
|
||
| func TestComputePollerHealthCondition(t *testing.T) { | ||
| tests := []struct { | ||
| name string | ||
| pollerHealth map[string]bool | ||
| unknown bool | ||
| wantStatus metav1.ConditionStatus | ||
| wantReason string | ||
| wantAffected []string | ||
| }{ | ||
| { | ||
| name: "nil map means never checked", | ||
| pollerHealth: nil, | ||
| unknown: false, | ||
| wantStatus: metav1.ConditionUnknown, | ||
| wantReason: temporaliov1alpha1.ReasonPollerStatusUnknown, | ||
| }, | ||
| { | ||
| name: "all queues healthy", | ||
| pollerHealth: map[string]bool{"tq-1": true, "tq-2": true}, | ||
| unknown: false, | ||
| wantStatus: metav1.ConditionTrue, | ||
| wantReason: temporaliov1alpha1.ReasonPollersHealthy, | ||
| }, | ||
| { | ||
| name: "one queue has no pollers", | ||
| pollerHealth: map[string]bool{"tq-1": true, "tq-2": false}, | ||
| unknown: false, | ||
| wantStatus: metav1.ConditionFalse, | ||
| wantReason: temporaliov1alpha1.ReasonNoActivePollers, | ||
| wantAffected: []string{"tq-2"}, | ||
| }, | ||
| { | ||
| name: "empty map with no fetch errors is healthy (no task queues to check)", | ||
| pollerHealth: map[string]bool{}, | ||
| unknown: false, | ||
| wantStatus: metav1.ConditionTrue, | ||
| wantReason: temporaliov1alpha1.ReasonPollersHealthy, | ||
| }, | ||
| { | ||
| name: "fetch error with no confirmed-unhealthy queue is unknown, not unhealthy", | ||
| pollerHealth: map[string]bool{"tq-1": true}, | ||
| unknown: true, | ||
| wantStatus: metav1.ConditionUnknown, | ||
| wantReason: temporaliov1alpha1.ReasonPollerStatusUnknown, | ||
| }, | ||
| { | ||
| name: "a confirmed no-poller queue wins over an unrelated fetch error", | ||
| pollerHealth: map[string]bool{"tq-1": false}, | ||
| unknown: true, | ||
| wantStatus: metav1.ConditionFalse, | ||
| wantReason: temporaliov1alpha1.ReasonNoActivePollers, | ||
| wantAffected: []string{"tq-1"}, | ||
| }, | ||
| } | ||
|
|
||
| for _, tt := range tests { | ||
| t.Run(tt.name, func(t *testing.T) { | ||
| status, reason, affected := computePollerHealthCondition(tt.pollerHealth, tt.unknown) | ||
| assert.Equal(t, tt.wantStatus, status) | ||
| assert.Equal(t, tt.wantReason, reason) | ||
| assert.Equal(t, tt.wantAffected, affected) | ||
| }) | ||
| } | ||
| } |
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Please change this to
ReasonActivePollers. The actual health of the poller isn't known :) Just that it's actively polling the server.Also, I have changed my mind about having this reason affect
ConditionReady. Instead, let's make this reason only impactConditionProgressing=Falseso that it matches the existingReasonWaitingForPollersthat is set onConditionProgressing=True.