diff --git a/internal/controller/node_controller.go b/internal/controller/node_controller.go index c6692af9..989fadb0 100644 --- a/internal/controller/node_controller.go +++ b/internal/controller/node_controller.go @@ -159,6 +159,8 @@ func (r *RuleReadinessController) processNodeAgainstAllRules(ctx context.Context r.recordNodeFailure(rule, node.Name, "EvaluationError", err.Error()) errs = append(errs, err) metrics.Failures.WithLabelValues(rule.Name, "EvaluationError").Inc() + } else { + r.clearNodeFailure(rule, node.Name) } // Persist the rule status @@ -425,6 +427,17 @@ func (r *RuleReadinessController) recordNodeFailure( rule.Status.FailedNodes = failedNodes } +// clearNodeFailure removes any failure record for a specific node from the rule status. +func (r *RuleReadinessController) clearNodeFailure(rule *readinessv1alpha1.NodeReadinessRule, nodeName string) { + var failedNodes []readinessv1alpha1.NodeFailure + for _, failure := range rule.Status.FailedNodes { + if failure.NodeName != nodeName { + failedNodes = append(failedNodes, failure) + } + } + rule.Status.FailedNodes = failedNodes +} + // SyncNodeStateMetrics synchronizes the NodesByState Prometheus metrics with the current rule status. func (r *RuleReadinessController) SyncNodeStateMetrics(ctx context.Context, rule *readinessv1alpha1.NodeReadinessRule) { var ready, notReady, bootstrapping float64 diff --git a/internal/controller/node_controller_unit_test.go b/internal/controller/node_controller_unit_test.go new file mode 100644 index 00000000..11211511 --- /dev/null +++ b/internal/controller/node_controller_unit_test.go @@ -0,0 +1,62 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "testing" + + . "github.com/onsi/gomega" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + readinessv1alpha1 "sigs.k8s.io/node-readiness-controller/api/v1alpha1" +) + +func TestClearNodeFailure(t *testing.T) { + g := NewWithT(t) + + c := &RuleReadinessController{} + + rule := &readinessv1alpha1.NodeReadinessRule{ + Status: readinessv1alpha1.NodeReadinessRuleStatus{ + FailedNodes: []readinessv1alpha1.NodeFailure{ + {NodeName: "node-1", Reason: "EvaluationError", LastEvaluationTime: metav1.Now()}, + {NodeName: "node-2", Reason: "EvaluationError", LastEvaluationTime: metav1.Now()}, + }, + }, + } + + c.clearNodeFailure(rule, "node-1") + + g.Expect(rule.Status.FailedNodes).To(HaveLen(1)) + g.Expect(rule.Status.FailedNodes[0].NodeName).To(Equal("node-2")) +} + +func TestRecordThenClearNodeFailure(t *testing.T) { + g := NewWithT(t) + + c := &RuleReadinessController{} + + rule := &readinessv1alpha1.NodeReadinessRule{} + + // A node fails evaluation and is recorded. + c.recordNodeFailure(rule, "node-1", "EvaluationError", "boom") + g.Expect(rule.Status.FailedNodes).To(HaveLen(1)) + + // The node later recovers; the failure record must be cleared. + c.clearNodeFailure(rule, "node-1") + g.Expect(rule.Status.FailedNodes).To(BeEmpty()) +}