diff --git a/pkg/controller/machinehealthcheck/machinehealthcheck_controller.go b/pkg/controller/machinehealthcheck/machinehealthcheck_controller.go index 74736266b9..911fd52c8c 100644 --- a/pkg/controller/machinehealthcheck/machinehealthcheck_controller.go +++ b/pkg/controller/machinehealthcheck/machinehealthcheck_controller.go @@ -28,10 +28,11 @@ import ( ) const ( - machineAnnotationKey = "machine.openshift.io/machine" - machineRebootAnnotationKey = "healthchecking.openshift.io/machine-remediation-reboot" - ownerControllerKind = "MachineSet" - remediationStrategyReboot = healthcheckingv1alpha1.RemediationStrategyType("reboot") + machineAnnotationKey = "machine.openshift.io/machine" + machineRebootAnnotationKey = "healthchecking.openshift.io/machine-remediation-reboot" + ownerControllerKind = "MachineSet" + remediationStrategyReboot = healthcheckingv1alpha1.RemediationStrategyType("reboot") + disableRemediationAnotationKey = "healthchecking.openshift.io/disabled" ) // Add creates a new MachineHealthCheck Controller and adds it to the Manager. The Manager will set fields on the Controller @@ -132,6 +133,12 @@ func (r *ReconcileMachineHealthCheck) Reconcile(request reconcile.Request) (reco } for _, hc := range allMachineHealthChecks.Items { + if hasMachineRemediationDisabled(machine) { + glog.Infof("Machine %s has a matching %s annotation set to , remediation is skipped.", + machineKey, disableRemediationAnotationKey) + continue + } + if hasMatchingLabels(&hc, machine) { glog.V(4).Infof("Machine %s has a matching machineHealthCheck: %s", machineKey, hc.Name) return remediate(r, hc.Spec.RemediationStrategy, machine) @@ -330,6 +337,11 @@ func hasMachineSetOwner(machine mapiv1.Machine) bool { return false } +func hasMachineRemediationDisabled(machine mapiv1.Machine) bool { + skipRemediation := machine.GetAnnotations()[disableRemediationAnotationKey] == "true" + return skipRemediation +} + func hasMatchingLabels(machineHealthCheck *healthcheckingv1alpha1.MachineHealthCheck, machine *mapiv1.Machine) bool { selector, err := metav1.LabelSelectorAsSelector(&machineHealthCheck.Spec.Selector) if err != nil { diff --git a/pkg/controller/machinehealthcheck/machinehealthcheck_controller_test.go b/pkg/controller/machinehealthcheck/machinehealthcheck_controller_test.go index 47d43600df..db461b34fa 100644 --- a/pkg/controller/machinehealthcheck/machinehealthcheck_controller_test.go +++ b/pkg/controller/machinehealthcheck/machinehealthcheck_controller_test.go @@ -332,6 +332,32 @@ func TestHasMachineSetOwner(t *testing.T) { } +func TestHasMachineRemediationDisabled(t *testing.T) { + machineWithRemediationDisabled := maotesting.NewMachine("machineWithRemediationDisabled", "node") + machineWithRemediationEnabled := maotesting.NewMachine("machineWithRemediationEnabled", "node") + machineWithRemediationDisabled.Annotations[disableRemediationAnotationKey] = "true" + + testsCases := []struct { + machine *mapiv1alpha1.Machine + expected bool + }{ + { + machine: machineWithRemediationDisabled, + expected: true, + }, + { + machine: machineWithRemediationEnabled, + expected: false, + }, + } + + for _, tc := range testsCases { + if got := hasMachineRemediationDisabled(*tc.machine); got != tc.expected { + t.Errorf("Test case: Machine %s. Expected: %t, got: %t", tc.machine.Name, tc.expected, got) + } + } +} + func TestUnhealthyForTooLong(t *testing.T) { nodeUnhealthyForTooLong := maotesting.NewNode("nodeUnhealthyForTooLong", false) nodeRecentlyUnhealthy := maotesting.NewNode("nodeRecentlyUnhealthy", false)