Bug 1850719: Measure container restarts over an interval instead of absolute

test/extended/operators/cluster.go

@@ -18,10 +18,11 @@ import (
 var _ = g.Describe("[Feature:Platform] Managed cluster should", func() {
 	defer g.GinkgoRecover()
 
-	g.It("have no crashlooping pods in core namespaces over two minutes", func() {
+	g.It("have no crashlooping pods in core namespaces over four minutes", func() {
 		c, err := e2e.LoadClientset()
 		o.Expect(err).NotTo(o.HaveOccurred())
 
+		restartingContainers := make(map[containerName]int)
 		podsWithProblems := make(map[string]*corev1.Pod)
 		var lastPending map[string]*corev1.Pod
 		wait.PollImmediate(5*time.Second, 2*time.Minute, func() (bool, error) {
@@ -63,7 +64,7 @@ var _ = g.Describe("[Feature:Platform] Managed cluster should", func() {
 				case hasCreateContainerError(pod):
 				case hasImagePullError(pod):
 				case isCrashLooping(pod):
-				case hasExcessiveRestarts(pod):
+				case hasExcessiveRestarts(pod, 2, restartingContainers):
 				case hasFailingContainer(pod):
 				default:
 					continue
@@ -172,10 +173,24 @@ func isCrashLooping(pod *corev1.Pod) bool {
 	return false
 }
 
-func hasExcessiveRestarts(pod *corev1.Pod) bool {
+type containerName struct {
+	namespace string
+	name      string
+	container string
+}
+
+func hasExcessiveRestarts(pod *corev1.Pod, excessiveCount int, counts map[containerName]int) bool {
 	for _, status := range append(append([]corev1.ContainerStatus{}, pod.Status.InitContainerStatuses...), pod.Status.ContainerStatuses...) {
-		if status.RestartCount > 5 {
-			pod.Status.Message = fmt.Sprintf("container %s has restarted more than 5 times", status.Name)
+		name := containerName{namespace: pod.Namespace, name: pod.Name, container: status.Name}
+		count, ok := counts[name]
+		if !ok {
+			counts[name] = int(status.RestartCount)
+			continue
+		}
+
+		current := int(status.RestartCount) - count
+		if current >= excessiveCount {
+			pod.Status.Message = fmt.Sprintf("container %s has restarted %d times (>= %d) within the allowed interval", status.Name, current, excessiveCount)
 			return true
 		}
 	}