From fe3822648dbfdf95c8eafb8aa3ba044924809142 Mon Sep 17 00:00:00 2001 From: Niko Oliveira Date: Mon, 22 Jan 2024 11:41:44 -0800 Subject: [PATCH] Increase tenacity wait in read_pod_logs We're still seeing issues in kubernetes based system tests. See context and discussions here: https://github.com/apache/airflow/pull/35504 --- airflow/providers/cncf/kubernetes/utils/pod_manager.py | 2 +- tests/providers/cncf/kubernetes/utils/test_pod_manager.py | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/airflow/providers/cncf/kubernetes/utils/pod_manager.py b/airflow/providers/cncf/kubernetes/utils/pod_manager.py index 0e736daa6aa76..281c82884a4e1 100644 --- a/airflow/providers/cncf/kubernetes/utils/pod_manager.py +++ b/airflow/providers/cncf/kubernetes/utils/pod_manager.py @@ -634,7 +634,7 @@ def container_is_terminated(self, pod: V1Pod, container_name: str) -> bool: remote_pod = self.read_pod(pod) return container_is_terminated(pod=remote_pod, container_name=container_name) - @tenacity.retry(stop=tenacity.stop_after_attempt(3), wait=tenacity.wait_exponential(), reraise=True) + @tenacity.retry(stop=tenacity.stop_after_attempt(6), wait=tenacity.wait_exponential(max=15), reraise=True) def read_pod_logs( self, pod: V1Pod, diff --git a/tests/providers/cncf/kubernetes/utils/test_pod_manager.py b/tests/providers/cncf/kubernetes/utils/test_pod_manager.py index fc09d6bb02e78..0300cd063947e 100644 --- a/tests/providers/cncf/kubernetes/utils/test_pod_manager.py +++ b/tests/providers/cncf/kubernetes/utils/test_pod_manager.py @@ -100,6 +100,9 @@ def test_read_pod_logs_retries_fails(self): BaseHTTPError("Boom"), BaseHTTPError("Boom"), BaseHTTPError("Boom"), + BaseHTTPError("Boom"), + BaseHTTPError("Boom"), + BaseHTTPError("Boom"), ] with pytest.raises(BaseHTTPError): self.pod_manager.read_pod_logs(pod=mock.sentinel, container_name="base")