From 6f61cdbdda0cd7234e675d58d105ea78c354266e Mon Sep 17 00:00:00 2001 From: Daniel Imberman Date: Tue, 3 Mar 2020 16:01:27 -0500 Subject: [PATCH] [AIRFLOW-6040] ReadTimoutError should not raise exception --- airflow/executors/kubernetes_executor.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/airflow/executors/kubernetes_executor.py b/airflow/executors/kubernetes_executor.py index bba82ee08fea6..984b5991be5c5 100644 --- a/airflow/executors/kubernetes_executor.py +++ b/airflow/executors/kubernetes_executor.py @@ -21,6 +21,7 @@ import json import multiprocessing import re +import time from queue import Empty, Queue # pylint: disable=unused-import from typing import Any, Dict, Optional, Tuple, Union @@ -29,6 +30,7 @@ from kubernetes import client, watch from kubernetes.client import Configuration from kubernetes.client.rest import ApiException +from urllib3.exceptions import ReadTimeoutError from airflow import settings from airflow.configuration import conf @@ -289,6 +291,10 @@ def run(self) -> None: try: self.resource_version = self._run(kube_client, self.resource_version, self.worker_uuid, self.kube_config) + except ReadTimeoutError: + self.log.warning("There was a timeout error accessing the Kube API. " + "Retrying request.", exc_info=True) + time.sleep(1) except Exception: self.log.exception('Unknown error in KubernetesJobWatcher. Failing') raise