From baa7f848249d4dfd2c489c46d62ec467ce87bc47 Mon Sep 17 00:00:00 2001 From: Enric Tobella Date: Wed, 6 Apr 2022 13:29:49 +0200 Subject: [PATCH] [IMP] queue_job: Recover gracefully when the CPU limit is reached --- queue_job/jobrunner/__init__.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/queue_job/jobrunner/__init__.py b/queue_job/jobrunner/__init__.py index 0b5f545885..688edb7149 100644 --- a/queue_job/jobrunner/__init__.py +++ b/queue_job/jobrunner/__init__.py @@ -54,6 +54,7 @@ def __init__(self, multi): super().__init__(multi) self.watchdog_timeout = None self.runner = QueueJobRunner.from_environ_or_config() + self._recover = False def sleep(self): pass @@ -64,10 +65,23 @@ def signal_handler(self, sig, frame): self.runner.stop() def process_work(self): + if self._recover: + _logger.info("WorkerJobRunner (%s) runner is reinitialized", self.pid) + self.runner = QueueJobRunner.from_environ_or_config() + self._recover = False _logger.debug("WorkerJobRunner (%s) starting up", self.pid) time.sleep(START_DELAY) self.runner.run() + def signal_time_expired_handler(self, n, stack): + _logger.info( + "Worker (%d) CPU time limit (%s) reached.Stop gracefully and recover", + self.pid, + config["limit_time_cpu"], + ) + self._recover = True + self.runner.stop() + runner_thread = None