From 34269ee9b39e5fb4e602ea12e57bf89ce0fc6d9a Mon Sep 17 00:00:00 2001 From: "Moises Lopez - https://www.vauxoo.com/" Date: Mon, 14 Feb 2022 21:03:18 -0600 Subject: [PATCH] [FIX] queue_job: runner - filedescriptor out of range in select Use the most efficient Selector implementation available on the current platform Odoo supports only SelectSelector but it is a little obsolete python >= 3.4 supports a new high-level library Selectors: - https://docs.python.org/es/3/library/selectors.html It could to auto-choose the following ones: - SelectSelector - PollSelector - EpollSelector - DevpollSelector - KqueueSelector Using the DefaultSelector class the most efficient implementation available on the current platform will be use: - https://docs.python.org/3/library/selectors.html#selectors.DefaultSelector It helps to support better the resources of the system Using SelectSelector you are not able to run workers >=255 If you set `ulimit -n 10240` and run `odoo-bin --workers=255` the following error is raised: Traceback (most recent call last): File "odoo/service/server.py", line 926, in run self.sleep() File "odoo/service/server.py", line 852, in sleep sel.select(self.beat) File "python3.8/lib/python3.8/selectors.py", line 323, in select r, w, _ = self._select(self._readers, self._writers, [], timeout) ValueError: filedescriptor out of range in select() But using PollSelector it is not reproduced even using more workers Most of platform supports PollSelector but using DefaultSelector we can be sure that even too old system are supported too And using this High-level library will allow to use the future new improvements e.g. Epoll has better performance improvements More info about: - https://devarea.com/linux-io-multiplexing-select-vs-poll-vs-epoll - https://github.com/redis/redis-py/issues/486 - https://github.com/odoo/odoo/pull/84684 --- queue_job/jobrunner/runner.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/queue_job/jobrunner/runner.py b/queue_job/jobrunner/runner.py index fb6e60e5be..2314510731 100644 --- a/queue_job/jobrunner/runner.py +++ b/queue_job/jobrunner/runner.py @@ -142,7 +142,7 @@ import datetime import logging import os -import select +import selectors import threading import time from contextlib import closing, contextmanager @@ -162,6 +162,8 @@ _logger = logging.getLogger(__name__) +select = selectors.DefaultSelector + # Unfortunately, it is not possible to extend the Odoo # server command line arguments, so we resort to environment variables @@ -485,10 +487,16 @@ def wait_notification(self): # probably a bug _logger.debug("select() timeout: %.2f sec", timeout) if timeout > 0: - conns, _, _ = select.select(conns, [], [], timeout) if conns and not self._stop: - for conn in conns: - conn.poll() + with select() as sel: + for conn in conns: + sel.register(conn, selectors.EVENT_READ) + events = sel.select(timeout=timeout) + for key, _mask in events: + if key.fileobj == self._stop_pipe[0]: + # stop-pipe is not a conn so doesn't need poll() + continue + key.fileobj.poll() def stop(self): _logger.info("graceful stop requested")