From 372c1cf189c0fe5c9d345a260d629313a980ff2d Mon Sep 17 00:00:00 2001 From: Leonardo Rochael Almeida Date: Mon, 2 Jan 2017 21:33:11 -0200 Subject: [PATCH 1/3] Allow configuring runner from config file In addition to the ODOO_CONNECTOR_CHANNELS environment variable, for backward compatibility, we can configure the runner channels through the ODOO_QUEUE_JOB_CHANNELS and in the odoo configuration file like this: [options-job_queue] channels = root:4 For backward compatibility, the section `[options-connector]` is also accepted. Moreover, it's now also possible to preload the module in the config file, like: [options] server_wide_modules = web,web_kanban,queue_job As such we can configure the job runner using only the Odoo config file. --- queue_job/README.rst | 29 +++++++++++--- queue_job/jobrunner/__init__.py | 7 +--- queue_job/jobrunner/runner.py | 70 ++++++++++++++++++++++++++++++--- 3 files changed, 90 insertions(+), 16 deletions(-) diff --git a/queue_job/README.rst b/queue_job/README.rst index fcf243a92a..bc33ac4e74 100644 --- a/queue_job/README.rst +++ b/queue_job/README.rst @@ -62,13 +62,32 @@ Be sure to have the ``requests`` library. Configuration ============= -* Set the following environment variables: +* Using environment variables and command line: - - ``ODOO_CONNECTOR_CHANNELS=root:4`` (or any other channels configuration) - - optional if ``xmlrpc_port`` is not set: ``ODOO_CONNECTOR_PORT=8069`` + * Adjust environment variables (optional): -* Start Odoo with ``--load=web,web_kanban,queue_job`` - and ``--workers`` greater than 1. [1]_ + - ``ODOO_QUEUE_JOB_CHANNELS=root:4`` + + - or any other channels configuration. The default is ``root:1`` + + - if ``xmlrpc_port`` is not set: ``ODOO_CONNECTOR_PORT=8069`` + + * Start Odoo with ``--load=web,web_kanban,queue_job`` + and ``--workers`` greater than 1. [1]_ + + +* Using the Odoo configuration file: + +.. code-block:: ini + + [options] + (...) + workers = 4 + server_wide_modules = web,web_kanban,queue_job + + (...) + [queue_job] + channels = root:4 * Confirm the runner is starting correctly by checking the odoo log file: diff --git a/queue_job/jobrunner/__init__.py b/queue_job/jobrunner/__init__.py index 484b91d6f1..b0b66d2a94 100644 --- a/queue_job/jobrunner/__init__.py +++ b/queue_job/jobrunner/__init__.py @@ -21,9 +21,6 @@ # Here we monkey patch the Odoo server to start the job runner thread # in the main server process (and not in forked workers). This is # very easy to deploy as we don't need another startup script. -# The drawback is that it is not possible to extend the Odoo -# server command line arguments, so we resort to environment variables -# to configure the runner (channels mostly). class QueueJobRunnerThread(Thread): @@ -31,10 +28,8 @@ class QueueJobRunnerThread(Thread): def __init__(self): Thread.__init__(self) self.daemon = True - # TODO: accept ODOO_CONNECTOR_CHANNELS or ODOO_QUEUE_JOB_CHANNELS port = os.environ.get('ODOO_CONNECTOR_PORT') or config['xmlrpc_port'] - channels = os.environ.get('ODOO_CONNECTOR_CHANNELS') - self.runner = QueueJobRunner(port or 8069, channels or 'root:1') + self.runner = QueueJobRunner(port or 8069) def run(self): # sleep a bit to let the workers start at ease diff --git a/queue_job/jobrunner/runner.py b/queue_job/jobrunner/runner.py index 544881f70f..cce8d9cba4 100644 --- a/queue_job/jobrunner/runner.py +++ b/queue_job/jobrunner/runner.py @@ -22,13 +22,52 @@ How to use it? -------------- -* Set the following environment variables: +* Optionally adjust your configuration through environment variables: - - ``ODOO_CONNECTOR_CHANNELS=root:4`` (or any other channels configuration) - - optional if ``xmlrpc_port`` is not set: ``ODOO_CONNECTOR_PORT=8069`` + - set ``ODOO_QUEUE_JOB_CHANNELS=root:4`` (or any other channels + configuration) if you don't want the default ``root:1``. + + - if ``xmlrpc-port`` is not set, you can set it for the jobrunner only with: + ``ODOO_CONNECTOR_PORT=8069``. + +* Alternatively, configure the channels through the Odoo configuration + file, like: + +.. code-block:: ini + + [queue_job] + channels = root:4 + +* Or, if using ``anybox.recipe.odoo``, add this to your buildout configuration: + +.. code-block:: ini + + [odoo] + recipe = anybox.recipe.odoo + (...) + queue_job.channels = root:4 * Start Odoo with ``--load=web,web_kanban,queue_job`` - and ``--workers`` greater than 1. [2]_ + and ``--workers`` greater than 1 [2]_, or set the ``server_wide_modules`` + option in The Odoo configuration file: + +.. code-block:: ini + + [options] + (...) + workers = 4 + server_wide_modules = web,web_kanban,queue_job + (...) + +* Or, if using ``anybox.recipe.odoo``: + +.. code-block:: ini + + [odoo] + recipe = anybox.recipe.odoo + (...) + options.workers = 4 + options.server_wide_modules = web,web_kanban,queue_job * Confirm the runner is starting correctly by checking the odoo log file: @@ -89,6 +128,7 @@ import requests import odoo +from odoo.tools import config from .channels import ChannelManager, PENDING, ENQUEUED, NOT_DONE @@ -98,6 +138,24 @@ _logger = logging.getLogger(__name__) +# Unfortunately, it is not possible to extend the Odoo +# server command line arguments, so we resort to environment variables +# to configure the runner (channels mostly). +# +# On the other hand, the odoo configuration file can be extended at will, +# so we check it in addition to the environment variables. + + +def _channels(): + return ( + os.environ.get('ODOO_QUEUE_JOB_CHANNELS') or + os.environ.get('ODOO_CONNECTOR_CHANNELS') or + config.misc.get("queue_job", {}).get("channels") or + config.misc.get("options-connector", {}).get("channels") or + "root:1" + ) + + def _datetime_to_epoch(dt): # important: this must return the same as postgresql # EXTRACT(EPOCH FROM TIMESTAMP dt) @@ -222,9 +280,11 @@ def set_job_enqueued(self, uuid): class QueueJobRunner(object): - def __init__(self, port=8069, channel_config_string='root:1'): + def __init__(self, port=8069, channel_config_string=None): self.port = port self.channel_manager = ChannelManager() + if channel_config_string is None: + channel_config_string = _channels() self.channel_manager.simple_configure(channel_config_string) self.db_by_name = {} self._stop = False From 25315f0203169a635cfd5910e8fbbe0952350dc7 Mon Sep 17 00:00:00 2001 From: Leonardo Rochael Almeida Date: Mon, 2 Jan 2017 21:33:43 -0200 Subject: [PATCH 2/3] Whitespace handling on channel configuration Ignore whitespace around values, and tolerate missing entries that would be caused by trailing commas or commented lines when the channel configuration is provided through the Odoo configuration file. Also, accept line breaks channel entry separators along with commas, which make the configuration file more readable. --- queue_job/jobrunner/channels.py | 44 ++++++++++++++++++++++++++++++--- 1 file changed, 40 insertions(+), 4 deletions(-) diff --git a/queue_job/jobrunner/channels.py b/queue_job/jobrunner/channels.py index 28f02e28b5..590dd4da51 100644 --- a/queue_job/jobrunner/channels.py +++ b/queue_job/jobrunner/channels.py @@ -518,6 +518,13 @@ def get_wakeup_time(self, wakeup_time=0): wakeup_time = child.get_wakeup_time(wakeup_time) return wakeup_time +def split_strip(s, sep, maxsplit=-1): + """Split string and strip each component. + + >>> ChannelManager.split_strip("foo: bar baz\\n: fred:", ":") + ['foo', 'bar baz', 'fred', ''] + """ + return [x.strip() for x in s.split(sep, maxsplit)] class ChannelManager(object): """ High level interface for channels @@ -618,7 +625,7 @@ def parse_simple_config(cls, config_string): """Parse a simple channels configuration string. The general form is as follow: - channel(.subchannel)*(:capacity(:key(=value)?)*)?,... + channel(.subchannel)*(:capacity(:key(=value)?)*)? [, ...] If capacity is absent, it defaults to 1. If a key is present without value, it gets True as value. @@ -640,11 +647,39 @@ def parse_simple_config(cls, config_string): [{'capacity': 1, 'name': 'root'}] >>> pp(ChannelManager.parse_simple_config('sub:2')) [{'capacity': 2, 'name': 'sub'}] + + It ignores whitespace around values, and drops empty entries which + would be generated by trailing commas, or commented lines on the Odoo + config file. + + >>> pp(ChannelManager.parse_simple_config(''' + ... root : 4, + ... , + ... foo bar:1: k=va lue, + ... ''')) + [{'capacity': 4, 'name': 'root'}, + {'capacity': 1, 'k': 'va lue', 'name': 'foo bar'}] + + It's also possible to replace commas with line breaks, which is more + readable if the channel configuration comes from the odoo config file. + + >>> pp(ChannelManager.parse_simple_config(''' + ... root : 4 + ... foo bar:1: k=va lue + ... baz + ... ''')) + [{'capacity': 4, 'name': 'root'}, + {'capacity': 1, 'k': 'va lue', 'name': 'foo bar'}, + {'capacity': 1, 'name': 'baz'}] """ res = [] - for channel_config_string in config_string.split(','): + config_string = config_string.replace("\n", ",") + for channel_config_string in split_strip(config_string, ','): + if not channel_config_string: + # ignore empty entries (commented lines, trailing commas) + continue config = {} - config_items = channel_config_string.split(':') + config_items = split_strip(channel_config_string, ':') name = config_items[0] if not name: raise ValueError('Invalid channel config %s: ' @@ -659,7 +694,7 @@ def parse_simple_config(cls, config_string): 'invalid capacity %s' % (config_string, capacity)) for config_item in config_items[2:]: - kv = config_item.split('=') + kv = split_strip(config_item, '=') if len(kv) == 1: k, v = kv[0], True elif len(kv) == 2: @@ -709,6 +744,7 @@ def get_channel_from_config(self, config): """ channel = self.get_channel_by_name(config['name'], autocreate=True) channel.configure(config) + _logger.info("Configured channel: %s", channel) return channel def get_channel_by_name(self, channel_name, autocreate=False): From 25bc846d62bb8552e937617dcfdb098c7492468b Mon Sep 17 00:00:00 2001 From: Leonardo Rochael Almeida Date: Tue, 3 Jan 2017 21:37:08 -0200 Subject: [PATCH 3/3] Remove all references to CONNECTOR Or replace them with references to queue_job. Also, rip off the barely useful modicum of backward compatibility. --- queue_job/README.rst | 2 +- queue_job/__manifest__.py | 2 +- queue_job/controllers/main.py | 6 ------ queue_job/jobrunner/__init__.py | 2 +- queue_job/jobrunner/runner.py | 4 +--- 5 files changed, 4 insertions(+), 12 deletions(-) diff --git a/queue_job/README.rst b/queue_job/README.rst index bc33ac4e74..241aabaed8 100644 --- a/queue_job/README.rst +++ b/queue_job/README.rst @@ -70,7 +70,7 @@ Configuration - or any other channels configuration. The default is ``root:1`` - - if ``xmlrpc_port`` is not set: ``ODOO_CONNECTOR_PORT=8069`` + - if ``xmlrpc_port`` is not set: ``ODOO_QUEUE_JOB_PORT=8069`` * Start Odoo with ``--load=web,web_kanban,queue_job`` and ``--workers`` greater than 1. [1]_ diff --git a/queue_job/__manifest__.py b/queue_job/__manifest__.py index df4fdae904..bcf0e0a6a1 100644 --- a/queue_job/__manifest__.py +++ b/queue_job/__manifest__.py @@ -5,7 +5,7 @@ {'name': 'Job Queue', 'version': '10.0.1.0.0', 'author': 'Camptocamp,ACSONE SA/NV,Odoo Community Association (OCA)', - 'website': 'http://odoo-connector.com', + 'website': 'https://github.com/OCA/queue/queue_job', 'license': 'AGPL-3', 'category': 'Generic Modules', 'depends': ['mail' diff --git a/queue_job/controllers/main.py b/queue_job/controllers/main.py index 9345dec008..bfaf3314eb 100644 --- a/queue_job/controllers/main.py +++ b/queue_job/controllers/main.py @@ -65,12 +65,6 @@ def _try_perform_job(self, env, job): http.request.env.cr.commit() _logger.debug('%s done', job) - @http.route('/connector/runjob', type='http', auth='none') - def old_runjob(self, db, job_uuid, **kw): - _logger.warning('/connector/runjob is deprecated, the new route is' - '/queue_job/runjob') - return self.runjob(db, job_uuid, **kw) - @http.route('/queue_job/runjob', type='http', auth='none') def runjob(self, db, job_uuid, **kw): http.request.session.db = db diff --git a/queue_job/jobrunner/__init__.py b/queue_job/jobrunner/__init__.py index b0b66d2a94..c48065c1d5 100644 --- a/queue_job/jobrunner/__init__.py +++ b/queue_job/jobrunner/__init__.py @@ -28,7 +28,7 @@ class QueueJobRunnerThread(Thread): def __init__(self): Thread.__init__(self) self.daemon = True - port = os.environ.get('ODOO_CONNECTOR_PORT') or config['xmlrpc_port'] + port = os.environ.get('ODOO_QUEUE_JOB_PORT') or config['xmlrpc_port'] self.runner = QueueJobRunner(port or 8069) def run(self): diff --git a/queue_job/jobrunner/runner.py b/queue_job/jobrunner/runner.py index cce8d9cba4..0270886003 100644 --- a/queue_job/jobrunner/runner.py +++ b/queue_job/jobrunner/runner.py @@ -28,7 +28,7 @@ configuration) if you don't want the default ``root:1``. - if ``xmlrpc-port`` is not set, you can set it for the jobrunner only with: - ``ODOO_CONNECTOR_PORT=8069``. + ``ODOO_QUEUE_JOB_PORT=8069``. * Alternatively, configure the channels through the Odoo configuration file, like: @@ -149,9 +149,7 @@ def _channels(): return ( os.environ.get('ODOO_QUEUE_JOB_CHANNELS') or - os.environ.get('ODOO_CONNECTOR_CHANNELS') or config.misc.get("queue_job", {}).get("channels") or - config.misc.get("options-connector", {}).get("channels") or "root:1" )