diff --git a/queue_job/README.rst b/queue_job/README.rst index fcf243a92a..241aabaed8 100644 --- a/queue_job/README.rst +++ b/queue_job/README.rst @@ -62,13 +62,32 @@ Be sure to have the ``requests`` library. Configuration ============= -* Set the following environment variables: +* Using environment variables and command line: - - ``ODOO_CONNECTOR_CHANNELS=root:4`` (or any other channels configuration) - - optional if ``xmlrpc_port`` is not set: ``ODOO_CONNECTOR_PORT=8069`` + * Adjust environment variables (optional): -* Start Odoo with ``--load=web,web_kanban,queue_job`` - and ``--workers`` greater than 1. [1]_ + - ``ODOO_QUEUE_JOB_CHANNELS=root:4`` + + - or any other channels configuration. The default is ``root:1`` + + - if ``xmlrpc_port`` is not set: ``ODOO_QUEUE_JOB_PORT=8069`` + + * Start Odoo with ``--load=web,web_kanban,queue_job`` + and ``--workers`` greater than 1. [1]_ + + +* Using the Odoo configuration file: + +.. code-block:: ini + + [options] + (...) + workers = 4 + server_wide_modules = web,web_kanban,queue_job + + (...) + [queue_job] + channels = root:4 * Confirm the runner is starting correctly by checking the odoo log file: diff --git a/queue_job/__manifest__.py b/queue_job/__manifest__.py index df4fdae904..bcf0e0a6a1 100644 --- a/queue_job/__manifest__.py +++ b/queue_job/__manifest__.py @@ -5,7 +5,7 @@ {'name': 'Job Queue', 'version': '10.0.1.0.0', 'author': 'Camptocamp,ACSONE SA/NV,Odoo Community Association (OCA)', - 'website': 'http://odoo-connector.com', + 'website': 'https://github.com/OCA/queue/queue_job', 'license': 'AGPL-3', 'category': 'Generic Modules', 'depends': ['mail' diff --git a/queue_job/controllers/main.py b/queue_job/controllers/main.py index 9345dec008..bfaf3314eb 100644 --- a/queue_job/controllers/main.py +++ b/queue_job/controllers/main.py @@ -65,12 +65,6 @@ def _try_perform_job(self, env, job): http.request.env.cr.commit() _logger.debug('%s done', job) - @http.route('/connector/runjob', type='http', auth='none') - def old_runjob(self, db, job_uuid, **kw): - _logger.warning('/connector/runjob is deprecated, the new route is' - '/queue_job/runjob') - return self.runjob(db, job_uuid, **kw) - @http.route('/queue_job/runjob', type='http', auth='none') def runjob(self, db, job_uuid, **kw): http.request.session.db = db diff --git a/queue_job/jobrunner/__init__.py b/queue_job/jobrunner/__init__.py index 484b91d6f1..c48065c1d5 100644 --- a/queue_job/jobrunner/__init__.py +++ b/queue_job/jobrunner/__init__.py @@ -21,9 +21,6 @@ # Here we monkey patch the Odoo server to start the job runner thread # in the main server process (and not in forked workers). This is # very easy to deploy as we don't need another startup script. -# The drawback is that it is not possible to extend the Odoo -# server command line arguments, so we resort to environment variables -# to configure the runner (channels mostly). class QueueJobRunnerThread(Thread): @@ -31,10 +28,8 @@ class QueueJobRunnerThread(Thread): def __init__(self): Thread.__init__(self) self.daemon = True - # TODO: accept ODOO_CONNECTOR_CHANNELS or ODOO_QUEUE_JOB_CHANNELS - port = os.environ.get('ODOO_CONNECTOR_PORT') or config['xmlrpc_port'] - channels = os.environ.get('ODOO_CONNECTOR_CHANNELS') - self.runner = QueueJobRunner(port or 8069, channels or 'root:1') + port = os.environ.get('ODOO_QUEUE_JOB_PORT') or config['xmlrpc_port'] + self.runner = QueueJobRunner(port or 8069) def run(self): # sleep a bit to let the workers start at ease diff --git a/queue_job/jobrunner/channels.py b/queue_job/jobrunner/channels.py index 28f02e28b5..590dd4da51 100644 --- a/queue_job/jobrunner/channels.py +++ b/queue_job/jobrunner/channels.py @@ -518,6 +518,13 @@ def get_wakeup_time(self, wakeup_time=0): wakeup_time = child.get_wakeup_time(wakeup_time) return wakeup_time +def split_strip(s, sep, maxsplit=-1): + """Split string and strip each component. + + >>> ChannelManager.split_strip("foo: bar baz\\n: fred:", ":") + ['foo', 'bar baz', 'fred', ''] + """ + return [x.strip() for x in s.split(sep, maxsplit)] class ChannelManager(object): """ High level interface for channels @@ -618,7 +625,7 @@ def parse_simple_config(cls, config_string): """Parse a simple channels configuration string. The general form is as follow: - channel(.subchannel)*(:capacity(:key(=value)?)*)?,... + channel(.subchannel)*(:capacity(:key(=value)?)*)? [, ...] If capacity is absent, it defaults to 1. If a key is present without value, it gets True as value. @@ -640,11 +647,39 @@ def parse_simple_config(cls, config_string): [{'capacity': 1, 'name': 'root'}] >>> pp(ChannelManager.parse_simple_config('sub:2')) [{'capacity': 2, 'name': 'sub'}] + + It ignores whitespace around values, and drops empty entries which + would be generated by trailing commas, or commented lines on the Odoo + config file. + + >>> pp(ChannelManager.parse_simple_config(''' + ... root : 4, + ... , + ... foo bar:1: k=va lue, + ... ''')) + [{'capacity': 4, 'name': 'root'}, + {'capacity': 1, 'k': 'va lue', 'name': 'foo bar'}] + + It's also possible to replace commas with line breaks, which is more + readable if the channel configuration comes from the odoo config file. + + >>> pp(ChannelManager.parse_simple_config(''' + ... root : 4 + ... foo bar:1: k=va lue + ... baz + ... ''')) + [{'capacity': 4, 'name': 'root'}, + {'capacity': 1, 'k': 'va lue', 'name': 'foo bar'}, + {'capacity': 1, 'name': 'baz'}] """ res = [] - for channel_config_string in config_string.split(','): + config_string = config_string.replace("\n", ",") + for channel_config_string in split_strip(config_string, ','): + if not channel_config_string: + # ignore empty entries (commented lines, trailing commas) + continue config = {} - config_items = channel_config_string.split(':') + config_items = split_strip(channel_config_string, ':') name = config_items[0] if not name: raise ValueError('Invalid channel config %s: ' @@ -659,7 +694,7 @@ def parse_simple_config(cls, config_string): 'invalid capacity %s' % (config_string, capacity)) for config_item in config_items[2:]: - kv = config_item.split('=') + kv = split_strip(config_item, '=') if len(kv) == 1: k, v = kv[0], True elif len(kv) == 2: @@ -709,6 +744,7 @@ def get_channel_from_config(self, config): """ channel = self.get_channel_by_name(config['name'], autocreate=True) channel.configure(config) + _logger.info("Configured channel: %s", channel) return channel def get_channel_by_name(self, channel_name, autocreate=False): diff --git a/queue_job/jobrunner/runner.py b/queue_job/jobrunner/runner.py index 544881f70f..0270886003 100644 --- a/queue_job/jobrunner/runner.py +++ b/queue_job/jobrunner/runner.py @@ -22,13 +22,52 @@ How to use it? -------------- -* Set the following environment variables: +* Optionally adjust your configuration through environment variables: - - ``ODOO_CONNECTOR_CHANNELS=root:4`` (or any other channels configuration) - - optional if ``xmlrpc_port`` is not set: ``ODOO_CONNECTOR_PORT=8069`` + - set ``ODOO_QUEUE_JOB_CHANNELS=root:4`` (or any other channels + configuration) if you don't want the default ``root:1``. + + - if ``xmlrpc-port`` is not set, you can set it for the jobrunner only with: + ``ODOO_QUEUE_JOB_PORT=8069``. + +* Alternatively, configure the channels through the Odoo configuration + file, like: + +.. code-block:: ini + + [queue_job] + channels = root:4 + +* Or, if using ``anybox.recipe.odoo``, add this to your buildout configuration: + +.. code-block:: ini + + [odoo] + recipe = anybox.recipe.odoo + (...) + queue_job.channels = root:4 * Start Odoo with ``--load=web,web_kanban,queue_job`` - and ``--workers`` greater than 1. [2]_ + and ``--workers`` greater than 1 [2]_, or set the ``server_wide_modules`` + option in The Odoo configuration file: + +.. code-block:: ini + + [options] + (...) + workers = 4 + server_wide_modules = web,web_kanban,queue_job + (...) + +* Or, if using ``anybox.recipe.odoo``: + +.. code-block:: ini + + [odoo] + recipe = anybox.recipe.odoo + (...) + options.workers = 4 + options.server_wide_modules = web,web_kanban,queue_job * Confirm the runner is starting correctly by checking the odoo log file: @@ -89,6 +128,7 @@ import requests import odoo +from odoo.tools import config from .channels import ChannelManager, PENDING, ENQUEUED, NOT_DONE @@ -98,6 +138,22 @@ _logger = logging.getLogger(__name__) +# Unfortunately, it is not possible to extend the Odoo +# server command line arguments, so we resort to environment variables +# to configure the runner (channels mostly). +# +# On the other hand, the odoo configuration file can be extended at will, +# so we check it in addition to the environment variables. + + +def _channels(): + return ( + os.environ.get('ODOO_QUEUE_JOB_CHANNELS') or + config.misc.get("queue_job", {}).get("channels") or + "root:1" + ) + + def _datetime_to_epoch(dt): # important: this must return the same as postgresql # EXTRACT(EPOCH FROM TIMESTAMP dt) @@ -222,9 +278,11 @@ def set_job_enqueued(self, uuid): class QueueJobRunner(object): - def __init__(self, port=8069, channel_config_string='root:1'): + def __init__(self, port=8069, channel_config_string=None): self.port = port self.channel_manager = ChannelManager() + if channel_config_string is None: + channel_config_string = _channels() self.channel_manager.simple_configure(channel_config_string) self.db_by_name = {} self._stop = False