Forward port config file setup from connector

queue_job/README.rst

@@ -62,13 +62,32 @@ Be sure to have the ``requests`` library.
 Configuration
 =============
 
-* Set the following environment variables:
+* Using environment variables and command line:
 
-  - ``ODOO_CONNECTOR_CHANNELS=root:4`` (or any other channels configuration)
-  - optional if ``xmlrpc_port`` is not set: ``ODOO_CONNECTOR_PORT=8069``
+ * Adjust environment variables (optional):
 
-* Start Odoo with ``--load=web,web_kanban,queue_job``
-  and ``--workers`` greater than 1. [1]_
+  - ``ODOO_QUEUE_JOB_CHANNELS=root:4``
+
+   - or any other channels configuration. The default is ``root:1``
+
+  - if ``xmlrpc_port`` is not set: ``ODOO_QUEUE_JOB_PORT=8069``
+
+ * Start Odoo with ``--load=web,web_kanban,queue_job``
+   and ``--workers`` greater than 1. [1]_
+
+
+* Using the Odoo configuration file:
+
+.. code-block:: ini
+
+  [options]
+  (...)
+  workers = 4
+  server_wide_modules = web,web_kanban,queue_job
+
+  (...)
+  [queue_job]
+  channels = root:4
 
 * Confirm the runner is starting correctly by checking the odoo log file:
 

queue_job/__manifest__.py

@@ -5,7 +5,7 @@
 {'name': 'Job Queue',
  'version': '10.0.1.0.0',
  'author': 'Camptocamp,ACSONE SA/NV,Odoo Community Association (OCA)',
- 'website': 'http://odoo-connector.com',
+ 'website': 'https://github.com/OCA/queue/queue_job',
  'license': 'AGPL-3',
  'category': 'Generic Modules',
  'depends': ['mail'

queue_job/controllers/main.py

@@ -65,12 +65,6 @@ def _try_perform_job(self, env, job):
         http.request.env.cr.commit()
         _logger.debug('%s done', job)
 
-    @http.route('/connector/runjob', type='http', auth='none')
-    def old_runjob(self, db, job_uuid, **kw):
-        _logger.warning('/connector/runjob is deprecated, the new route is'
-                        '/queue_job/runjob')
-        return self.runjob(db, job_uuid, **kw)
-
     @http.route('/queue_job/runjob', type='http', auth='none')
     def runjob(self, db, job_uuid, **kw):
         http.request.session.db = db

queue_job/jobrunner/__init__.py

@@ -21,20 +21,15 @@
 # Here we monkey patch the Odoo server to start the job runner thread
 # in the main server process (and not in forked workers). This is
 # very easy to deploy as we don't need another startup script.
-# The drawback is that it is not possible to extend the Odoo
-# server command line arguments, so we resort to environment variables
-# to configure the runner (channels mostly).
 
 
 class QueueJobRunnerThread(Thread):
 
     def __init__(self):
         Thread.__init__(self)
         self.daemon = True
-        # TODO: accept ODOO_CONNECTOR_CHANNELS or ODOO_QUEUE_JOB_CHANNELS
-        port = os.environ.get('ODOO_CONNECTOR_PORT') or config['xmlrpc_port']
-        channels = os.environ.get('ODOO_CONNECTOR_CHANNELS')
-        self.runner = QueueJobRunner(port or 8069, channels or 'root:1')
+        port = os.environ.get('ODOO_QUEUE_JOB_PORT') or config['xmlrpc_port']
+        self.runner = QueueJobRunner(port or 8069)
 
     def run(self):
         # sleep a bit to let the workers start at ease

queue_job/jobrunner/channels.py

@@ -518,6 +518,13 @@ def get_wakeup_time(self, wakeup_time=0):
             wakeup_time = child.get_wakeup_time(wakeup_time)
         return wakeup_time
 
+def split_strip(s, sep, maxsplit=-1):
+    """Split string and strip each component.
+
+    >>> ChannelManager.split_strip("foo: bar baz\\n: fred:", ":")
+    ['foo', 'bar baz', 'fred', '']
+    """
+    return [x.strip() for x in s.split(sep, maxsplit)]
 
 class ChannelManager(object):
     """ High level interface for channels
@@ -618,7 +625,7 @@ def parse_simple_config(cls, config_string):
         """Parse a simple channels configuration string.
 
         The general form is as follow:
-        channel(.subchannel)*(:capacity(:key(=value)?)*)?,...
+        channel(.subchannel)*(:capacity(:key(=value)?)*)? [, ...]
 
         If capacity is absent, it defaults to 1.
         If a key is present without value, it gets True as value.
@@ -640,11 +647,39 @@ def parse_simple_config(cls, config_string):
         [{'capacity': 1, 'name': 'root'}]
         >>> pp(ChannelManager.parse_simple_config('sub:2'))
         [{'capacity': 2, 'name': 'sub'}]
+
+        It ignores whitespace around values, and drops empty entries which
+        would be generated by trailing commas, or commented lines on the Odoo
+        config file.
+
+        >>> pp(ChannelManager.parse_simple_config('''
+        ...     root : 4,
+        ...     ,
+        ...     foo bar:1: k=va lue,
+        ... '''))
+        [{'capacity': 4, 'name': 'root'},
+         {'capacity': 1, 'k': 'va lue', 'name': 'foo bar'}]
+
+        It's also possible to replace commas with line breaks, which is more
+        readable if the channel configuration comes from the odoo config file.
+
+        >>> pp(ChannelManager.parse_simple_config('''
+        ...     root : 4
+        ...     foo bar:1: k=va lue
+        ...     baz
+        ... '''))
+        [{'capacity': 4, 'name': 'root'},
+         {'capacity': 1, 'k': 'va lue', 'name': 'foo bar'},
+         {'capacity': 1, 'name': 'baz'}]
         """
         res = []
-        for channel_config_string in config_string.split(','):
+        config_string = config_string.replace("\n", ",")
+        for channel_config_string in split_strip(config_string, ','):
+            if not channel_config_string:
+                # ignore empty entries (commented lines, trailing commas)
+                continue
             config = {}
-            config_items = channel_config_string.split(':')
+            config_items = split_strip(channel_config_string, ':')
             name = config_items[0]
             if not name:
                 raise ValueError('Invalid channel config %s: '
@@ -659,7 +694,7 @@ def parse_simple_config(cls, config_string):
                                      'invalid capacity %s' %
                                      (config_string, capacity))
                 for config_item in config_items[2:]:
-                    kv = config_item.split('=')
+                    kv = split_strip(config_item, '=')
                     if len(kv) == 1:
                         k, v = kv[0], True
                     elif len(kv) == 2:
@@ -709,6 +744,7 @@ def get_channel_from_config(self, config):
         """
         channel = self.get_channel_by_name(config['name'], autocreate=True)
         channel.configure(config)
+        _logger.info("Configured channel: %s", channel)
         return channel
 
     def get_channel_by_name(self, channel_name, autocreate=False):

queue_job/jobrunner/runner.py

@@ -22,13 +22,52 @@
 How to use it?
 --------------
 
-* Set the following environment variables:
+* Optionally adjust your configuration through environment variables:
 
-  - ``ODOO_CONNECTOR_CHANNELS=root:4`` (or any other channels configuration)
-  - optional if ``xmlrpc_port`` is not set: ``ODOO_CONNECTOR_PORT=8069``
+  - set ``ODOO_QUEUE_JOB_CHANNELS=root:4`` (or any other channels
+    configuration) if you don't want the default ``root:1``.
+
+  - if ``xmlrpc-port`` is not set, you can set it for the jobrunner only with:
+    ``ODOO_QUEUE_JOB_PORT=8069``.
+
+* Alternatively, configure the channels through the Odoo configuration
+  file, like:
+
+.. code-block:: ini
+
+  [queue_job]
+  channels = root:4
+
+* Or, if using ``anybox.recipe.odoo``, add this to your buildout configuration:
+
+.. code-block:: ini
+
+  [odoo]
+  recipe = anybox.recipe.odoo
+  (...)
+  queue_job.channels = root:4
 
 * Start Odoo with ``--load=web,web_kanban,queue_job``
-  and ``--workers`` greater than 1. [2]_
+  and ``--workers`` greater than 1 [2]_, or set the ``server_wide_modules``
+  option in The Odoo configuration file:
+
+.. code-block:: ini
+
+  [options]
+  (...)
+  workers = 4
+  server_wide_modules = web,web_kanban,queue_job
+  (...)
+
+* Or, if using ``anybox.recipe.odoo``:
+
+.. code-block:: ini
+
+  [odoo]
+  recipe = anybox.recipe.odoo
+  (...)
+  options.workers = 4
+  options.server_wide_modules = web,web_kanban,queue_job
 
 * Confirm the runner is starting correctly by checking the odoo log file:
 
@@ -89,6 +128,7 @@
 import requests
 
 import odoo
+from odoo.tools import config
 
 from .channels import ChannelManager, PENDING, ENQUEUED, NOT_DONE
 
@@ -98,6 +138,22 @@
 _logger = logging.getLogger(__name__)
 
 
+# Unfortunately, it is not possible to extend the Odoo
+# server command line arguments, so we resort to environment variables
+# to configure the runner (channels mostly).
+#
+# On the other hand, the odoo configuration file can be extended at will,
+# so we check it in addition to the environment variables.
+
+
+def _channels():
+    return (
+        os.environ.get('ODOO_QUEUE_JOB_CHANNELS') or
+        config.misc.get("queue_job", {}).get("channels") or
+        "root:1"
+    )
+
+
 def _datetime_to_epoch(dt):
     # important: this must return the same as postgresql
     # EXTRACT(EPOCH FROM TIMESTAMP dt)
@@ -222,9 +278,11 @@ def set_job_enqueued(self, uuid):
 
 class QueueJobRunner(object):
 
-    def __init__(self, port=8069, channel_config_string='root:1'):
+    def __init__(self, port=8069, channel_config_string=None):
         self.port = port
         self.channel_manager = ChannelManager()
+        if channel_config_string is None:
+            channel_config_string = _channels()
         self.channel_manager.simple_configure(channel_config_string)
         self.db_by_name = {}
         self._stop = False