[13.0] Improve queue_job result filtering and data display

queue_job/__manifest__.py

@@ -3,7 +3,7 @@
 
 {
     "name": "Job Queue",
-    "version": "13.0.3.7.1",
+    "version": "13.0.3.8.0",
     "author": "Camptocamp,ACSONE SA/NV,Odoo Community Association (OCA)",
     "website": "https://github.com/OCA/queue/queue_job",
     "license": "LGPL-3",

queue_job/controllers/main.py

@@ -100,21 +100,36 @@ def retry_postpone(job, message, seconds=None):
             # retries are exhausted
             env.cr.rollback()
 
-        except (FailedJobError, Exception):
+        except (FailedJobError, Exception) as orig_exception:
             buff = StringIO()
             traceback.print_exc(file=buff)
-            _logger.error(buff.getvalue())
+            traceback_txt = buff.getvalue()
+            _logger.error(traceback_txt)
             job.env.clear()
             with odoo.api.Environment.manage():
                 with odoo.registry(job.env.cr.dbname).cursor() as new_cr:
                     job.env = job.env(cr=new_cr)
-                    job.set_failed(exc_info=buff.getvalue())
+                    vals = self._get_failure_values(job, traceback_txt, orig_exception)
+                    job.set_failed(**vals)
                     job.store()
                     new_cr.commit()
+                    buff.close()
             raise
 
         return ""
 
+    def _get_failure_values(self, job, traceback_txt, orig_exception):
+        """Collect relevant data from exception."""
+        exception_name = orig_exception.__class__.__name__
+        if hasattr(orig_exception, "__module__"):
+            exception_name = orig_exception.__module__ + "." + exception_name
+        exc_message = getattr(orig_exception, "name", str(orig_exception))
+        return {
+            "exc_info": traceback_txt,
+            "exc_name": exception_name,
+            "exc_message": exc_message,
+        }
+
     @http.route("/queue_job/create_test_job", type="http", auth="user")
     def create_test_job(
         self, priority=None, max_retries=None, channel=None, description="Test job"

queue_job/job.py

@@ -214,6 +214,14 @@ class Job(object):
 
         A description of the result (for humans).
 
+    .. attribute:: exc_name
+
+        Exception error name when the job failed.
+
+    .. attribute:: exc_message
+
+        Exception error message when the job failed.
+
     .. attribute:: exc_info
 
         Exception information (traceback) when the job failed.
@@ -478,6 +486,8 @@ def __init__(
         self.date_done = None
 
         self.result = None
+        self.exc_name = None
+        self.exc_message = None
         self.exc_info = None
 
         if "company_id" in env.context:
@@ -518,11 +528,29 @@ def perform(self):
 
     def store(self):
         """Store the Job"""
+        job_model = self.env["queue.job"]
+        # The sentinel is used to prevent edition sensitive fields (such as
+        # method_name) from RPC methods.
+        edit_sentinel = job_model.EDIT_SENTINEL
+
+        db_record = self.db_record()
+        if db_record:
+            db_record.with_context(_job_edit_sentinel=edit_sentinel).write(
+                self._store_values()
+            )
+        else:
+            job_model.with_context(_job_edit_sentinel=edit_sentinel).sudo().create(
+                self._store_values(create=True)
+            )
+
+    def _store_values(self, create=False):
         vals = {
             "state": self.state,
             "priority": self.priority,
             "retry": self.retry,
             "max_retries": self.max_retries,
+            "exc_name": self.exc_name,
+            "exc_message": self.exc_message,
             "exc_info": self.exc_info,
             "company_id": self.company_id,
             "result": str(self.result) if self.result else False,
@@ -548,15 +576,7 @@ def store(self):
         if self.identity_key:
             vals["identity_key"] = self.identity_key
 
-        job_model = self.env["queue.job"]
-        # The sentinel is used to prevent edition sensitive fields (such as
-        # method_name) from RPC methods.
-        edit_sentinel = job_model.EDIT_SENTINEL
-
-        db_record = self.db_record()
-        if db_record:
-            db_record.with_context(_job_edit_sentinel=edit_sentinel).write(vals)
-        else:
+        if create:
             vals.update(
                 {
                     "user_id": self.env.uid,
@@ -576,7 +596,24 @@ def store(self):
                     "kwargs": self.kwargs,
                 }
             )
-            job_model.with_context(_job_edit_sentinel=edit_sentinel).sudo().create(vals)
+
+        vals_from_model = self._store_values_from_model()
+        # Sanitize values: make sure you cannot screw core values
+        vals_from_model = {k: v for k, v in vals_from_model.items() if k not in vals}
+        vals.update(vals_from_model)
+        return vals
+
+    def _store_values_from_model(self):
+        vals = {}
+        value_handlers_candidates = (
+            "_job_store_values_for_" + self.method_name,
+            "_job_store_values",
+        )
+        for candidate in value_handlers_candidates:
+            handler = getattr(self.recordset, candidate, None)
+            if handler is not None:
+                vals = handler(self)
+        return vals
 
     @property
     def func_string(self):
@@ -694,15 +731,17 @@ def set_started(self):
 
     def set_done(self, result=None):
         self.state = DONE
+        self.exc_name = None
         self.exc_info = None
         self.date_done = datetime.now()
         if result is not None:
             self.result = result
 
-    def set_failed(self, exc_info=None):
+    def set_failed(self, **kw):
         self.state = FAILED
-        if exc_info is not None:
-            self.exc_info = exc_info
+        for k, v in kw.items():
+            if v is not None:
+                setattr(self, k, v)
 
     def __repr__(self):
         return "<Job %s, priority:%d>" % (self.uuid, self.priority)
@@ -734,6 +773,7 @@ def postpone(self, result=None, seconds=None):
         """
         eta_seconds = self._get_retry_seconds(seconds)
         self.eta = timedelta(seconds=eta_seconds)
+        self.exc_name = None
         self.exc_info = None
         if result is not None:
             self.result = result

queue_job/migrations/13.0.3.8.0/post-migration.py

@@ -0,0 +1,47 @@
+# License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl.html)
+
+import logging
+
+from odoo import SUPERUSER_ID, api
+
+_logger = logging.getLogger(__name__)
+
+
+def migrate(cr, version):
+    with api.Environment.manage():
+        env = api.Environment(cr, SUPERUSER_ID, {})
+        _logger.info("Computing exception name for failed jobs")
+        _compute_jobs_new_values(env)
+
+
+def _compute_jobs_new_values(env):
+    for job in env["queue.job"].search(
+        [("state", "=", "failed"), ("exc_info", "!=", False)]
+    ):
+        exception_details = _get_exception_details(job)
+        if exception_details:
+            job.update(exception_details)
+
+
+def _get_exception_details(job):
+    for line in reversed(job.exc_info.splitlines()):
+        if _find_exception(line):
+            name, msg = line.split(":", 1)
+            return {
+                "exc_name": name.strip(),
+                "exc_message": msg.strip("()', \""),
+            }
+
+
+def _find_exception(line):
+    # Just a list of common errors.
+    # If you want to target others, add your own migration step for your db.
+    exceptions = (
+        "Error:",  # catch all well named exceptions
+        # other live instance errors found
+        "requests.exceptions.MissingSchema",
+        "botocore.errorfactory.NoSuchKey",
+    )
+    for exc in exceptions:
+        if exc in line:
+            return exc

queue_job/models/base.py

@@ -6,7 +6,7 @@
 import logging
 import os
 
-from odoo import models
+from odoo import api, models
 
 from ..job import DelayableRecordset
 
@@ -200,3 +200,19 @@ def auto_delay_wrapper(self, *args, **kwargs):
 
         origin = getattr(self, method_name)
         return functools.update_wrapper(auto_delay_wrapper, origin)
+
+    @api.model
+    def _job_store_values(self, job):
+        """Hook for manipulating job stored values.
+
+        You can define a more specific hook for a job function
+        by defining a method name with this pattern:
+
+            `_queue_job_store_values_${func_name}`
+
+        NOTE: values will be stored only if they match stored fields on `queue.job`.
+
+        :param job: current queue_job.job.Job instance.
+        :return: dictionary for setting job values.
+        """
+        return {}

queue_job/models/queue_job.py

@@ -72,6 +72,8 @@ class QueueJob(models.Model):
 
     state = fields.Selection(STATES, readonly=True, required=True, index=True)
     priority = fields.Integer()
+    exc_name = fields.Char(string="Exception", readonly=True)
+    exc_message = fields.Char(string="Exception Message", readonly=True)
     exc_info = fields.Text(string="Exception Info", readonly=True)
     result = fields.Text(readonly=True)
 

queue_job/views/queue_job_views.xml

@@ -70,20 +70,25 @@
                             > If the max. retries is 0, the number of retries is infinite.</span>
                         </div>
                     </group>
+                    <group
+                        name="exc_info"
+                        string="Exception Information"
+                        attrs="{'invisible': [('exc_info', '=', False)]}"
+                        colspan="4"
+                    >
+                        <div id="exc_name" colspan="4">
+                            <label for="exc_name" string="Exception:" />
+                            <field name="exc_name" class="oe_inline" />
+                        </div>
+                        <field colspan="4" nolabel="1" name="exc_info" />
+                    </group>
                     <group
                         name="result"
                         string="Result"
                         attrs="{'invisible': [('result', '=', False)]}"
                     >
                         <field nolabel="1" name="result" />
                     </group>
-                    <group
-                        name="exc_info"
-                        string="Exception Information"
-                        attrs="{'invisible': [('exc_info', '=', False)]}"
-                    >
-                        <field nolabel="1" name="exc_info" />
-                    </group>
                 </sheet>
                 <div class="oe_chatter">
                     <field name="message_follower_ids" widget="mail_followers" />
@@ -106,10 +111,12 @@
                 <field name="name" />
                 <field name="model_name" />
                 <field name="state" />
-                <field name="eta" />
                 <field name="date_created" />
+                <field name="eta" />
                 <field name="date_done" />
                 <field name="exec_time" widget="float_time" />
+                <field name="exc_name" />
+                <field name="exc_message" />
                 <field name="uuid" />
                 <field name="channel" />
                 <field name="company_id" groups="base.group_multi_company" />
@@ -148,6 +155,10 @@
                 <field name="channel" />
                 <field name="job_function_id" />
                 <field name="model_name" />
+                <field name="exc_name" />
+                <field name="exc_message" />
+                <field name="exc_info" />
+                <field name="result" />
                 <field
                     name="company_id"
                     groups="base.group_multi_company"
@@ -195,6 +206,16 @@
                         string="Model"
                         context="{'group_by': 'model_name'}"
                     />
+                    <filter
+                        name="group_by_exc_name"
+                        string="Exception"
+                        context="{'group_by': 'exc_name'}"
+                    />
+                    <filter
+                        name="group_by_exc_message"
+                        string="Exception message"
+                        context="{'group_by': 'exc_message'}"
+                    />
                 </group>
             </search>
         </field>

test_queue_job/models/test_models.py

@@ -10,6 +10,8 @@ class QueueJob(models.Model):
 
     _inherit = "queue.job"
 
+    additional_info = fields.Char()
+
     def testing_related_method(self, **kwargs):
         return self, kwargs
 
@@ -88,6 +90,12 @@ def _register_hook(self):
         )
         return super()._register_hook()
 
+    def _job_store_values(self, job):
+        value = "JUST_TESTING"
+        if job.state == "failed":
+            value += "_BUT_FAILED"
+        return {"additional_info": value}
+
 
 class TestQueueChannel(models.Model):
 

test_queue_job/tests/test_job.py

@@ -163,9 +163,15 @@ def test_set_done(self):
 
     def test_set_failed(self):
         job_a = Job(self.method)
-        job_a.set_failed(exc_info="failed test")
+        job_a.set_failed(
+            exc_info="failed test",
+            exc_name="FailedTest",
+            exc_message="Sadly this job failed",
+        )
         self.assertEquals(job_a.state, FAILED)
         self.assertEquals(job_a.exc_info, "failed test")
+        self.assertEquals(job_a.exc_name, "FailedTest")
+        self.assertEquals(job_a.exc_message, "Sadly this job failed")
 
     def test_postpone(self):
         job_a = Job(self.method)
@@ -184,6 +190,16 @@ def test_store(self):
         stored = self.queue_job.search([("uuid", "=", test_job.uuid)])
         self.assertEqual(len(stored), 1)
 
+    def test_store_extra_data(self):
+        test_job = Job(self.method)
+        test_job.store()
+        stored = self.queue_job.search([("uuid", "=", test_job.uuid)])
+        self.assertEqual(stored.additional_info, "JUST_TESTING")
+        test_job.set_failed(exc_info="failed test", exc_name="FailedTest")
+        test_job.store()
+        stored.invalidate_cache()
+        self.assertEqual(stored.additional_info, "JUST_TESTING_BUT_FAILED")
+
     def test_read(self):
         eta = datetime.now() + timedelta(hours=5)
         test_job = Job(