From a277a7043c391ec59641198d15dd1cc5324c5825 Mon Sep 17 00:00:00 2001 From: Holger Brunn Date: Thu, 6 May 2021 07:42:17 +0200 Subject: [PATCH 1/2] [IMP] queue_job: migrate args and kwargs from func column --- .../migrations/10.0.1.0.0/end-migration.py | 6 ++++ .../migrations/10.0.1.0.0/post-migration.py | 36 +++++++++++++++++++ 2 files changed, 42 insertions(+) diff --git a/queue_job/migrations/10.0.1.0.0/end-migration.py b/queue_job/migrations/10.0.1.0.0/end-migration.py index 8e300fb518..ab2650e78f 100644 --- a/queue_job/migrations/10.0.1.0.0/end-migration.py +++ b/queue_job/migrations/10.0.1.0.0/end-migration.py @@ -3,6 +3,7 @@ # License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl.html) from odoo import api, SUPERUSER_ID +from odoo.addons.queue_job.job import DONE def migrate(cr, version): @@ -28,3 +29,8 @@ def migrate(cr, version): method.im_class._name, method.__name__, ), }) + # recompute func_string after other addons have adapted their + # args/kwargs/record_ids + records = QueueJob.search([('state', 'not in', [DONE])]) + records._recompute_todo(QueueJob._fields['func_string']) + records.recompute() diff --git a/queue_job/migrations/10.0.1.0.0/post-migration.py b/queue_job/migrations/10.0.1.0.0/post-migration.py index d92848911e..613bea06d8 100644 --- a/queue_job/migrations/10.0.1.0.0/post-migration.py +++ b/queue_job/migrations/10.0.1.0.0/post-migration.py @@ -1,6 +1,15 @@ # -*- coding: utf-8 -*- # Copyright 2018 Tecnativa - Pedro M. Baeza # License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl.html) +import json +import logging +from cPickle import Unpickler +from StringIO import StringIO +from odoo.addons.queue_job.job import DONE +from odoo.addons.queue_job.fields import JobEncoder + + +_logger = logging.getLogger(__name__) def migrate(cr, version): @@ -15,3 +24,30 @@ def migrate(cr, version): UPDATE queue_job SET method_name = reverse(split_part(reverse(func_name), '.', 1))""" ) + migrate_args_kwargs(cr) + + +def migrate_args_kwargs(cr): + """Extract args and kwargs from v9 func column, which is the tuple + (func_name, args, kwargs) pickled in a binary string. Ignore done + jobs for performance reasons""" + cr.execute( + 'select id, func from queue_job where state not in %s', + (tuple([DONE]),), + ) + for _id, func in cr.fetchall(): + try: + func_name, args, kwargs = Unpickler(StringIO(func)).load() + except Exception: + _logger.exception( + 'Failed to parse func column for queue_job#%s', _id, + ) + continue + cr.execute( + 'update queue_job set args=%s, kwargs=%s where id=%s', + ( + json.dumps(args, cls=JobEncoder), + json.dumps(kwargs, cls=JobEncoder), + _id, + ), + ) From 999fe453be2661fb6bc0fb2fd3cc51b15c2c97a6 Mon Sep 17 00:00:00 2001 From: Stefan Rijnhart Date: Sat, 22 May 2021 08:21:27 +0200 Subject: [PATCH 2/2] [RFR] Failsafe and verbose --- .../migrations/10.0.1.0.0/end-migration.py | 24 +++++++++++++++---- .../migrations/10.0.1.0.0/post-migration.py | 17 +++++++++---- 2 files changed, 33 insertions(+), 8 deletions(-) diff --git a/queue_job/migrations/10.0.1.0.0/end-migration.py b/queue_job/migrations/10.0.1.0.0/end-migration.py index ab2650e78f..74f2484456 100644 --- a/queue_job/migrations/10.0.1.0.0/end-migration.py +++ b/queue_job/migrations/10.0.1.0.0/end-migration.py @@ -1,9 +1,9 @@ # -*- coding: utf-8 -*- # Copyright 2017-2018 Tecnativa - Pedro M. Baeza # License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl.html) +import logging from odoo import api, SUPERUSER_ID -from odoo.addons.queue_job.job import DONE def migrate(cr, version): @@ -13,6 +13,7 @@ def migrate(cr, version): if not version: return env = api.Environment(cr, SUPERUSER_ID, {}) + logger = logging.getLogger("odoo.addons.queue.migrations") QueueJob = env['queue.job'] groups = QueueJob.read_group( [], ['model_name', 'method_name'], ['model_name', 'method_name'], @@ -31,6 +32,21 @@ def migrate(cr, version): }) # recompute func_string after other addons have adapted their # args/kwargs/record_ids - records = QueueJob.search([('state', 'not in', [DONE])]) - records._recompute_todo(QueueJob._fields['func_string']) - records.recompute() + records = env["queue.job"].search([]) + total = len(records) + count = 0 + error = 0 + for record in records: + count += 1 + if not count % 1000: + logger.info("Recomputing func_string of job %s of %s", + count, total) + try: + record._compute_func_string() + except KeyError: # unknown model or non-compliant arguments + error += 1 + logger.debug("Could not recompute func_string of queue.job#%s", + record.id) + + if error: + logger.warning("Could not recompute func_string of %s jobs", error) diff --git a/queue_job/migrations/10.0.1.0.0/post-migration.py b/queue_job/migrations/10.0.1.0.0/post-migration.py index 613bea06d8..30b906f6ef 100644 --- a/queue_job/migrations/10.0.1.0.0/post-migration.py +++ b/queue_job/migrations/10.0.1.0.0/post-migration.py @@ -5,7 +5,6 @@ import logging from cPickle import Unpickler from StringIO import StringIO -from odoo.addons.queue_job.job import DONE from odoo.addons.queue_job.fields import JobEncoder @@ -32,16 +31,24 @@ def migrate_args_kwargs(cr): (func_name, args, kwargs) pickled in a binary string. Ignore done jobs for performance reasons""" cr.execute( - 'select id, func from queue_job where state not in %s', - (tuple([DONE]),), + 'select id, func from queue_job' ) - for _id, func in cr.fetchall(): + records = cr.fetchall() + total = len(records) + count = 0 + error = 0 + for _id, func in records: + count += 1 + if not count % 1000: + _logger.info("Parsing arguments of job %s of %s", + count, total) try: func_name, args, kwargs = Unpickler(StringIO(func)).load() except Exception: _logger.exception( 'Failed to parse func column for queue_job#%s', _id, ) + error += 1 continue cr.execute( 'update queue_job set args=%s, kwargs=%s where id=%s', @@ -51,3 +58,5 @@ def migrate_args_kwargs(cr): _id, ), ) + if error: + _logger.warning("Could not parse arguments of %s jobs", error)