[12.0] [WIP] database_cleanup: migration to 12.0

.travis.yml

@@ -17,8 +17,11 @@ env:
 
   matrix:
   - LINT_CHECK="1"
-  - TESTS="1" ODOO_REPO="OCA/OCB"
-  - TESTS="1" ODOO_REPO="OCA/OCB" MAKEPOT="1"
+  - TESTS="1" ODOO_REPO="OCA/OCB" EXCLUDE="database_cleanup"
+  - TESTS="1" ODOO_REPO="OCA/OCB" INCLUDE="database_cleanup"
+  - TESTS="1" ODOO_REPO="odoo/odoo" EXCLUDE="database_cleanup" MAKEPOT="1"
+  - TESTS="1" ODOO_REPO="odoo/odoo" INCLUDE="database_cleanup" MAKEPOT="1"
+
 
 install:
   - git clone --depth=1 https://github.com/OCA/maintainer-quality-tools.git ${HOME}/maintainer-quality-tools

database_cleanup/README.rst

@@ -0,0 +1,69 @@
+.. image:: https://img.shields.io/badge/licence-AGPL--3-blue.svg
+    :target: http://www.gnu.org/licenses/agpl-3.0-standalone.html
+    :alt: License: AGPL-3
+
+================
+Database cleanup
+================
+
+Clean your Odoo database from remnants of modules, models, columns and
+tables left by uninstalled modules (prior to 7.0) or a homebrew database
+upgrade to a new major version of Odoo.
+
+Caution! This module is potentially harmful and can *easily* destroy the
+integrity of your data. Do not use if you are not entirely comfortable
+with the technical details of the Odoo data model of *all* the modules
+that have ever been installed on your database, and do not purge any module,
+model, column or table if you do not know exactly what you are doing.
+
+Usage
+=====
+
+After installation of this module, go to the Settings menu -> Technical ->
+Database cleanup. This menu is only available to members of the *Access Rights*
+group. Go through the modules, models, columns and tables
+entries under this menu (in that order) and find out if there is orphaned data
+in your database. You can either delete entries by line, or sweep all entries
+in one big step (if you are *really* confident).
+
+.. image:: https://odoo-community.org/website/image/ir.attachment/5784_f2813bd/datas
+    :alt: Try me on Runbot
+    :target: https://runbot.odoo-community.org/runbot/149/11.0
+
+Bug Tracker
+===========
+
+Bugs are tracked on `GitHub Issues <https://github.com/OCA/database_cleanup/issues>`_.
+In case of trouble, please check there if your issue has already been reported.
+If you spotted it first, help us smashing it by providing a detailed and welcomed feedback.
+
+Credits
+=======
+
+Images
+------
+
+* Odoo Community Association: `Icon <https://github.com/OCA/maintainer-tools/blob/master/template/module/static/description/icon.svg>`_.
+
+Contributors
+------------
+
+* Stefan Rijnhart <stefan@opener.amsterdam>
+* Holger Brunn <hbrunn@therp.nl>
+
+Do not contact contributors directly about help with questions or problems concerning this addon, but use the `community mailing list <mailto:community@mail.odoo.com>`_ or the `appropriate specialized mailinglist <https://odoo-community.org/groups>`_ for help, and the bug tracker linked in `Bug Tracker`_ above for technical issues.
+
+Maintainer
+----------
+
+.. image:: https://odoo-community.org/logo.png
+   :alt: Odoo Community Association
+   :target: https://odoo-community.org
+
+This module is maintained by the OCA.
+
+OCA, or the Odoo Community Association, is a nonprofit organization whose
+mission is to support the collaborative development of Odoo features and
+promote its widespread use.
+
+To contribute to this module, please visit https://odoo-community.org.

database_cleanup/__init__.py

@@ -0,0 +1,3 @@
+# Copyright 2014-2016 Therp BV <http://therp.nl>
+# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html).
+from . import models

database_cleanup/__openerp__.py

@@ -0,0 +1,23 @@
+# Copyright 2014-2016 Therp BV <http://therp.nl>
+# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html).
+{
+    'name': 'Database cleanup',
+    'version': '12.0.1.0.0',
+    'author': "Therp BV,Odoo Community Association (OCA)",
+    'depends': ['base'],
+    'license': 'AGPL-3',
+    'category': 'Tools',
+    'data': [
+        "views/purge_wizard.xml",
+        'views/purge_menus.xml',
+        'views/purge_modules.xml',
+        'views/purge_models.xml',
+        'views/purge_columns.xml',
+        'views/purge_tables.xml',
+        'views/purge_data.xml',
+        "views/create_indexes.xml",
+        'views/purge_properties.xml',
+        'views/menu.xml',
+    ],
+    'installable': True,
+}

database_cleanup/identifier_adapter.py

@@ -0,0 +1,24 @@
+# Copyright 2016 Therp BV <http://therp.nl>
+# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html).
+from psycopg2.extensions import ISQLQuote
+
+
+class IdentifierAdapter(ISQLQuote):
+    def __init__(self, identifier, quote=True):
+        self.quote = quote
+        self.identifier = identifier
+
+    def __conform__(self, protocol):
+        if protocol == ISQLQuote:
+            return self
+
+    def getquoted(self):
+        def is_identifier_char(c):
+            return c.isalnum() or c in ['_', '$']
+
+        format_string = '"%s"'
+        if not self.quote:
+            format_string = '%s'
+        return format_string % ''.join(
+            filter(is_identifier_char, self.identifier)
+        )

database_cleanup/models/__init__.py

@@ -0,0 +1,9 @@
+from . import purge_wizard
+from . import purge_modules
+from . import purge_models
+from . import purge_columns
+from . import purge_tables
+from . import purge_data
+from . import purge_menus
+from . import create_indexes
+from . import purge_properties

database_cleanup/models/create_indexes.py

@@ -0,0 +1,82 @@
+# Copyright 2017 Therp BV <http://therp.nl>
+# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html).
+# pylint: disable=consider-merging-classes-inherited
+from ..identifier_adapter import IdentifierAdapter
+from odoo import api, fields, models
+
+
+class CreateIndexesLine(models.TransientModel):
+    _inherit = 'cleanup.purge.line'
+    _name = 'cleanup.create_indexes.line'
+
+    purged = fields.Boolean('Created')
+    wizard_id = fields.Many2one('cleanup.create_indexes.wizard')
+    field_id = fields.Many2one('ir.model.fields', required=True)
+
+    @api.multi
+    def purge(self):
+        tables = set()
+        for field in self.mapped('field_id'):
+            model = self.env[field.model]
+            name = '%s_%s_index' % (model._table, field.name)
+            self.env.cr.execute(
+                'create index %s ON %s (%s)',
+                (
+                    IdentifierAdapter(name, quote=False),
+                    IdentifierAdapter(model._table),
+                    IdentifierAdapter(field.name),
+                ),
+            )
+            tables.add(model._table)
+        for table in tables:
+            self.env.cr.execute(
+                'analyze %s', (IdentifierAdapter(model._table),)
+            )
+        self.write({
+            'purged': True,
+        })
+
+
+class CreateIndexesWizard(models.TransientModel):
+    _inherit = 'cleanup.purge.wizard'
+    _name = 'cleanup.create_indexes.wizard'
+    _description = 'Create indexes'
+
+    purge_line_ids = fields.One2many(
+        'cleanup.create_indexes.line', 'wizard_id',
+    )
+
+    @api.multi
+    def find(self):
+        res = list()
+        for field in self.env['ir.model.fields'].search([
+                ('index', '=', True),
+        ]):
+            if field.model not in self.env.registry:
+                continue
+            model = self.env[field.model]
+            name = '%s_%s_index' % (model._table, field.name)
+            self.env.cr.execute(
+                'select indexname from pg_indexes '
+                'where indexname=%s and tablename=%s',
+                (name, model._table)
+            )
+            if self.env.cr.rowcount:
+                continue
+
+            self.env.cr.execute(
+                'select a.attname '
+                'from pg_attribute a '
+                'join pg_class c on a.attrelid=c.oid '
+                'join pg_tables t on t.tablename=c.relname '
+                'where attname=%s and c.relname=%s',
+                (field.name, model._table,)
+            )
+            if not self.env.cr.rowcount:
+                continue
+
+            res.append((0, 0, {
+                'name': '%s.%s' % (field.model, field.name),
+                'field_id': field.id,
+            }))
+        return res

database_cleanup/models/purge_columns.py

@@ -0,0 +1,132 @@
+# Copyright 2014-2016 Therp BV <http://therp.nl>
+# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html).
+# pylint: disable=consider-merging-classes-inherited
+from odoo import _, api, fields, models
+from odoo.exceptions import UserError
+from ..identifier_adapter import IdentifierAdapter
+
+
+class CleanupPurgeLineColumn(models.TransientModel):
+    _inherit = 'cleanup.purge.line'
+    _name = 'cleanup.purge.line.column'
+    _description = 'Purge Column Wizard Lines'
+
+    model_id = fields.Many2one('ir.model', 'Model', required=True,
+                               ondelete='CASCADE')
+    wizard_id = fields.Many2one(
+        'cleanup.purge.wizard.column', 'Purge Wizard', readonly=True)
+
+    @api.multi
+    def purge(self):
+        """
+        Unlink columns upon manual confirmation.
+        """
+        if self:
+            objs = self
+        else:
+            objs = self.env['cleanup.purge.line.column']\
+                .browse(self._context.get('active_ids'))
+        for line in objs:
+            if line.purged:
+                continue
+            model_pool = self.env[line.model_id.model]
+            # Check whether the column actually still exists.
+            # Inheritance such as stock.picking.in from stock.picking
+            # can lead to double attempts at removal
+            self.env.cr.execute(
+                'SELECT count(attname) FROM pg_attribute '
+                'WHERE attrelid = '
+                '( SELECT oid FROM pg_class WHERE relname = %s ) '
+                'AND attname = %s',
+                (model_pool._table, line.name))
+            if not self.env.cr.fetchone()[0]:
+                continue
+
+            self.logger.info(
+                'Dropping column %s from table %s',
+                line.name, model_pool._table)
+            self.env.cr.execute(
+                'ALTER TABLE %s DROP COLUMN %s',
+                (
+                    IdentifierAdapter(model_pool._table),
+                    IdentifierAdapter(line.name)
+                ))
+            line.write({'purged': True})
+            # we need this commit because the ORM will deadlock if
+            # we still have a pending transaction
+            self.env.cr.commit()  # pylint: disable=invalid-commit
+        return True
+
+
+class CleanupPurgeWizardColumn(models.TransientModel):
+    _inherit = 'cleanup.purge.wizard'
+    _name = 'cleanup.purge.wizard.column'
+    _description = 'Purge columns'
+
+    # List of known columns in use without corresponding fields
+    # Format: {table: [fields]}
+    blacklist = {
+        'wkf_instance': ['uid'],  # lp:1277899
+        'res_users': ['password', 'password_crypt'],
+    }
+
+    @api.model
+    def get_orphaned_columns(self, model_pools):
+        """
+        From openobject-server/openerp/osv/orm.py
+        Iterate on the database columns to identify columns
+        of fields which have been removed
+        """
+        columns = list(set([
+            column.name
+            for model_pool in model_pools
+            for column in model_pool._fields.values()
+            if not (column.compute is not None and not column.store)
+        ]))
+        columns += models.MAGIC_COLUMNS
+        columns += self.blacklist.get(model_pools[0]._table, [])
+
+        self.env.cr.execute(
+            "SELECT a.attname FROM pg_class c, pg_attribute a "
+            "WHERE c.relname=%s AND c.oid=a.attrelid AND a.attisdropped=False "
+            "AND pg_catalog.format_type(a.atttypid, a.atttypmod) "
+            "NOT IN ('cid', 'tid', 'oid', 'xid') "
+            "AND a.attname NOT IN %s",
+            (model_pools[0]._table, tuple(columns)))
+        return [column for column, in self.env.cr.fetchall()]
+
+    @api.model
+    def find(self):
+        """
+        Search for columns that are not in the corresponding model.
+
+        Group models by table to prevent false positives for columns
+        that are only in some of the models sharing the same table.
+        Example of this is 'sale_id' not being a field of stock.picking.in
+        """
+        res = []
+
+        # mapping of tables to tuples (model id, [pool1, pool2, ...])
+        table2model = {}
+
+        for model in self.env['ir.model'].search([]):
+            if model.model not in self.env:
+                continue
+            model_pool = self.env[model.model]
+            if not model_pool._auto:
+                continue
+            table2model.setdefault(
+                model_pool._table, (model.id, [])
+            )[1].append(model_pool)
+
+        for table, model_spec in table2model.items():
+            for column in self.get_orphaned_columns(model_spec[1]):
+                res.append((0, 0, {
+                            'name': column,
+                            'model_id': model_spec[0]}))
+        if not res:
+            raise UserError(_('No orphaned columns found'))
+        return res
+
+    purge_line_ids = fields.One2many(
+        'cleanup.purge.line.column', 'wizard_id', 'Columns to purge')