[WiP] Introduce uuids to power ImportExportMixin

.pre-commit-config.yaml

@@ -15,8 +15,24 @@
 # limitations under the License.
 #
 repos:
-  - repo: https://github.com/ambv/black
+-   repo: https://github.com/ambv/black
     rev: stable
     hooks:
-      - id: black
+    -   id: black
         language_version: python3.6
+
+-   repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v2.2.3
+    hooks:
+    -   id: trailing-whitespace
+    -   id: end-of-file-fixer
+    -   id: check-docstring-first
+    -   id: check-json
+    -   id: check-added-large-files
+    -   id: check-yaml
+    -   id: debug-statements
+
+-   repo: https://gitlab.com/pycqa/flake8
+    rev: 3.7.1
+    hooks:
+    -   id: flake8

superset/connectors/base/models.py

@@ -22,11 +22,11 @@
 from sqlalchemy.orm import foreign, relationship
 
 from superset.models.core import Slice
-from superset.models.helpers import AuditMixinNullable, ImportMixin
+from superset.models.helpers import AuditMixinNullable, ExportImportMixin
 from superset.utils import core as utils
 
 
-class BaseDatasource(AuditMixinNullable, ImportMixin):
+class BaseDatasource(AuditMixinNullable, ExportImportMixin):
     """A common interface to objects that are queryable
     (tables and datasources)"""
 
@@ -330,7 +330,7 @@ def update_from_object(self, obj):
         )
 
 
-class BaseColumn(AuditMixinNullable, ImportMixin):
+class BaseColumn(AuditMixinNullable, ExportImportMixin):
     """Interface for column"""
 
     __tablename__ = None  # {connector_name}_column
@@ -347,6 +347,7 @@ class BaseColumn(AuditMixinNullable, ImportMixin):
 
     # [optional] Set this to support import/export functionality
     export_fields = []
+    export_ordering = "column_name"
 
     def __repr__(self):
         return self.column_name
@@ -399,7 +400,7 @@ def data(self):
         return {s: getattr(self, s) for s in attrs if hasattr(self, s)}
 
 
-class BaseMetric(AuditMixinNullable, ImportMixin):
+class BaseMetric(AuditMixinNullable, ExportImportMixin):
 
     """Interface for Metrics"""
 
@@ -414,6 +415,8 @@ class BaseMetric(AuditMixinNullable, ImportMixin):
     d3format = Column(String(128))
     warning_text = Column(Text)
 
+    export_ordering = "metric_name"
+
     """
     The interface should also declare a datasource relationship pointing
     to a derivative of BaseDatasource, along with a FK

superset/connectors/connector_registry.py

@@ -40,6 +40,11 @@ def get_datasource(cls, datasource_type, datasource_id, session):
             .first()
         )
 
+    @classmethod
+    def get_datasource_by_uuid(cls, session, source_type, uuid):
+        source_class = ConnectorRegistry.sources[source_type]
+        return session.query(source_class).filter_by(uuid=uuid).one()
+
     @classmethod
     def get_all_datasources(cls, session):
         datasources = []

superset/connectors/druid/models.py

@@ -63,7 +63,7 @@
 from superset import conf, db, security_manager
 from superset.connectors.base.models import BaseColumn, BaseDatasource, BaseMetric
 from superset.exceptions import MetricPermException, SupersetException
-from superset.models.helpers import AuditMixinNullable, ImportMixin, QueryResult
+from superset.models.helpers import AuditMixinNullable, ExportImportMixin, QueryResult
 from superset.utils import core as utils, import_datasource
 from superset.utils.core import DimSelector, DTTM_ALIAS, flasher
 
@@ -97,7 +97,7 @@ def __init__(self, name, post_aggregator):
         self.post_aggregator = post_aggregator
 
 
-class DruidCluster(Model, AuditMixinNullable, ImportMixin):
+class DruidCluster(Model, AuditMixinNullable, ExportImportMixin):
 
     """ORM object referencing the Druid clusters"""
 

superset/connectors/sqla/models.py

@@ -331,6 +331,7 @@ class SqlaTable(Model, BaseDatasource):
     ]
     export_parent = "database"
     export_children = ["metrics", "columns"]
+    export_ordering = "table_name"
 
     sqla_aggregations = {
         "COUNT_DISTINCT": lambda column_name: sa.func.COUNT(sa.distinct(column_name)),

superset/migrations/versions/38f7d2edcb84_uuids.py

@@ -0,0 +1,142 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""Adds uuid columns to all classes with ImportExportMixin: dashboards, datasources, dbs, slices, tables
+Revision ID: e5200a951e62
+Revises: e9df189e5c7e
+Create Date: 2019-05-08 13:42:48.479145
+"""
+import uuid
+
+from alembic import op
+from sqlalchemy import CHAR, Column, Integer
+from sqlalchemy.ext.declarative import declarative_base
+
+from superset import db
+from superset.utils.sqla import get_uuid, uuid_sqla_column
+
+# revision identifiers, used by Alembic.
+revision = "38f7d2edcb84"
+down_revision = "b4a38aa87893"
+
+Base = declarative_base()
+
+
+class ImportExportMixin:
+    id = Column(Integer, primary_key=True)
+    uuid = uuid_sqla_column
+
+
+class Dashboard(Base, ImportExportMixin):
+    __tablename__ = "dashboards"
+
+
+class Datasource(Base, ImportExportMixin):
+    __tablename__ = "datasources"
+
+
+class Database(Base, ImportExportMixin):
+    __tablename__ = "dbs"
+
+
+class DruidCluster(Base, ImportExportMixin):
+    __tablename__ = "clusters"
+
+
+class DruidMetric(Base, ImportExportMixin):
+    __tablename__ = "metrics"
+
+
+class DruidColumn(Base, ImportExportMixin):
+    __tablename__ = "columns"
+
+
+class Slice(Base, ImportExportMixin):
+    __tablename__ = "slices"
+
+
+class SqlaTable(Base, ImportExportMixin):
+    __tablename__ = "tables"
+
+
+class SqlMetric(Base, ImportExportMixin):
+    __tablename__ = "sql_metrics"
+
+
+class TableColumn(Base, ImportExportMixin):
+    __tablename__ = "table_columns"
+
+
+def batch_commit(iterable, mutator, session, batch_size=100):
+    count = len(iterable)
+    for i, obj in enumerate(iterable):
+        mutator(obj)
+        session.merge(obj)
+        if i % 100 == 0:
+            session.commit()
+            print(f"uuid assigned to {i} out of {count}")
+    session.commit()
+    print(f"Done! Assigned {count} uuids")
+
+
+def default_mutator(obj):
+    obj.uuid = get_uuid()
+
+
+def upgrade():
+    bind = op.get_bind()
+    session = db.Session(bind=bind)
+    db_type = session.bind.dialect.name
+
+    def add_uuid_column(tbl_name, _type):
+        """Add a uuid column to a given table"""
+        print(f"Add uuid column to table '{tbl_name}'")
+        with op.batch_alter_table(tbl_name) as batch_op:
+            batch_op.add_column(Column("uuid", CHAR(36), default=get_uuid))
+        batch_commit(session.query(_type).all(), default_mutator, session)
+
+    add_uuid_column("dashboards", Dashboard)
+    add_uuid_column("datasources", Datasource)
+    add_uuid_column("dbs", Database)
+    add_uuid_column("clusters", DruidCluster)
+    add_uuid_column("metrics", DruidMetric)
+    add_uuid_column("columns", DruidColumn)
+    add_uuid_column("slices", Slice)
+    add_uuid_column("sql_metrics", SqlMetric)
+    add_uuid_column("tables", SqlaTable)
+    add_uuid_column("table_columns", TableColumn)
+
+    session.close()
+
+
+def downgrade():
+    for tbl in [
+        "dashboards",
+        "datasources",
+        "dbs",
+        "clusters",
+        "metrics",
+        "columns",
+        "slices",
+        "sql_metrics",
+        "tables",
+        "table_columns",
+    ]:
+        try:
+            with op.batch_alter_table(tbl) as batch_op:
+                batch_op.drop_column("uuid")
+        except Exception:
+            pass

superset/models/core.py

@@ -56,7 +56,7 @@
 from superset import app, db, db_engine_specs, security_manager
 from superset.connectors.connector_registry import ConnectorRegistry
 from superset.legacy import update_time_range
-from superset.models.helpers import AuditMixinNullable, ImportMixin
+from superset.models.helpers import AuditMixinNullable, ExportImportMixin
 from superset.models.tags import ChartUpdater, DashboardUpdater, FavStarUpdater
 from superset.models.user_attributes import UserAttribute
 from superset.utils import cache as cache_util, core as utils
@@ -151,7 +151,7 @@ class CssTemplate(Model, AuditMixinNullable):
 )
 
 
-class Slice(Model, AuditMixinNullable, ImportMixin):
+class Slice(Model, AuditMixinNullable, ExportImportMixin):
 
     """A slice is essentially a report or a view on data"""
 
@@ -176,6 +176,8 @@ class Slice(Model, AuditMixinNullable, ImportMixin):
         "params",
         "cache_timeout",
     )
+    export_fields_json = ["params"]
+    export_ordering = "slice_name"
 
     def __repr__(self):
         return self.slice_name or str(self.id)
@@ -407,7 +409,7 @@ def url(self):
 )
 
 
-class Dashboard(Model, AuditMixinNullable, ImportMixin):
+class Dashboard(Model, AuditMixinNullable, ExportImportMixin):
 
     """The dashboard object!"""
 
@@ -430,6 +432,8 @@ class Dashboard(Model, AuditMixinNullable, ImportMixin):
         "css",
         "slug",
     )
+    export_fields_json = ("position_json", "json_metadata")
+    export_ordering = "dashboard_title"
 
     def __repr__(self):
         return self.dashboard_title or str(self.id)
@@ -695,7 +699,7 @@ def export_dashboards(cls, dashboard_ids):
         )
 
 
-class Database(Model, AuditMixinNullable, ImportMixin):
+class Database(Model, AuditMixinNullable, ExportImportMixin):
 
     """An ORM object that stores Database related information"""
 
@@ -744,6 +748,7 @@ class Database(Model, AuditMixinNullable, ImportMixin):
         "extra",
     )
     export_children = ["tables"]
+    export_ordering = "database_name"
 
     def __repr__(self):
         return self.verbose_name if self.verbose_name else self.database_name