diff --git a/client/app/components/queries/SchemaData.jsx b/client/app/components/queries/SchemaData.jsx index b17600676e..bd984df811 100644 --- a/client/app/components/queries/SchemaData.jsx +++ b/client/app/components/queries/SchemaData.jsx @@ -30,13 +30,20 @@ class SchemaData extends React.PureComponent { dataIndex: 'type', width: 400, key: 'type', - }, { - title: 'Example', - dataIndex: 'example', - width: 400, - key: 'example', }]; + const hasExample = + this.props.tableMetadata.some(columnMetadata => columnMetadata.example); + + if (hasExample) { + columns.push({ + title: 'Example', + dataIndex: 'example', + width: 400, + key: 'example', + }); + } + return (
{{column.name}} - ({{column.type}})
diff --git a/redash/models/__init__.py b/redash/models/__init__.py index 03a03557de..c5b2dcf1f4 100644 --- a/redash/models/__init__.py +++ b/redash/models/__init__.py @@ -206,7 +206,22 @@ def delete(self): def get_schema(self): schema = [] + columns_by_table_id = {} tables = TableMetadata.query.filter(TableMetadata.data_source_id == self.id).all() + columns = ColumnMetadata.query.all() + + for column in columns: + if not column.exists: + continue + + columns_by_table_id.setdefault(column.table_id, []).append({ + 'key': column.id, + 'name': column.name, + 'type': column.type, + 'exists': column.exists, + 'example': column.example + }) + for table in tables: if not table.exists: continue @@ -216,14 +231,9 @@ def get_schema(self): 'exists': table.exists, 'hasColumnMetadata': table.column_metadata, 'columns': []} - columns = ColumnMetadata.query.filter(ColumnMetadata.table_id == table.id) - table_info['columns'] = sorted([{ - 'key': column.id, - 'name': column.name, - 'type': column.type, - 'exists': column.exists, - 'example': column.example - } for column in columns if column.exists == True], key=itemgetter('name')) + + table_info['columns'] = sorted( + columns_by_table_id.get(table.id, []), key=itemgetter('name')) schema.append(table_info) return sorted(schema, key=itemgetter('name')) diff --git a/redash/query_runner/athena.py b/redash/query_runner/athena.py index 252fb37927..15b973d65b 100644 --- a/redash/query_runner/athena.py +++ b/redash/query_runner/athena.py @@ -143,9 +143,18 @@ def __get_schema_from_glue(self): table_name = '%s.%s' % (database['Name'], table['Name']) if table_name not in schema: column = [columns['Name'] for columns in table['StorageDescriptor']['Columns']] - schema[table_name] = {'name': table_name, 'columns': column} + metadata = [{ + "name": column_data['Name'], + "type": column_data['Type'] + } for column_data in table['StorageDescriptor']['Columns']] + schema[table_name] = {'name': table_name, 'columns': column, 'metadata': metadata} for partition in table.get('PartitionKeys', []): schema[table_name]['columns'].append(partition['Name']) + schema[table_name]['metadata'].append({ + "name": partition['Name'], + "type": partition['Type'] + }) + return schema.values() def get_schema(self, get_stats=False): diff --git a/redash/tasks/queries.py b/redash/tasks/queries.py index d7d4e9d433..dbac37d290 100644 --- a/redash/tasks/queries.py +++ b/redash/tasks/queries.py @@ -378,7 +378,7 @@ def refresh_schema(data_source_id): "column_metadata": "metadata" in table } new_column_names[table_name] = table['columns'] - new_column_metadata[table_name] = table['metadata'] + new_column_metadata[table_name] = table.get('metadata', None) insert_or_update_table_metadata(ds, existing_tables_set, table_data) models.db.session.flush() diff --git a/tests/query_runner/test_athena.py b/tests/query_runner/test_athena.py index fe444de64f..7c7a139a13 100644 --- a/tests/query_runner/test_athena.py +++ b/tests/query_runner/test_athena.py @@ -72,7 +72,11 @@ def test_external_table(self): {'DatabaseName': 'test1'}, ) with self.stubber: - assert query_runner.get_schema() == [{'columns': ['row_id'], 'name': 'test1.jdbc_table'}] + assert query_runner.get_schema() == [{ + 'columns': ['row_id'], + 'name': 'test1.jdbc_table', + 'metadata': [{'type': 'int', 'name': 'row_id'}] + }] def test_partitioned_table(self): """ @@ -118,7 +122,11 @@ def test_partitioned_table(self): {'DatabaseName': 'test1'}, ) with self.stubber: - assert query_runner.get_schema() == [{'columns': ['sk', 'category'], 'name': 'test1.partitioned_table'}] + assert query_runner.get_schema() == [{ + 'columns': ['sk', 'category'], + 'name': 'test1.partitioned_table', + 'metadata': [{'type': 'int', 'name': 'sk'}, {'type': 'int', 'name': 'category'}] + }] def test_view(self): query_runner = Athena({'glue': True, 'region': 'mars-east-1'}) @@ -150,7 +158,11 @@ def test_view(self): {'DatabaseName': 'test1'}, ) with self.stubber: - assert query_runner.get_schema() == [{'columns': ['sk'], 'name': 'test1.view'}] + assert query_runner.get_schema() == [{ + 'columns': ['sk'], + 'name': 'test1.view', + 'metadata': [{'type': 'int', 'name': 'sk'}] + }] def test_dodgy_table_does_not_break_schema_listing(self): """ @@ -187,4 +199,8 @@ def test_dodgy_table_does_not_break_schema_listing(self): {'DatabaseName': 'test1'}, ) with self.stubber: - assert query_runner.get_schema() == [{'columns': ['region'], 'name': 'test1.csv'}] + assert query_runner.get_schema() == [{ + 'columns': ['region'], + 'name': 'test1.csv', + 'metadata': [{'type': 'string', 'name': 'region'}] + }]