diff --git a/requirements.txt b/requirements.txt index 3b88de610364..0789c1b370b5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -54,6 +54,7 @@ numpy==1.15.2 # via pandas pandas==0.23.4 parsedatetime==2.0.0 pathlib2==2.3.0 +pillow==5.4.1 polyline==1.3.2 prison==0.1.0 # via flask-appbuilder py==1.7.0 # via retry diff --git a/setup.py b/setup.py index fc91ea484d59..a4d963026957 100644 --- a/setup.py +++ b/setup.py @@ -92,6 +92,7 @@ def get_git_sha(): 'pandas>=0.18.0, <0.24.0', # `pandas`>=0.24.0 changes datetimelike API 'parsedatetime', 'pathlib2', + 'pillow==5.4.1', 'polyline', 'pydruid>=0.5.2', 'python-dateutil', diff --git a/superset/__init__.py b/superset/__init__.py index 6971dc9d4c8a..6f37478b45e4 100644 --- a/superset/__init__.py +++ b/superset/__init__.py @@ -115,6 +115,10 @@ def get_manifest(): if conf.get('SILENCE_FAB'): logging.getLogger('flask_appbuilder').setLevel(logging.ERROR) +logging.getLogger('urllib3').setLevel(logging.ERROR) +logging.getLogger('selenium').setLevel(logging.ERROR) +logging.getLogger('PIL').setLevel(logging.ERROR) + if app.debug: app.logger.setLevel(logging.DEBUG) # pylint: disable=no-member else: @@ -135,6 +139,7 @@ def get_manifest(): cache = setup_cache(app, conf.get('CACHE_CONFIG')) tables_cache = setup_cache(app, conf.get('TABLE_NAMES_CACHE_CONFIG')) +thumbnail_cache = setup_cache(app, conf.get('THUMBNAIL_CACHE_CONFIG')) migrate = Migrate(app, db, directory=APP_DIR + '/migrations') diff --git a/superset/assets/images/no-image.png b/superset/assets/images/no-image.png new file mode 100644 index 000000000000..0371c4733d49 Binary files /dev/null and b/superset/assets/images/no-image.png differ diff --git a/superset/assets/package-lock.json b/superset/assets/package-lock.json index da886a0c95d1..6047df5d0417 100644 --- a/superset/assets/package-lock.json +++ b/superset/assets/package-lock.json @@ -4953,6 +4953,11 @@ "integrity": "sha1-3DQxT05nkxgJP8dgJyUl+UvyXBY=", "dev": true }, + "batch-processor": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/batch-processor/-/batch-processor-1.0.0.tgz", + "integrity": "sha1-dclcMrdI4IUNEMKxaPa9vpiRrOg=" + }, "bcrypt-pbkdf": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz", @@ -5470,6 +5475,11 @@ "resolved": "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz", "integrity": "sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw=" }, + "chain-function": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/chain-function/-/chain-function-1.0.1.tgz", + "integrity": "sha512-SxltgMwL9uCko5/ZCLiyG2B7R9fY4pDZUw7hJ4MhirdjBLosoDqkWABi3XMucddHdLiFJMb7PD2MZifZriuMTg==" + }, "chalk": { "version": "1.1.3", "resolved": "http://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz", @@ -7375,6 +7385,14 @@ "resolved": "https://registry.npmjs.org/earcut/-/earcut-2.1.4.tgz", "integrity": "sha512-ttRjmPD5oaTtXOoxhFp9aZvMB14kBjapYaiBuzBB1elOgSLU9P2Ev86G2OClBg+uspUXERsIzXKpUWweH2K4Xg==" }, + "easy-css-transform-builder": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/easy-css-transform-builder/-/easy-css-transform-builder-0.0.2.tgz", + "integrity": "sha1-pUFmenkZ4X9n2CsR08tV/dVDMCI=", + "requires": { + "invariant": "^2.2.2" + } + }, "ecc-jsbn": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz", @@ -7402,6 +7420,14 @@ "integrity": "sha512-En051LMzMl3/asMWGZEtU808HOoVWIpmmZx1Ep8N+TT9e7z/X8RcLeBU2kLSNLGQ+5SuKELzMx+MVuTBXk6Q9w==", "dev": true }, + "element-resize-detector": { + "version": "1.1.15", + "resolved": "https://registry.npmjs.org/element-resize-detector/-/element-resize-detector-1.1.15.tgz", + "integrity": "sha512-16/5avDegXlUxytGgaumhjyQoM6hpp5j3+L79sYq5hlXfTNRy5WMMuTVWkZU3egp/CokCmTmvf18P3KeB57Iog==", + "requires": { + "batch-processor": "^1.0.0" + } + }, "elliptic": { "version": "6.4.1", "resolved": "https://registry.npmjs.org/elliptic/-/elliptic-6.4.1.tgz", @@ -8108,6 +8134,11 @@ "integrity": "sha1-Qa4u62XvpiJorr/qg6x9eSmbCIc=", "dev": true }, + "ev-emitter": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/ev-emitter/-/ev-emitter-1.1.1.tgz", + "integrity": "sha512-ipiDYhdQSCZ4hSbX4rMW+XzNKMD1prg/sTvoVmSLkuQ1MVlwjJQQA+sW8tMYR3BLUr9KjodFV4pvzunvRhd33Q==" + }, "eventemitter3": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-3.1.0.tgz", @@ -8160,6 +8191,11 @@ "strip-eof": "^1.0.0" } }, + "exenv": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/exenv/-/exenv-1.2.2.tgz", + "integrity": "sha1-KueOhdmJQVhnCwPUe+wfA72Ru50=" + }, "exif-parser": { "version": "0.1.12", "resolved": "https://registry.npmjs.org/exif-parser/-/exif-parser-0.1.12.tgz", @@ -10718,6 +10754,14 @@ "dev": true, "optional": true }, + "imagesloaded": { + "version": "4.1.4", + "resolved": "https://registry.npmjs.org/imagesloaded/-/imagesloaded-4.1.4.tgz", + "integrity": "sha512-ltiBVcYpc/TYTF5nolkMNsnREHW+ICvfQ3Yla2Sgr71YFwQ86bDwV9hgpFhFtrGPuwEx5+LqOHIrdXBdoWwwsA==", + "requires": { + "ev-emitter": "^1.0.0" + } + }, "immutable": { "version": "3.8.2", "resolved": "https://registry.npmjs.org/immutable/-/immutable-3.8.2.tgz", @@ -17685,6 +17729,17 @@ "js-search": "^1.3.1" } }, + "react-sizeme": { + "version": "2.6.7", + "resolved": "https://registry.npmjs.org/react-sizeme/-/react-sizeme-2.6.7.tgz", + "integrity": "sha512-xCjPoBP5jmeW58TxIkcviMZqabZis7tTvDFWf0/Wa5XCgVWQTIe74NQBes2N1Kmp64GRLkpm60BaP0kk+v8aCQ==", + "requires": { + "element-resize-detector": "^1.1.15", + "invariant": "^2.2.4", + "shallowequal": "^1.1.0", + "throttle-debounce": "^2.1.0" + } + }, "react-sortable-hoc": { "version": "0.8.4", "resolved": "https://registry.npmjs.org/react-sortable-hoc/-/react-sortable-hoc-0.8.4.tgz", @@ -17714,6 +17769,36 @@ "react-style-proptype": "^3.0.0" } }, + "react-stack-grid": { + "version": "0.7.1", + "resolved": "https://registry.npmjs.org/react-stack-grid/-/react-stack-grid-0.7.1.tgz", + "integrity": "sha512-Fw7qMt5Rd9wQpNCnvK4Gi+ry/nL5rKfzP2hGsw5/DZxArEMk60VoDLy68Uskq09l6wk7qb2w7P2+lNzSd9mYEw==", + "requires": { + "easy-css-transform-builder": "^0.0.2", + "exenv": "^1.2.1", + "imagesloaded": "^4.1.1", + "inline-style-prefixer": "^3.0.6", + "invariant": "^2.2.2", + "prop-types": "^15.5.10", + "react-sizeme": "^2.2.0", + "react-transition-group": "^1.2.0", + "shallowequal": "^1.0.1" + }, + "dependencies": { + "react-transition-group": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/react-transition-group/-/react-transition-group-1.2.1.tgz", + "integrity": "sha512-CWaL3laCmgAFdxdKbhhps+c0HRGF4c+hdM4H23+FI1QBNUyx/AMeIJGWorehPNSaKnQNOAxL7PQmqMu78CDj3Q==", + "requires": { + "chain-function": "^1.0.0", + "dom-helpers": "^3.2.0", + "loose-envify": "^1.3.1", + "prop-types": "^15.5.6", + "warning": "^3.0.0" + } + } + } + }, "react-sticky": { "version": "6.0.3", "resolved": "https://registry.npmjs.org/react-sticky/-/react-sticky-6.0.3.tgz", @@ -20271,6 +20356,11 @@ "integrity": "sha1-iQN8vJLFarGJJua6TLsgDhVnKmo=", "dev": true }, + "throttle-debounce": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/throttle-debounce/-/throttle-debounce-2.1.0.tgz", + "integrity": "sha512-AOvyNahXQuU7NN+VVvOOX+uW6FPaWdAOdRP5HfwYxAfCzXTFKRMoIMk+n+po318+ktcChx+F1Dd91G3YHeMKyg==" + }, "through": { "version": "2.3.8", "resolved": "http://registry.npmjs.org/through/-/through-2.3.8.tgz", diff --git a/superset/assets/package.json b/superset/assets/package.json index 52e82b12e01a..cccc2de63cac 100644 --- a/superset/assets/package.json +++ b/superset/assets/package.json @@ -127,6 +127,7 @@ "react-select-fast-filter-options": "^0.2.1", "react-sortable-hoc": "^0.8.3", "react-split": "^2.0.4", + "react-stack-grid": "^0.7.1", "react-sticky": "^6.0.2", "react-syntax-highlighter": "^7.0.4", "react-transition-group": "^2.5.3", diff --git a/superset/assets/spec/javascripts/welcome/DashboardTable_spec.jsx b/superset/assets/spec/javascripts/welcome/DashboardTable_spec.jsx index e989f9a207ae..28c1adda9e2c 100644 --- a/superset/assets/spec/javascripts/welcome/DashboardTable_spec.jsx +++ b/superset/assets/spec/javascripts/welcome/DashboardTable_spec.jsx @@ -23,7 +23,7 @@ import configureStore from 'redux-mock-store'; import fetchMock from 'fetch-mock'; import { Table } from 'reactable-arc'; -import DashboardTable from '../../../src/welcome/DashboardTable'; +import DashboardCardTable from '../../../src/welcome/DashboardCardTable'; import Loading from '../../../src/components/Loading'; // store needed for withToasts(TableLoader) @@ -39,10 +39,10 @@ fetchMock.get(dashboardsEndpoint, { result: mockDashboards }); function setup() { // use mount because data fetching is triggered on mount - return mount(, { context: { store } }); + return mount(, { context: { store } }); } -describe('DashboardTable', () => { +describe('DashboardCardTable', () => { beforeEach(fetchMock.resetHistory); it('renders a Loading initially', () => { diff --git a/superset/assets/spec/javascripts/welcome/Welcome_spec.jsx b/superset/assets/spec/javascripts/welcome/Welcome_spec.jsx index 8a18b299955e..a83728264b38 100644 --- a/superset/assets/spec/javascripts/welcome/Welcome_spec.jsx +++ b/superset/assets/spec/javascripts/welcome/Welcome_spec.jsx @@ -33,6 +33,6 @@ describe('Welcome', () => { const wrapper = shallow(); expect(wrapper.find(Tab)).toHaveLength(3); expect(wrapper.find(Panel)).toHaveLength(3); - expect(wrapper.find(Row)).toHaveLength(3); + expect(wrapper.find(Row)).toHaveLength(2); }); }); diff --git a/superset/assets/src/components/Card.css b/superset/assets/src/components/Card.css new file mode 100644 index 000000000000..b70e828ac75b --- /dev/null +++ b/superset/assets/src/components/Card.css @@ -0,0 +1,58 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +.card { + background-color: white; + border-radius: 5px; + box-shadow: 0 1px 1px 0 rgba(60,64,67,.16), 0 1px 3px 1px rgba(60,64,67,.25); + overflow: hidden; + transition: box-shadow 0.5s ease-in-out; +} +.card:hover { + box-shadow: 0 2px 2px 0 rgba(60,64,67,.30), 0 2px 4px 2px rgba(60,64,67,.55); +} +.card-header { + padding-top: 5px; + padding-left: 15px; + padding-right: 15px; + padding-bottom: 0px; +} +.card-body { + padding-top: 5px; + padding-left: 15px; + padding-right: 15px; + padding-bottom: 15px; +} +.card-body .text-muted{ + color: #888; +} + +.card-header .fa { + margin-top: 10px; + padding-left: 5px; + padding-right: 5px; +} + +.card img { + border-top: 1px solid #EEE; + border-bottom: 1px solid #EEE; +} +.card-title { + white-space: nowrap; + width: 100px; +} diff --git a/superset/assets/src/components/Card.jsx b/superset/assets/src/components/Card.jsx new file mode 100644 index 000000000000..ea7d7e65b632 --- /dev/null +++ b/superset/assets/src/components/Card.jsx @@ -0,0 +1,102 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import React from 'react'; +import PropTypes from 'prop-types'; +import { Dropdown, Fade } from 'react-bootstrap'; + +import ToggleWrapper from './ToggleWrapper'; +import './Card.css'; + +const propTypes = { + title: PropTypes.string.isRequired, + body: PropTypes.node.isRequired, + imageSource: PropTypes.string.isRequired, + cardWidth: PropTypes.number, + imageAspectRatio: PropTypes.number, + dropdownMenu: PropTypes.node.isRequired, + onTitleClick: PropTypes.func, +}; +const defaultProps = { + onTitleClick: () => {}, + imageAspectRatio: 4/3, +}; + +export default class Card extends React.PureComponent { + constructor(props) { + super(props); + this.state = { + hovered: false, + }; + } + renderActionDropdown() { + return ( + + + + + + + {this.props.dropdownMenu} + ); + } + render() { + const { + body, + cardWidth, + imageAspectRatio, + imageSource, + onTitleClick, + title, + } = this.props; + const imgHeight = cardWidth / imageAspectRatio; + return ( +
this.setState({ hovered: true })} + onMouseOut={() => this.setState({ hovered: false })} + > +
+
onTitleClick()} + > +
{title}
+
+
+ {this.renderActionDropdown()} +
+
+
onTitleClick()}> + +
+
+ {body} +
+
+ ); + } +} +Card.propTypes = propTypes; +Card.defaultProps = defaultProps; diff --git a/superset/assets/src/components/CardTable.css b/superset/assets/src/components/CardTable.css new file mode 100644 index 000000000000..8c7fb0cb6588 --- /dev/null +++ b/superset/assets/src/components/CardTable.css @@ -0,0 +1,45 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +.CardTable { + min-height: 500px; +} +.CardTable .input-search { + margin-top: 25px; + width: 300px; +} +.CardTable .card-table-toggle .fa { + margin-top: .2em; + margin-left: 0; +} +.CardTable .controls-right { + margin-top: 25px; +} + +.table .td-thumb img { + float: left; + width: 100px; +} + +.table .td-thumb { + padding: 0px; +} + +.table .td-thumb span { + padding: 8px; +} diff --git a/superset/assets/src/components/CardTable.jsx b/superset/assets/src/components/CardTable.jsx new file mode 100644 index 000000000000..53425580e34d --- /dev/null +++ b/superset/assets/src/components/CardTable.jsx @@ -0,0 +1,128 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import React from 'react'; +import PropTypes from 'prop-types'; +import { Button, ButtonGroup, FormControl } from 'react-bootstrap'; +import StackGrid from 'react-stack-grid'; +import { t } from '@superset-ui/translation'; + +import Loading from '../components/Loading'; +import withToasts from '../messageToasts/enhancers/withToasts'; +import './CardTable.css'; + +const propTypes = { + renderTable: PropTypes.func.isRequired, + renderCards: PropTypes.func.isRequired, + cardWidth: PropTypes.number, + onSearchChange: PropTypes.func.isRequired, + title: PropTypes.string.isRequired, + addDangerToast: PropTypes.func.isRequired, + items: PropTypes.array, + loading: PropTypes.bool.isRequired, + showCardCount: PropTypes.number, + emptyMessage: PropTypes.node, +}; + +const defaultProps = { + cardWidth: 250, + emptyMessage: t('No data'), +}; + +class CardTable extends React.PureComponent { + constructor(props) { + super(props); + this.state = { + search: '', + showTable: false, + }; + this.onSearchChange = this.onSearchChange.bind(this); + } + onSearchChange(event) { + const search = event.target.value; + this.setState({ search }); + this.props.onSearchChange(search); + } + renderCards() { + return ( + + {this.props.renderCards()} + ); + } + renderList() { + if (this.state.showTable) { + return this.props.renderTable(this.props.items); + } + return this.renderCards(); + } + render() { + const { showTable } = this.state; + const { loading, items, emptyMessage } = this.props; + if (loading) { + return ; + } + if (items && items.length === 0) { + return emptyMessage; + } + return ( +
+
+
+

{this.props.title}

+
+
+ + + + +
+
+ +
+
+
+ {this.renderList()} +
); + } +} +CardTable.propTypes = propTypes; +export default withToasts(CardTable); diff --git a/superset/assets/src/components/ChartCard.jsx b/superset/assets/src/components/ChartCard.jsx new file mode 100644 index 000000000000..299f219ec007 --- /dev/null +++ b/superset/assets/src/components/ChartCard.jsx @@ -0,0 +1,104 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import React from 'react'; +import PropTypes from 'prop-types'; +import { Dropdown, MenuItem } from 'react-bootstrap'; +import { t } from '@superset-ui/translation'; +import Dialog from 'react-bootstrap-dialog'; + +import Card from './Card.jsx'; + +const propTypes = { + chart: PropTypes.object.isRequired, + onDelete: PropTypes.func.isRequired, + cardWidth: PropTypes.number.isRequired, +}; + + +export default class ChartCard extends React.PureComponent { + constructor(props) { + super(props); + this.openChart = this.openChart.bind(this); + this.deleteDialog = this.deleteDialog.bind(this); + } + openChart() { + window.open(this.props.chart.slice_url); + } + deleteDialog() { + const onOk = () => { + this.props.onDelete(this.props.chart); + this.dialog.hide(); + }; + this.dialog.show({ + title: t('Delete Chart'), + body: t('Are you sure you want to delete this chart?'), + actions: [ + Dialog.CancelAction(diag => diag.hide()), + Dialog.DefaultAction('Ok', onOk, 'btn-danger'), + ], + bsSize: 'small', + onHide: (dialog) => { + dialog.hide(); + }, + }); + } + renderDropdownMenu() { + const { chart } = this.props; + return ( + + + {t('Edit chart metadata')} + + + {t('Delete Chart')} + + + ); + } + renderCardBody() { + const { chart } = this.props; + return ( + + Modified {chart.changed_on_humanized} +
+ + {t('Created by')} {chart.created_by_name || t('N/A')} + + { + this.dialog = el; + }} + /> + + ); + } + render() { + const { chart } = this.props; + return ( + ); + } +} +ChartCard.propTypes = propTypes; diff --git a/superset/assets/src/components/DashboardCard.jsx b/superset/assets/src/components/DashboardCard.jsx new file mode 100644 index 000000000000..3f883213d00a --- /dev/null +++ b/superset/assets/src/components/DashboardCard.jsx @@ -0,0 +1,103 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import React from 'react'; +import PropTypes from 'prop-types'; +import { Dropdown, MenuItem } from 'react-bootstrap'; +import { t } from '@superset-ui/translation'; +import Dialog from 'react-bootstrap-dialog'; + +import Card from './Card.jsx'; + +const propTypes = { + cardWidth: PropTypes.number.isRequired, + dashboard: PropTypes.object.isRequired, +}; + + +export default class DashboardCard extends React.PureComponent { + constructor(props) { + super(props); + this.openDashboard = this.openDashboard.bind(this); + this.deleteDialog = this.deleteDialog.bind(this); + } + openDashboard() { + window.open(this.props.dashboard.url); + } + deleteDialog() { + const onOk = () => { + this.props.onDelete(this.props.dashboard); + this.dialog.hide(); + }; + this.dialog.show({ + title: t('Delete dashboard'), + body: t('Are you sure you want to delete this dashboard?'), + actions: [ + Dialog.CancelAction(diag => diag.hide()), + Dialog.DefaultAction('Ok', onOk, 'btn-danger'), + ], + bsSize: 'small', + onHide: (dialog) => { + dialog.hide(); + }, + }); + } + renderDropdownMenu() { + const { dashboard } = this.props; + return ( + + + {t('Edit dashboard metadata')} + + + {t('Delete Dashboard')} + + + ); + } + renderCardBody() { + const { dashboard } = this.props; + return ( + + Modified {dashboard.changed_on_humanized} +
+ + {t('Created by')} {dashboard.created_by_name || t('N/A')} + + { + this.dialog = el; + }} + /> + + ); + } + render() { + const { dashboard } = this.props; + return ( + ); + } +} +DashboardCard.propTypes = propTypes; diff --git a/superset/assets/src/components/ToggleWrapper.jsx b/superset/assets/src/components/ToggleWrapper.jsx new file mode 100644 index 000000000000..b1ec6ee7c53f --- /dev/null +++ b/superset/assets/src/components/ToggleWrapper.jsx @@ -0,0 +1,53 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/** + * Allows any component to act as a Dropdown toggle + * for instance a simple . + * inspired from https://react-bootstrap.github.io/components/dropdowns/ + * "Custom Dropdown Components" section + */ +import React from 'react'; +import PropTypes from 'prop-types'; + +const propTypes = { + onClick: PropTypes.func, + children: PropTypes.node, +}; +export default class ToggleWrapper extends React.Component { + constructor(props, context) { + super(props, context); + + this.handleClick = this.handleClick.bind(this); + } + + handleClick(e) { + e.preventDefault(); + this.props.onClick(e); + } + + render() { + return ( + + {this.props.children} + + ); + } +} +ToggleWrapper.propTypes = propTypes; diff --git a/superset/assets/src/dashboard/components/MissingChart.jsx b/superset/assets/src/dashboard/components/MissingChart.jsx index 34d3d2da8c16..65e2be37bc69 100644 --- a/superset/assets/src/dashboard/components/MissingChart.jsx +++ b/superset/assets/src/dashboard/components/MissingChart.jsx @@ -29,9 +29,9 @@ const propTypes = { export default function MissingChart({ height }) { return (
-
- -
+
+ +
{t( 'There is no chart definition associated with this component, could it have been deleted?', diff --git a/superset/assets/src/welcome/ChartCardTable.jsx b/superset/assets/src/welcome/ChartCardTable.jsx new file mode 100644 index 000000000000..1acc07c87833 --- /dev/null +++ b/superset/assets/src/welcome/ChartCardTable.jsx @@ -0,0 +1,177 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import React from 'react'; +import PropTypes from 'prop-types'; +import { t } from '@superset-ui/translation'; +import { Table, Tr, Td, unsafe } from 'reactable-arc'; +import { SupersetClient } from '@superset-ui/connection'; + +import withToasts from '../messageToasts/enhancers/withToasts'; +import CardTable from '../components/CardTable'; +import ChartCard from '../components/ChartCard'; +import '../../stylesheets/reactable-pagination.css'; + +const propTypes = { + showTable: PropTypes.bool, + addDangerToast: PropTypes.func.isRequired, +}; + +const CHART_WIDTH = 250; + +class ChartCardTable extends React.PureComponent { + constructor(props) { + super(props); + this.state = { + search: '', + loading: true, + charts: null, + filteredCharts: null, + }; + this.renderTable = this.renderTable.bind(this); + this.onSearchChange = this.onSearchChange.bind(this); + this.renderCards = this.renderCards.bind(this); + this.deleteChart = this.deleteChart.bind(this); + } + componentDidMount() { + const endpoint = ( + '/sliceasync/api/read?' + + '_oc_ChartModelViewAsync=changed_on' + + '&_od_ChartModelViewAsync=desc' + ); + SupersetClient.get({ endpoint }) + .then(({ json }) => { + const charts = json.result; + this.setState({ + charts, + loading: false, + filteredCharts: charts, + }); + }) + .catch(() => { + this.props.addDangerToast(t('An error occurred while fetching data')); + this.setState({ charts: null, loading: false }); + }); + } + deleteChart(chart) { + const endpoint = `/chart/api/delete/${chart.id}`; + SupersetClient.delete({ endpoint }) + .then(({ json }) => { + this.removeChart(chart); + }) + .catch(() => { + this.props.addDangerToast(t('An error occurred while deleting the chart')); + }); + } + onSearchChange(search) { + const filteredCharts = this.filterCharts(search); + this.setState({ search, filteredCharts }); + } + removeChart(chart) { + const filter = o => o.id !== chart.id; + const charts = this.state.charts.filter(filter); + const filteredCharts = this.state.filteredCharts.filter(filter); + this.setState({ charts, filteredCharts }); + } + getDatasourceName(chart) { + const o = chart.datasource_data_summary; + return o.schema ? `${o.schema}.${o.datasource_name}` : o.datasource_name; + } + filterCharts(searchText) { + const { charts } = this.state; + if (!searchText) { + return charts; + } + const lcaseSearchText = searchText.toLowerCase(); + return charts.filter(o => o.slice_name.toLowerCase().indexOf(lcaseSearchText) >= 0); + } + renderDatasourceLink(chart) { + const o = chart.datasource_data_summary; + const url = o.explore_url; + const name = this.getDatasourceName(chart); + return ( + {name} + ); + } + renderTable() { + const { filteredCharts } = this.state; + return ( + + {filteredCharts.map(o => ( + + + + + + ))} +
+ + + {o.slice_name} + + + {this.renderDatasourceLink(o)} + + {unsafe(o.creator)} + + {unsafe(o.modified)} +
+ ); + } + renderCards() { + return this.state.filteredCharts.map(chart => ( + + )); + } + + render() { + return ( + + ); + } +} + +ChartCardTable.propTypes = propTypes; +export default withToasts(ChartCardTable); diff --git a/superset/assets/src/welcome/DashboardCardTable.jsx b/superset/assets/src/welcome/DashboardCardTable.jsx new file mode 100644 index 000000000000..9dc2b08344f4 --- /dev/null +++ b/superset/assets/src/welcome/DashboardCardTable.jsx @@ -0,0 +1,159 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import React from 'react'; +import PropTypes from 'prop-types'; +import { t } from '@superset-ui/translation'; +import { Table, Tr, Td, unsafe } from 'reactable-arc'; +import { SupersetClient } from '@superset-ui/connection'; + +import withToasts from '../messageToasts/enhancers/withToasts'; +import CardTable from '../components/CardTable'; +import DashboardCard from '../components/DashboardCard'; +import '../../stylesheets/reactable-pagination.css'; + +const propTypes = { + showTable: PropTypes.bool, + addDangerToast: PropTypes.func.isRequired, +}; + +const CARD_WIDTH = 250; + +class DashboardCardTable extends React.PureComponent { + constructor(props) { + super(props); + this.state = { + search: '', + dashboards: null, + loading: true, + filteredDashboards: null, + }; + this.renderTable = this.renderTable.bind(this); + this.onSearchChange = this.onSearchChange.bind(this); + this.onDelete = this.onDelete.bind(this); + this.renderCards = this.renderCards.bind(this); + this.deleteDashboard = this.deleteDashboard.bind(this); + } + componentDidMount() { + const endpoint = ( + '/dashboardasync/api/read?' + + '_oc_DashboardModelViewAsync=changed_on' + + '&_od_DashboardModelViewAsync=desc' + ); + SupersetClient.get({ endpoint }) + .then(({ json }) => { + const dashboards = json.result; + this.setState({ + dashboards, + loading: false, + filteredDashboards: dashboards, + }); + }) + .catch(() => { + this.props.addDangerToast(t('An error occurred while fetching data')); + this.setState({ dashboards: null, loading: false }); + }); + } + onSearchChange(search) { + const filteredDashboards = this.filterDashboards(search); + this.setState({ search, filteredDashboards }); + } + onDelete(dashboard) { + const filter = o => o.id !== dashboard.id; + const dashboards = this.state.dashboards.filter(filter); + const filteredDashboards = this.state.filteredDashboards.filter(filter); + this.setState({ dashboards, filteredDashboards }); + } + deleteDashboard(dashboard) { + const endpoint = `/dashboard/api/delete/${dashboard.id}`; + SupersetClient.delete({ endpoint }) + .then(({ json }) => { + this.onDelete(dashboard); + }) + .catch(() => { + this.props.addDangerToast(t('An error occurred while deleting the dashboard')); + }); + } + filterDashboards(searchText) { + const { dashboards } = this.state; + if (!searchText) { + return dashboards; + } + const lcaseSearchText = searchText.toLowerCase(); + return dashboards.filter(o => o.dashboard_title.toLowerCase().indexOf(lcaseSearchText) >= 0); + } + renderTable() { + return ( + + {this.state.filteredDashboards.map(o => ( + + + + + ))} +
+ + + {o.dashboard_title} + + + {unsafe(o.creator)} + + {unsafe(o.modified)} +
+ ); + } + renderCards() { + return this.state.filteredDashboards.map(dashboard => ( + + )); + } + + render() { + return ( + + ); + } +} + +DashboardCardTable.propTypes = propTypes; +export default withToasts(DashboardCardTable); diff --git a/superset/assets/src/welcome/DashboardTable.jsx b/superset/assets/src/welcome/DashboardTable.jsx deleted file mode 100644 index a3d689aec932..000000000000 --- a/superset/assets/src/welcome/DashboardTable.jsx +++ /dev/null @@ -1,93 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -import React from 'react'; -import PropTypes from 'prop-types'; -import { Table, Tr, Td, unsafe } from 'reactable-arc'; -import { SupersetClient } from '@superset-ui/connection'; -import { t } from '@superset-ui/translation'; - -import withToasts from '../messageToasts/enhancers/withToasts'; -import Loading from '../components/Loading'; -import '../../stylesheets/reactable-pagination.css'; - -const propTypes = { - search: PropTypes.string, - addDangerToast: PropTypes.func.isRequired, -}; - -class DashboardTable extends React.PureComponent { - constructor(props) { - super(props); - this.state = { - dashboards: [], - }; - } - - componentDidMount() { - SupersetClient.get({ - endpoint: '/dashboardasync/api/read?_oc_DashboardModelViewAsync=changed_on&_od_DashboardModelViewAsync=desc', - }) - .then(({ json }) => { - this.setState({ dashboards: json.result }); - }) - .catch(() => { - this.props.addDangerToast(t('An error occurred while fethching Dashboards')); - }); - } - - render() { - if (this.state.dashboards.length > 0) { - return ( - - {this.state.dashboards.map(o => ( - - - - - ))} -
- {o.dashboard_title} - - {unsafe(o.creator)} - - {unsafe(o.modified)} -
- ); - } - - return ; - } -} - -DashboardTable.propTypes = propTypes; - -export default withToasts(DashboardTable); diff --git a/superset/assets/src/welcome/Welcome.jsx b/superset/assets/src/welcome/Welcome.jsx index db1a632fd5f3..a1ca7dc6c698 100644 --- a/superset/assets/src/welcome/Welcome.jsx +++ b/superset/assets/src/welcome/Welcome.jsx @@ -18,51 +18,33 @@ */ import React from 'react'; import PropTypes from 'prop-types'; -import { Panel, Row, Col, Tabs, Tab, FormControl } from 'react-bootstrap'; +import { Panel, Row, Col, Tabs, Tab } from 'react-bootstrap'; import { t } from '@superset-ui/translation'; import RecentActivity from '../profile/components/RecentActivity'; import Favorites from '../profile/components/Favorites'; -import DashboardTable from './DashboardTable'; +import DashboardCardTable from './DashboardCardTable'; +import ChartCardTable from './ChartCardTable'; const propTypes = { user: PropTypes.object.isRequired, }; export default class Welcome extends React.PureComponent { - constructor(props) { - super(props); - this.state = { - search: '', - }; - this.onSearchChange = this.onSearchChange.bind(this); - } - onSearchChange(event) { - this.setState({ search: event.target.value }); - } render() { return (
- -

{t('Dashboards')}

- - - - -
- + + + + + + - +

{t('Recently Viewed')}

@@ -71,7 +53,7 @@ export default class Welcome extends React.PureComponent { - +

{t('Favorites')}

diff --git a/superset/assets/stylesheets/less/cosmo/variables.less b/superset/assets/stylesheets/less/cosmo/variables.less index 7b979c224525..dc61b52cbb13 100644 --- a/superset/assets/stylesheets/less/cosmo/variables.less +++ b/superset/assets/stylesheets/less/cosmo/variables.less @@ -29,8 +29,8 @@ @gray-darker: lighten(@gray-base, 13.5%); @gray-dark: lighten(@gray-base, 20%); @gray: lighten(@gray-base, 33.5%); -@gray-light: lighten(@gray-base, 70%); -@gray-lighter: lighten(@gray-base, 95%); +@gray-light: lighten(@gray-base, 80%); +@gray-lighter: lighten(@gray-base, 90%); @brand-primary: #00A699; @brand-success: #4AC15F; diff --git a/superset/assets/stylesheets/superset.less b/superset/assets/stylesheets/superset.less index d25dc6b73ccf..de9bb8a6c1b1 100644 --- a/superset/assets/stylesheets/superset.less +++ b/superset/assets/stylesheets/superset.less @@ -348,6 +348,9 @@ table.table-no-hover tr:hover { .m-t-5 { margin-top: 5px; } +.m-t-8 { + margin-top: 8px; +} .m-t-10 { margin-top: 10px; } diff --git a/superset/cli.py b/superset/cli.py index 6691b0148f8f..0e58dae969c3 100755 --- a/superset/cli.py +++ b/superset/cli.py @@ -29,6 +29,8 @@ from superset import ( app, appbuilder, data, db, security_manager, ) +from superset.models import core as models +from superset.tasks.thumbnails import cache_chart_thumbnail, cache_dashboard_thumbnail from superset.utils import ( core as utils, dashboard_import_export, dict_import_export) @@ -347,6 +349,29 @@ def flower(port, address): Popen(cmd, shell=True).wait() +''' +@app.cli.command() +def test_email(): + """Test that email is configured""" + from superset.utils.email import send_email_smtp + send_email_smtp( + to='max@preset.io', + subject='it works!', + html_content='

SUPER

', + config=app.config, + ) + from superset.tasks.schedules import deliver_dashboard, deliver_slice + from superset.models.schedules import EmailDeliveryType, SliceEmailReportFormat + #dashboard = db.session.query(models.Dashboard).filter_by(id=2).one() + #deliver_dashboard(dashboard, [('max@preset.io', None)], EmailDeliveryType.inline) + + slc = db.session.query(models.Slice).filter_by(id=76).one() + deliver_slice( + slc, [('max@preset.io', None)], + SliceEmailReportFormat.visualization, EmailDeliveryType.inline) +''' + + @app.cli.command() def load_test_users(): """ @@ -358,6 +383,57 @@ def load_test_users(): load_test_users_run() +@app.cli.command() +@click.option( + '--asynchronous', '-a', is_flag=True, default=False, + help='Trigger commands to run remotely on a worker') +@click.option( + '--dashboards_only', '-d', is_flag=True, default=False, + help='Only process dashboards') +@click.option( + '--charts_only', '-c', is_flag=True, default=False, + help='Only process charts') +@click.option( + '--force', '-f', is_flag=True, default=False, + help='Force refresh, even if previously cached') +@click.option('--id', '-i', multiple=True) +def compute_thumbnails(asynchronous, dashboards_only, charts_only, force, id): + """Compute thumbnails""" + if not charts_only: + query = db.session.query(models.Dashboard) + if id: + query = query.filter(models.Dashboard.id.in_(id)) + dashboards = query.all() + count = len(dashboards) + for i, dash in enumerate(dashboards): + if asynchronous: + func = cache_dashboard_thumbnail.delay + action = 'Triggering' + else: + func = cache_dashboard_thumbnail + action = 'Processing' + msg = f'{action} dashboard "{dash.dashboard_title}" ({i+1}/{count})' + click.secho(msg, fg='green') + func(dash.id, force=force) + + if not dashboards_only: + query = db.session.query(models.Slice) + if id: + query = query.filter(models.Slice.id.in_(id)) + slices = query.all() + count = len(slices) + for i, slc in enumerate(slices): + if asynchronous: + func = cache_chart_thumbnail.delay + action = 'Triggering' + else: + func = cache_chart_thumbnail + action = 'Processing' + msg = f'{action} chart "{slc.slice_name}" ({i+1}/{count})' + click.secho(msg, fg='green') + func(slc.id, force=force) + + def load_test_users_run(): """ Loads admin, alpha, and gamma user for testing purposes diff --git a/superset/config.py b/superset/config.py index 7d6ca33e5f9a..0fc768237d43 100644 --- a/superset/config.py +++ b/superset/config.py @@ -238,6 +238,7 @@ CACHE_DEFAULT_TIMEOUT = 60 * 60 * 24 CACHE_CONFIG = {'CACHE_TYPE': 'null'} TABLE_NAMES_CACHE_CONFIG = {'CACHE_TYPE': 'null'} +THUMBNAIL_CACHE_CONFIG = {'CACHE_TYPE': 'null'} # CORS Options ENABLE_CORS = False @@ -370,6 +371,8 @@ # you'll want to use a proper broker as specified here: # http://docs.celeryproject.org/en/latest/getting-started/brokers/index.html +CELERYD_LOG_LEVEL = 'DEBUG' + class CeleryConfig(object): BROKER_URL = 'sqla+sqlite:///celerydb.sqlite' @@ -584,7 +587,9 @@ class CeleryConfig(object): # Window size - this will impact the rendering of the data WEBDRIVER_WINDOW = { 'dashboard': (1600, 2000), - 'slice': (3000, 1200), + 'slice': (800, 600), + 'thumbnail_chart': (800, 600), + 'thumbnail_dashboard': (800, 600), } # Any config options to be passed as-is to the webdriver diff --git a/superset/connectors/base/models.py b/superset/connectors/base/models.py index 57db0ad4684d..cccd87cf8a05 100644 --- a/superset/connectors/base/models.py +++ b/superset/connectors/base/models.py @@ -162,6 +162,16 @@ def short_data(self): def select_star(self): pass + @property + def data_summary(self): + return { + 'datasource_name': self.datasource_name, + 'type': self.type, + 'schema': self.schema, + 'id': self.id, + 'explore_url': self.explore_url, + } + @property def data(self): """Data representation of the datasource sent to the frontend""" diff --git a/superset/models/core.py b/superset/models/core.py index b379af7caba2..b527f97248de 100644 --- a/superset/models/core.py +++ b/superset/models/core.py @@ -51,6 +51,7 @@ from superset.models.helpers import AuditMixinNullable, ImportMixin from superset.models.tags import ChartUpdater, DashboardUpdater, FavStarUpdater from superset.models.user_attributes import UserAttribute +from superset.tasks.thumbnails import cache_chart_thumbnail, cache_dashboard_thumbnail from superset.utils import ( cache as cache_util, core as utils, @@ -175,6 +176,28 @@ def cls_model(self): def datasource(self): return self.get_datasource + @property + def datasource_data_summary(self): + return self.datasource.data_summary + + @property + def thumbnail_url(self): + # SHA here is to force bypassing the browser cache when chart has changed + sha = utils.md5_hex(self.params, 6) + return f'/thumb/chart/{self.id}/{sha}/' + + @property + def thumbnail_img(self): + return Markup(f'') + + @property + def thumbnail_link(self): + return Markup(f""" + + {self.thumbnail_img} + + """) + def clone(self): return Slice( slice_name=self.slice_name, @@ -247,6 +270,7 @@ def data(self): 'modified': self.modified(), 'changed_on_humanized': self.changed_on_humanized, 'changed_on': self.changed_on.isoformat(), + 'thumbnail_url': self.thumbnail_url, } @property @@ -375,8 +399,13 @@ def url(self): ) -sqla.event.listen(Slice, 'before_insert', set_related_perm) -sqla.event.listen(Slice, 'before_update', set_related_perm) +def event_after_slice_changed(mapper, connection, target): + set_related_perm(mapper, connection, target) + cache_chart_thumbnail.delay(target.id, force=True) + + +sqla.event.listen(Slice, 'before_insert', event_after_slice_changed) +sqla.event.listen(Slice, 'before_update', event_after_slice_changed) dashboard_slices = Table( @@ -469,6 +498,7 @@ def data(self): 'dashboard_title': self.dashboard_title, 'slug': self.slug, 'slices': [slc.data for slc in self.slices], + 'thumbnail_url': self.thumbnail_url, 'position_json': positions, } @@ -657,6 +687,32 @@ def export_dashboards(cls, dashboard_ids): 'datasources': eager_datasources, }, cls=utils.DashboardEncoder, indent=4) + @property + def thumbnail_url(self): + # SHA here is to force bypassing the browser cache when chart has changed + sha = utils.md5_hex(self.position_json, 6) + return f'/thumb/dashboard/{self.id}/{sha}/' + + @property + def thumbnail_img(self): + return Markup(f'') + + @property + def thumbnail_link(self): + return Markup(f""" + + {self.thumbnail_img} + + """) + + +def event_after_dashboard_changed(mapper, connection, target): + cache_dashboard_thumbnail.delay(target.id, force=True) + + +sqla.event.listen(Dashboard, 'before_insert', event_after_dashboard_changed) +sqla.event.listen(Dashboard, 'before_update', event_after_dashboard_changed) + class Database(Model, AuditMixinNullable, ImportMixin): diff --git a/superset/models/helpers.py b/superset/models/helpers.py index 78b438d9f8f1..fd8d2f6dabc4 100644 --- a/superset/models/helpers.py +++ b/superset/models/helpers.py @@ -272,8 +272,13 @@ def _user_link(self, user): return Markup('{}'.format(url, escape(user) or '')) def changed_by_name(self): - if self.created_by: - return escape('{}'.format(self.created_by)) + if self.changed_by: + return escape('{}'.format(self.changed_by)) + return '' + + def created_by_name(self): + if self.changed_by: + return escape('{}'.format(self.changed_by)) return '' @renders('created_by') diff --git a/superset/tasks/cache.py b/superset/tasks/cache.py index 831bb6642914..acd3caddbd72 100644 --- a/superset/tasks/cache.py +++ b/superset/tasks/cache.py @@ -27,7 +27,6 @@ from sqlalchemy import and_, func from superset import app, db -from superset.models.core import Dashboard, Log, Slice from superset.models.tags import Tag, TaggedObject from superset.tasks.celery_app import app as celery_app from superset.utils.core import parse_human_datetime @@ -135,6 +134,7 @@ class DummyStrategy(Strategy): def get_urls(self): session = db.create_scoped_session() + from superset.models.core import Slice charts = session.query(Slice).all() return [get_url({'form_data': get_form_data(chart.id)}) for chart in charts] @@ -169,6 +169,7 @@ def get_urls(self): urls = [] session = db.create_scoped_session() + from superset.models.core import Dashboard, Log records = ( session .query(Log.dashboard_id, func.count(Log.dashboard_id)) @@ -240,6 +241,7 @@ def get_urls(self): )) .all() ) + from superset.models.core import Dashboard dash_ids = [tagged_object.object_id for tagged_object in tagged_objects] tagged_dashboards = ( session @@ -262,6 +264,7 @@ def get_urls(self): .all() ) chart_ids = [tagged_object.object_id for tagged_object in tagged_objects] + from superset.models.core import Slice tagged_charts = ( session .query(Slice) diff --git a/superset/tasks/schedules.py b/superset/tasks/schedules.py index b4ca9f41bdc0..b92d6083153b 100644 --- a/superset/tasks/schedules.py +++ b/superset/tasks/schedules.py @@ -22,23 +22,15 @@ from datetime import datetime, timedelta from email.utils import make_msgid, parseaddr import logging -import time - import croniter from dateutil.tz import tzlocal -from flask import render_template, Response, session, url_for +from flask import render_template, url_for from flask_babel import gettext as __ -from flask_login import login_user import requests -from retry.api import retry_call -from selenium.common.exceptions import WebDriverException -from selenium.webdriver import chrome, firefox import simplejson as json from six.moves import urllib -from werkzeug.utils import parse_cookie -# Superset framework imports from superset import app, db, security_manager from superset.models.schedules import ( EmailDeliveryType, @@ -47,47 +39,48 @@ SliceEmailReportFormat, ) from superset.tasks.celery_app import app as celery_app -from superset.utils.core import ( - get_email_address_list, - send_email_smtp, +from superset.utils import core as utils +from superset.utils.selenium import ( + DashboardScreenshot, + get_auth_cookies, + SliceScreenshot, ) # Globals -config = app.config logging.getLogger('tasks.email_reports').setLevel(logging.INFO) -# Time in seconds, we will wait for the page to load and render -PAGE_RENDER_WAIT = 30 - - EmailContent = namedtuple('EmailContent', ['body', 'data', 'images']) def _get_recipients(schedule): - bcc = config.get('EMAIL_REPORT_BCC_ADDRESS', None) + bcc = app.config.get('EMAIL_REPORT_BCC_ADDRESS', None) if schedule.deliver_as_group: to = schedule.recipients yield (to, bcc) else: - for to in get_email_address_list(schedule.recipients): + for to in utils.get_email_address_list(schedule.recipients): yield (to, bcc) -def _deliver_email(schedule, subject, email): - for (to, bcc) in _get_recipients(schedule): - send_email_smtp( - to, subject, email.body, config, +def _deliver_email(recipients, subject, email): + config = app.config + dryrun = config.get('SCHEDULED_EMAIL_DEBUG_MODE') + for (to, bcc) in recipients: + utils.send_email_smtp( + to, subject, email.body, + config, data=email.data, images=email.images, bcc=bcc, mime_subtype='related', - dryrun=config.get('SCHEDULED_EMAIL_DEBUG_MODE'), + dryrun=dryrun, ) -def _generate_mail_content(schedule, screenshot, name, url): - if schedule.delivery_type == EmailDeliveryType.attachment: +def _generate_mail_content(delivery_type, screenshot, name, url): + config = app.config + if delivery_type == EmailDeliveryType.attachment: images = None data = { 'screenshot.png': screenshot, @@ -97,7 +90,9 @@ def _generate_mail_content(schedule, screenshot, name, url): name=name, url=url, ) - elif schedule.delivery_type == EmailDeliveryType.inline: + else: + # Implicit: delivery_type == EmailDeliveryType.inline: + # Get the domain from the 'From' address .. # and make a message id without the < > in the ends domain = parseaddr(config.get('SMTP_MAIL_FROM'))[1].split('@')[1] @@ -118,135 +113,33 @@ def _generate_mail_content(schedule, screenshot, name, url): return EmailContent(body, data, images) -def _get_auth_cookies(): - # Login with the user specified to get the reports - with app.test_request_context(): - user = security_manager.find_user(config.get('EMAIL_REPORTS_USER')) - login_user(user) - - # A mock response object to get the cookie information from - response = Response() - app.session_interface.save_session(app, session, response) - - cookies = [] - - # Set the cookies in the driver - for name, value in response.headers: - if name.lower() == 'set-cookie': - cookie = parse_cookie(value) - cookies.append(cookie['session']) - - return cookies - - def _get_url_path(view, **kwargs): with app.test_request_context(): return urllib.parse.urljoin( - str(config.get('WEBDRIVER_BASEURL')), + str(app.config.get('WEBDRIVER_BASEURL')), url_for(view, **kwargs), ) -def create_webdriver(): - # Create a webdriver for use in fetching reports - if config.get('EMAIL_REPORTS_WEBDRIVER') == 'firefox': - driver_class = firefox.webdriver.WebDriver - options = firefox.options.Options() - elif config.get('EMAIL_REPORTS_WEBDRIVER') == 'chrome': - driver_class = chrome.webdriver.WebDriver - options = chrome.options.Options() - - options.add_argument('--headless') - - # Prepare args for the webdriver init - kwargs = dict( - options=options, - ) - kwargs.update(config.get('WEBDRIVER_CONFIGURATION')) - - # Initialize the driver - driver = driver_class(**kwargs) - - # Some webdrivers need an initial hit to the welcome URL - # before we set the cookie - welcome_url = _get_url_path('Superset.welcome') - - # Hit the welcome URL and check if we were asked to login - driver.get(welcome_url) - elements = driver.find_elements_by_id('loginbox') - - # This indicates that we were not prompted for a login box. - if not elements: - return driver - - # Set the cookies in the driver - for cookie in _get_auth_cookies(): - info = dict(name='session', value=cookie) - driver.add_cookie(info) - - return driver - - -def destroy_webdriver(driver): - """ - Destroy a driver - """ - - # This is some very flaky code in selenium. Hence the retries - # and catch-all exceptions - try: - retry_call(driver.close, tries=2) - except Exception: - pass - try: - driver.quit() - except Exception: - pass - - -def deliver_dashboard(schedule): +def deliver_dashboard(dashboard, recipients, delivery_type): """ Given a schedule, delivery the dashboard as an email report """ - dashboard = schedule.dashboard - - dashboard_url = _get_url_path( - 'Superset.dashboard', - dashboard_id=dashboard.id, - ) + config = app.config - # Create a driver, fetch the page, wait for the page to render - driver = create_webdriver() window = config.get('WEBDRIVER_WINDOW')['dashboard'] - driver.set_window_size(*window) - driver.get(dashboard_url) - time.sleep(PAGE_RENDER_WAIT) - - # Set up a function to retry once for the element. - # This is buggy in certain selenium versions with firefox driver - get_element = getattr(driver, 'find_element_by_class_name') - element = retry_call( - get_element, - fargs=['grid-container'], - tries=2, - delay=PAGE_RENDER_WAIT, - ) - - try: - screenshot = element.screenshot_as_png - except WebDriverException: - # Some webdrivers do not support screenshots for elements. - # In such cases, take a screenshot of the entire page. - screenshot = driver.screenshot() # pylint: disable=no-member - finally: - destroy_webdriver(driver) + user = security_manager.find_user(config.get('EMAIL_REPORTS_USER')) + with app.app_context(): + screenshot = DashboardScreenshot(id=dashboard.id) + img = screenshot.get_thumb( + user=user, window_size=window, thumb_size=window) # Generate the email body and attachments email = _generate_mail_content( - schedule, - screenshot, + delivery_type, + img, dashboard.dashboard_title, - dashboard_url, + screenshot.url, ) subject = __( @@ -255,11 +148,11 @@ def deliver_dashboard(schedule): title=dashboard.dashboard_title, ) - _deliver_email(schedule, subject, email) + _deliver_email(recipients, subject, email) -def _get_slice_data(schedule): - slc = schedule.slice +def _get_slice_data(slc, delivery_type): + config = app.config slice_url = _get_url_path( 'Superset.explore_json', @@ -274,8 +167,10 @@ def _get_slice_data(schedule): ) cookies = {} - for cookie in _get_auth_cookies(): - cookies['session'] = cookie + user = security_manager.find_user(config.get('EMAIL_REPORTS_USER')) + with app.app_context(): + for cookie in get_auth_cookies(user): + cookies['session'] = cookie response = requests.get(slice_url, cookies=cookies) response.raise_for_status() @@ -283,7 +178,7 @@ def _get_slice_data(schedule): # TODO: Move to the csv module rows = [r.split(b',') for r in response.content.splitlines()] - if schedule.delivery_type == EmailDeliveryType.inline: + if delivery_type == EmailDeliveryType.inline: data = None # Parse the csv file and generate HTML @@ -297,7 +192,7 @@ def _get_slice_data(schedule): link=url, ) - elif schedule.delivery_type == EmailDeliveryType.attachment: + elif delivery_type == EmailDeliveryType.attachment: data = { __('%(name)s.csv', name=slc.slice_name): response.content, } @@ -310,67 +205,48 @@ def _get_slice_data(schedule): return EmailContent(body, data, None) -def _get_slice_visualization(schedule): - slc = schedule.slice - - # Create a driver, fetch the page, wait for the page to render - driver = create_webdriver() - window = config.get('WEBDRIVER_WINDOW')['slice'] - driver.set_window_size(*window) - - slice_url = _get_url_path( +def _get_slice_visualization(slc, delivery_type): + config = app.config + url = _get_url_path( 'Superset.slice', slice_id=slc.id, ) + window = config.get('WEBDRIVER_WINDOW')['slice'] + user = security_manager.find_user(config.get('EMAIL_REPORTS_USER')) - driver.get(slice_url) - time.sleep(PAGE_RENDER_WAIT) - - # Set up a function to retry once for the element. - # This is buggy in certain selenium versions with firefox driver - element = retry_call( - driver.find_element_by_class_name, - fargs=['chart-container'], - tries=2, - delay=PAGE_RENDER_WAIT, - ) - - try: - screenshot = element.screenshot_as_png - except WebDriverException: - # Some webdrivers do not support screenshots for elements. - # In such cases, take a screenshot of the entire page. - screenshot = driver.screenshot() # pylint: disable=no-member - finally: - destroy_webdriver(driver) + with app.app_context(): + screenshot = SliceScreenshot(id=slc.id) + img = screenshot.get_thumb( + user=user, window_size=window, thumb_size=window) # Generate the email body and attachments return _generate_mail_content( - schedule, - screenshot, + delivery_type, + img, slc.slice_name, - slice_url, + url, ) -def deliver_slice(schedule): +def deliver_slice(slc, recipients, email_format, delivery_type): """ Given a schedule, delivery the slice as an email report """ - if schedule.email_format == SliceEmailReportFormat.data: - email = _get_slice_data(schedule) - elif schedule.email_format == SliceEmailReportFormat.visualization: - email = _get_slice_visualization(schedule) + config = app.config + if email_format == SliceEmailReportFormat.data: + email = _get_slice_data(slc, delivery_type) + elif email_format == SliceEmailReportFormat.visualization: + email = _get_slice_visualization(slc, delivery_type) else: raise RuntimeError('Unknown email report format') subject = __( '%(prefix)s %(title)s', prefix=config.get('EMAIL_REPORTS_SUBJECT_PREFIX'), - title=schedule.slice.slice_name, + title=slc.slice_name, ) - _deliver_email(schedule, subject, email) + _deliver_email(recipients, subject, email) @celery_app.task(name='email_reports.send', bind=True, soft_time_limit=300) @@ -389,9 +265,15 @@ def schedule_email_report(task, report_type, schedule_id, recipients=None): schedule.recipients = recipients if report_type == ScheduleType.dashboard.value: - deliver_dashboard(schedule) + deliver_dashboard( + schedule.dashboard, _get_recipients(schedule), schedule.delivery_type) elif report_type == ScheduleType.slice.value: - deliver_slice(schedule) + deliver_slice( + schedule.slice, + _get_recipients(schedule), + schedule.email_format, + schedule.delivery_type, + ) else: raise RuntimeError('Unknown report type') @@ -443,6 +325,7 @@ def schedule_window(report_type, start_at, stop_at, resolution): @celery_app.task(name='email_reports.schedule_hourly') def schedule_hourly(): """ Celery beat job meant to be invoked hourly """ + config = app.config if not config.get('ENABLE_SCHEDULED_EMAIL_REPORTS'): logging.info('Scheduled email reports not enabled in config') diff --git a/superset/tasks/thumbnails.py b/superset/tasks/thumbnails.py new file mode 100644 index 000000000000..a4cbd46a5d6a --- /dev/null +++ b/superset/tasks/thumbnails.py @@ -0,0 +1,43 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# pylint: disable=C,R,W + +"""Utility functions used across Superset""" + +import logging + +from superset import app, security_manager, thumbnail_cache +from superset.tasks.celery_app import app as celery_app +from superset.utils.selenium import DashboardScreenshot, SliceScreenshot + + +@celery_app.task(name='cache_chart_thumbnail', soft_time_limit=300) +def cache_chart_thumbnail(chart_id, force=False): + with app.app_context(): + logging.info(f'Caching chart {chart_id}') + screenshot = SliceScreenshot(id=chart_id) + user = security_manager.find_user('Admin') + screenshot.compute_and_cache(user=user, cache=thumbnail_cache, force=force) + + +@celery_app.task(name='cache_dashboard_thumbnail', soft_time_limit=300) +def cache_dashboard_thumbnail(dashboard_id, force=False): + with app.app_context(): + logging.info(f'Caching dashboard {dashboard_id}') + screenshot = DashboardScreenshot(id=dashboard_id) + user = security_manager.find_user('Admin') + screenshot.compute_and_cache(user=user, cache=thumbnail_cache, force=force) diff --git a/superset/utils/selenium.py b/superset/utils/selenium.py new file mode 100644 index 000000000000..0e70ed241590 --- /dev/null +++ b/superset/utils/selenium.py @@ -0,0 +1,297 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# pylint: disable=C,R,W +from io import BytesIO +import logging +import time +import urllib + +from flask import current_app, request, Response, session, url_for +from flask_login import login_user +from PIL import Image +from retry.api import retry_call +from selenium.common.exceptions import TimeoutException, WebDriverException +from selenium.webdriver import chrome, firefox +from selenium.webdriver.common.by import By +from selenium.webdriver.support import expected_conditions as EC +from selenium.webdriver.support.ui import WebDriverWait +from werkzeug.utils import parse_cookie + +# Time in seconds, we will wait for the page to load and render +SELENIUM_CHECK_INTERVAL = 2 +SELENIUM_RETRIES = 5 +SELENIUM_HEADSTART = 3 + + +def headless_url(path): + return urllib.parse.urljoin( + current_app.config.get('WEBDRIVER_BASEURL'), + path, + ) + + +def get_url_path(view, **kwargs): + with current_app.test_request_context(): + return headless_url(url_for(view, **kwargs)) + + +class BaseScreenshot: + thumbnail_type = None + orm_class = None + element = None + window_size = (800, 600) + thumb_size = (400, 300) + + def __init__(self, id): + self.id = id + self.screenshot = None + + @property + def cache_key(self): + return f'thumb__{self.thumbnail_type}__{self.id}' + + @property + def url(self): + raise NotImplementedError() + + def fetch_screenshot(self, user, window_size=None): + window_size = window_size or self.window_size + self.screenshot = get_png_from_url( + self.url, window_size, self.element, user=user) + return self.screenshot + + def get_thumb_as_bytes(self, *args, **kwargs): + payload = self.get_thumb(*args, **kwargs) + return BytesIO(payload) + + def get_from_cache(self, cache): + payload = cache.get(self.cache_key) + if payload: + return BytesIO(payload) + + def compute_and_cache( + self, user, cache, window_size=None, thumb_size=None, + force=True): + cache_key = self.cache_key + if not force and cache.get(cache_key): + logging.info('Thumb already cached, skipping...') + return + window_size = window_size or self.window_size + thumb_size = thumb_size or self.thumb_size + logging.info(f'Processing url for thumbnail: {cache_key}') + + payload = None + + # Assuming all sorts of things can go wrong with Selenium + try: + payload = self.fetch_screenshot(window_size=window_size, user=user) + except Exception as e: + logging.error('Failed at generating thumbnail') + logging.exception(e) + + if payload and window_size != thumb_size: + try: + payload = self.resize_image(payload, size=thumb_size) + except Exception as e: + logging.error('Failed at resizing thumbnail') + logging.exception(e) + payload = None + + if payload and cache: + logging.info(f'Caching thumbnail: {cache_key}') + cache.set(cache_key, payload) + + return payload + + def get_thumb( + self, user, + window_size=None, thumb_size=None, + cache=None, + ): + payload = None + cache_key = self.cache_key + window_size = window_size or self.window_size + thumb_size = thumb_size or self.thumb_size + if cache: + payload = cache.get(cache_key) + if not payload: + payload = self.compute_and_cache(user, cache, window_size, thumb_size) + else: + logging.info(f'Loaded thumbnail from cache: {cache_key}') + return payload + + @classmethod + def resize_image(cls, img_bytes, output='png', size=None, crop=True): + size = size or cls.thumb_size + img = Image.open(BytesIO(img_bytes)) + logging.debug(f'Selenium image size: {img.size}') + if crop and img.size[1] != cls.window_size[1]: + desired_ratio = float(cls.window_size[1]) / cls.window_size[0] + desired_width = int(img.size[0] * desired_ratio) + logging.debug(f'Cropping to: {img.size[0]}*{desired_width}') + img = img.crop((0, 0, img.size[0], desired_width)) + logging.debug(f'Resizing to {size}') + img = img.resize(size, Image.ANTIALIAS) + new_img = BytesIO() + if output != 'png': + img = img.convert('RGB') + img.save(new_img, output) + new_img.seek(0) + return new_img.read() + + +class SliceScreenshot(BaseScreenshot): + thumbnail_type = 'slice' + element = 'chart-container' + window_size = (600, int(600 * 0.75)) + thumb_size = (300, int(300 * 0.75)) + + @property + def url(self): + return get_url_path( + 'Superset.slice', + slice_id=self.id, + standalone='true', + ) + + +class DashboardScreenshot(BaseScreenshot): + thumbnail_type = 'dashboard' + element = 'grid-container' + window_size = (1600, int(1600 * 0.75)) + thumb_size = (400, int(400 * 0.75)) + + @property + def url(self): + return get_url_path( + 'Superset.dashboard', + dashboard_id=self.id, + ) + + +def _destroy_webdriver(driver): + """Destroy a driver""" + # This is some very flaky code in selenium. Hence the retries + # and catch-all exceptions + try: + retry_call(driver.close, tries=2) + except Exception: + pass + try: + driver.quit() + except Exception: + pass + + +def get_auth_cookies(user): + # Login with the user specified to get the reports + with current_app.test_request_context(): + login_user(user) + + # A mock response object to get the cookie information from + response = Response() + current_app.session_interface.save_session(current_app, session, response) + + cookies = [] + + # Set the cookies in the driver + for name, value in response.headers: + if name.lower() == 'set-cookie': + cookie = parse_cookie(value) + cookies.append(cookie['session']) + return cookies + + +def create_webdriver(user=None, webdriver='chrome', window=None): + """Creates a selenium webdriver + + If no user is specified, we use the current request's context""" + # Create a webdriver for use in fetching reports + if webdriver == 'firefox': + driver_class = firefox.webdriver.WebDriver + options = firefox.options.Options() + else: + # webdriver == 'chrome': + driver_class = chrome.webdriver.WebDriver + options = chrome.options.Options() + arg = f'--window-size={window[0]},{window[1]}' + options.add_argument(arg) + + options.add_argument('--headless') + + # Prepare args for the webdriver init + kwargs = dict( + options=options, + ) + kwargs.update(current_app.config.get('WEBDRIVER_CONFIGURATION')) + + logging.info('Init selenium driver') + driver = driver_class(**kwargs) + + # Setting cookies requires doing a request first + driver.get(headless_url('/login/')) + + if user: + # Set the cookies in the driver + for cookie in get_auth_cookies(user): + info = dict(name='session', value=cookie) + driver.add_cookie(info) + elif request.cookies: + cookies = request.cookies + for k, v in cookies.items(): + cookie = dict(name=k, value=v) + driver.add_cookie(cookie) + return driver + + +def get_png_from_url( + url, + window, + element, + user, + webdriver='chrome', + retries=SELENIUM_RETRIES, +): + driver = create_webdriver(user, webdriver, window) + driver.set_window_size(*window) + driver.get(url) + img = None + logging.debug(f'Sleeping for {SELENIUM_HEADSTART} seconds') + time.sleep(SELENIUM_HEADSTART) + try: + logging.debug(f'Wait for the presence of {element}') + element = WebDriverWait(driver, 10).until( + EC.presence_of_element_located((By.CLASS_NAME, element)), + ) + logging.debug(f'Wait for .loading to be done') + WebDriverWait(driver, 60).until_not( + EC.presence_of_all_elements_located( + (By.CLASS_NAME, 'loading'), + ), + ) + logging.info('Taking a PNG screenshot') + img = element.screenshot_as_png + except TimeoutException: + logging.error('Selenium timed out') + except WebDriverException as e: + logging.exception(e) + # Some webdrivers do not support screenshots for elements. + # In such cases, take a screenshot of the entire page. + img = driver.screenshot() # pylint: disable=no-member + finally: + _destroy_webdriver(driver) + return img diff --git a/superset/views/__init__.py b/superset/views/__init__.py index 380ea6ec93f1..db726188342f 100644 --- a/superset/views/__init__.py +++ b/superset/views/__init__.py @@ -19,6 +19,7 @@ from . import core # noqa from . import sql_lab # noqa from . import dashboard # noqa +from . import thumbnails # noqa from . import annotations # noqa from . import datasource # noqa from . import schedules # noqa diff --git a/superset/views/core.py b/superset/views/core.py index d8a3692c2440..8015c748b589 100755 --- a/superset/views/core.py +++ b/superset/views/core.py @@ -58,7 +58,7 @@ from superset.sql_validators import get_validator_by_name from superset.utils import core as utils from superset.utils import dashboard_import_export -from superset.utils.dates import now_as_float +from superset.utils.dates import EPOCH, now_as_float from superset.utils.decorators import etag_cache from .base import ( api, BaseSupersetView, @@ -499,7 +499,9 @@ class SliceModelView(SupersetModelView, DeleteMixin): # noqa 'slice_name', 'description', 'viz_type', 'datasource_name', 'owners', ) list_columns = [ - 'slice_link', 'viz_type', 'datasource_link', 'creator', 'modified'] + 'slice_link', 'viz_type', 'datasource_link', + 'creator', 'modified', + ] order_columns = ['viz_type', 'datasource_link', 'modified'] edit_columns = [ 'slice_name', 'description', 'viz_type', 'owners', 'dashboards', @@ -576,7 +578,9 @@ class SliceAsync(SliceModelView): # noqa route_base = '/sliceasync' list_columns = [ 'id', 'slice_link', 'viz_type', 'slice_name', - 'creator', 'modified', 'icons', 'changed_on_humanized', + 'creator', 'modified', 'icons', 'thumbnail_url', + 'slice_url', 'created_by_name', 'changed_on', + 'datasource_data_summary', 'changed_on_humanized', ] label_columns = { 'icons': ' ', @@ -709,7 +713,8 @@ class DashboardModelViewAsync(DashboardModelView): # noqa route_base = '/dashboardasync' list_columns = [ 'id', 'dashboard_link', 'creator', 'modified', 'dashboard_title', - 'changed_on', 'url', 'changed_by_name', + 'changed_on', 'url', 'changed_by_name', 'created_by_name', + 'thumbnail_url', 'changed_on_humanized', ] label_columns = { 'dashboard_link': _('Dashboard'), @@ -2797,7 +2802,7 @@ def queries(self, last_updated_ms): last_updated_ms_int = int(float(last_updated_ms)) if last_updated_ms else 0 # UTC date time, same that is stored in the DB. - last_updated_dt = utils.EPOCH + timedelta(seconds=last_updated_ms_int / 1000) + last_updated_dt = EPOCH + timedelta(seconds=last_updated_ms_int / 1000) sql_queries = ( db.session.query(Query) @@ -2923,7 +2928,7 @@ def profile(self, username): 'superset/basic.html', title=_("%(user)s's profile", user=username), entry='profile', - bootstrap_data=json.dumps(payload, default=utils.json_iso_dttm_ser), + bootstrap_data=json.dumps(payload, default=utils.utils.json_iso_dttm_ser), ) @has_access diff --git a/superset/views/schedules.py b/superset/views/schedules.py index 6fdae7726521..a7afda6e56bc 100644 --- a/superset/views/schedules.py +++ b/superset/views/schedules.py @@ -37,10 +37,7 @@ SliceEmailSchedule, ) from superset.tasks.schedules import schedule_email_report -from superset.utils.core import ( - get_email_address_list, - json_iso_dttm_ser, -) +from superset.utils.core import get_email_address_list, json_iso_dttm_ser from superset.views.core import json_success from .base import DeleteMixin, SupersetModelView diff --git a/superset/views/thumbnails.py b/superset/views/thumbnails.py new file mode 100644 index 000000000000..281ea17c9eb7 --- /dev/null +++ b/superset/views/thumbnails.py @@ -0,0 +1,38 @@ +# pylint: disable=C,R,W +from flask import redirect, send_file +from flask_appbuilder import expose +from flask_appbuilder.security.decorators import has_access + +from superset import app, appbuilder, thumbnail_cache +from superset.utils.selenium import DashboardScreenshot, SliceScreenshot +from .base import BaseSupersetView + +config = app.config +NO_IMAGE_SRC = '/static/assets/images/no-image.png' + + +class Thumb(BaseSupersetView): + """Base views for thumbnails""" + @expose('/chart///') + @has_access + def chart(self, slice_id, sha=None): + """Returns an thumbnail for a given chart, uses cache if possible""" + # TODO security + screenshot = SliceScreenshot(id=slice_id) + img = screenshot.get_from_cache(thumbnail_cache) + if not img: + return redirect(NO_IMAGE_SRC) + return send_file(img, mimetype='image/png') + + @expose('/dashboard///') + @has_access + def dashboard(self, dashboard_id, sha=None): + """Returns an thumbnail for a given dash, uses cache if possible""" + screenshot = DashboardScreenshot(id=dashboard_id) + img = screenshot.get_from_cache(thumbnail_cache) + if not img: + return redirect(NO_IMAGE_SRC) + return send_file(img, mimetype='image/png') + + +appbuilder.add_view_no_menu(Thumb) diff --git a/tests/access_tests.py b/tests/access_tests.py index 0bc1743cfe1f..cc0848280587 100644 --- a/tests/access_tests.py +++ b/tests/access_tests.py @@ -330,7 +330,7 @@ def test_clean_requests_after_schema_grant(self): session.commit() - @mock.patch('superset.utils.core.send_MIME_email') + @mock.patch('superset.utils.email.send_MIME_email') def test_approve(self, mock_send_mime): if app.config.get('ENABLE_ACCESS_REQUEST'): session = db.session diff --git a/tests/email_tests.py b/tests/email_tests.py index 7e1830ccac5b..25da9cce4529 100644 --- a/tests/email_tests.py +++ b/tests/email_tests.py @@ -25,7 +25,7 @@ from unittest import mock from superset import app -from superset.utils import core as utils +from superset.utils.email import send_email_smtp, send_MIME_email from .utils import read_fixture send_email_test = mock.Mock() @@ -35,12 +35,12 @@ class EmailSmtpTest(unittest.TestCase): def setUp(self): app.config['smtp_ssl'] = False - @mock.patch('superset.utils.core.send_MIME_email') + @mock.patch('superset.utils.email.send_MIME_email') def test_send_smtp(self, mock_send_mime): attachment = tempfile.NamedTemporaryFile() attachment.write(b'attachment') attachment.seek(0) - utils.send_email_smtp( + send_email_smtp( 'to', 'subject', 'content', app.config, files=[attachment.name]) assert mock_send_mime.called call_args = mock_send_mime.call_args[0] @@ -54,9 +54,9 @@ def test_send_smtp(self, mock_send_mime): mimeapp = MIMEApplication('attachment') assert msg.get_payload()[-1].get_payload() == mimeapp.get_payload() - @mock.patch('superset.utils.core.send_MIME_email') + @mock.patch('superset.utils.email.send_MIME_email') def test_send_smtp_data(self, mock_send_mime): - utils.send_email_smtp( + send_email_smtp( 'to', 'subject', 'content', app.config, data={'1.txt': b'data'}) assert mock_send_mime.called call_args = mock_send_mime.call_args[0] @@ -70,10 +70,10 @@ def test_send_smtp_data(self, mock_send_mime): mimeapp = MIMEApplication('data') assert msg.get_payload()[-1].get_payload() == mimeapp.get_payload() - @mock.patch('superset.utils.core.send_MIME_email') + @mock.patch('superset.utils.email.send_MIME_email') def test_send_smtp_inline_images(self, mock_send_mime): image = read_fixture('sample.png') - utils.send_email_smtp( + send_email_smtp( 'to', 'subject', 'content', app.config, images=dict(blah=image)) assert mock_send_mime.called call_args = mock_send_mime.call_args[0] @@ -87,12 +87,12 @@ def test_send_smtp_inline_images(self, mock_send_mime): mimeapp = MIMEImage(image) assert msg.get_payload()[-1].get_payload() == mimeapp.get_payload() - @mock.patch('superset.utils.core.send_MIME_email') + @mock.patch('superset.utils.email.send_MIME_email') def test_send_bcc_smtp(self, mock_send_mime): attachment = tempfile.NamedTemporaryFile() attachment.write(b'attachment') attachment.seek(0) - utils.send_email_smtp( + send_email_smtp( 'to', 'subject', 'content', app.config, files=[attachment.name], cc='cc', bcc='bcc') assert mock_send_mime.called @@ -112,7 +112,7 @@ def test_send_mime(self, mock_smtp, mock_smtp_ssl): mock_smtp.return_value = mock.Mock() mock_smtp_ssl.return_value = mock.Mock() msg = MIMEMultipart() - utils.send_MIME_email('from', 'to', msg, app.config, dryrun=False) + send_MIME_email('from', 'to', msg, app.config, dryrun=False) mock_smtp.assert_called_with( app.config.get('SMTP_HOST'), app.config.get('SMTP_PORT'), @@ -132,7 +132,7 @@ def test_send_mime_ssl(self, mock_smtp, mock_smtp_ssl): app.config['SMTP_SSL'] = True mock_smtp.return_value = mock.Mock() mock_smtp_ssl.return_value = mock.Mock() - utils.send_MIME_email( + send_MIME_email( 'from', 'to', MIMEMultipart(), app.config, dryrun=False) assert not mock_smtp.called mock_smtp_ssl.assert_called_with( @@ -147,7 +147,7 @@ def test_send_mime_noauth(self, mock_smtp, mock_smtp_ssl): app.config['SMTP_PASSWORD'] = None mock_smtp.return_value = mock.Mock() mock_smtp_ssl.return_value = mock.Mock() - utils.send_MIME_email( + send_MIME_email( 'from', 'to', MIMEMultipart(), app.config, dryrun=False) assert not mock_smtp_ssl.called mock_smtp.assert_called_with( @@ -159,7 +159,7 @@ def test_send_mime_noauth(self, mock_smtp, mock_smtp_ssl): @mock.patch('smtplib.SMTP_SSL') @mock.patch('smtplib.SMTP') def test_send_mime_dryrun(self, mock_smtp, mock_smtp_ssl): - utils.send_MIME_email( + send_MIME_email( 'from', 'to', MIMEMultipart(), app.config, dryrun=True) assert not mock_smtp.called assert not mock_smtp_ssl.called diff --git a/tests/schedules_test.py b/tests/schedules_test.py index b0d311ab89e9..96748ece1782 100644 --- a/tests/schedules_test.py +++ b/tests/schedules_test.py @@ -21,7 +21,7 @@ from flask_babel import gettext as __ from selenium.common.exceptions import WebDriverException -from superset import app, db +from superset import app, db, security_manager from superset.models.core import Dashboard, Slice from superset.models.schedules import ( DashboardEmailSchedule, @@ -30,11 +30,11 @@ SliceEmailSchedule, ) from superset.tasks.schedules import ( - create_webdriver, deliver_dashboard, deliver_slice, next_schedules, ) +from superset.utils.selenium import create_webdriver from .utils import read_fixture @@ -149,19 +149,20 @@ def test_complex_schedule(self): self.assertEqual(schedules[59], datetime.strptime('2018-03-30 17:40:00', fmt)) self.assertEqual(schedules[60], datetime.strptime('2018-05-04 17:10:00', fmt)) - @patch('superset.tasks.schedules.firefox.webdriver.WebDriver') + @patch('superset.utils.selenium.firefox.webdriver.WebDriver') def test_create_driver(self, mock_driver_class): mock_driver = Mock() mock_driver_class.return_value = mock_driver mock_driver.find_elements_by_id.side_effect = [True, False] - create_webdriver() - create_webdriver() + alpha_user = security_manager.find_user(username='alpha') + with app.app_context(): + create_webdriver(alpha_user, webdriver='firefox') mock_driver.add_cookie.assert_called_once() - @patch('superset.tasks.schedules.firefox.webdriver.WebDriver') + @patch('superset.utils.selenium.firefox.webdriver.WebDriver') @patch('superset.tasks.schedules.send_email_smtp') - @patch('superset.tasks.schedules.time') + @patch('superset.utils.selenium.time') def test_deliver_dashboard_inline(self, mtime, send_email_smtp, driver_class): element = Mock() driver = Mock() @@ -182,9 +183,9 @@ def test_deliver_dashboard_inline(self, mtime, send_email_smtp, driver_class): driver.screenshot.assert_not_called() send_email_smtp.assert_called_once() - @patch('superset.tasks.schedules.firefox.webdriver.WebDriver') + @patch('superset.utils.selenium.firefox.webdriver.WebDriver') @patch('superset.tasks.schedules.send_email_smtp') - @patch('superset.tasks.schedules.time') + @patch('superset.utils.selenium.time') def test_deliver_dashboard_as_attachment(self, mtime, send_email_smtp, driver_class): element = Mock() driver = Mock() @@ -213,9 +214,9 @@ def test_deliver_dashboard_as_attachment(self, mtime, send_email_smtp, driver_cl element.screenshot_as_png, ) - @patch('superset.tasks.schedules.firefox.webdriver.WebDriver') + @patch('superset.utils.selenium.firefox.webdriver.WebDriver') @patch('superset.tasks.schedules.send_email_smtp') - @patch('superset.tasks.schedules.time') + @patch('superset.utils.selenium.time') def test_dashboard_chrome_like(self, mtime, send_email_smtp, driver_class): # Test functionality for chrome driver which does not support # element snapshots @@ -236,6 +237,7 @@ def test_dashboard_chrome_like(self, mtime, send_email_smtp, driver_class): id=self.dashboard_schedule).all()[0] deliver_dashboard(schedule) + mtime.sleep.assert_called_once() driver.screenshot.assert_called_once() send_email_smtp.assert_called_once() @@ -246,9 +248,9 @@ def test_dashboard_chrome_like(self, mtime, send_email_smtp, driver_class): driver.screenshot.return_value, ) - @patch('superset.tasks.schedules.firefox.webdriver.WebDriver') + @patch('superset.utils.selenium.firefox.webdriver.WebDriver') @patch('superset.tasks.schedules.send_email_smtp') - @patch('superset.tasks.schedules.time') + @patch('superset.utils.selenium.time') def test_deliver_email_options(self, mtime, send_email_smtp, driver_class): element = Mock() driver = Mock() @@ -277,9 +279,9 @@ def test_deliver_email_options(self, mtime, send_email_smtp, driver_class): self.assertEquals(send_email_smtp.call_count, 2) self.assertEquals(send_email_smtp.call_args[1]['bcc'], self.BCC) - @patch('superset.tasks.schedules.firefox.webdriver.WebDriver') + @patch('superset.utils.selenium.firefox.webdriver.WebDriver') @patch('superset.tasks.schedules.send_email_smtp') - @patch('superset.tasks.schedules.time') + @patch('superset.utils.selenium.time') def test_deliver_slice_inline_image(self, mtime, send_email_smtp, driver_class): element = Mock() driver = Mock() @@ -308,9 +310,9 @@ def test_deliver_slice_inline_image(self, mtime, send_email_smtp, driver_class): element.screenshot_as_png, ) - @patch('superset.tasks.schedules.firefox.webdriver.WebDriver') + @patch('superset.utils.selenium.firefox.webdriver.WebDriver') @patch('superset.tasks.schedules.send_email_smtp') - @patch('superset.tasks.schedules.time') + @patch('superset.utils.selenium.time') def test_deliver_slice_attachment(self, mtime, send_email_smtp, driver_class): element = Mock() driver = Mock() diff --git a/tests/sqllab_tests.py b/tests/sqllab_tests.py index dc86866a9960..5c328d4d6725 100644 --- a/tests/sqllab_tests.py +++ b/tests/sqllab_tests.py @@ -25,7 +25,8 @@ from superset.dataframe import SupersetDataFrame from superset.db_engine_specs import BaseEngineSpec from superset.models.sql_lab import Query -from superset.utils.core import datetime_to_epoch, get_main_database +from superset.utils.core import get_main_database +from superset.utils.dates import datetime_to_epoch from .base_tests import SupersetTestCase diff --git a/tests/utils_tests.py b/tests/utils_tests.py index a39631b83227..ed95e0244351 100644 --- a/tests/utils_tests.py +++ b/tests/utils_tests.py @@ -24,13 +24,9 @@ from superset.exceptions import SupersetException from superset.utils.core import ( - base_json_conv, convert_legacy_filters_into_adhoc, datetime_f, get_since_until, - json_int_dttm_ser, - json_iso_dttm_ser, - JSONEncodedDict, memoized, merge_extra_filters, merge_request_params, @@ -40,6 +36,7 @@ zlib_compress, zlib_decompress_to_string, ) +from superset.utils.json import base_json_conv, json_int_dttm_ser, json_iso_dttm_ser def mock_parse_human_datetime(s): @@ -509,15 +506,6 @@ def test_datetime_f(self): datetime_f(datetime(a, b, c)), '00:00:00', ) - def test_json_encoded_obj(self): - obj = {'a': 5, 'b': ['a', 'g', 5]} - val = '{"a": 5, "b": ["a", "g", 5]}' - jsonObj = JSONEncodedDict() - resp = jsonObj.process_bind_param(obj, 'dialect') - self.assertIn('"a": 5', resp) - self.assertIn('"b": ["a", "g", 5]', resp) - self.assertEquals(jsonObj.process_result_value(val, 'dialect'), obj) - def test_validate_json(self): invalid = '{"a": 5, "b": [1, 5, ["g", "h]]}' with self.assertRaises(SupersetException):