From bebba03135d9689e1bf5efe9e841a1dad3a2fd26 Mon Sep 17 00:00:00 2001 From: Maxime Beauchemin Date: Sun, 6 Jan 2019 13:42:50 -0800 Subject: [PATCH 1/2] Introducing Thumbnails! --- requirements.txt | 1 + setup.py | 1 + superset/__init__.py | 3 + superset/assets/images/no-image.png | Bin 0 -> 1860 bytes superset/assets/package-lock.json | 90 ++++++ superset/assets/package.json | 1 + .../welcome/DashboardTable_spec.jsx | 6 +- .../spec/javascripts/welcome/Welcome_spec.jsx | 2 +- superset/assets/src/components/Card.css | 58 ++++ superset/assets/src/components/Card.jsx | 102 ++++++ superset/assets/src/components/CardTable.css | 45 +++ superset/assets/src/components/CardTable.jsx | 128 ++++++++ superset/assets/src/components/ChartCard.jsx | 104 ++++++ .../assets/src/components/DashboardCard.jsx | 103 ++++++ .../assets/src/components/ToggleWrapper.jsx | 53 ++++ .../assets/src/welcome/ChartCardTable.jsx | 177 +++++++++++ .../assets/src/welcome/DashboardCardTable.jsx | 159 ++++++++++ .../assets/src/welcome/DashboardTable.jsx | 93 ------ superset/assets/src/welcome/Welcome.jsx | 40 +-- .../stylesheets/less/cosmo/variables.less | 4 +- superset/assets/stylesheets/superset.less | 3 + superset/cli.py | 69 ++++ superset/config.py | 7 +- superset/connectors/base/models.py | 10 + superset/models/core.py | 63 +++- superset/models/helpers.py | 9 +- superset/sql_lab.py | 2 +- superset/tasks/cache.py | 4 +- superset/tasks/schedules.py | 255 ++++----------- superset/tasks/thumbnails.py | 43 +++ superset/utils/selenium.py | 295 ++++++++++++++++++ superset/views/__init__.py | 1 + superset/views/base.py | 5 +- superset/views/core.py | 22 +- superset/views/schedules.py | 6 +- superset/views/thumbnails.py | 38 +++ superset/viz.py | 7 +- tests/access_tests.py | 2 +- tests/email_tests.py | 26 +- tests/schedules_test.py | 36 ++- tests/sqllab_tests.py | 3 +- tests/utils_tests.py | 14 +- 42 files changed, 1706 insertions(+), 384 deletions(-) create mode 100644 superset/assets/images/no-image.png create mode 100644 superset/assets/src/components/Card.css create mode 100644 superset/assets/src/components/Card.jsx create mode 100644 superset/assets/src/components/CardTable.css create mode 100644 superset/assets/src/components/CardTable.jsx create mode 100644 superset/assets/src/components/ChartCard.jsx create mode 100644 superset/assets/src/components/DashboardCard.jsx create mode 100644 superset/assets/src/components/ToggleWrapper.jsx create mode 100644 superset/assets/src/welcome/ChartCardTable.jsx create mode 100644 superset/assets/src/welcome/DashboardCardTable.jsx delete mode 100644 superset/assets/src/welcome/DashboardTable.jsx create mode 100644 superset/tasks/thumbnails.py create mode 100644 superset/utils/selenium.py create mode 100644 superset/views/thumbnails.py diff --git a/requirements.txt b/requirements.txt index 3b88de610364..0789c1b370b5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -54,6 +54,7 @@ numpy==1.15.2 # via pandas pandas==0.23.4 parsedatetime==2.0.0 pathlib2==2.3.0 +pillow==5.4.1 polyline==1.3.2 prison==0.1.0 # via flask-appbuilder py==1.7.0 # via retry diff --git a/setup.py b/setup.py index fc91ea484d59..a4d963026957 100644 --- a/setup.py +++ b/setup.py @@ -92,6 +92,7 @@ def get_git_sha(): 'pandas>=0.18.0, <0.24.0', # `pandas`>=0.24.0 changes datetimelike API 'parsedatetime', 'pathlib2', + 'pillow==5.4.1', 'polyline', 'pydruid>=0.5.2', 'python-dateutil', diff --git a/superset/__init__.py b/superset/__init__.py index 6971dc9d4c8a..e45cf91b1fe6 100644 --- a/superset/__init__.py +++ b/superset/__init__.py @@ -115,6 +115,8 @@ def get_manifest(): if conf.get('SILENCE_FAB'): logging.getLogger('flask_appbuilder').setLevel(logging.ERROR) +logging.getLogger('selenium').setLevel(logging.ERROR) + if app.debug: app.logger.setLevel(logging.DEBUG) # pylint: disable=no-member else: @@ -135,6 +137,7 @@ def get_manifest(): cache = setup_cache(app, conf.get('CACHE_CONFIG')) tables_cache = setup_cache(app, conf.get('TABLE_NAMES_CACHE_CONFIG')) +thumbnail_cache = setup_cache(app, conf.get('THUMBNAIL_CACHE_CONFIG')) migrate = Migrate(app, db, directory=APP_DIR + '/migrations') diff --git a/superset/assets/images/no-image.png b/superset/assets/images/no-image.png new file mode 100644 index 0000000000000000000000000000000000000000..0371c4733d49bad77f07783d7529868e097d9524 GIT binary patch literal 1860 zcmeAS@N?(olHy`uVBq!ia0y~yV4MKNIvi|3k+<8Q9%5i%=k|1Q45_&F_U=nRrD&0c zi*^5}7tKo-KBy?AFJa#2Zs}(4!{6mWMpAM#w=hUAi%-` z!N+VBu4ZiwTN&ar{qx6<50~>WChlS0v~OSB>=`o*V$VH(ym-D5&;&siMn?w+2as8e zf&xITEFu9CF+dwmI6K^ut-fb-{rdIVzsvOX^pIu2e)iqk!U7bGo8%^BZM|ivP&j`{{3M|G%(-*_ zeqPVd4RYoeONEC&e(V6cv$S+;zWIgvyjauoH*d}a2GoSF4J}?vKfV3>?Afzd^L6i? zPj|SLJ3luk2k1YhN)8XDZGF393%`7NN{YcP%_3=QBOxFdE zR!s3@ym|k=IM5kxrV2}gR({-&c&lT{&$H`i%$zxM9ykOX9gJiq>=5wx_ph%`+8gZh z==|j)$}=xt4hH%v;R@5Hy?g(@-l)D-X;RtiRh5sbML=e0D2R)T>*`h z4-@PZ^}N$9k>;@LXGSd0Jb?*}EKBZY?&uIC+_`fBk0;*o_pJ7t4)h3z$9(2X zSzj;O6dJwX@OYxn&KNOJTnbDmTivhYS7X-+^6Zm_C6To$g?ah;Ko2%1vZ>^+?$5JJ z3I`?5CH{;jU%yg*d$t5@l^v_f`qlj@kM8RO9mHdyP#IUNvU$^{pn9V?plXQ(hervA zDppy)zx;0V4RC1v+R|$J3VS91=dmS%6Dls`%~>9x0Z#BG(gK(3P?STmBv1&f{0LYT zM+eZ^3JSpBYG?o^Vh$!HAdjO%5a>0aGN40oG@& zl_A<<-OEl_R#tutZfh{O2P$760F)UJ7?kM+1VH5k0y6+hPMn1&i3J_J$R|=Ca{a)_ jNNXM`^-;tB>-B+Km{vYK@>myGJ2H5>`njxgN@xNAM;r-$ literal 0 HcmV?d00001 diff --git a/superset/assets/package-lock.json b/superset/assets/package-lock.json index da886a0c95d1..6047df5d0417 100644 --- a/superset/assets/package-lock.json +++ b/superset/assets/package-lock.json @@ -4953,6 +4953,11 @@ "integrity": "sha1-3DQxT05nkxgJP8dgJyUl+UvyXBY=", "dev": true }, + "batch-processor": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/batch-processor/-/batch-processor-1.0.0.tgz", + "integrity": "sha1-dclcMrdI4IUNEMKxaPa9vpiRrOg=" + }, "bcrypt-pbkdf": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz", @@ -5470,6 +5475,11 @@ "resolved": "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz", "integrity": "sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw=" }, + "chain-function": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/chain-function/-/chain-function-1.0.1.tgz", + "integrity": "sha512-SxltgMwL9uCko5/ZCLiyG2B7R9fY4pDZUw7hJ4MhirdjBLosoDqkWABi3XMucddHdLiFJMb7PD2MZifZriuMTg==" + }, "chalk": { "version": "1.1.3", "resolved": "http://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz", @@ -7375,6 +7385,14 @@ "resolved": "https://registry.npmjs.org/earcut/-/earcut-2.1.4.tgz", "integrity": "sha512-ttRjmPD5oaTtXOoxhFp9aZvMB14kBjapYaiBuzBB1elOgSLU9P2Ev86G2OClBg+uspUXERsIzXKpUWweH2K4Xg==" }, + "easy-css-transform-builder": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/easy-css-transform-builder/-/easy-css-transform-builder-0.0.2.tgz", + "integrity": "sha1-pUFmenkZ4X9n2CsR08tV/dVDMCI=", + "requires": { + "invariant": "^2.2.2" + } + }, "ecc-jsbn": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz", @@ -7402,6 +7420,14 @@ "integrity": "sha512-En051LMzMl3/asMWGZEtU808HOoVWIpmmZx1Ep8N+TT9e7z/X8RcLeBU2kLSNLGQ+5SuKELzMx+MVuTBXk6Q9w==", "dev": true }, + "element-resize-detector": { + "version": "1.1.15", + "resolved": "https://registry.npmjs.org/element-resize-detector/-/element-resize-detector-1.1.15.tgz", + "integrity": "sha512-16/5avDegXlUxytGgaumhjyQoM6hpp5j3+L79sYq5hlXfTNRy5WMMuTVWkZU3egp/CokCmTmvf18P3KeB57Iog==", + "requires": { + "batch-processor": "^1.0.0" + } + }, "elliptic": { "version": "6.4.1", "resolved": "https://registry.npmjs.org/elliptic/-/elliptic-6.4.1.tgz", @@ -8108,6 +8134,11 @@ "integrity": "sha1-Qa4u62XvpiJorr/qg6x9eSmbCIc=", "dev": true }, + "ev-emitter": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/ev-emitter/-/ev-emitter-1.1.1.tgz", + "integrity": "sha512-ipiDYhdQSCZ4hSbX4rMW+XzNKMD1prg/sTvoVmSLkuQ1MVlwjJQQA+sW8tMYR3BLUr9KjodFV4pvzunvRhd33Q==" + }, "eventemitter3": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-3.1.0.tgz", @@ -8160,6 +8191,11 @@ "strip-eof": "^1.0.0" } }, + "exenv": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/exenv/-/exenv-1.2.2.tgz", + "integrity": "sha1-KueOhdmJQVhnCwPUe+wfA72Ru50=" + }, "exif-parser": { "version": "0.1.12", "resolved": "https://registry.npmjs.org/exif-parser/-/exif-parser-0.1.12.tgz", @@ -10718,6 +10754,14 @@ "dev": true, "optional": true }, + "imagesloaded": { + "version": "4.1.4", + "resolved": "https://registry.npmjs.org/imagesloaded/-/imagesloaded-4.1.4.tgz", + "integrity": "sha512-ltiBVcYpc/TYTF5nolkMNsnREHW+ICvfQ3Yla2Sgr71YFwQ86bDwV9hgpFhFtrGPuwEx5+LqOHIrdXBdoWwwsA==", + "requires": { + "ev-emitter": "^1.0.0" + } + }, "immutable": { "version": "3.8.2", "resolved": "https://registry.npmjs.org/immutable/-/immutable-3.8.2.tgz", @@ -17685,6 +17729,17 @@ "js-search": "^1.3.1" } }, + "react-sizeme": { + "version": "2.6.7", + "resolved": "https://registry.npmjs.org/react-sizeme/-/react-sizeme-2.6.7.tgz", + "integrity": "sha512-xCjPoBP5jmeW58TxIkcviMZqabZis7tTvDFWf0/Wa5XCgVWQTIe74NQBes2N1Kmp64GRLkpm60BaP0kk+v8aCQ==", + "requires": { + "element-resize-detector": "^1.1.15", + "invariant": "^2.2.4", + "shallowequal": "^1.1.0", + "throttle-debounce": "^2.1.0" + } + }, "react-sortable-hoc": { "version": "0.8.4", "resolved": "https://registry.npmjs.org/react-sortable-hoc/-/react-sortable-hoc-0.8.4.tgz", @@ -17714,6 +17769,36 @@ "react-style-proptype": "^3.0.0" } }, + "react-stack-grid": { + "version": "0.7.1", + "resolved": "https://registry.npmjs.org/react-stack-grid/-/react-stack-grid-0.7.1.tgz", + "integrity": "sha512-Fw7qMt5Rd9wQpNCnvK4Gi+ry/nL5rKfzP2hGsw5/DZxArEMk60VoDLy68Uskq09l6wk7qb2w7P2+lNzSd9mYEw==", + "requires": { + "easy-css-transform-builder": "^0.0.2", + "exenv": "^1.2.1", + "imagesloaded": "^4.1.1", + "inline-style-prefixer": "^3.0.6", + "invariant": "^2.2.2", + "prop-types": "^15.5.10", + "react-sizeme": "^2.2.0", + "react-transition-group": "^1.2.0", + "shallowequal": "^1.0.1" + }, + "dependencies": { + "react-transition-group": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/react-transition-group/-/react-transition-group-1.2.1.tgz", + "integrity": "sha512-CWaL3laCmgAFdxdKbhhps+c0HRGF4c+hdM4H23+FI1QBNUyx/AMeIJGWorehPNSaKnQNOAxL7PQmqMu78CDj3Q==", + "requires": { + "chain-function": "^1.0.0", + "dom-helpers": "^3.2.0", + "loose-envify": "^1.3.1", + "prop-types": "^15.5.6", + "warning": "^3.0.0" + } + } + } + }, "react-sticky": { "version": "6.0.3", "resolved": "https://registry.npmjs.org/react-sticky/-/react-sticky-6.0.3.tgz", @@ -20271,6 +20356,11 @@ "integrity": "sha1-iQN8vJLFarGJJua6TLsgDhVnKmo=", "dev": true }, + "throttle-debounce": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/throttle-debounce/-/throttle-debounce-2.1.0.tgz", + "integrity": "sha512-AOvyNahXQuU7NN+VVvOOX+uW6FPaWdAOdRP5HfwYxAfCzXTFKRMoIMk+n+po318+ktcChx+F1Dd91G3YHeMKyg==" + }, "through": { "version": "2.3.8", "resolved": "http://registry.npmjs.org/through/-/through-2.3.8.tgz", diff --git a/superset/assets/package.json b/superset/assets/package.json index 52e82b12e01a..cccc2de63cac 100644 --- a/superset/assets/package.json +++ b/superset/assets/package.json @@ -127,6 +127,7 @@ "react-select-fast-filter-options": "^0.2.1", "react-sortable-hoc": "^0.8.3", "react-split": "^2.0.4", + "react-stack-grid": "^0.7.1", "react-sticky": "^6.0.2", "react-syntax-highlighter": "^7.0.4", "react-transition-group": "^2.5.3", diff --git a/superset/assets/spec/javascripts/welcome/DashboardTable_spec.jsx b/superset/assets/spec/javascripts/welcome/DashboardTable_spec.jsx index e989f9a207ae..28c1adda9e2c 100644 --- a/superset/assets/spec/javascripts/welcome/DashboardTable_spec.jsx +++ b/superset/assets/spec/javascripts/welcome/DashboardTable_spec.jsx @@ -23,7 +23,7 @@ import configureStore from 'redux-mock-store'; import fetchMock from 'fetch-mock'; import { Table } from 'reactable-arc'; -import DashboardTable from '../../../src/welcome/DashboardTable'; +import DashboardCardTable from '../../../src/welcome/DashboardCardTable'; import Loading from '../../../src/components/Loading'; // store needed for withToasts(TableLoader) @@ -39,10 +39,10 @@ fetchMock.get(dashboardsEndpoint, { result: mockDashboards }); function setup() { // use mount because data fetching is triggered on mount - return mount(, { context: { store } }); + return mount(, { context: { store } }); } -describe('DashboardTable', () => { +describe('DashboardCardTable', () => { beforeEach(fetchMock.resetHistory); it('renders a Loading initially', () => { diff --git a/superset/assets/spec/javascripts/welcome/Welcome_spec.jsx b/superset/assets/spec/javascripts/welcome/Welcome_spec.jsx index 8a18b299955e..a83728264b38 100644 --- a/superset/assets/spec/javascripts/welcome/Welcome_spec.jsx +++ b/superset/assets/spec/javascripts/welcome/Welcome_spec.jsx @@ -33,6 +33,6 @@ describe('Welcome', () => { const wrapper = shallow(); expect(wrapper.find(Tab)).toHaveLength(3); expect(wrapper.find(Panel)).toHaveLength(3); - expect(wrapper.find(Row)).toHaveLength(3); + expect(wrapper.find(Row)).toHaveLength(2); }); }); diff --git a/superset/assets/src/components/Card.css b/superset/assets/src/components/Card.css new file mode 100644 index 000000000000..b70e828ac75b --- /dev/null +++ b/superset/assets/src/components/Card.css @@ -0,0 +1,58 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +.card { + background-color: white; + border-radius: 5px; + box-shadow: 0 1px 1px 0 rgba(60,64,67,.16), 0 1px 3px 1px rgba(60,64,67,.25); + overflow: hidden; + transition: box-shadow 0.5s ease-in-out; +} +.card:hover { + box-shadow: 0 2px 2px 0 rgba(60,64,67,.30), 0 2px 4px 2px rgba(60,64,67,.55); +} +.card-header { + padding-top: 5px; + padding-left: 15px; + padding-right: 15px; + padding-bottom: 0px; +} +.card-body { + padding-top: 5px; + padding-left: 15px; + padding-right: 15px; + padding-bottom: 15px; +} +.card-body .text-muted{ + color: #888; +} + +.card-header .fa { + margin-top: 10px; + padding-left: 5px; + padding-right: 5px; +} + +.card img { + border-top: 1px solid #EEE; + border-bottom: 1px solid #EEE; +} +.card-title { + white-space: nowrap; + width: 100px; +} diff --git a/superset/assets/src/components/Card.jsx b/superset/assets/src/components/Card.jsx new file mode 100644 index 000000000000..4b4a986b7938 --- /dev/null +++ b/superset/assets/src/components/Card.jsx @@ -0,0 +1,102 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import React from 'react'; +import PropTypes from 'prop-types'; +import { Dropdown, Fade } from 'react-bootstrap'; + +import ToggleWrapper from './ToggleWrapper'; +import './Card.css'; + +const propTypes = { + title: PropTypes.string.isRequired, + body: PropTypes.node.isRequired, + imageSource: PropTypes.string.isRequired, + cardWidth: PropTypes.number, + imageAspectRatio: PropTypes.number, + dropdownMenu: PropTypes.node.isRequired, + onTitleClick: PropTypes.func, +}; +const defaultProps = { + onTitleClick: () => {}, + imageAspectRatio: 3/2, +}; + +export default class Card extends React.PureComponent { + constructor(props) { + super(props); + this.state = { + hovered: false, + }; + } + renderActionDropdown() { + return ( + + + + + + + {this.props.dropdownMenu} + ); + } + render() { + const { + body, + cardWidth, + imageAspectRatio, + imageSource, + onTitleClick, + title, + } = this.props; + const imgHeight = cardWidth / imageAspectRatio; + return ( +
this.setState({ hovered: true })} + onMouseOut={() => this.setState({ hovered: false })} + > +
+
onTitleClick()} + > +
{title}
+
+
+ {this.renderActionDropdown()} +
+
+
onTitleClick()}> + +
+
+ {body} +
+
+ ); + } +} +Card.propTypes = propTypes; +Card.defaultProps = defaultProps; diff --git a/superset/assets/src/components/CardTable.css b/superset/assets/src/components/CardTable.css new file mode 100644 index 000000000000..8c7fb0cb6588 --- /dev/null +++ b/superset/assets/src/components/CardTable.css @@ -0,0 +1,45 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +.CardTable { + min-height: 500px; +} +.CardTable .input-search { + margin-top: 25px; + width: 300px; +} +.CardTable .card-table-toggle .fa { + margin-top: .2em; + margin-left: 0; +} +.CardTable .controls-right { + margin-top: 25px; +} + +.table .td-thumb img { + float: left; + width: 100px; +} + +.table .td-thumb { + padding: 0px; +} + +.table .td-thumb span { + padding: 8px; +} diff --git a/superset/assets/src/components/CardTable.jsx b/superset/assets/src/components/CardTable.jsx new file mode 100644 index 000000000000..53425580e34d --- /dev/null +++ b/superset/assets/src/components/CardTable.jsx @@ -0,0 +1,128 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import React from 'react'; +import PropTypes from 'prop-types'; +import { Button, ButtonGroup, FormControl } from 'react-bootstrap'; +import StackGrid from 'react-stack-grid'; +import { t } from '@superset-ui/translation'; + +import Loading from '../components/Loading'; +import withToasts from '../messageToasts/enhancers/withToasts'; +import './CardTable.css'; + +const propTypes = { + renderTable: PropTypes.func.isRequired, + renderCards: PropTypes.func.isRequired, + cardWidth: PropTypes.number, + onSearchChange: PropTypes.func.isRequired, + title: PropTypes.string.isRequired, + addDangerToast: PropTypes.func.isRequired, + items: PropTypes.array, + loading: PropTypes.bool.isRequired, + showCardCount: PropTypes.number, + emptyMessage: PropTypes.node, +}; + +const defaultProps = { + cardWidth: 250, + emptyMessage: t('No data'), +}; + +class CardTable extends React.PureComponent { + constructor(props) { + super(props); + this.state = { + search: '', + showTable: false, + }; + this.onSearchChange = this.onSearchChange.bind(this); + } + onSearchChange(event) { + const search = event.target.value; + this.setState({ search }); + this.props.onSearchChange(search); + } + renderCards() { + return ( + + {this.props.renderCards()} + ); + } + renderList() { + if (this.state.showTable) { + return this.props.renderTable(this.props.items); + } + return this.renderCards(); + } + render() { + const { showTable } = this.state; + const { loading, items, emptyMessage } = this.props; + if (loading) { + return ; + } + if (items && items.length === 0) { + return emptyMessage; + } + return ( +
+
+
+

{this.props.title}

+
+
+ + + + +
+
+ +
+
+
+ {this.renderList()} +
); + } +} +CardTable.propTypes = propTypes; +export default withToasts(CardTable); diff --git a/superset/assets/src/components/ChartCard.jsx b/superset/assets/src/components/ChartCard.jsx new file mode 100644 index 000000000000..299f219ec007 --- /dev/null +++ b/superset/assets/src/components/ChartCard.jsx @@ -0,0 +1,104 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import React from 'react'; +import PropTypes from 'prop-types'; +import { Dropdown, MenuItem } from 'react-bootstrap'; +import { t } from '@superset-ui/translation'; +import Dialog from 'react-bootstrap-dialog'; + +import Card from './Card.jsx'; + +const propTypes = { + chart: PropTypes.object.isRequired, + onDelete: PropTypes.func.isRequired, + cardWidth: PropTypes.number.isRequired, +}; + + +export default class ChartCard extends React.PureComponent { + constructor(props) { + super(props); + this.openChart = this.openChart.bind(this); + this.deleteDialog = this.deleteDialog.bind(this); + } + openChart() { + window.open(this.props.chart.slice_url); + } + deleteDialog() { + const onOk = () => { + this.props.onDelete(this.props.chart); + this.dialog.hide(); + }; + this.dialog.show({ + title: t('Delete Chart'), + body: t('Are you sure you want to delete this chart?'), + actions: [ + Dialog.CancelAction(diag => diag.hide()), + Dialog.DefaultAction('Ok', onOk, 'btn-danger'), + ], + bsSize: 'small', + onHide: (dialog) => { + dialog.hide(); + }, + }); + } + renderDropdownMenu() { + const { chart } = this.props; + return ( + + + {t('Edit chart metadata')} + + + {t('Delete Chart')} + + + ); + } + renderCardBody() { + const { chart } = this.props; + return ( + + Modified {chart.changed_on_humanized} +
+ + {t('Created by')} {chart.created_by_name || t('N/A')} + + { + this.dialog = el; + }} + /> + + ); + } + render() { + const { chart } = this.props; + return ( + ); + } +} +ChartCard.propTypes = propTypes; diff --git a/superset/assets/src/components/DashboardCard.jsx b/superset/assets/src/components/DashboardCard.jsx new file mode 100644 index 000000000000..3f883213d00a --- /dev/null +++ b/superset/assets/src/components/DashboardCard.jsx @@ -0,0 +1,103 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import React from 'react'; +import PropTypes from 'prop-types'; +import { Dropdown, MenuItem } from 'react-bootstrap'; +import { t } from '@superset-ui/translation'; +import Dialog from 'react-bootstrap-dialog'; + +import Card from './Card.jsx'; + +const propTypes = { + cardWidth: PropTypes.number.isRequired, + dashboard: PropTypes.object.isRequired, +}; + + +export default class DashboardCard extends React.PureComponent { + constructor(props) { + super(props); + this.openDashboard = this.openDashboard.bind(this); + this.deleteDialog = this.deleteDialog.bind(this); + } + openDashboard() { + window.open(this.props.dashboard.url); + } + deleteDialog() { + const onOk = () => { + this.props.onDelete(this.props.dashboard); + this.dialog.hide(); + }; + this.dialog.show({ + title: t('Delete dashboard'), + body: t('Are you sure you want to delete this dashboard?'), + actions: [ + Dialog.CancelAction(diag => diag.hide()), + Dialog.DefaultAction('Ok', onOk, 'btn-danger'), + ], + bsSize: 'small', + onHide: (dialog) => { + dialog.hide(); + }, + }); + } + renderDropdownMenu() { + const { dashboard } = this.props; + return ( + + + {t('Edit dashboard metadata')} + + + {t('Delete Dashboard')} + + + ); + } + renderCardBody() { + const { dashboard } = this.props; + return ( + + Modified {dashboard.changed_on_humanized} +
+ + {t('Created by')} {dashboard.created_by_name || t('N/A')} + + { + this.dialog = el; + }} + /> + + ); + } + render() { + const { dashboard } = this.props; + return ( + ); + } +} +DashboardCard.propTypes = propTypes; diff --git a/superset/assets/src/components/ToggleWrapper.jsx b/superset/assets/src/components/ToggleWrapper.jsx new file mode 100644 index 000000000000..b1ec6ee7c53f --- /dev/null +++ b/superset/assets/src/components/ToggleWrapper.jsx @@ -0,0 +1,53 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/** + * Allows any component to act as a Dropdown toggle + * for instance a simple . + * inspired from https://react-bootstrap.github.io/components/dropdowns/ + * "Custom Dropdown Components" section + */ +import React from 'react'; +import PropTypes from 'prop-types'; + +const propTypes = { + onClick: PropTypes.func, + children: PropTypes.node, +}; +export default class ToggleWrapper extends React.Component { + constructor(props, context) { + super(props, context); + + this.handleClick = this.handleClick.bind(this); + } + + handleClick(e) { + e.preventDefault(); + this.props.onClick(e); + } + + render() { + return ( + + {this.props.children} + + ); + } +} +ToggleWrapper.propTypes = propTypes; diff --git a/superset/assets/src/welcome/ChartCardTable.jsx b/superset/assets/src/welcome/ChartCardTable.jsx new file mode 100644 index 000000000000..1acc07c87833 --- /dev/null +++ b/superset/assets/src/welcome/ChartCardTable.jsx @@ -0,0 +1,177 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import React from 'react'; +import PropTypes from 'prop-types'; +import { t } from '@superset-ui/translation'; +import { Table, Tr, Td, unsafe } from 'reactable-arc'; +import { SupersetClient } from '@superset-ui/connection'; + +import withToasts from '../messageToasts/enhancers/withToasts'; +import CardTable from '../components/CardTable'; +import ChartCard from '../components/ChartCard'; +import '../../stylesheets/reactable-pagination.css'; + +const propTypes = { + showTable: PropTypes.bool, + addDangerToast: PropTypes.func.isRequired, +}; + +const CHART_WIDTH = 250; + +class ChartCardTable extends React.PureComponent { + constructor(props) { + super(props); + this.state = { + search: '', + loading: true, + charts: null, + filteredCharts: null, + }; + this.renderTable = this.renderTable.bind(this); + this.onSearchChange = this.onSearchChange.bind(this); + this.renderCards = this.renderCards.bind(this); + this.deleteChart = this.deleteChart.bind(this); + } + componentDidMount() { + const endpoint = ( + '/sliceasync/api/read?' + + '_oc_ChartModelViewAsync=changed_on' + + '&_od_ChartModelViewAsync=desc' + ); + SupersetClient.get({ endpoint }) + .then(({ json }) => { + const charts = json.result; + this.setState({ + charts, + loading: false, + filteredCharts: charts, + }); + }) + .catch(() => { + this.props.addDangerToast(t('An error occurred while fetching data')); + this.setState({ charts: null, loading: false }); + }); + } + deleteChart(chart) { + const endpoint = `/chart/api/delete/${chart.id}`; + SupersetClient.delete({ endpoint }) + .then(({ json }) => { + this.removeChart(chart); + }) + .catch(() => { + this.props.addDangerToast(t('An error occurred while deleting the chart')); + }); + } + onSearchChange(search) { + const filteredCharts = this.filterCharts(search); + this.setState({ search, filteredCharts }); + } + removeChart(chart) { + const filter = o => o.id !== chart.id; + const charts = this.state.charts.filter(filter); + const filteredCharts = this.state.filteredCharts.filter(filter); + this.setState({ charts, filteredCharts }); + } + getDatasourceName(chart) { + const o = chart.datasource_data_summary; + return o.schema ? `${o.schema}.${o.datasource_name}` : o.datasource_name; + } + filterCharts(searchText) { + const { charts } = this.state; + if (!searchText) { + return charts; + } + const lcaseSearchText = searchText.toLowerCase(); + return charts.filter(o => o.slice_name.toLowerCase().indexOf(lcaseSearchText) >= 0); + } + renderDatasourceLink(chart) { + const o = chart.datasource_data_summary; + const url = o.explore_url; + const name = this.getDatasourceName(chart); + return ( + {name} + ); + } + renderTable() { + const { filteredCharts } = this.state; + return ( + + {filteredCharts.map(o => ( + + + + + + ))} +
+ + + {o.slice_name} + + + {this.renderDatasourceLink(o)} + + {unsafe(o.creator)} + + {unsafe(o.modified)} +
+ ); + } + renderCards() { + return this.state.filteredCharts.map(chart => ( + + )); + } + + render() { + return ( + + ); + } +} + +ChartCardTable.propTypes = propTypes; +export default withToasts(ChartCardTable); diff --git a/superset/assets/src/welcome/DashboardCardTable.jsx b/superset/assets/src/welcome/DashboardCardTable.jsx new file mode 100644 index 000000000000..9dc2b08344f4 --- /dev/null +++ b/superset/assets/src/welcome/DashboardCardTable.jsx @@ -0,0 +1,159 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import React from 'react'; +import PropTypes from 'prop-types'; +import { t } from '@superset-ui/translation'; +import { Table, Tr, Td, unsafe } from 'reactable-arc'; +import { SupersetClient } from '@superset-ui/connection'; + +import withToasts from '../messageToasts/enhancers/withToasts'; +import CardTable from '../components/CardTable'; +import DashboardCard from '../components/DashboardCard'; +import '../../stylesheets/reactable-pagination.css'; + +const propTypes = { + showTable: PropTypes.bool, + addDangerToast: PropTypes.func.isRequired, +}; + +const CARD_WIDTH = 250; + +class DashboardCardTable extends React.PureComponent { + constructor(props) { + super(props); + this.state = { + search: '', + dashboards: null, + loading: true, + filteredDashboards: null, + }; + this.renderTable = this.renderTable.bind(this); + this.onSearchChange = this.onSearchChange.bind(this); + this.onDelete = this.onDelete.bind(this); + this.renderCards = this.renderCards.bind(this); + this.deleteDashboard = this.deleteDashboard.bind(this); + } + componentDidMount() { + const endpoint = ( + '/dashboardasync/api/read?' + + '_oc_DashboardModelViewAsync=changed_on' + + '&_od_DashboardModelViewAsync=desc' + ); + SupersetClient.get({ endpoint }) + .then(({ json }) => { + const dashboards = json.result; + this.setState({ + dashboards, + loading: false, + filteredDashboards: dashboards, + }); + }) + .catch(() => { + this.props.addDangerToast(t('An error occurred while fetching data')); + this.setState({ dashboards: null, loading: false }); + }); + } + onSearchChange(search) { + const filteredDashboards = this.filterDashboards(search); + this.setState({ search, filteredDashboards }); + } + onDelete(dashboard) { + const filter = o => o.id !== dashboard.id; + const dashboards = this.state.dashboards.filter(filter); + const filteredDashboards = this.state.filteredDashboards.filter(filter); + this.setState({ dashboards, filteredDashboards }); + } + deleteDashboard(dashboard) { + const endpoint = `/dashboard/api/delete/${dashboard.id}`; + SupersetClient.delete({ endpoint }) + .then(({ json }) => { + this.onDelete(dashboard); + }) + .catch(() => { + this.props.addDangerToast(t('An error occurred while deleting the dashboard')); + }); + } + filterDashboards(searchText) { + const { dashboards } = this.state; + if (!searchText) { + return dashboards; + } + const lcaseSearchText = searchText.toLowerCase(); + return dashboards.filter(o => o.dashboard_title.toLowerCase().indexOf(lcaseSearchText) >= 0); + } + renderTable() { + return ( + + {this.state.filteredDashboards.map(o => ( + + + + + ))} +
+ + + {o.dashboard_title} + + + {unsafe(o.creator)} + + {unsafe(o.modified)} +
+ ); + } + renderCards() { + return this.state.filteredDashboards.map(dashboard => ( + + )); + } + + render() { + return ( + + ); + } +} + +DashboardCardTable.propTypes = propTypes; +export default withToasts(DashboardCardTable); diff --git a/superset/assets/src/welcome/DashboardTable.jsx b/superset/assets/src/welcome/DashboardTable.jsx deleted file mode 100644 index a3d689aec932..000000000000 --- a/superset/assets/src/welcome/DashboardTable.jsx +++ /dev/null @@ -1,93 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -import React from 'react'; -import PropTypes from 'prop-types'; -import { Table, Tr, Td, unsafe } from 'reactable-arc'; -import { SupersetClient } from '@superset-ui/connection'; -import { t } from '@superset-ui/translation'; - -import withToasts from '../messageToasts/enhancers/withToasts'; -import Loading from '../components/Loading'; -import '../../stylesheets/reactable-pagination.css'; - -const propTypes = { - search: PropTypes.string, - addDangerToast: PropTypes.func.isRequired, -}; - -class DashboardTable extends React.PureComponent { - constructor(props) { - super(props); - this.state = { - dashboards: [], - }; - } - - componentDidMount() { - SupersetClient.get({ - endpoint: '/dashboardasync/api/read?_oc_DashboardModelViewAsync=changed_on&_od_DashboardModelViewAsync=desc', - }) - .then(({ json }) => { - this.setState({ dashboards: json.result }); - }) - .catch(() => { - this.props.addDangerToast(t('An error occurred while fethching Dashboards')); - }); - } - - render() { - if (this.state.dashboards.length > 0) { - return ( - - {this.state.dashboards.map(o => ( - - - - - ))} -
- {o.dashboard_title} - - {unsafe(o.creator)} - - {unsafe(o.modified)} -
- ); - } - - return ; - } -} - -DashboardTable.propTypes = propTypes; - -export default withToasts(DashboardTable); diff --git a/superset/assets/src/welcome/Welcome.jsx b/superset/assets/src/welcome/Welcome.jsx index db1a632fd5f3..a1ca7dc6c698 100644 --- a/superset/assets/src/welcome/Welcome.jsx +++ b/superset/assets/src/welcome/Welcome.jsx @@ -18,51 +18,33 @@ */ import React from 'react'; import PropTypes from 'prop-types'; -import { Panel, Row, Col, Tabs, Tab, FormControl } from 'react-bootstrap'; +import { Panel, Row, Col, Tabs, Tab } from 'react-bootstrap'; import { t } from '@superset-ui/translation'; import RecentActivity from '../profile/components/RecentActivity'; import Favorites from '../profile/components/Favorites'; -import DashboardTable from './DashboardTable'; +import DashboardCardTable from './DashboardCardTable'; +import ChartCardTable from './ChartCardTable'; const propTypes = { user: PropTypes.object.isRequired, }; export default class Welcome extends React.PureComponent { - constructor(props) { - super(props); - this.state = { - search: '', - }; - this.onSearchChange = this.onSearchChange.bind(this); - } - onSearchChange(event) { - this.setState({ search: event.target.value }); - } render() { return (
- -

{t('Dashboards')}

- - - - -
- + + + + + + - +

{t('Recently Viewed')}

@@ -71,7 +53,7 @@ export default class Welcome extends React.PureComponent { - +

{t('Favorites')}

diff --git a/superset/assets/stylesheets/less/cosmo/variables.less b/superset/assets/stylesheets/less/cosmo/variables.less index 7b979c224525..dc61b52cbb13 100644 --- a/superset/assets/stylesheets/less/cosmo/variables.less +++ b/superset/assets/stylesheets/less/cosmo/variables.less @@ -29,8 +29,8 @@ @gray-darker: lighten(@gray-base, 13.5%); @gray-dark: lighten(@gray-base, 20%); @gray: lighten(@gray-base, 33.5%); -@gray-light: lighten(@gray-base, 70%); -@gray-lighter: lighten(@gray-base, 95%); +@gray-light: lighten(@gray-base, 80%); +@gray-lighter: lighten(@gray-base, 90%); @brand-primary: #00A699; @brand-success: #4AC15F; diff --git a/superset/assets/stylesheets/superset.less b/superset/assets/stylesheets/superset.less index d25dc6b73ccf..de9bb8a6c1b1 100644 --- a/superset/assets/stylesheets/superset.less +++ b/superset/assets/stylesheets/superset.less @@ -348,6 +348,9 @@ table.table-no-hover tr:hover { .m-t-5 { margin-top: 5px; } +.m-t-8 { + margin-top: 8px; +} .m-t-10 { margin-top: 10px; } diff --git a/superset/cli.py b/superset/cli.py index 6691b0148f8f..30e0aa4015ab 100755 --- a/superset/cli.py +++ b/superset/cli.py @@ -29,6 +29,8 @@ from superset import ( app, appbuilder, data, db, security_manager, ) +from superset.models import core as models +from superset.tasks.thumbnails import cache_chart_thumbnail, cache_dashboard_thumbnail from superset.utils import ( core as utils, dashboard_import_export, dict_import_export) @@ -347,6 +349,29 @@ def flower(port, address): Popen(cmd, shell=True).wait() +''' +@app.cli.command() +def test_email(): + """Test that email is configured""" + from superset.utils.email import send_email_smtp + send_email_smtp( + to='max@preset.io', + subject='it works!', + html_content='

SUPER

', + config=app.config, + ) + from superset.tasks.schedules import deliver_dashboard, deliver_slice + from superset.models.schedules import EmailDeliveryType, SliceEmailReportFormat + #dashboard = db.session.query(models.Dashboard).filter_by(id=2).one() + #deliver_dashboard(dashboard, [('max@preset.io', None)], EmailDeliveryType.inline) + + slc = db.session.query(models.Slice).filter_by(id=76).one() + deliver_slice( + slc, [('max@preset.io', None)], + SliceEmailReportFormat.visualization, EmailDeliveryType.inline) +''' + + @app.cli.command() def load_test_users(): """ @@ -358,6 +383,50 @@ def load_test_users(): load_test_users_run() +@app.cli.command() +@click.option( + '--asynchronous', '-a', is_flag=True, default=False, + help='Trigger commands to run remotely on a worker') +@click.option( + '--dashboards_only', '-d', is_flag=True, default=False, + help='Only process dashboards') +@click.option( + '--charts_only', '-c', is_flag=True, default=False, + help='Only process charts') +@click.option( + '--force', '-f', is_flag=True, default=False, + help='Force refresh, even if previously cached') +def compute_thumbnails(asynchronous, dashboards_only, charts_only, force): + """Compute thumbnails""" + if not charts_only: + dashboards = db.session.query(models.Dashboard).all() + count = len(dashboards) + for i, dash in enumerate(dashboards): + if asynchronous: + func = cache_dashboard_thumbnail.delay + action = 'Triggering' + else: + func = cache_dashboard_thumbnail + action = 'Processing' + msg = f'{action} dashboard "{dash.dashboard_title}" ({i+1}/{count})' + click.secho(msg, fg='green') + func(dash.id, force=force) + + if not dashboards_only: + slices = db.session.query(models.Slice).all() + count = len(slices) + for i, slc in enumerate(slices): + if asynchronous: + func = cache_chart_thumbnail.delay + action = 'Triggering' + else: + func = cache_chart_thumbnail + action = 'Processing' + msg = f'{action} chart "{slc.slice_name}" ({i+1}/{count})' + click.secho(msg, fg='green') + func(slc.id, force=force) + + def load_test_users_run(): """ Loads admin, alpha, and gamma user for testing purposes diff --git a/superset/config.py b/superset/config.py index 7d6ca33e5f9a..0fc768237d43 100644 --- a/superset/config.py +++ b/superset/config.py @@ -238,6 +238,7 @@ CACHE_DEFAULT_TIMEOUT = 60 * 60 * 24 CACHE_CONFIG = {'CACHE_TYPE': 'null'} TABLE_NAMES_CACHE_CONFIG = {'CACHE_TYPE': 'null'} +THUMBNAIL_CACHE_CONFIG = {'CACHE_TYPE': 'null'} # CORS Options ENABLE_CORS = False @@ -370,6 +371,8 @@ # you'll want to use a proper broker as specified here: # http://docs.celeryproject.org/en/latest/getting-started/brokers/index.html +CELERYD_LOG_LEVEL = 'DEBUG' + class CeleryConfig(object): BROKER_URL = 'sqla+sqlite:///celerydb.sqlite' @@ -584,7 +587,9 @@ class CeleryConfig(object): # Window size - this will impact the rendering of the data WEBDRIVER_WINDOW = { 'dashboard': (1600, 2000), - 'slice': (3000, 1200), + 'slice': (800, 600), + 'thumbnail_chart': (800, 600), + 'thumbnail_dashboard': (800, 600), } # Any config options to be passed as-is to the webdriver diff --git a/superset/connectors/base/models.py b/superset/connectors/base/models.py index 57db0ad4684d..cccd87cf8a05 100644 --- a/superset/connectors/base/models.py +++ b/superset/connectors/base/models.py @@ -162,6 +162,16 @@ def short_data(self): def select_star(self): pass + @property + def data_summary(self): + return { + 'datasource_name': self.datasource_name, + 'type': self.type, + 'schema': self.schema, + 'id': self.id, + 'explore_url': self.explore_url, + } + @property def data(self): """Data representation of the datasource sent to the frontend""" diff --git a/superset/models/core.py b/superset/models/core.py index b379af7caba2..aa17c4dff30f 100644 --- a/superset/models/core.py +++ b/superset/models/core.py @@ -51,10 +51,12 @@ from superset.models.helpers import AuditMixinNullable, ImportMixin from superset.models.tags import ChartUpdater, DashboardUpdater, FavStarUpdater from superset.models.user_attributes import UserAttribute +from superset.tasks.thumbnails import cache_chart_thumbnail, cache_dashboard_thumbnail from superset.utils import ( cache as cache_util, core as utils, ) +from superset.utils.json import json_dumps_w_dates from superset.viz import viz_types from urllib import parse # noqa @@ -175,6 +177,28 @@ def cls_model(self): def datasource(self): return self.get_datasource + @property + def datasource_data_summary(self): + return self.datasource.data_summary + + @property + def thumbnail_url(self): + # SHA here is to force bypassing the browser cache when chart has changed + sha = utils.md5_hex(self.params, 6) + return f'/thumb/chart/{self.id}/{sha}/' + + @property + def thumbnail_img(self): + return Markup(f'') + + @property + def thumbnail_link(self): + return Markup(f""" + + {self.thumbnail_img} + + """) + def clone(self): return Slice( slice_name=self.slice_name, @@ -247,6 +271,7 @@ def data(self): 'modified': self.modified(), 'changed_on_humanized': self.changed_on_humanized, 'changed_on': self.changed_on.isoformat(), + 'thumbnail_url': self.thumbnail_url, } @property @@ -375,8 +400,13 @@ def url(self): ) -sqla.event.listen(Slice, 'before_insert', set_related_perm) -sqla.event.listen(Slice, 'before_update', set_related_perm) +def event_after_slice_changed(mapper, connection, target): + set_related_perm(mapper, connection, target) + cache_chart_thumbnail.delay(target.id, force=True) + + +sqla.event.listen(Slice, 'before_insert', event_after_slice_changed) +sqla.event.listen(Slice, 'before_update', event_after_slice_changed) dashboard_slices = Table( @@ -469,6 +499,7 @@ def data(self): 'dashboard_title': self.dashboard_title, 'slug': self.slug, 'slices': [slc.data for slc in self.slices], + 'thumbnail_url': self.thumbnail_url, 'position_json': positions, } @@ -657,6 +688,32 @@ def export_dashboards(cls, dashboard_ids): 'datasources': eager_datasources, }, cls=utils.DashboardEncoder, indent=4) + @property + def thumbnail_url(self): + # SHA here is to force bypassing the browser cache when chart has changed + sha = utils.md5_hex(self.position_json, 6) + return f'/thumb/dashboard/{self.id}/{sha}/' + + @property + def thumbnail_img(self): + return Markup(f'') + + @property + def thumbnail_link(self): + return Markup(f""" + + {self.thumbnail_img} + + """) + + +def event_after_dashboard_changed(mapper, connection, target): + cache_dashboard_thumbnail.delay(target.id, force=True) + + +sqla.event.listen(Dashboard, 'before_insert', event_after_dashboard_changed) +sqla.event.listen(Dashboard, 'before_update', event_after_dashboard_changed) + class Database(Model, AuditMixinNullable, ImportMixin): @@ -889,7 +946,7 @@ def _log_query(sql): for k, v in df.dtypes.items(): if v.type == numpy.object_ and needs_conversion(df[k]): - df[k] = df[k].apply(utils.json_dumps_w_dates) + df[k] = df[k].apply(json_dumps_w_dates) return df def compile_sqla_query(self, qry, schema=None): diff --git a/superset/models/helpers.py b/superset/models/helpers.py index 78b438d9f8f1..fd8d2f6dabc4 100644 --- a/superset/models/helpers.py +++ b/superset/models/helpers.py @@ -272,8 +272,13 @@ def _user_link(self, user): return Markup('{}'.format(url, escape(user) or '')) def changed_by_name(self): - if self.created_by: - return escape('{}'.format(self.created_by)) + if self.changed_by: + return escape('{}'.format(self.changed_by)) + return '' + + def created_by_name(self): + if self.changed_by: + return escape('{}'.format(self.changed_by)) return '' @renders('created_by') diff --git a/superset/sql_lab.py b/superset/sql_lab.py index 86e171ba44d7..7e89a5cd08a8 100644 --- a/superset/sql_lab.py +++ b/superset/sql_lab.py @@ -34,13 +34,13 @@ from superset.sql_parse import ParsedQuery from superset.tasks.celery_app import app as celery_app from superset.utils.core import ( - json_iso_dttm_ser, QueryStatus, sources, zlib_compress, ) from superset.utils.dates import now_as_float from superset.utils.decorators import stats_timing +from superset.utils.json import json_iso_dttm_ser config = app.config stats_logger = config.get('STATS_LOGGER') diff --git a/superset/tasks/cache.py b/superset/tasks/cache.py index 831bb6642914..dd58dfa909ed 100644 --- a/superset/tasks/cache.py +++ b/superset/tasks/cache.py @@ -27,7 +27,6 @@ from sqlalchemy import and_, func from superset import app, db -from superset.models.core import Dashboard, Log, Slice from superset.models.tags import Tag, TaggedObject from superset.tasks.celery_app import app as celery_app from superset.utils.core import parse_human_datetime @@ -169,6 +168,7 @@ def get_urls(self): urls = [] session = db.create_scoped_session() + from superset.models.core import Dashboard, Log records = ( session .query(Log.dashboard_id, func.count(Log.dashboard_id)) @@ -182,6 +182,7 @@ def get_urls(self): .all() ) dash_ids = [record.dashboard_id for record in records] + from superset.models.core import Dashboard dashboards = ( session .query(Dashboard) @@ -262,6 +263,7 @@ def get_urls(self): .all() ) chart_ids = [tagged_object.object_id for tagged_object in tagged_objects] + from superset.models.core import Slice tagged_charts = ( session .query(Slice) diff --git a/superset/tasks/schedules.py b/superset/tasks/schedules.py index b4ca9f41bdc0..029024acf063 100644 --- a/superset/tasks/schedules.py +++ b/superset/tasks/schedules.py @@ -22,23 +22,15 @@ from datetime import datetime, timedelta from email.utils import make_msgid, parseaddr import logging -import time - import croniter from dateutil.tz import tzlocal -from flask import render_template, Response, session, url_for +from flask import render_template, url_for from flask_babel import gettext as __ -from flask_login import login_user import requests -from retry.api import retry_call -from selenium.common.exceptions import WebDriverException -from selenium.webdriver import chrome, firefox import simplejson as json from six.moves import urllib -from werkzeug.utils import parse_cookie -# Superset framework imports from superset import app, db, security_manager from superset.models.schedules import ( EmailDeliveryType, @@ -47,24 +39,21 @@ SliceEmailReportFormat, ) from superset.tasks.celery_app import app as celery_app -from superset.utils.core import ( - get_email_address_list, - send_email_smtp, +from superset.utils.email import get_email_address_list, send_email_smtp +from superset.utils.selenium import ( + DashboardScreenshot, + get_auth_cookies, + SliceScreenshot, ) # Globals -config = app.config logging.getLogger('tasks.email_reports').setLevel(logging.INFO) -# Time in seconds, we will wait for the page to load and render -PAGE_RENDER_WAIT = 30 - - EmailContent = namedtuple('EmailContent', ['body', 'data', 'images']) def _get_recipients(schedule): - bcc = config.get('EMAIL_REPORT_BCC_ADDRESS', None) + bcc = app.config.get('EMAIL_REPORT_BCC_ADDRESS', None) if schedule.deliver_as_group: to = schedule.recipients @@ -74,20 +63,24 @@ def _get_recipients(schedule): yield (to, bcc) -def _deliver_email(schedule, subject, email): - for (to, bcc) in _get_recipients(schedule): +def _deliver_email(recipients, subject, email): + config = app.config + dryrun = config.get('SCHEDULED_EMAIL_DEBUG_MODE') + for (to, bcc) in recipients: send_email_smtp( - to, subject, email.body, config, + to, subject, email.body, + config, data=email.data, images=email.images, bcc=bcc, mime_subtype='related', - dryrun=config.get('SCHEDULED_EMAIL_DEBUG_MODE'), + dryrun=dryrun, ) -def _generate_mail_content(schedule, screenshot, name, url): - if schedule.delivery_type == EmailDeliveryType.attachment: +def _generate_mail_content(delivery_type, screenshot, name, url): + config = app.config + if delivery_type == EmailDeliveryType.attachment: images = None data = { 'screenshot.png': screenshot, @@ -97,7 +90,9 @@ def _generate_mail_content(schedule, screenshot, name, url): name=name, url=url, ) - elif schedule.delivery_type == EmailDeliveryType.inline: + else: + # Implicit: delivery_type == EmailDeliveryType.inline: + # Get the domain from the 'From' address .. # and make a message id without the < > in the ends domain = parseaddr(config.get('SMTP_MAIL_FROM'))[1].split('@')[1] @@ -118,135 +113,33 @@ def _generate_mail_content(schedule, screenshot, name, url): return EmailContent(body, data, images) -def _get_auth_cookies(): - # Login with the user specified to get the reports - with app.test_request_context(): - user = security_manager.find_user(config.get('EMAIL_REPORTS_USER')) - login_user(user) - - # A mock response object to get the cookie information from - response = Response() - app.session_interface.save_session(app, session, response) - - cookies = [] - - # Set the cookies in the driver - for name, value in response.headers: - if name.lower() == 'set-cookie': - cookie = parse_cookie(value) - cookies.append(cookie['session']) - - return cookies - - def _get_url_path(view, **kwargs): with app.test_request_context(): return urllib.parse.urljoin( - str(config.get('WEBDRIVER_BASEURL')), + str(app.config.get('WEBDRIVER_BASEURL')), url_for(view, **kwargs), ) -def create_webdriver(): - # Create a webdriver for use in fetching reports - if config.get('EMAIL_REPORTS_WEBDRIVER') == 'firefox': - driver_class = firefox.webdriver.WebDriver - options = firefox.options.Options() - elif config.get('EMAIL_REPORTS_WEBDRIVER') == 'chrome': - driver_class = chrome.webdriver.WebDriver - options = chrome.options.Options() - - options.add_argument('--headless') - - # Prepare args for the webdriver init - kwargs = dict( - options=options, - ) - kwargs.update(config.get('WEBDRIVER_CONFIGURATION')) - - # Initialize the driver - driver = driver_class(**kwargs) - - # Some webdrivers need an initial hit to the welcome URL - # before we set the cookie - welcome_url = _get_url_path('Superset.welcome') - - # Hit the welcome URL and check if we were asked to login - driver.get(welcome_url) - elements = driver.find_elements_by_id('loginbox') - - # This indicates that we were not prompted for a login box. - if not elements: - return driver - - # Set the cookies in the driver - for cookie in _get_auth_cookies(): - info = dict(name='session', value=cookie) - driver.add_cookie(info) - - return driver - - -def destroy_webdriver(driver): - """ - Destroy a driver - """ - - # This is some very flaky code in selenium. Hence the retries - # and catch-all exceptions - try: - retry_call(driver.close, tries=2) - except Exception: - pass - try: - driver.quit() - except Exception: - pass - - -def deliver_dashboard(schedule): +def deliver_dashboard(dashboard, recipients, delivery_type): """ Given a schedule, delivery the dashboard as an email report """ - dashboard = schedule.dashboard - - dashboard_url = _get_url_path( - 'Superset.dashboard', - dashboard_id=dashboard.id, - ) + config = app.config - # Create a driver, fetch the page, wait for the page to render - driver = create_webdriver() window = config.get('WEBDRIVER_WINDOW')['dashboard'] - driver.set_window_size(*window) - driver.get(dashboard_url) - time.sleep(PAGE_RENDER_WAIT) - - # Set up a function to retry once for the element. - # This is buggy in certain selenium versions with firefox driver - get_element = getattr(driver, 'find_element_by_class_name') - element = retry_call( - get_element, - fargs=['grid-container'], - tries=2, - delay=PAGE_RENDER_WAIT, - ) - - try: - screenshot = element.screenshot_as_png - except WebDriverException: - # Some webdrivers do not support screenshots for elements. - # In such cases, take a screenshot of the entire page. - screenshot = driver.screenshot() # pylint: disable=no-member - finally: - destroy_webdriver(driver) + user = security_manager.find_user(config.get('EMAIL_REPORTS_USER')) + with app.app_context(): + screenshot = DashboardScreenshot(id=dashboard.id) + img = screenshot.get_thumb( + user=user, window_size=window, thumb_size=window) # Generate the email body and attachments email = _generate_mail_content( - schedule, - screenshot, + delivery_type, + img, dashboard.dashboard_title, - dashboard_url, + screenshot.url, ) subject = __( @@ -255,11 +148,11 @@ def deliver_dashboard(schedule): title=dashboard.dashboard_title, ) - _deliver_email(schedule, subject, email) + _deliver_email(recipients, subject, email) -def _get_slice_data(schedule): - slc = schedule.slice +def _get_slice_data(slc, delivery_type): + config = app.config slice_url = _get_url_path( 'Superset.explore_json', @@ -274,8 +167,10 @@ def _get_slice_data(schedule): ) cookies = {} - for cookie in _get_auth_cookies(): - cookies['session'] = cookie + user = security_manager.find_user(config.get('EMAIL_REPORTS_USER')) + with app.app_context(): + for cookie in get_auth_cookies(user): + cookies['session'] = cookie response = requests.get(slice_url, cookies=cookies) response.raise_for_status() @@ -283,7 +178,7 @@ def _get_slice_data(schedule): # TODO: Move to the csv module rows = [r.split(b',') for r in response.content.splitlines()] - if schedule.delivery_type == EmailDeliveryType.inline: + if delivery_type == EmailDeliveryType.inline: data = None # Parse the csv file and generate HTML @@ -297,7 +192,7 @@ def _get_slice_data(schedule): link=url, ) - elif schedule.delivery_type == EmailDeliveryType.attachment: + elif delivery_type == EmailDeliveryType.attachment: data = { __('%(name)s.csv', name=slc.slice_name): response.content, } @@ -310,67 +205,48 @@ def _get_slice_data(schedule): return EmailContent(body, data, None) -def _get_slice_visualization(schedule): - slc = schedule.slice - - # Create a driver, fetch the page, wait for the page to render - driver = create_webdriver() - window = config.get('WEBDRIVER_WINDOW')['slice'] - driver.set_window_size(*window) - - slice_url = _get_url_path( +def _get_slice_visualization(slc, delivery_type): + config = app.config + url = _get_url_path( 'Superset.slice', slice_id=slc.id, ) + window = config.get('WEBDRIVER_WINDOW')['slice'] + user = security_manager.find_user(config.get('EMAIL_REPORTS_USER')) - driver.get(slice_url) - time.sleep(PAGE_RENDER_WAIT) - - # Set up a function to retry once for the element. - # This is buggy in certain selenium versions with firefox driver - element = retry_call( - driver.find_element_by_class_name, - fargs=['chart-container'], - tries=2, - delay=PAGE_RENDER_WAIT, - ) - - try: - screenshot = element.screenshot_as_png - except WebDriverException: - # Some webdrivers do not support screenshots for elements. - # In such cases, take a screenshot of the entire page. - screenshot = driver.screenshot() # pylint: disable=no-member - finally: - destroy_webdriver(driver) + with app.app_context(): + screenshot = SliceScreenshot(id=slc.id) + img = screenshot.get_thumb( + user=user, window_size=window, thumb_size=window) # Generate the email body and attachments return _generate_mail_content( - schedule, - screenshot, + delivery_type, + img, slc.slice_name, - slice_url, + url, ) -def deliver_slice(schedule): +def deliver_slice(slc, recipients, email_format, delivery_type): """ Given a schedule, delivery the slice as an email report """ - if schedule.email_format == SliceEmailReportFormat.data: - email = _get_slice_data(schedule) - elif schedule.email_format == SliceEmailReportFormat.visualization: - email = _get_slice_visualization(schedule) + config = app.config + if email_format == SliceEmailReportFormat.data: + email = _get_slice_data(slc, delivery_type) + elif email_format == SliceEmailReportFormat.visualization: + email = _get_slice_visualization(slc, delivery_type) else: raise RuntimeError('Unknown email report format') subject = __( '%(prefix)s %(title)s', prefix=config.get('EMAIL_REPORTS_SUBJECT_PREFIX'), - title=schedule.slice.slice_name, + title=slc.slice_name, ) - _deliver_email(schedule, subject, email) + _deliver_email(recipients, subject, email) @celery_app.task(name='email_reports.send', bind=True, soft_time_limit=300) @@ -389,9 +265,15 @@ def schedule_email_report(task, report_type, schedule_id, recipients=None): schedule.recipients = recipients if report_type == ScheduleType.dashboard.value: - deliver_dashboard(schedule) + deliver_dashboard( + schedule.dashboard, _get_recipients(schedule), schedule.delivery_type) elif report_type == ScheduleType.slice.value: - deliver_slice(schedule) + deliver_slice( + schedule.slice, + _get_recipients(schedule), + schedule.email_format, + schedule.delivery_type, + ) else: raise RuntimeError('Unknown report type') @@ -443,6 +325,7 @@ def schedule_window(report_type, start_at, stop_at, resolution): @celery_app.task(name='email_reports.schedule_hourly') def schedule_hourly(): """ Celery beat job meant to be invoked hourly """ + config = app.config if not config.get('ENABLE_SCHEDULED_EMAIL_REPORTS'): logging.info('Scheduled email reports not enabled in config') diff --git a/superset/tasks/thumbnails.py b/superset/tasks/thumbnails.py new file mode 100644 index 000000000000..a4cbd46a5d6a --- /dev/null +++ b/superset/tasks/thumbnails.py @@ -0,0 +1,43 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# pylint: disable=C,R,W + +"""Utility functions used across Superset""" + +import logging + +from superset import app, security_manager, thumbnail_cache +from superset.tasks.celery_app import app as celery_app +from superset.utils.selenium import DashboardScreenshot, SliceScreenshot + + +@celery_app.task(name='cache_chart_thumbnail', soft_time_limit=300) +def cache_chart_thumbnail(chart_id, force=False): + with app.app_context(): + logging.info(f'Caching chart {chart_id}') + screenshot = SliceScreenshot(id=chart_id) + user = security_manager.find_user('Admin') + screenshot.compute_and_cache(user=user, cache=thumbnail_cache, force=force) + + +@celery_app.task(name='cache_dashboard_thumbnail', soft_time_limit=300) +def cache_dashboard_thumbnail(dashboard_id, force=False): + with app.app_context(): + logging.info(f'Caching dashboard {dashboard_id}') + screenshot = DashboardScreenshot(id=dashboard_id) + user = security_manager.find_user('Admin') + screenshot.compute_and_cache(user=user, cache=thumbnail_cache, force=force) diff --git a/superset/utils/selenium.py b/superset/utils/selenium.py new file mode 100644 index 000000000000..c6e5013f2152 --- /dev/null +++ b/superset/utils/selenium.py @@ -0,0 +1,295 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# pylint: disable=C,R,W +from io import BytesIO +import logging +import time +import urllib + +from flask import current_app, request, Response, session, url_for +from flask_login import login_user +from PIL import Image +from retry.api import retry_call +from selenium.common.exceptions import WebDriverException +from selenium.webdriver import chrome, firefox +from selenium.webdriver.common.by import By +from selenium.webdriver.support import expected_conditions as EC +from selenium.webdriver.support.ui import WebDriverWait +from werkzeug.utils import parse_cookie + +# Time in seconds, we will wait for the page to load and render +SELENIUM_CHECK_INTERVAL = 2 +SELENIUM_RETRIES = 5 +SELENIUM_HEADSTART = 3 + + +def headless_url(path): + return urllib.parse.urljoin( + current_app.config.get('WEBDRIVER_BASEURL'), + path, + ) + + +def get_url_path(view, **kwargs): + with current_app.test_request_context(): + return headless_url(url_for(view, **kwargs)) + + +class BaseScreenshot: + thumbnail_type = None + orm_class = None + element = None + window_size = (800, 600) + thumb_size = (400, 300) + + def __init__(self, id): + self.id = id + self.screenshot = None + + @property + def cache_key(self): + return f'thumb__{self.thumbnail_type}__{self.id}' + + @property + def url(self): + raise NotImplementedError() + + def fetch_screenshot(self, user, window_size=None): + window_size = window_size or self.window_size + self.screenshot = get_png_from_url( + self.url, window_size, self.element, user=user) + return self.screenshot + + def get_thumb_as_bytes(self, *args, **kwargs): + payload = self.get_thumb(*args, **kwargs) + return BytesIO(payload) + + def get_from_cache(self, cache): + payload = cache.get(self.cache_key) + if payload: + return BytesIO(payload) + + def compute_and_cache( + self, user, cache, window_size=None, thumb_size=None, + force=True): + cache_key = self.cache_key + if not force and cache.get(cache_key): + logging.info('Thumb already cached, skipping...') + return + window_size = window_size or self.window_size + thumb_size = thumb_size or self.thumb_size + logging.info(f'Processing url for thumbnail: {cache_key}') + + payload = None + + # Assuming all sorts of things can go wrong with Selenium + try: + payload = self.fetch_screenshot(window_size=window_size, user=user) + except Exception as e: + logging.error('Failed at generating thumbnail') + logging.exception(e) + + if payload and window_size != thumb_size: + try: + payload = self.resize_image(payload, size=thumb_size) + except Exception as e: + logging.error('Failed at resizing thumbnail') + logging.exception(e) + payload = None + + if payload and cache: + logging.info(f'Caching thumbnail: {cache_key}') + cache.set(cache_key, payload) + + return payload + + def get_thumb( + self, user, + window_size=None, thumb_size=None, + cache=None, + ): + payload = None + cache_key = self.cache_key + window_size = window_size or self.window_size + thumb_size = thumb_size or self.thumb_size + if cache: + payload = cache.get(cache_key) + if not payload: + payload = self.compute_and_cache(user, cache, window_size, thumb_size) + else: + logging.info(f'Loaded thumbnail from cache: {cache_key}') + return payload + + @classmethod + def resize_image(cls, img_bytes, output='png', size=None, crop=True): + size = size or cls.thumb_size + img = Image.open(BytesIO(img_bytes)) + logging.debug(f'Selenium image size: {img.size}') + if crop and img.size[1] > cls.window_size[1]: + desired_ratio = float(cls.window_size[1]) / cls.window_size[0] + desired_width = int(img.size[0] * desired_ratio) + logging.debug(f'Cropping to: {img.size[0]}*{desired_width}') + img = img.crop((0, 0, img.size[0], desired_width)) + logging.debug(f'Resizing to {size}') + img = img.resize(size, Image.ANTIALIAS) + new_img = BytesIO() + if output != 'png': + img = img.convert('RGB') + img.save(new_img, output) + new_img.seek(0) + return new_img.read() + + +class SliceScreenshot(BaseScreenshot): + thumbnail_type = 'slice' + element = 'chart-container' + window_size = (600, int(600 * 0.75)) + thumb_size = (300, int(300 * 0.75)) + + @property + def url(self): + return get_url_path( + 'Superset.slice', + slice_id=self.id, + standalone='true', + ) + + +class DashboardScreenshot(BaseScreenshot): + thumbnail_type = 'dashboard' + element = 'grid-container' + window_size = (1600, int(1600 * 0.75)) + thumb_size = (400, int(400 * 0.75)) + + @property + def url(self): + return get_url_path( + 'Superset.dashboard', + dashboard_id=self.id, + ) + + +def _destroy_webdriver(driver): + """Destroy a driver""" + # This is some very flaky code in selenium. Hence the retries + # and catch-all exceptions + try: + retry_call(driver.close, tries=2) + except Exception: + pass + try: + driver.quit() + except Exception: + pass + + +def get_auth_cookies(user): + # Login with the user specified to get the reports + with current_app.test_request_context(): + login_user(user) + + # A mock response object to get the cookie information from + response = Response() + current_app.session_interface.save_session(current_app, session, response) + + cookies = [] + + # Set the cookies in the driver + for name, value in response.headers: + if name.lower() == 'set-cookie': + cookie = parse_cookie(value) + cookies.append(cookie['session']) + return cookies + + +def create_webdriver(user=None, webdriver='chrome', window=None): + """Creates a selenium webdriver + + If no user is specified, we use the current request's context""" + # Create a webdriver for use in fetching reports + if webdriver == 'firefox': + driver_class = firefox.webdriver.WebDriver + options = firefox.options.Options() + else: + # webdriver == 'chrome': + driver_class = chrome.webdriver.WebDriver + options = chrome.options.Options() + arg = f'--window-size={window[0]},{window[1]}' + options.add_argument(arg) + + options.add_argument('--headless') + + # Prepare args for the webdriver init + kwargs = dict( + options=options, + ) + kwargs.update(current_app.config.get('WEBDRIVER_CONFIGURATION')) + + logging.info('Init selenium driver') + driver = driver_class(**kwargs) + + # Setting cookies requires doing a request first + driver.get(headless_url('/login/')) + + if user: + # Set the cookies in the driver + for cookie in get_auth_cookies(user): + info = dict(name='session', value=cookie) + driver.add_cookie(info) + elif request.cookies: + cookies = request.cookies + for k, v in cookies.items(): + cookie = dict(name=k, value=v) + driver.add_cookie(cookie) + return driver + + +def get_png_from_url( + url, + window, + element, + user, + webdriver='chrome', + retries=SELENIUM_RETRIES, +): + driver = create_webdriver(user, webdriver, window) + driver.set_window_size(*window) + driver.get(url) + img = None + logging.debug(f'Sleeping for {SELENIUM_HEADSTART} seconds') + time.sleep(SELENIUM_HEADSTART) + try: + logging.debug(f'Wait for the presence of {element}') + element = WebDriverWait(driver, 10).until( + EC.presence_of_element_located((By.CLASS_NAME, element)), + ) + logging.debug(f'Wait for .loading to be done') + WebDriverWait(driver, 60).until_not( + EC.presence_of_all_elements_located( + (By.CLASS_NAME, 'loading'), + ), + ) + logging.info('Taking a PNG screenshot') + img = element.screenshot_as_png + except WebDriverException as e: + logging.exception(e) + # Some webdrivers do not support screenshots for elements. + # In such cases, take a screenshot of the entire page. + img = driver.screenshot() # pylint: disable=no-member + finally: + _destroy_webdriver(driver) + return img diff --git a/superset/views/__init__.py b/superset/views/__init__.py index 380ea6ec93f1..db726188342f 100644 --- a/superset/views/__init__.py +++ b/superset/views/__init__.py @@ -19,6 +19,7 @@ from . import core # noqa from . import sql_lab # noqa from . import dashboard # noqa +from . import thumbnails # noqa from . import annotations # noqa from . import datasource # noqa from . import schedules # noqa diff --git a/superset/views/base.py b/superset/views/base.py index 2d14622ea893..28df8791792b 100644 --- a/superset/views/base.py +++ b/superset/views/base.py @@ -39,6 +39,7 @@ from superset.exceptions import SupersetException, SupersetSecurityException from superset.translations.utils import get_language_pack from superset.utils import core as utils +from superset.utils.json import json_int_dttm_ser, json_iso_dttm_ser FRONTEND_CONF_KEYS = ( 'SUPERSET_WEBSERVER_TIMEOUT', @@ -71,7 +72,7 @@ def json_error_response(msg=None, status=500, stacktrace=None, payload=None, lin payload['link'] = link return Response( - json.dumps(payload, default=utils.json_iso_dttm_ser, ignore_nan=True), + json.dumps(payload, default=json_iso_dttm_ser, ignore_nan=True), status=status, mimetype='application/json') @@ -149,7 +150,7 @@ class BaseSupersetView(BaseView): def json_response(self, obj, status=200): return Response( - json.dumps(obj, default=utils.json_int_dttm_ser, ignore_nan=True), + json.dumps(obj, default=json_int_dttm_ser, ignore_nan=True), status=status, mimetype='application/json') diff --git a/superset/views/core.py b/superset/views/core.py index d8a3692c2440..fc58a86457e8 100755 --- a/superset/views/core.py +++ b/superset/views/core.py @@ -58,8 +58,9 @@ from superset.sql_validators import get_validator_by_name from superset.utils import core as utils from superset.utils import dashboard_import_export -from superset.utils.dates import now_as_float +from superset.utils.dates import EPOCH, now_as_float from superset.utils.decorators import etag_cache + from .base import ( api, BaseSupersetView, check_ownership, @@ -499,7 +500,9 @@ class SliceModelView(SupersetModelView, DeleteMixin): # noqa 'slice_name', 'description', 'viz_type', 'datasource_name', 'owners', ) list_columns = [ - 'slice_link', 'viz_type', 'datasource_link', 'creator', 'modified'] + 'slice_link', 'viz_type', 'datasource_link', + 'creator', 'modified', + ] order_columns = ['viz_type', 'datasource_link', 'modified'] edit_columns = [ 'slice_name', 'description', 'viz_type', 'owners', 'dashboards', @@ -576,7 +579,9 @@ class SliceAsync(SliceModelView): # noqa route_base = '/sliceasync' list_columns = [ 'id', 'slice_link', 'viz_type', 'slice_name', - 'creator', 'modified', 'icons', 'changed_on_humanized', + 'creator', 'modified', 'icons', 'thumbnail_url', + 'slice_url', 'created_by_name', 'changed_on', + 'datasource_data_summary', 'changed_on_humanized', ] label_columns = { 'icons': ' ', @@ -709,7 +714,8 @@ class DashboardModelViewAsync(DashboardModelView): # noqa route_base = '/dashboardasync' list_columns = [ 'id', 'dashboard_link', 'creator', 'modified', 'dashboard_title', - 'changed_on', 'url', 'changed_by_name', + 'changed_on', 'url', 'changed_by_name', 'created_by_name', + 'thumbnail_url', 'changed_on_humanized' ] label_columns = { 'dashboard_link': _('Dashboard'), @@ -1976,7 +1982,7 @@ def created_dashboards(self, user_id): 'dttm': o.changed_on, } for o in qry.all()] return json_success( - json.dumps(payload, default=utils.json_int_dttm_ser)) + json.dumps(payload, default=json_int_dttm_ser)) @api @has_access_api @@ -2015,7 +2021,7 @@ def user_slices(self, user_id=None): 'viz_type': o.Slice.viz_type, } for o in qry.all()] return json_success( - json.dumps(payload, default=utils.json_int_dttm_ser)) + json.dumps(payload, default=json_int_dttm_ser)) @api @has_access_api @@ -2797,7 +2803,7 @@ def queries(self, last_updated_ms): last_updated_ms_int = int(float(last_updated_ms)) if last_updated_ms else 0 # UTC date time, same that is stored in the DB. - last_updated_dt = utils.EPOCH + timedelta(seconds=last_updated_ms_int / 1000) + last_updated_dt = EPOCH + timedelta(seconds=last_updated_ms_int / 1000) sql_queries = ( db.session.query(Query) @@ -2923,7 +2929,7 @@ def profile(self, username): 'superset/basic.html', title=_("%(user)s's profile", user=username), entry='profile', - bootstrap_data=json.dumps(payload, default=utils.json_iso_dttm_ser), + bootstrap_data=json.dumps(payload, default=utils.utils.json_iso_dttm_ser), ) @has_access diff --git a/superset/views/schedules.py b/superset/views/schedules.py index 6fdae7726521..95db908f79d9 100644 --- a/superset/views/schedules.py +++ b/superset/views/schedules.py @@ -37,10 +37,8 @@ SliceEmailSchedule, ) from superset.tasks.schedules import schedule_email_report -from superset.utils.core import ( - get_email_address_list, - json_iso_dttm_ser, -) +from superset.utils.email import get_email_address_list +from superset.utils.json import json_iso_dttm_ser from superset.views.core import json_success from .base import DeleteMixin, SupersetModelView diff --git a/superset/views/thumbnails.py b/superset/views/thumbnails.py new file mode 100644 index 000000000000..281ea17c9eb7 --- /dev/null +++ b/superset/views/thumbnails.py @@ -0,0 +1,38 @@ +# pylint: disable=C,R,W +from flask import redirect, send_file +from flask_appbuilder import expose +from flask_appbuilder.security.decorators import has_access + +from superset import app, appbuilder, thumbnail_cache +from superset.utils.selenium import DashboardScreenshot, SliceScreenshot +from .base import BaseSupersetView + +config = app.config +NO_IMAGE_SRC = '/static/assets/images/no-image.png' + + +class Thumb(BaseSupersetView): + """Base views for thumbnails""" + @expose('/chart///') + @has_access + def chart(self, slice_id, sha=None): + """Returns an thumbnail for a given chart, uses cache if possible""" + # TODO security + screenshot = SliceScreenshot(id=slice_id) + img = screenshot.get_from_cache(thumbnail_cache) + if not img: + return redirect(NO_IMAGE_SRC) + return send_file(img, mimetype='image/png') + + @expose('/dashboard///') + @has_access + def dashboard(self, dashboard_id, sha=None): + """Returns an thumbnail for a given dash, uses cache if possible""" + screenshot = DashboardScreenshot(id=dashboard_id) + img = screenshot.get_from_cache(thumbnail_cache) + if not img: + return redirect(NO_IMAGE_SRC) + return send_file(img, mimetype='image/png') + + +appbuilder.add_view_no_menu(Thumb) diff --git a/superset/viz.py b/superset/viz.py index a6864d32312c..61561f5adcba 100644 --- a/superset/viz.py +++ b/superset/viz.py @@ -55,6 +55,7 @@ merge_extra_filters, to_adhoc, ) +from superset.utils.json import json_int_dttm_ser, json_iso_dttm_ser config = app.config @@ -333,7 +334,7 @@ def cache_timeout(self): def get_json(self): return json.dumps( self.get_payload(), - default=utils.json_int_dttm_ser, ignore_nan=True) + default=json_int_dttm_ser, ignore_nan=True) def cache_key(self, query_obj, **extra): """ @@ -460,7 +461,7 @@ def get_df_payload(self, query_obj=None, **kwargs): def json_dumps(self, obj, sort_keys=False): return json.dumps( obj, - default=utils.json_int_dttm_ser, + default=json_int_dttm_ser, ignore_nan=True, sort_keys=sort_keys, ) @@ -595,7 +596,7 @@ def get_data(self, df): def json_dumps(self, obj, sort_keys=False): return json.dumps( obj, - default=utils.json_iso_dttm_ser, + default=json_iso_dttm_ser, sort_keys=sort_keys, ignore_nan=True) diff --git a/tests/access_tests.py b/tests/access_tests.py index 0bc1743cfe1f..cc0848280587 100644 --- a/tests/access_tests.py +++ b/tests/access_tests.py @@ -330,7 +330,7 @@ def test_clean_requests_after_schema_grant(self): session.commit() - @mock.patch('superset.utils.core.send_MIME_email') + @mock.patch('superset.utils.email.send_MIME_email') def test_approve(self, mock_send_mime): if app.config.get('ENABLE_ACCESS_REQUEST'): session = db.session diff --git a/tests/email_tests.py b/tests/email_tests.py index 7e1830ccac5b..25da9cce4529 100644 --- a/tests/email_tests.py +++ b/tests/email_tests.py @@ -25,7 +25,7 @@ from unittest import mock from superset import app -from superset.utils import core as utils +from superset.utils.email import send_email_smtp, send_MIME_email from .utils import read_fixture send_email_test = mock.Mock() @@ -35,12 +35,12 @@ class EmailSmtpTest(unittest.TestCase): def setUp(self): app.config['smtp_ssl'] = False - @mock.patch('superset.utils.core.send_MIME_email') + @mock.patch('superset.utils.email.send_MIME_email') def test_send_smtp(self, mock_send_mime): attachment = tempfile.NamedTemporaryFile() attachment.write(b'attachment') attachment.seek(0) - utils.send_email_smtp( + send_email_smtp( 'to', 'subject', 'content', app.config, files=[attachment.name]) assert mock_send_mime.called call_args = mock_send_mime.call_args[0] @@ -54,9 +54,9 @@ def test_send_smtp(self, mock_send_mime): mimeapp = MIMEApplication('attachment') assert msg.get_payload()[-1].get_payload() == mimeapp.get_payload() - @mock.patch('superset.utils.core.send_MIME_email') + @mock.patch('superset.utils.email.send_MIME_email') def test_send_smtp_data(self, mock_send_mime): - utils.send_email_smtp( + send_email_smtp( 'to', 'subject', 'content', app.config, data={'1.txt': b'data'}) assert mock_send_mime.called call_args = mock_send_mime.call_args[0] @@ -70,10 +70,10 @@ def test_send_smtp_data(self, mock_send_mime): mimeapp = MIMEApplication('data') assert msg.get_payload()[-1].get_payload() == mimeapp.get_payload() - @mock.patch('superset.utils.core.send_MIME_email') + @mock.patch('superset.utils.email.send_MIME_email') def test_send_smtp_inline_images(self, mock_send_mime): image = read_fixture('sample.png') - utils.send_email_smtp( + send_email_smtp( 'to', 'subject', 'content', app.config, images=dict(blah=image)) assert mock_send_mime.called call_args = mock_send_mime.call_args[0] @@ -87,12 +87,12 @@ def test_send_smtp_inline_images(self, mock_send_mime): mimeapp = MIMEImage(image) assert msg.get_payload()[-1].get_payload() == mimeapp.get_payload() - @mock.patch('superset.utils.core.send_MIME_email') + @mock.patch('superset.utils.email.send_MIME_email') def test_send_bcc_smtp(self, mock_send_mime): attachment = tempfile.NamedTemporaryFile() attachment.write(b'attachment') attachment.seek(0) - utils.send_email_smtp( + send_email_smtp( 'to', 'subject', 'content', app.config, files=[attachment.name], cc='cc', bcc='bcc') assert mock_send_mime.called @@ -112,7 +112,7 @@ def test_send_mime(self, mock_smtp, mock_smtp_ssl): mock_smtp.return_value = mock.Mock() mock_smtp_ssl.return_value = mock.Mock() msg = MIMEMultipart() - utils.send_MIME_email('from', 'to', msg, app.config, dryrun=False) + send_MIME_email('from', 'to', msg, app.config, dryrun=False) mock_smtp.assert_called_with( app.config.get('SMTP_HOST'), app.config.get('SMTP_PORT'), @@ -132,7 +132,7 @@ def test_send_mime_ssl(self, mock_smtp, mock_smtp_ssl): app.config['SMTP_SSL'] = True mock_smtp.return_value = mock.Mock() mock_smtp_ssl.return_value = mock.Mock() - utils.send_MIME_email( + send_MIME_email( 'from', 'to', MIMEMultipart(), app.config, dryrun=False) assert not mock_smtp.called mock_smtp_ssl.assert_called_with( @@ -147,7 +147,7 @@ def test_send_mime_noauth(self, mock_smtp, mock_smtp_ssl): app.config['SMTP_PASSWORD'] = None mock_smtp.return_value = mock.Mock() mock_smtp_ssl.return_value = mock.Mock() - utils.send_MIME_email( + send_MIME_email( 'from', 'to', MIMEMultipart(), app.config, dryrun=False) assert not mock_smtp_ssl.called mock_smtp.assert_called_with( @@ -159,7 +159,7 @@ def test_send_mime_noauth(self, mock_smtp, mock_smtp_ssl): @mock.patch('smtplib.SMTP_SSL') @mock.patch('smtplib.SMTP') def test_send_mime_dryrun(self, mock_smtp, mock_smtp_ssl): - utils.send_MIME_email( + send_MIME_email( 'from', 'to', MIMEMultipart(), app.config, dryrun=True) assert not mock_smtp.called assert not mock_smtp_ssl.called diff --git a/tests/schedules_test.py b/tests/schedules_test.py index b0d311ab89e9..96748ece1782 100644 --- a/tests/schedules_test.py +++ b/tests/schedules_test.py @@ -21,7 +21,7 @@ from flask_babel import gettext as __ from selenium.common.exceptions import WebDriverException -from superset import app, db +from superset import app, db, security_manager from superset.models.core import Dashboard, Slice from superset.models.schedules import ( DashboardEmailSchedule, @@ -30,11 +30,11 @@ SliceEmailSchedule, ) from superset.tasks.schedules import ( - create_webdriver, deliver_dashboard, deliver_slice, next_schedules, ) +from superset.utils.selenium import create_webdriver from .utils import read_fixture @@ -149,19 +149,20 @@ def test_complex_schedule(self): self.assertEqual(schedules[59], datetime.strptime('2018-03-30 17:40:00', fmt)) self.assertEqual(schedules[60], datetime.strptime('2018-05-04 17:10:00', fmt)) - @patch('superset.tasks.schedules.firefox.webdriver.WebDriver') + @patch('superset.utils.selenium.firefox.webdriver.WebDriver') def test_create_driver(self, mock_driver_class): mock_driver = Mock() mock_driver_class.return_value = mock_driver mock_driver.find_elements_by_id.side_effect = [True, False] - create_webdriver() - create_webdriver() + alpha_user = security_manager.find_user(username='alpha') + with app.app_context(): + create_webdriver(alpha_user, webdriver='firefox') mock_driver.add_cookie.assert_called_once() - @patch('superset.tasks.schedules.firefox.webdriver.WebDriver') + @patch('superset.utils.selenium.firefox.webdriver.WebDriver') @patch('superset.tasks.schedules.send_email_smtp') - @patch('superset.tasks.schedules.time') + @patch('superset.utils.selenium.time') def test_deliver_dashboard_inline(self, mtime, send_email_smtp, driver_class): element = Mock() driver = Mock() @@ -182,9 +183,9 @@ def test_deliver_dashboard_inline(self, mtime, send_email_smtp, driver_class): driver.screenshot.assert_not_called() send_email_smtp.assert_called_once() - @patch('superset.tasks.schedules.firefox.webdriver.WebDriver') + @patch('superset.utils.selenium.firefox.webdriver.WebDriver') @patch('superset.tasks.schedules.send_email_smtp') - @patch('superset.tasks.schedules.time') + @patch('superset.utils.selenium.time') def test_deliver_dashboard_as_attachment(self, mtime, send_email_smtp, driver_class): element = Mock() driver = Mock() @@ -213,9 +214,9 @@ def test_deliver_dashboard_as_attachment(self, mtime, send_email_smtp, driver_cl element.screenshot_as_png, ) - @patch('superset.tasks.schedules.firefox.webdriver.WebDriver') + @patch('superset.utils.selenium.firefox.webdriver.WebDriver') @patch('superset.tasks.schedules.send_email_smtp') - @patch('superset.tasks.schedules.time') + @patch('superset.utils.selenium.time') def test_dashboard_chrome_like(self, mtime, send_email_smtp, driver_class): # Test functionality for chrome driver which does not support # element snapshots @@ -236,6 +237,7 @@ def test_dashboard_chrome_like(self, mtime, send_email_smtp, driver_class): id=self.dashboard_schedule).all()[0] deliver_dashboard(schedule) + mtime.sleep.assert_called_once() driver.screenshot.assert_called_once() send_email_smtp.assert_called_once() @@ -246,9 +248,9 @@ def test_dashboard_chrome_like(self, mtime, send_email_smtp, driver_class): driver.screenshot.return_value, ) - @patch('superset.tasks.schedules.firefox.webdriver.WebDriver') + @patch('superset.utils.selenium.firefox.webdriver.WebDriver') @patch('superset.tasks.schedules.send_email_smtp') - @patch('superset.tasks.schedules.time') + @patch('superset.utils.selenium.time') def test_deliver_email_options(self, mtime, send_email_smtp, driver_class): element = Mock() driver = Mock() @@ -277,9 +279,9 @@ def test_deliver_email_options(self, mtime, send_email_smtp, driver_class): self.assertEquals(send_email_smtp.call_count, 2) self.assertEquals(send_email_smtp.call_args[1]['bcc'], self.BCC) - @patch('superset.tasks.schedules.firefox.webdriver.WebDriver') + @patch('superset.utils.selenium.firefox.webdriver.WebDriver') @patch('superset.tasks.schedules.send_email_smtp') - @patch('superset.tasks.schedules.time') + @patch('superset.utils.selenium.time') def test_deliver_slice_inline_image(self, mtime, send_email_smtp, driver_class): element = Mock() driver = Mock() @@ -308,9 +310,9 @@ def test_deliver_slice_inline_image(self, mtime, send_email_smtp, driver_class): element.screenshot_as_png, ) - @patch('superset.tasks.schedules.firefox.webdriver.WebDriver') + @patch('superset.utils.selenium.firefox.webdriver.WebDriver') @patch('superset.tasks.schedules.send_email_smtp') - @patch('superset.tasks.schedules.time') + @patch('superset.utils.selenium.time') def test_deliver_slice_attachment(self, mtime, send_email_smtp, driver_class): element = Mock() driver = Mock() diff --git a/tests/sqllab_tests.py b/tests/sqllab_tests.py index dc86866a9960..5c328d4d6725 100644 --- a/tests/sqllab_tests.py +++ b/tests/sqllab_tests.py @@ -25,7 +25,8 @@ from superset.dataframe import SupersetDataFrame from superset.db_engine_specs import BaseEngineSpec from superset.models.sql_lab import Query -from superset.utils.core import datetime_to_epoch, get_main_database +from superset.utils.core import get_main_database +from superset.utils.dates import datetime_to_epoch from .base_tests import SupersetTestCase diff --git a/tests/utils_tests.py b/tests/utils_tests.py index a39631b83227..ed95e0244351 100644 --- a/tests/utils_tests.py +++ b/tests/utils_tests.py @@ -24,13 +24,9 @@ from superset.exceptions import SupersetException from superset.utils.core import ( - base_json_conv, convert_legacy_filters_into_adhoc, datetime_f, get_since_until, - json_int_dttm_ser, - json_iso_dttm_ser, - JSONEncodedDict, memoized, merge_extra_filters, merge_request_params, @@ -40,6 +36,7 @@ zlib_compress, zlib_decompress_to_string, ) +from superset.utils.json import base_json_conv, json_int_dttm_ser, json_iso_dttm_ser def mock_parse_human_datetime(s): @@ -509,15 +506,6 @@ def test_datetime_f(self): datetime_f(datetime(a, b, c)), '00:00:00', ) - def test_json_encoded_obj(self): - obj = {'a': 5, 'b': ['a', 'g', 5]} - val = '{"a": 5, "b": ["a", "g", 5]}' - jsonObj = JSONEncodedDict() - resp = jsonObj.process_bind_param(obj, 'dialect') - self.assertIn('"a": 5', resp) - self.assertIn('"b": ["a", "g", 5]', resp) - self.assertEquals(jsonObj.process_result_value(val, 'dialect'), obj) - def test_validate_json(self): invalid = '{"a": 5, "b": [1, 5, ["g", "h]]}' with self.assertRaises(SupersetException): From 686f69b5b28b0053ff2571c71604e4b253d030a4 Mon Sep 17 00:00:00 2001 From: Maxime Beauchemin Date: Wed, 24 Apr 2019 21:30:22 -0700 Subject: [PATCH 2/2] tweaks --- superset/__init__.py | 2 ++ superset/assets/src/components/Card.jsx | 2 +- .../src/dashboard/components/MissingChart.jsx | 6 +++--- superset/cli.py | 13 ++++++++++--- superset/models/core.py | 3 +-- superset/sql_lab.py | 2 +- superset/tasks/cache.py | 3 ++- superset/tasks/schedules.py | 6 +++--- superset/utils/selenium.py | 6 ++++-- superset/views/base.py | 5 ++--- superset/views/core.py | 7 +++---- superset/views/schedules.py | 3 +-- superset/viz.py | 7 +++---- 13 files changed, 36 insertions(+), 29 deletions(-) diff --git a/superset/__init__.py b/superset/__init__.py index e45cf91b1fe6..6f37478b45e4 100644 --- a/superset/__init__.py +++ b/superset/__init__.py @@ -115,7 +115,9 @@ def get_manifest(): if conf.get('SILENCE_FAB'): logging.getLogger('flask_appbuilder').setLevel(logging.ERROR) +logging.getLogger('urllib3').setLevel(logging.ERROR) logging.getLogger('selenium').setLevel(logging.ERROR) +logging.getLogger('PIL').setLevel(logging.ERROR) if app.debug: app.logger.setLevel(logging.DEBUG) # pylint: disable=no-member diff --git a/superset/assets/src/components/Card.jsx b/superset/assets/src/components/Card.jsx index 4b4a986b7938..ea7d7e65b632 100644 --- a/superset/assets/src/components/Card.jsx +++ b/superset/assets/src/components/Card.jsx @@ -34,7 +34,7 @@ const propTypes = { }; const defaultProps = { onTitleClick: () => {}, - imageAspectRatio: 3/2, + imageAspectRatio: 4/3, }; export default class Card extends React.PureComponent { diff --git a/superset/assets/src/dashboard/components/MissingChart.jsx b/superset/assets/src/dashboard/components/MissingChart.jsx index 34d3d2da8c16..65e2be37bc69 100644 --- a/superset/assets/src/dashboard/components/MissingChart.jsx +++ b/superset/assets/src/dashboard/components/MissingChart.jsx @@ -29,9 +29,9 @@ const propTypes = { export default function MissingChart({ height }) { return (
-
- -
+
+ +
{t( 'There is no chart definition associated with this component, could it have been deleted?', diff --git a/superset/cli.py b/superset/cli.py index 30e0aa4015ab..0e58dae969c3 100755 --- a/superset/cli.py +++ b/superset/cli.py @@ -396,10 +396,14 @@ def load_test_users(): @click.option( '--force', '-f', is_flag=True, default=False, help='Force refresh, even if previously cached') -def compute_thumbnails(asynchronous, dashboards_only, charts_only, force): +@click.option('--id', '-i', multiple=True) +def compute_thumbnails(asynchronous, dashboards_only, charts_only, force, id): """Compute thumbnails""" if not charts_only: - dashboards = db.session.query(models.Dashboard).all() + query = db.session.query(models.Dashboard) + if id: + query = query.filter(models.Dashboard.id.in_(id)) + dashboards = query.all() count = len(dashboards) for i, dash in enumerate(dashboards): if asynchronous: @@ -413,7 +417,10 @@ def compute_thumbnails(asynchronous, dashboards_only, charts_only, force): func(dash.id, force=force) if not dashboards_only: - slices = db.session.query(models.Slice).all() + query = db.session.query(models.Slice) + if id: + query = query.filter(models.Slice.id.in_(id)) + slices = query.all() count = len(slices) for i, slc in enumerate(slices): if asynchronous: diff --git a/superset/models/core.py b/superset/models/core.py index aa17c4dff30f..b527f97248de 100644 --- a/superset/models/core.py +++ b/superset/models/core.py @@ -56,7 +56,6 @@ cache as cache_util, core as utils, ) -from superset.utils.json import json_dumps_w_dates from superset.viz import viz_types from urllib import parse # noqa @@ -946,7 +945,7 @@ def _log_query(sql): for k, v in df.dtypes.items(): if v.type == numpy.object_ and needs_conversion(df[k]): - df[k] = df[k].apply(json_dumps_w_dates) + df[k] = df[k].apply(utils.json_dumps_w_dates) return df def compile_sqla_query(self, qry, schema=None): diff --git a/superset/sql_lab.py b/superset/sql_lab.py index 7e89a5cd08a8..86e171ba44d7 100644 --- a/superset/sql_lab.py +++ b/superset/sql_lab.py @@ -34,13 +34,13 @@ from superset.sql_parse import ParsedQuery from superset.tasks.celery_app import app as celery_app from superset.utils.core import ( + json_iso_dttm_ser, QueryStatus, sources, zlib_compress, ) from superset.utils.dates import now_as_float from superset.utils.decorators import stats_timing -from superset.utils.json import json_iso_dttm_ser config = app.config stats_logger = config.get('STATS_LOGGER') diff --git a/superset/tasks/cache.py b/superset/tasks/cache.py index dd58dfa909ed..acd3caddbd72 100644 --- a/superset/tasks/cache.py +++ b/superset/tasks/cache.py @@ -134,6 +134,7 @@ class DummyStrategy(Strategy): def get_urls(self): session = db.create_scoped_session() + from superset.models.core import Slice charts = session.query(Slice).all() return [get_url({'form_data': get_form_data(chart.id)}) for chart in charts] @@ -182,7 +183,6 @@ def get_urls(self): .all() ) dash_ids = [record.dashboard_id for record in records] - from superset.models.core import Dashboard dashboards = ( session .query(Dashboard) @@ -241,6 +241,7 @@ def get_urls(self): )) .all() ) + from superset.models.core import Dashboard dash_ids = [tagged_object.object_id for tagged_object in tagged_objects] tagged_dashboards = ( session diff --git a/superset/tasks/schedules.py b/superset/tasks/schedules.py index 029024acf063..b92d6083153b 100644 --- a/superset/tasks/schedules.py +++ b/superset/tasks/schedules.py @@ -39,7 +39,7 @@ SliceEmailReportFormat, ) from superset.tasks.celery_app import app as celery_app -from superset.utils.email import get_email_address_list, send_email_smtp +from superset.utils import core as utils from superset.utils.selenium import ( DashboardScreenshot, get_auth_cookies, @@ -59,7 +59,7 @@ def _get_recipients(schedule): to = schedule.recipients yield (to, bcc) else: - for to in get_email_address_list(schedule.recipients): + for to in utils.get_email_address_list(schedule.recipients): yield (to, bcc) @@ -67,7 +67,7 @@ def _deliver_email(recipients, subject, email): config = app.config dryrun = config.get('SCHEDULED_EMAIL_DEBUG_MODE') for (to, bcc) in recipients: - send_email_smtp( + utils.send_email_smtp( to, subject, email.body, config, data=email.data, diff --git a/superset/utils/selenium.py b/superset/utils/selenium.py index c6e5013f2152..0e70ed241590 100644 --- a/superset/utils/selenium.py +++ b/superset/utils/selenium.py @@ -24,7 +24,7 @@ from flask_login import login_user from PIL import Image from retry.api import retry_call -from selenium.common.exceptions import WebDriverException +from selenium.common.exceptions import TimeoutException, WebDriverException from selenium.webdriver import chrome, firefox from selenium.webdriver.common.by import By from selenium.webdriver.support import expected_conditions as EC @@ -139,7 +139,7 @@ def resize_image(cls, img_bytes, output='png', size=None, crop=True): size = size or cls.thumb_size img = Image.open(BytesIO(img_bytes)) logging.debug(f'Selenium image size: {img.size}') - if crop and img.size[1] > cls.window_size[1]: + if crop and img.size[1] != cls.window_size[1]: desired_ratio = float(cls.window_size[1]) / cls.window_size[0] desired_width = int(img.size[0] * desired_ratio) logging.debug(f'Cropping to: {img.size[0]}*{desired_width}') @@ -285,6 +285,8 @@ def get_png_from_url( ) logging.info('Taking a PNG screenshot') img = element.screenshot_as_png + except TimeoutException: + logging.error('Selenium timed out') except WebDriverException as e: logging.exception(e) # Some webdrivers do not support screenshots for elements. diff --git a/superset/views/base.py b/superset/views/base.py index 28df8791792b..2d14622ea893 100644 --- a/superset/views/base.py +++ b/superset/views/base.py @@ -39,7 +39,6 @@ from superset.exceptions import SupersetException, SupersetSecurityException from superset.translations.utils import get_language_pack from superset.utils import core as utils -from superset.utils.json import json_int_dttm_ser, json_iso_dttm_ser FRONTEND_CONF_KEYS = ( 'SUPERSET_WEBSERVER_TIMEOUT', @@ -72,7 +71,7 @@ def json_error_response(msg=None, status=500, stacktrace=None, payload=None, lin payload['link'] = link return Response( - json.dumps(payload, default=json_iso_dttm_ser, ignore_nan=True), + json.dumps(payload, default=utils.json_iso_dttm_ser, ignore_nan=True), status=status, mimetype='application/json') @@ -150,7 +149,7 @@ class BaseSupersetView(BaseView): def json_response(self, obj, status=200): return Response( - json.dumps(obj, default=json_int_dttm_ser, ignore_nan=True), + json.dumps(obj, default=utils.json_int_dttm_ser, ignore_nan=True), status=status, mimetype='application/json') diff --git a/superset/views/core.py b/superset/views/core.py index fc58a86457e8..8015c748b589 100755 --- a/superset/views/core.py +++ b/superset/views/core.py @@ -60,7 +60,6 @@ from superset.utils import dashboard_import_export from superset.utils.dates import EPOCH, now_as_float from superset.utils.decorators import etag_cache - from .base import ( api, BaseSupersetView, check_ownership, @@ -715,7 +714,7 @@ class DashboardModelViewAsync(DashboardModelView): # noqa list_columns = [ 'id', 'dashboard_link', 'creator', 'modified', 'dashboard_title', 'changed_on', 'url', 'changed_by_name', 'created_by_name', - 'thumbnail_url', 'changed_on_humanized' + 'thumbnail_url', 'changed_on_humanized', ] label_columns = { 'dashboard_link': _('Dashboard'), @@ -1982,7 +1981,7 @@ def created_dashboards(self, user_id): 'dttm': o.changed_on, } for o in qry.all()] return json_success( - json.dumps(payload, default=json_int_dttm_ser)) + json.dumps(payload, default=utils.json_int_dttm_ser)) @api @has_access_api @@ -2021,7 +2020,7 @@ def user_slices(self, user_id=None): 'viz_type': o.Slice.viz_type, } for o in qry.all()] return json_success( - json.dumps(payload, default=json_int_dttm_ser)) + json.dumps(payload, default=utils.json_int_dttm_ser)) @api @has_access_api diff --git a/superset/views/schedules.py b/superset/views/schedules.py index 95db908f79d9..a7afda6e56bc 100644 --- a/superset/views/schedules.py +++ b/superset/views/schedules.py @@ -37,8 +37,7 @@ SliceEmailSchedule, ) from superset.tasks.schedules import schedule_email_report -from superset.utils.email import get_email_address_list -from superset.utils.json import json_iso_dttm_ser +from superset.utils.core import get_email_address_list, json_iso_dttm_ser from superset.views.core import json_success from .base import DeleteMixin, SupersetModelView diff --git a/superset/viz.py b/superset/viz.py index 61561f5adcba..a6864d32312c 100644 --- a/superset/viz.py +++ b/superset/viz.py @@ -55,7 +55,6 @@ merge_extra_filters, to_adhoc, ) -from superset.utils.json import json_int_dttm_ser, json_iso_dttm_ser config = app.config @@ -334,7 +333,7 @@ def cache_timeout(self): def get_json(self): return json.dumps( self.get_payload(), - default=json_int_dttm_ser, ignore_nan=True) + default=utils.json_int_dttm_ser, ignore_nan=True) def cache_key(self, query_obj, **extra): """ @@ -461,7 +460,7 @@ def get_df_payload(self, query_obj=None, **kwargs): def json_dumps(self, obj, sort_keys=False): return json.dumps( obj, - default=json_int_dttm_ser, + default=utils.json_int_dttm_ser, ignore_nan=True, sort_keys=sort_keys, ) @@ -596,7 +595,7 @@ def get_data(self, df): def json_dumps(self, obj, sort_keys=False): return json.dumps( obj, - default=json_iso_dttm_ser, + default=utils.json_iso_dttm_ser, sort_keys=sort_keys, ignore_nan=True)